cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Added a Docker Machine dynamic inventory plugin (#54946)

Browse source 
* Added my Docker Machine dynamic inventory plugin (from https://github.com/ximon18/ansible-docker-machine-inventory-plugin) to begin the process of proposing it for inclusion in Ansible core. There are no integration tests yet. The docker_swarm inventory plugin has such tests but has some concerning note in its 'aliases' file about disabling docker due to test instability and also I wouldn't know at his point how to get Docker Machine installed on the integration test platform.
This commit is contained in:
ximon18 2019-08-08 00:14:14 +02:00 committed by Brian Coca
parent 3a3727d200
commit 2cca9176a7
13 changed files with 448 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

256
lib/ansible/plugins/inventory/docker_machine.py Normal file
View file

@ -0,0 +1,256 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Ximon Eighteen <ximon.eighteen@gmail.com>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
DOCUMENTATION = '''
name: docker_machine
plugin_type: inventory
author: Ximon Eighteen (@ximon18)
short_description: Docker Machine inventory source
requirements:
- L(Docker Machine,https://docs.docker.com/machine/)
extends_documentation_fragment:
- constructed
description:
- Get inventory hosts from Docker Machine.
- Uses a YAML configuration file that ends with docker_machine.(yml|yaml).
- The plugin sets standard host variables C(ansible_host), C(ansible_port), C(ansible_user) and C(ansible_ssh_private_key).
- The plugin stores the Docker Machine 'env' output variables in I(dm_) prefixed host variables.
options:
plugin:
description: token that ensures this is a source file for the C(docker_machine) plugin.
required: yes
choices: ['docker_machine']
daemon_env:
description:
- Whether docker daemon connection environment variables should be fetched, and how to behave if they cannot be fetched.
- With C(require) and C(require-silently), fetch them and skip any host for which they cannot be fetched.
A warning will be issued for any skipped host if the choice is C(require).
- With C(optional) and C(optional-silently), fetch them and not skip hosts for which they cannot be fetched.
A warning will be issued for hosts where they cannot be fetched if the choice is C(optional).
- With C(skip), do not attempt to fetch the docker daemon connection environment variables.
- If fetched successfully, the variables will be prefixed with I(dm_) and stored as host variables.
type: str
choices:
- require
- require-silently
- optional
- optional-silently
- skip
default: require
running_required:
description: when true, hosts which Docker Machine indicates are in a state other than C(running) will be skipped.
type: bool
default: yes
verbose_output:
description: when true, include all available nodes metadata (e.g. Image, Region, Size) as a JSON object named C(docker_machine_node_attributes).
type: bool
default: yes
'''
EXAMPLES = '''
# Minimal example
plugin: docker_machine
# Example using constructed features to create a group per Docker Machine driver
# (https://docs.docker.com/machine/drivers/), e.g.:
# $ docker-machine create --driver digitalocean ... mymachine
# $ ansible-inventory -i ./path/to/docker-machine.yml --host=mymachine
# {
# ...
# "digitalocean": {
# "hosts": [
# "mymachine"
# ]
# ...
# }
strict: no
keyed_groups:
- separator: ''
key: docker_machine_node_attributes.DriverName
# Example grouping hosts by Digital Machine tag
strict: no
keyed_groups:
- prefix: tag
key: 'dm_tags'
# Example using compose to override the default SSH behaviour of asking the user to accept the remote host key
compose:
ansible_ssh_common_args: '"-o StrictHostKeyChecking=accept-new"'
'''
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_native
from ansible.module_utils._text import to_text
from ansible.module_utils.common.process import get_bin_path
from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable
from ansible.utils.display import Display
import json
import re
import subprocess
display = Display()
class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
''' Host inventory parser for ansible using Docker machine as source. '''
NAME = 'docker_machine'
DOCKER_MACHINE_PATH = None
def _run_command(self, args):
if not self.DOCKER_MACHINE_PATH:
try:
self.DOCKER_MACHINE_PATH = get_bin_path('docker-machine', required=True)
except ValueError as e:
raise AnsibleError('Unable to locate the docker-machine binary.', orig_exc=e)
command = [self.DOCKER_MACHINE_PATH]
command.extend(args)
display.debug('Executing command {0}'.format(command))
try:
result = subprocess.check_output(command)
except subprocess.CalledProcessError as e:
display.warning('Exception {0} caught while executing command {1}, this was the original exception: {2}'.format(type(e).__name__, command, e))
raise e
return to_text(result).strip()
def _get_docker_daemon_variables(self, machine_name):
'''
Capture settings from Docker Machine that would be needed to connect to the remote Docker daemon installed on
the Docker Machine remote host. Note: passing '--shell=sh' is a workaround for 'Error: Unknown shell'.
'''
try:
env_lines = self._run_command(['env', '--shell=sh', machine_name]).splitlines()
except subprocess.CalledProcessError:
# This can happen when the machine is created but provisioning is incomplete
return []
# example output of docker-machine env --shell=sh:
# export DOCKER_TLS_VERIFY="1"
# export DOCKER_HOST="tcp://134.209.204.160:2376"
# export DOCKER_CERT_PATH="/root/.docker/machine/machines/routinator"
# export DOCKER_MACHINE_NAME="routinator"
# # Run this command to configure your shell:
# # eval $(docker-machine env --shell=bash routinator)
# capture any of the DOCKER_xxx variables that were output and create Ansible host vars
# with the same name and value but with a dm_ name prefix.
vars = []
for line in env_lines:
match = re.search('(DOCKER_[^=]+)="([^"]+)"', line)
if match:
env_var_name = match.group(1)
env_var_value = match.group(2)
vars.append((env_var_name, env_var_value))
return vars
def _get_machine_names(self):
# Filter out machines that are not in the Running state as we probably can't do anything useful actions
# with them.
ls_command = ['ls', '-q']
if self.get_option('running_required'):
ls_command.extend(['--filter', 'state=Running'])
try:
ls_lines = self._run_command(ls_command)
except subprocess.CalledProcessError:
return []
return ls_lines.splitlines()
def _inspect_docker_machine_host(self, node):
try:
inspect_lines = self._run_command(['inspect', self.node])
except subprocess.CalledProcessError:
return None
return json.loads(inspect_lines)
def _should_skip_host(self, machine_name, env_var_tuples, daemon_env):
if not env_var_tuples:
warning_prefix = 'Unable to fetch Docker daemon env vars from Docker Machine for host {0}'.format(machine_name)
if daemon_env in ('require', 'require-silently'):
if daemon_env == 'require':
display.warning('{0}: host will be skipped'.format(warning_prefix))
return True
else: # 'optional', 'optional-silently'
if daemon_env == 'optional':
display.warning('{0}: host will lack dm_DOCKER_xxx variables'.format(warning_prefix))
return False
def _populate(self):
daemon_env = self.get_option('daemon_env')
try:
for self.node in self._get_machine_names():
self.node_attrs = self._inspect_docker_machine_host(self.node)
if not self.node_attrs:
continue
machine_name = self.node_attrs['Driver']['MachineName']
# query `docker-machine env` to obtain remote Docker daemon connection settings in the form of commands
# that could be used to set environment variables to influence a local Docker client:
if daemon_env == 'skip':
env_var_tuples = []
else:
env_var_tuples = self._get_docker_daemon_variables(machine_name)
if self._should_skip_host(machine_name, env_var_tuples, daemon_env):
continue
# add an entry in the inventory for this host
self.inventory.add_host(machine_name)
# set standard Ansible remote host connection settings to details captured from `docker-machine`
# see: https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html
self.inventory.set_variable(machine_name, 'ansible_host', self.node_attrs['Driver']['IPAddress'])
self.inventory.set_variable(machine_name, 'ansible_port', self.node_attrs['Driver']['SSHPort'])
self.inventory.set_variable(machine_name, 'ansible_user', self.node_attrs['Driver']['SSHUser'])
self.inventory.set_variable(machine_name, 'ansible_ssh_private_key_file', self.node_attrs['Driver']['SSHKeyPath'])
# set variables based on Docker Machine tags
tags = self.node_attrs['Driver'].get('Tags') or ''
self.inventory.set_variable(machine_name, 'dm_tags', tags)
# set variables based on Docker Machine env variables
for kv in env_var_tuples:
self.inventory.set_variable(machine_name, 'dm_{0}'.format(kv[0]), kv[1])
if self.get_option('verbose_output'):
self.inventory.set_variable(machine_name, 'docker_machine_node_attributes', self.node_attrs)
# Use constructed if applicable
strict = self.get_option('strict')
# Composed variables
self._set_composite_vars(self.get_option('compose'), self.node_attrs, machine_name, strict=strict)
# Complex groups based on jinja2 conditionals, hosts that meet the conditional are added to group
self._add_host_to_composed_groups(self.get_option('groups'), self.node_attrs, machine_name, strict=strict)
# Create groups based on variable values and add the corresponding hosts to it
self._add_host_to_keyed_groups(self.get_option('keyed_groups'), self.node_attrs, machine_name, strict=strict)
except Exception as e:
raise AnsibleError('Unable to fetch hosts from Docker Machine, this was the original exception: %s' %
to_native(e), orig_exc=e)
def verify_file(self, path):
"""Return the possibility of a file being consumable by this plugin."""
return (
super(InventoryModule, self).verify_file(path) and
path.endswith((self.NAME + '.yaml', self.NAME + '.yml')))
def parse(self, inventory, loader, path, cache=True):
super(InventoryModule, self).parse(inventory, loader, path, cache)
self._read_config_data(path)
self._populate()

8
test/integration/targets/inventory_docker_machine/aliases Normal file
View file

@ -0,0 +1,8 @@
shippable/posix/group2
skip/osx
skip/freebsd
destructive
skip/docker # We need SSH access to the VM and need to be able to let
# docker-machine install docker in the VM. This won't work
# with tests running in docker containers.
needs/root

20
test/integration/targets/inventory_docker_machine/docker-machine Normal file
View file

@ -0,0 +1,20 @@
#!/usr/bin/env bash
# Mock Docker Machine wrapper for testing purposes
[ "$MOCK_ERROR_IN" == "$1" ] && echo >&2 "Mock Docker Machine error" && exit 1
case $1 in
env)
cat <<'EOF'
export DOCKER_TLS_VERIFY="1"
export DOCKER_HOST="tcp://134.209.204.160:2376"
export DOCKER_CERT_PATH="/root/.docker/machine/machines/routinator"
export DOCKER_MACHINE_NAME="routinator"
# Run this command to configure your shell:
# eval $(docker-machine env --shell=bash routinator)
EOF
;;
*)
/usr/bin/docker-machine $*
;;
esac

1
test/integration/targets/inventory_docker_machine/inventory_1.docker_machine.yml Normal file
View file

@ -0,0 +1 @@
plugin: docker_machine

2
test/integration/targets/inventory_docker_machine/inventory_2.docker_machine.yml Normal file
View file

@ -0,0 +1,2 @@
plugin: docker_machine
daemon_env: require

2
test/integration/targets/inventory_docker_machine/inventory_3.docker_machine.yml Normal file
View file

@ -0,0 +1,2 @@
plugin: docker_machine
daemon_env: optional

3
test/integration/targets/inventory_docker_machine/meta/main.yml Normal file
View file

@ -0,0 +1,3 @@
---
dependencies:
- setup_docker

18
test/integration/targets/inventory_docker_machine/playbooks/pre-setup.yml Normal file
View file

@ -0,0 +1,18 @@
---
- hosts: 127.0.0.1
connection: local
tasks:
- name: Setup docker
include_role:
name: setup_docker
# There seems to be no better way to install docker-machine. At least I couldn't find any packages for RHEL7/8.
- name: Download docker-machine binary
vars:
docker_machine_version: "0.16.1"
get_url:
url: "https://github.com/docker/machine/releases/download/v{{ docker_machine_version }}/docker-machine-{{ ansible_system }}-{{ ansible_userspace_architecture }}"
dest: /tmp/docker-machine
- name: Install docker-machine binary
command: install /tmp/docker-machine /usr/bin/docker-machine
become: yes

11
test/integration/targets/inventory_docker_machine/playbooks/setup.yml Normal file
View file

@ -0,0 +1,11 @@
---
- hosts: 127.0.0.1
connection: local
tasks:
- name: Request Docker Machine to use this machine as a generic VM
command: "docker-machine --debug create \
--driver generic \
--generic-ip-address=localhost \
--generic-ssh-key {{ lookup('env', 'HOME') }}/.ssh/id_rsa \
--generic-ssh-user root \
vm"

6
test/integration/targets/inventory_docker_machine/playbooks/teardown.yml Normal file
View file

@ -0,0 +1,6 @@
---
- hosts: 127.0.0.1
connection: local
tasks:
- name: Request Docker Machine to remove this machine as a generic VM
command: "docker-machine rm vm -f"

50
test/integration/targets/inventory_docker_machine/playbooks/test_inventory_1.yml Normal file
View file

@ -0,0 +1,50 @@
- hosts: 127.0.0.1
gather_facts: no
tasks:
- name: sanity check Docker Machine output
vars:
dm_ls_format: !unsafe '{{.Name}} | {{.DriverName}} | {{.State}} | {{.URL}} | {{.Error}}'
success_regex: "^vm | [^|]+ | Running | tcp://.+ |$"
command: docker-machine ls --format '{{ dm_ls_format }}'
register: result
failed_when: result.rc != 0 or result.stdout is not match(success_regex)
- name: verify Docker Machine ip
command: docker-machine ip vm
register: result
failed_when: result.rc != 0 or result.stdout != hostvars['vm'].ansible_host
- name: verify Docker Machine env
command: docker-machine env --shell=sh vm
register: result
- debug: var=result.stdout
- assert:
that:
- "'DOCKER_TLS_VERIFY=\"{{ hostvars['vm'].dm_DOCKER_TLS_VERIFY }}\"' in result.stdout"
- "'DOCKER_HOST=\"{{ hostvars['vm'].dm_DOCKER_HOST }}\"' in result.stdout"
- "'DOCKER_CERT_PATH=\"{{ hostvars['vm'].dm_DOCKER_CERT_PATH }}\"' in result.stdout"
- "'DOCKER_MACHINE_NAME=\"{{ hostvars['vm'].dm_DOCKER_MACHINE_NAME }}\"' in result.stdout"
- hosts: vm
gather_facts: no
tasks:
- name: do something to verify that accept-new ssh setting was applied by the docker-machine inventory plugin
raw: uname -a
register: result
- debug: var=result.stdout
- hosts: 127.0.0.1
gather_facts: no
environment:
DOCKER_CERT_PATH: "{{ hostvars['vm'].dm_DOCKER_CERT_PATH }}"
DOCKER_HOST: "{{ hostvars['vm'].dm_DOCKER_HOST }}"
DOCKER_MACHINE_NAME: "{{ hostvars['vm'].dm_DOCKER_MACHINE_NAME }}"
DOCKER_TLS_VERIFY: "{{ hostvars['vm'].dm_DOCKER_TLS_VERIFY }}"
tasks:
- name: run a Docker container on the target Docker Machine host to verify that Docker daemon connection settings from the docker-machine inventory plugin work as expected
docker_container:
name: test
image: hello-world:latest

68
test/integration/targets/inventory_docker_machine/runme.sh Executable file
View file

@ -0,0 +1,68 @@
#!/usr/bin/env bash
SCRIPT_DIR=$(dirname "$0")
echo "Who am I: $(whoami)"
echo "Home: ${HOME}"
echo "PWD: $(pwd)"
echo "Script dir: ${SCRIPT_DIR}"
# restrict Ansible just to our inventory plugin, to prevent inventory data being matched by the test but being provided
# by some other dynamic inventory provider
export ANSIBLE_INVENTORY_ENABLED=docker_machine
[[ -n "$DEBUG" || -n "$ANSIBLE_DEBUG" ]] && set -x
set -euo pipefail
SAVED_PATH="$PATH"
cleanup() {
PATH="${SAVED_PATH}"
echo "Cleanup"
ansible-playbook -i teardown.docker_machine.yml playbooks/teardown.yml
echo "Done"
}
trap cleanup INT TERM EXIT
echo "Pre-setup (install docker, docker-machine)"
ANSIBLE_ROLES_PATH=.. ansible-playbook playbooks/pre-setup.yml
echo "Print docker-machine version"
docker-machine --version
echo "Check preconditions"
# Host should NOT be known to Ansible before the test starts
ansible-inventory -i inventory_1.docker_machine.yml --host vm >/dev/null && exit 1
echo "Test that the docker_machine inventory plugin is being loaded"
ANSIBLE_DEBUG=yes ansible-inventory -i inventory_1.docker_machine.yml --list | grep -F "Loading InventoryModule 'docker_machine'"
echo "Setup"
ansible-playbook playbooks/setup.yml
echo "Test docker_machine inventory 1"
ansible-playbook -i inventory_1.docker_machine.yml playbooks/test_inventory_1.yml
echo "Activate Docker Machine mock"
PATH=${SCRIPT_DIR}:$PATH
echo "Test docker_machine inventory 2: daemon_env=require daemon env success=yes"
ansible-inventory -i inventory_2.docker_machine.yml --list
echo "Test docker_machine inventory 2: daemon_env=require daemon env success=no"
export MOCK_ERROR_IN=env
ansible-inventory -i inventory_2.docker_machine.yml --list
unset MOCK_ERROR_IN
echo "Test docker_machine inventory 3: daemon_env=optional daemon env success=yes"
ansible-inventory -i inventory_3.docker_machine.yml --list
echo "Test docker_machine inventory 3: daemon_env=optional daemon env success=no"
export MOCK_ERROR_IN=env
ansible-inventory -i inventory_2.docker_machine.yml --list
unset MOCK_ERROR_IN
echo "Deactivate Docker Machine mock"
PATH="${SAVED_PATH}"

3
test/integration/targets/inventory_docker_machine/teardown.docker_machine.yml Normal file
View file

@ -0,0 +1,3 @@
plugin: docker_machine
daemon_env: skip
running_required: no