From 530e54b3e44becd9ad631a6500e4232ccadac1af Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 22:38:51 -0500 Subject: [PATCH] Fix multiprocessing pool usage and remove stackoverflow hack --- README.md | 12 ++++--- bin/ansible-inventory | 80 ----------------------------------------- lib/ansible/__init__.py | 29 ++++----------- 3 files changed, 15 insertions(+), 106 deletions(-) delete mode 100755 bin/ansible-inventory diff --git a/README.md b/README.md index 71fc7f50157..e58f1946510 100644 --- a/README.md +++ b/README.md @@ -85,10 +85,14 @@ The API is simple and returns basic datastructures. ) data = runner.run() - { - 'xyz.example.com' : [ 'any kind of datastructure is returnable' ], - 'foo.example.com' : None, # failed to connect, - ... + { + 'successful' : { + 'xyz.example.com' : [ 'any kind of datastructure is returnable' ], + 'foo.example.com' : [ '...' ] + }, + 'failed' : { + 'bar.example.com' : [ 'failure message' ] + } } Additional options to Runner include the number of forks, hostname diff --git a/bin/ansible-inventory b/bin/ansible-inventory deleted file mode 100755 index bb8cce72957..00000000000 --- a/bin/ansible-inventory +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2012 Michael DeHaan -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR -# ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from optparse import OptionParser -import json -import os -import ansible - -DEFAULT_HOST_LIST = '/etc/ansible/hosts' -DEFAULT_MODULE_PATH = '/usr/share/ansible' -DEFAULT_MODULE_NAME = 'ping' -DEFAULT_PATTERN = '*' -DEFAULT_FORKS = 3 -DEFAULT_MODULE_ARGS = '' -DEFAULT_REMOTE_USER = 'root' - -class Cli(object): - - def __init__(self): - pass - - def runner(self): - parser = OptionParser() - parser.add_option("-H", "--host-list", dest="host_list", - help="path to hosts list", default=DEFAULT_HOST_LIST) - parser.add_option("-L", "--library", dest="module_path", - help="path to module library", default=DEFAULT_MODULE_PATH) - parser.add_option("-f", "--forks", dest="forks", - help="level of parallelism", default=DEFAULT_FORKS) - parser.add_option("-n", "--name", dest="module_name", - help="module name to execute", default=DEFAULT_MODULE_NAME) - parser.add_option("-a", "--args", dest="module_args", - help="module arguments", default=DEFAULT_MODULE_ARGS) - parser.add_option("-p", "--pattern", dest="pattern", - help="hostname pattern", default=DEFAULT_PATTERN) - parser.add_option("-u", "--remote-user", dest="remote_user", - help="remote username", default=DEFAULT_REMOTE_USER) - - options, args = parser.parse_args() - - # TODO: more shell like splitting on module_args would - # be a good idea - - return ansible.Runner( - module_name=options.module_name, - module_path=options.module_path, - module_args=options.module_args.split(' '), - remote_user=options.remote_user, - host_list=options.host_list, - forks=options.forks, - pattern=options.pattern, - verbose=False, - ) - -if __name__ == '__main__': - - result = Cli().runner().run() - print json.dumps(result, sort_keys=True, indent=4) - - diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index d5c0b95c73e..420b3fe0395 100755 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -20,7 +20,7 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import fnmatch -from multiprocessing import Process, Pipe +import multiprocessing from itertools import izip import os import json @@ -39,24 +39,9 @@ DEFAULT_MODULE_ARGS = '' DEFAULT_TIMEOUT = 60 DEFAULT_REMOTE_USER = 'root' -class Pooler(object): - - # credit: http://stackoverflow.com/questions/3288595/multiprocessing-using-pool-map-on-a-function-defined-in-a-class - - @classmethod - def spawn(cls, f): - def fun(pipe,x): - pipe.send(f(x)) - pipe.close() - return fun - - @classmethod - def parmap(cls, f, X): - pipe=[Pipe() for x in X] - proc=[Process(target=cls.spawn(f),args=(c,x)) for x,(p,c) in izip(X,pipe)] - [p.start() for p in proc] - [p.join() for p in proc] - return [p.recv() for (p,c) in pipe] +def _executor_hook(x): + (runner, host) = x + return runner._executor(host) class Runner(object): @@ -166,9 +151,9 @@ class Runner(object): def run(self): ''' xfer & run module on all matched hosts ''' hosts = [ h for h in self.host_list if self._matches(h) ] - def executor(x): - return self._executor(x) - results = Pooler.parmap(executor, hosts) + pool = multiprocessing.Pool(self.forks) + hosts = [ (self,x) for x in hosts ] + results = pool.map(_executor_hook, hosts) results2 = { "successful" : {}, "failed" : {}