Fix multiprocessing pool usage and remove stackoverflow hack
This commit is contained in:
Michael DeHaan
parent
4608a93de5
commit
530e54b3e4
3 changed files with 15 additions and 106 deletions
10
README.md
10
README.md
|
|
@ -86,9 +86,13 @@ The API is simple and returns basic datastructures.
|
|||
data = runner.run()
|
||||
|
||||
{
|
||||
'xyz.example.com' : [ 'any kind of datastructure is returnable' ],
|
||||
'foo.example.com' : None, # failed to connect,
|
||||
...
|
||||
'successful' : {
|
||||
'xyz.example.com' : [ 'any kind of datastructure is returnable' ],
|
||||
'foo.example.com' : [ '...' ]
|
||||
},
|
||||
'failed' : {
|
||||
'bar.example.com' : [ 'failure message' ]
|
||||
}
|
||||
}
|
||||
|
||||
Additional options to Runner include the number of forks, hostname
|
||||
|
|
|
|||
|
|
@ -1,80 +0,0 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
# Copyright (c) 2012 Michael DeHaan <michael.dehaan@gmail.com>
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person
|
||||
# obtaining a copy of this software and associated documentation
|
||||
# files (the "Software"), to deal in the Software without restriction,
|
||||
# including without limitation the rights to use, copy, modify, merge,
|
||||
# publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
# and to permit persons to whom the Software is furnished to do so,
|
||||
# subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be
|
||||
# included in all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
|
||||
# ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
|
||||
# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
from optparse import OptionParser
|
||||
import json
|
||||
import os
|
||||
import ansible
|
||||
|
||||
DEFAULT_HOST_LIST = '/etc/ansible/hosts'
|
||||
DEFAULT_MODULE_PATH = '/usr/share/ansible'
|
||||
DEFAULT_MODULE_NAME = 'ping'
|
||||
DEFAULT_PATTERN = '*'
|
||||
DEFAULT_FORKS = 3
|
||||
DEFAULT_MODULE_ARGS = ''
|
||||
DEFAULT_REMOTE_USER = 'root'
|
||||
|
||||
class Cli(object):
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def runner(self):
|
||||
parser = OptionParser()
|
||||
parser.add_option("-H", "--host-list", dest="host_list",
|
||||
help="path to hosts list", default=DEFAULT_HOST_LIST)
|
||||
parser.add_option("-L", "--library", dest="module_path",
|
||||
help="path to module library", default=DEFAULT_MODULE_PATH)
|
||||
parser.add_option("-f", "--forks", dest="forks",
|
||||
help="level of parallelism", default=DEFAULT_FORKS)
|
||||
parser.add_option("-n", "--name", dest="module_name",
|
||||
help="module name to execute", default=DEFAULT_MODULE_NAME)
|
||||
parser.add_option("-a", "--args", dest="module_args",
|
||||
help="module arguments", default=DEFAULT_MODULE_ARGS)
|
||||
parser.add_option("-p", "--pattern", dest="pattern",
|
||||
help="hostname pattern", default=DEFAULT_PATTERN)
|
||||
parser.add_option("-u", "--remote-user", dest="remote_user",
|
||||
help="remote username", default=DEFAULT_REMOTE_USER)
|
||||
|
||||
options, args = parser.parse_args()
|
||||
|
||||
# TODO: more shell like splitting on module_args would
|
||||
# be a good idea
|
||||
|
||||
return ansible.Runner(
|
||||
module_name=options.module_name,
|
||||
module_path=options.module_path,
|
||||
module_args=options.module_args.split(' '),
|
||||
remote_user=options.remote_user,
|
||||
host_list=options.host_list,
|
||||
forks=options.forks,
|
||||
pattern=options.pattern,
|
||||
verbose=False,
|
||||
)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
result = Cli().runner().run()
|
||||
print json.dumps(result, sort_keys=True, indent=4)
|
||||
|
||||
|
||||
|
|
@ -20,7 +20,7 @@
|
|||
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
import fnmatch
|
||||
from multiprocessing import Process, Pipe
|
||||
import multiprocessing
|
||||
from itertools import izip
|
||||
import os
|
||||
import json
|
||||
|
|
@ -39,24 +39,9 @@ DEFAULT_MODULE_ARGS = ''
|
|||
DEFAULT_TIMEOUT = 60
|
||||
DEFAULT_REMOTE_USER = 'root'
|
||||
|
||||
class Pooler(object):
|
||||
|
||||
# credit: http://stackoverflow.com/questions/3288595/multiprocessing-using-pool-map-on-a-function-defined-in-a-class
|
||||
|
||||
@classmethod
|
||||
def spawn(cls, f):
|
||||
def fun(pipe,x):
|
||||
pipe.send(f(x))
|
||||
pipe.close()
|
||||
return fun
|
||||
|
||||
@classmethod
|
||||
def parmap(cls, f, X):
|
||||
pipe=[Pipe() for x in X]
|
||||
proc=[Process(target=cls.spawn(f),args=(c,x)) for x,(p,c) in izip(X,pipe)]
|
||||
[p.start() for p in proc]
|
||||
[p.join() for p in proc]
|
||||
return [p.recv() for (p,c) in pipe]
|
||||
def _executor_hook(x):
|
||||
(runner, host) = x
|
||||
return runner._executor(host)
|
||||
|
||||
class Runner(object):
|
||||
|
||||
|
|
@ -166,9 +151,9 @@ class Runner(object):
|
|||
def run(self):
|
||||
''' xfer & run module on all matched hosts '''
|
||||
hosts = [ h for h in self.host_list if self._matches(h) ]
|
||||
def executor(x):
|
||||
return self._executor(x)
|
||||
results = Pooler.parmap(executor, hosts)
|
||||
pool = multiprocessing.Pool(self.forks)
|
||||
hosts = [ (self,x) for x in hosts ]
|
||||
results = pool.map(_executor_hook, hosts)
|
||||
results2 = {
|
||||
"successful" : {},
|
||||
"failed" : {}
|
||||
|
|
|
|||
Loading…
Reference in a new issue