From efc2aebd88ab77c5383b4f14eed08bec6ae0830a Mon Sep 17 00:00:00 2001 From: Steven McDonald Date: Sun, 25 Sep 2011 23:49:54 +1000 Subject: [PATCH] initialise repo --- INSTALL | 10 ++++ LICENSE | 24 +++++++++ README | 42 +++++++++++++++ auto.py | 73 +++++++++++++++++++++++++ config.py | 15 ++++++ constants.py | 19 +++++++ logging.conf | 29 ++++++++++ magiclink.py | 126 +++++++++++++++++++++++++++++++++++++++++++ modules/__init__.py | 0 modules/base.py | 35 ++++++++++++ modules/orchestra.py | 88 ++++++++++++++++++++++++++++++ modules/test.py | 65 ++++++++++++++++++++++ mudpuppy.py | 123 ++++++++++++++++++++++++++++++++++++++++++ tests.py | 117 ++++++++++++++++++++++++++++++++++++++++ util.py | 126 +++++++++++++++++++++++++++++++++++++++++++ 15 files changed, 892 insertions(+) create mode 100644 INSTALL create mode 100644 LICENSE create mode 100644 README create mode 100644 auto.py create mode 100644 config.py create mode 100644 constants.py create mode 100644 logging.conf create mode 100644 magiclink.py create mode 100644 modules/__init__.py create mode 100644 modules/base.py create mode 100644 modules/orchestra.py create mode 100644 modules/test.py create mode 100755 mudpuppy.py create mode 100755 tests.py create mode 100644 util.py diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..50f54ab --- /dev/null +++ b/INSTALL @@ -0,0 +1,10 @@ +- First, get make-magic up and running. It's available from + https://github.com/anchor/make-magic + +- Install python2.6 or newer + +- Install 'requests' following from PyPi using easy_install or pip. e.g.: + + pip install requests + +- Read the README to get started diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7eca0f5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2011, Anchor Systems Pty Ltd +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Anchor Systems Pty Ltd nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL ANCHOR SYSTEMS PTY LTD BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README b/README new file mode 100644 index 0000000..69b7468 --- /dev/null +++ b/README @@ -0,0 +1,42 @@ +mudpuppy is a Python based automation agent/client for make-magic + +It allows you to write independant 'modules' in Python code, to automate +items in a make-magic task. When run, mudpuppy will poll a make-magic +server for tasks that are outstanding, and will complete any items with +which it has a module for. Once the module has run successfully, mudpuppy +will tell make-magic that the item has been completed, and then will look +for more work to do. + +There are some example modules in modules/test.py. The simplest case is: + +class TestModule(Automatia): + '''Test item automation model + Successfuly automates a specific item + ''' + can_handle = ('TestModuleItem',) + + def do_magic(self): + print "Hello world" + +which will automate an item with the name 'TestModuleItem' by printing +"Hello world" to stdout, and then returning successfully (allowing the +item to be marked as complete). + +In config.py you can define both the API URL of your make-magic server, +as well as a list of modules to load. Only modules listed in the +installed_items config item will by loaded by mudpuppy. + +There is also a helper base class that allows you to easily kick off +an Orchestra job, wait for it to complete, and optionally update the +make-magic task state from the key/value pairs returned by Orchestra. +This makes it very easy to integrate Orchestra with make-magic/mudpuppy +with very little work. Look in modules/test.py for some examples for +doing this. See https://github.com/anchor/orchestra for more information +about Orchestra. + +Although make-magic is designed to be able to be used with almost no python +knowledge, mudpuppy is specifically designed to automate items with python +code; Feel free to implement agents such as this in any other language you +wish. + +For more information on make-magic, see https://github.com/anchor/make-magic diff --git a/auto.py b/auto.py new file mode 100644 index 0000000..207841a --- /dev/null +++ b/auto.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import config +import modules +import logging + +registered_modules = [] + +class ModuleLoadException(Exception): pass + +def import_class(absolutename): + name = absolutename.split('.') + assert (len(name) >= 2) + modbase = ".".join(name[:-1]) + classname = name[-1] + m = __import__(modbase, fromlist=[classname]) + return getattr(m,classname) + +def register_module(module_name): + m = import_class(module_name) + + if m in registered_modules: + logging.exception("Module already registered: "+str(module_name)) + raise ModuleLoadException('Module already registered: %s' % (module_name)) + registered_modules.append(m) + +def load_modules(): + '''load in all modules and register them + returns the names of the modules loaded + ''' + for module_name in config.installed_modules: + module_name = 'modules.'+module_name + logging.debug("Loading module "+module_name) + register_module(module_name) + return registered_modules + +def get_modules_for_item(item_state, task_metadata): + '''return the module classes to automate an item or empty list if one isn't found + This only returns the classes, and does not instansiate them. They should not be instansiated + until they are about to be run, as the constructor may throw an exception that must be + caught. + ''' + return [mod for mod in registered_modules if mod.can_automate_item(item_state,task_metadata)] + +# Helper modules not for day to day use + +def scan_for_mudpuppy_modules(pkgname='modules'): + '''returns a list of Automatia subclasses in the modules directory + Warning: this actually imports the things + ''' + classes = set() + import pkgutil + import modules.base + for pymodname in [name for _,name,_ in pkgutil.iter_modules([pkgname])]: + try: + if pymodname == 'test': continue + pymod = __import__(pkgname+'.'+pymodname) + pymod = getattr(pymod, pymodname) + for key in dir(pymod): + if key[:1] == '_': continue + obj = getattr(pymod, key) + if type(obj) == type and issubclass(obj, modules.base.Automatia): + if len(obj.can_handle) == 0: continue ## Abstract class + classes.add("%s.%s" % (pymodname,key)) + except Exception, err: + logging.error("! Error importing %s.%s: %s" % (pkgname,str(pymodname).ljust(20), err)) + return classes + +def get_uninstalled_mudpuppy_modules(): + '''get a list of mudpuppy modules not listed in the config''' + allmods = scan_for_mudpuppy_modules() + ourmods = set(config.installed_modules) + return allmods - ourmods diff --git a/config.py b/config.py new file mode 100644 index 0000000..84a5cd5 --- /dev/null +++ b/config.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +mudpuppy_api_url = 'http://localhost:4554/' + +orchestra_socket = '/var/run/conductor.sock' +default_orchestra_target = 'localhost' # see modules.orchestra + +installed_modules = ( + 'test.TestAlwaysAutomates', + ) + +try: + from localconfig import * +except: + pass diff --git a/constants.py b/constants.py new file mode 100644 index 0000000..942ee7e --- /dev/null +++ b/constants.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +'''constants for talking to make-magic''' + +# Item states +# +# These are the same as core.bits.Item.allowed_states in make-magic + +INCOMPLETE = 'INCOMPLETE' +FAILED = 'FAILED' +IN_PROGRESS = 'IN_PROGRESS' +COMPLETE = 'COMPLETE' + + +# Token name used for checking to see if we did a state change +# This must be the same across all agents talking to a make-magic server +# +# This is the same as in make-magic's mclient.py stub shell client +CHANGE_STATE_TOKEN = '_change_state_token' diff --git a/logging.conf b/logging.conf new file mode 100644 index 0000000..11b32af --- /dev/null +++ b/logging.conf @@ -0,0 +1,29 @@ +[loggers] +keys=root + +[handlers] +keys=fileHandler,stderrHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=DEBUG +#handlers=fileHandler +handlers=stderrHandler + +[handler_fileHandler] +class=FileHandler +level=DEBUG +formatter=simpleFormatter +args=('/tmp/mudpuppy.log',) + +[handler_stderrHandler] +class=StreamHandler +level=DEBUG +formatter=simpleFormatter +args=(sys.stderr,) + +[formatter_simpleFormatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s +datefmt= diff --git a/magiclink.py b/magiclink.py new file mode 100644 index 0000000..d1f665d --- /dev/null +++ b/magiclink.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python + +'''Access the make-magic HTTP API + +contains both make-magic HTTP API access and the policy for dealing with the same +''' +import requests +import json +import random + +from constants import * +import config + +class MagicError(Exception): + def __init__(self, error, description): + self.response,self.content = response,content + def __str__(self): + return "MudpuppyError: %s: %s" %(str(self.response), str(self.content)) + +class MagicAPI(object): + '''Talk to the make-magic API over HTTP + ''' + ## HTTP Specific + # + def json_http_request(self, method, url, decode_response = True, data=None): + '''Make a HTTP request, and demarshal the HTTP response + + if POST is given, the data is marshalled to JSON put in the + HTTP request body + ''' + assert url[:4] == 'http' + headers = {'User-Agent': 'mudpuppy MagicAPI'} + if method == 'POST': + json_data = json.dumps(data) + headers['Content-Type'] = 'application/json' + response = requests.request(method, url, headers=headers, data=json_data) + else: + response = requests.request(method, url) + + if response.status_code == None: + # Didn't manage to get a HTTP response + response.raise_for_status() + + if response.status_code != 200: + # Got an error, but hopefully make-magic gave us more information + try: + jsondata = json.loads(response.content) + raise MagicError(jsondata['error'], jsondata['message']) + except: + # Couldn't marshal. Raise a less interesting error. + response.raise_for_status() + + # Yay! good response. Try and marshal to JSON and return + if decode_response: + return json.loads(response.content) + else: + return response.content + + def full_url(self, relpath): + '''return the full URL from a relative API path''' + if config.mudpuppy_api_url[-1] == '/': + config.mudpuppy_api_url = config.mudpuppy_api_url[:-1] + return "%s/%s" % (config.mudpuppy_api_url,relpath) + + def json_http_get(self, relpath, decode_response=True): + return self.json_http_request('GET', self.full_url(relpath), decode_response) + def json_http_post(self, relpath, data, decode_response=True): + return self.json_http_request('POST', self.full_url(relpath), decode_response, data) + def json_http_delete(self, relpath, decode_response=True): + return self.json_http_request('DELETE', self.full_url(relpath), decode_response) + + ## API to expose + # + def get_tasks(self): + '''return a list of all task UUIDs''' + return self.json_http_get('task') + def get_task(self,uuid): + '''return all data for a task''' + return self.json_http_get('task/%s' % (str(uuid),)) + def get_task_metadata(self,uuid): + '''return metadata for a task''' + return self.json_http_get('task/%s/metadata' % (str(uuid),)) + def update_task_metadata(self,uuid, updatedict): + '''update metadata for a task''' + return self.json_http_post('task/%s/metadata' % (str(uuid),),updatedict) + def get_available_items(self,uuid): + '''return all items in a task that are currently ready to be automated''' + return self.json_http_get('task/%s/available' % (str(uuid),)) + def update_item(self,uuid,itemname, updatedict): + '''update data for a specific item''' + return self.json_http_post('task/%s/%s' % (str(uuid),str(itemname)),updatedict) + def get_item(self,uuid,itemname): + '''return data for a specific item''' + return self.json_http_get('task/%s/%s' % (str(uuid),str(itemname))) + def get_item_state(self,uuid,itemname): + '''return item state for a specific item''' + return self.json_http_get('task/%s/%s/state' % (str(uuid),str(itemname)),decode_response=False) + def create_task(self,taskdatadict): + '''create a new task + mudpuppy shouldn't have to do this ever but it is here for + completeness and testing. Unless you want to automatically + create tasks from a task, in which case, power to you. + ''' + return self.json_http_post('task/create', taskdatadict) + def delete_task(self,uuid): + '''delete a task + mudpuppy REALLY shoudn't be doing this, but is here for testing + and completeness. Unless you want to automatically delete + tasks from a task, in which case I'll be hiding behind this rock + ''' + return self.json_http_delete('task/%s' % (str(uuid),)) + +class MagicLink(MagicAPI): + '''wrap the make-magic API while adding some of the logic for dealing with it + ''' + def update_item_state(self, uuid, item, old_state, new_state): + '''atomically update the item state, failing if we don't manage to + + returns True iff the state was changed from old_state to new_state and this call made the change + ''' + token = random.randint(1,2**48) + item_state_update = {"state": new_state, "onlyif": dict(state=old_state)} + item_state_update[CHANGE_STATE_TOKEN] = token + + new_item = self.update_item(uuid, item, item_state_update) + return new_item.get('state') == new_state and new_item.get(CHANGE_STATE_TOKEN) == token diff --git a/modules/__init__.py b/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/base.py b/modules/base.py new file mode 100644 index 0000000..3de9285 --- /dev/null +++ b/modules/base.py @@ -0,0 +1,35 @@ +class CannotAutomateException(Exception): + '''signal from an Automatia that we can't automate this item + This is NOT an error. It's just saying that the item cannot + be processed by this module. + + This should ONLY be raised if the system state was not changed + by the module. + ''' + +class Automatia(object): + '''Base class for anything that can automate an item''' + can_handle = [] + + def __init__(self, item_state, task_metadata): + self.item_state = item_state + self.task_metadata = task_metadata + + module_name = property(lambda self: self.__class__.__name__) + + @classmethod + def can_automate_item(cls, item_state, task_metadata): + '''return if this module can automate an item given item state and task metadata + + default implementation to return True iff item name is in + cls.can_handle + + This must be a class method as modules can count on + only being instantiated if they are going to be used + to attempt to automate something + ''' + return item_state['name'] in cls.can_handle + + def do_magic(self): + '''automate a build step''' + raise NotImplementedError(self.module_name+'.do_magic()') diff --git a/modules/orchestra.py b/modules/orchestra.py new file mode 100644 index 0000000..1542f0f --- /dev/null +++ b/modules/orchestra.py @@ -0,0 +1,88 @@ +from modules.base import Automatia +import config +import util + +try: import json +except: import simplejson as json + +class OrchestraAutomatia(Automatia): + """Abstract helper module to wrap an Orchestra score. + + Use this helper to implement any item that can be directly + mapped to a single Orchestra score. + + Override do_magic() if you want to do more than just wrap a single + score (almost multiple scores should almost certainly be instead made + multiple items in make-magic) + + """ + score_name = None + score_scope = 'one' + score_target = config.default_orchestra_target + score_args = {} + score_timeout = None + + def do_magic(self): + """Execute an Orchestra score.""" + if self.score_name == None: + raise NotImplementedError(self.module_name + '.score_name') + + self.execute_score( + self.score_name, + self.score_scope, + self.score_target, + self.score_args, + self.score_timeout) + + @staticmethod + def execute_score(name, scope='one', + target=config.default_orchestra_target, args={}, + timeout=None): + """Execute an Orchestra score, block for completion, then return + the result as a (job_id, result) tuple. + + Will raise ``util.OrchestraError`` on any Orchestra failure. + + """ + oc = util.OrchestraClient() + job_id = oc.submit_job(name, scope, target, args) + result = oc.wait_for_completion(job_id, timeout=timeout) + + if result['Status'] == 'OK': + return (job_id, result, ) + else: + raise util.OrchestraError(repr(result)) + +class OrchestraMetadataAutomatia(OrchestraAutomatia): + '''Helper module to wrap an Orchestra score and update task metadata from the result + ''' + + # Mapping from orchestra response variables to metadata key names + orchestra_response_map = None + + def get_variable_from_results(self, results, resultvarname): + '''look through orchestra results and find the value keyed on resultvarname + + FIXME: THIS SHOULD BE IN util.py + pre: results status is OK and players have completed the score + ''' + # response looks like: + # {u'Status': u'OK', u'Players': {u'orchestra.player.hostname.example': {u'Status': u'OK', u'Response': {u'echo': u'{"PATH": "/usr/bin:/usr/sbin:/bin:/sbin", "PWD": "/var/lib/service/player", "IFS": " \\t\\n", "ORC_foo": "bar", "ORC_fish": "heads"}'}}}} + assert results['Status'] == 'OK' + for player,result in results['Players'].items(): + if result['Status'] != 'OK': continue + for score,playerresponse in result['Response'].items(): + playerresponse = json.loads(playerresponse) # UGH. This should NOT be in here FIXME: Move unrolling to OrchestraClient + if playerresponse.has_key(resultvarname): + return playerresponse[resultvarname] + return KeyError("Variable '"+resultvarname+"' not returned by any Orchestra player") + + def do_magic(self): + if self.orchestra_response_map == None: + raise NotImplementedError(str(self.module_name+'.orchestra_response_map')) + results = self.execute_score(self.score_args) + metadata_update = {} + for response_var,metadata_key in self.orchestra_response_map.items(): + task_metadata_update[metadata_key] = self.get_variable_from_results(results, response_var) + + return dict(task_metadata_update=metadata_update) diff --git a/modules/test.py b/modules/test.py new file mode 100644 index 0000000..efed0a0 --- /dev/null +++ b/modules/test.py @@ -0,0 +1,65 @@ +'''modules for use in unit testing''' + +from modules.base import Automatia, CannotAutomateException +from modules.orchestra import OrchestraAutomatia, OrchestraMetadataAutomatia + +class TestModule(Automatia): + '''Test item automation model + Successfuly automates a specific item + ''' + can_handle = ('TestModuleItem',) + + def do_magic(self): + return # Returning nothing is fine. The item will still succeed + +class TestModuleUnimplemented(Automatia): + '''Test item automation model + + Raises NotImplementedError when mudpuppy tries to call do_magic + ''' + can_handle = ('TestUnimplementedModuleItem',) + +class TestAlwaysAutomates(Automatia): + '''Test item that always automates anything it sees + also adds stuff to both the task metadata and the state metadata + ''' + @classmethod + def can_automate_item(cls, item_state, task_metadata): + return True + + def do_magic(self): + itemstuff = { 'guess what': ['TestAlwaysAutomates was here', {'fish': 'are cool'}], 'foo': 'bar' } + taskstuff = { 'TestAlwaysAutomates last touched': self.item_state['name'] } + return dict( task_metadata_update=taskstuff, item_metadata_update=itemstuff ) + +class TestFakeIP(Automatia): + '''Test item automation model to add task metadata + Adds a dummy IP address to the task metadata + ''' + can_handle = ('TestFakeIPitem',) + + def do_magic(self): + return dict( task_metadata_update={'ip': '10.0.0.1'} ) + +class TestCannotAutomate(Automatia): + can_handle = ('TestCannotAutomate item',) + def do_magic(self): + nyan = "NYAN NYAN NYAN NYAN NYAN NYAN NYAN NYAN NYAN" + raise CannotAutomateException(nyan) + +class TestOrchestraBad(OrchestraAutomatia): + can_handle = ('TestOrchestraBadItem',) + +class TestOrchestra(OrchestraAutomatia): + can_handle = ('TestOrchestraItem',) + score_name = 'helloworld' + +class TestOrchestraMetadata(OrchestraMetadataAutomatia): + can_handle = ('TestOrchestraMetadataItem',) + + # echo score returns score args passed with ORC_ prepended to the key names + score_name = 'echo' + + # get return variables from orchestra and put into into task metadata keys + orchestra_response_map = { 'ORC_fish': "fish", "ORC_foo": "taskfoo" } + score_args = { 'fish':'heads', 'foo':'bar' } diff --git a/mudpuppy.py b/mudpuppy.py new file mode 100755 index 0000000..a1f272a --- /dev/null +++ b/mudpuppy.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python + +'''A very, very simple mudpuppy implementation +''' + +import os +import sys +import auto +import util +import config +import time +import traceback +import signal + +from constants import * +from modules.base import CannotAutomateException +from magiclink import MagicLink + +import logging +import logging.config +logging.config.fileConfig(os.path.join(os.path.dirname(os.path.abspath(__file__)),'logging.conf')) + +exit_gracefully = False + +def sighup_handler(signum, frame): + global exit_gracefully + exit_gracefully = True + print "got SIGHUP" +signal.signal(signal.SIGHUP, sighup_handler) + + +def automate_task(uuid): + '''find an item that we can do in a task and automate it + + At the moment we return as soon as we automate a single item successfully + + returns True if we managed to automate anything + ''' + + api = MagicLink() + + automated_something = False + + for item in api.get_available_items(uuid): + item_name = item['name'] + if automated_something: break + + # Needs to be at least as new as the available list + task_metadata = api.get_task_metadata(uuid) + for module in auto.get_modules_for_item(item, task_metadata): + if automated_something: break + # Lock + if not api.update_item_state(uuid, item_name, INCOMPLETE, IN_PROGRESS): + break # Couldn't lock item. Try the next + logging.info('Set %s/%s to IN_PROGRESS' %(uuid,item_name)) + + # Attempt to run + try: + builder = module(item, task_metadata) + result = builder.do_magic() + if result != None and result.has_key('task_metadata_update'): + api.update_task_metadata(uuid, result['task_metadata_update']) + logging.debug('updating task metadata for %s with: %s' %(uuid,result['task_metadata_update'])) + if result != None and result.has_key('item_metadata_update'): + api.update_item(uuid, item_name, result['item_metadata_update']) + logging.debug('updating item metadata for %s/%s with: %s' %(uuid,item_name,result['item_metadata_update'])) + except CannotAutomateException: + # Unlock and try the next builder + logging.debug('Module threw CANNOT_AUTOMATE. Continuing with other modules if there are there', exc_info=1) + logging.info('Setting %s/%s to INCOMPLETE' %(uuid,item_name)) + worked = api.update_item_state(uuid, item_name, IN_PROGRESS, INCOMPLETE) + if not worked: + logging.error("Couldn't set item state from IN_PROGRESS to FAILED! Task may be in inconsistant state") + return False + continue + except Exception, e: + logging.error('Module threw an exception. Setting the item to FAILED', exc_info=1) + worked = api.update_item_state(uuid, item_name, IN_PROGRESS, FAILED) + if not worked: + logging.error("Couldn't set item state from IN_PROGRESS TO FAILED! Task may be in inconsistant state") + return False + automated_something = True + break + + logging.info('Module finished. Setting %s/%s to COMPLETE' %(uuid,item['name'])) + automated_something = True + worked = api.update_item_state(uuid, item_name, IN_PROGRESS, COMPLETE) + if not worked: + logging.error("Couldn't set item state from IN_PROGRESS TO COMPLETE! Task may be in inconsistant state") + return False + + return automated_something + +def mudpuppy_main(): + logging.config.fileConfig(os.path.join(os.path.dirname(os.path.abspath(__file__)),'logging.conf')) + + api = MagicLink() + + # Load in the modules that do the real work + auto.load_modules() + + logging.info("Started. Waiting for things to do.") + while not exit_gracefully: + tasks = api.get_tasks() + + automated_something = False + for uuid in tasks: + automated = automate_task(uuid) + automated_something |= automated + + if not automated_something: + # Didn't manage to change the state of anything + # so wait at least a second before hammering + # the server again + time.sleep(1) + continue + + logging.debug("Polling make-magic for serverbuilds for us to automate") + logging.info("Exiting gracefully after SIGHUP") + + +if __name__ == '__main__': + mudpuppy_main() diff --git a/tests.py b/tests.py new file mode 100755 index 0000000..8d01ba8 --- /dev/null +++ b/tests.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python + +import unittest2 as unittest +import config +import auto +import sys +import util +from modules.base import CannotAutomateException + +import magiclink +import constants + +import logging +logging.basicConfig(level=logging.DEBUG) + +'''tests for mudpuppy''' + +class OrchestraTests(unittest.TestCase): + def test_orchestra_client(self): + oc = util.OrchestraClient() + jobid = oc.submit_job('madworld','one',config.default_orchestra_target) + status = oc.wait_for_completion(jobid) + self.assertEquals(status['Status'], 'FAIL') + jobid = oc.submit_job('helloworld','one',config.default_orchestra_target) + status = oc.wait_for_completion(jobid) + self.assertEquals(status['Status'], 'OK') + +class ModuleTests(unittest.TestCase): + def setUp(self): + reload(auto) + def tearDown(self): + reload(auto) + def test_1_load_modules(self): + config.installed_modules = ('test.TestModule', ) + modules = auto.load_modules() + self.assertEquals(len(modules),1) + + # Shouldn't allow duplication of loads + self.assertRaises(auto.ModuleLoadException, auto.load_modules) + def test_allocation_modules(self): + config.installed_modules = ('test.TestModule', ) + modules = auto.load_modules() + + _itemdata = {'name': 'FakeItem'} + bots = auto.get_modules_for_item(_itemdata,{}) + self.assertEquals(bots, []) + + _itemdata = {'name': 'TestModuleItem'} + bots = auto.get_modules_for_item(_itemdata,{}) + for botclass in bots: + self.assertIsNotNone(botclass) + bot = botclass(_itemdata,{}) + self.assertEquals(bot.module_name, 'TestModule') + def test_question_modules(self): + config.installed_modules = ('test.TestFakeIP', ) + modules = auto.load_modules() + _item_state = {'name': 'TestFakeIPitem'} + bots = auto.get_modules_for_item(_item_state,{}) + for botclass in bots: + bot = botclass(_item_state,{}) + self.assertIsNotNone(bot) + self.assertEquals(bot.module_name, 'TestFakeIP') + self.assertEquals(bot.do_magic(), {'task_metadata_update': {'ip': '10.0.0.1'}}) + def test_module_generated_cannot_automate(self): + config.installed_modules = ('test.TestCannotAutomate', ) + modules = auto.load_modules() + _item_state = {'name': 'TestCannotAutomateItem'} + bots = auto.get_modules_for_item(_item_state,{}) + for botclass in bots: + self.assertIsNotNone(botclass) + bot = botclass(_item_state, {}) + self.assertRaises(CannotAutomateException, bot.do_magic) + +class MagicAPITests(unittest.TestCase): + def test_000_get_tasks(self): + api = magiclink.MagicAPI() + tasks = api.get_tasks() + self.assertIsInstance(tasks, list) + def test_create_delete_tasks(self): + api = magiclink.MagicAPI() + taskdata = api.create_task(dict(requirements=[], description="UNITTEST EPHEMERAL TEST TASK", automate=False)) + uuid = taskdata['metadata']['uuid'] + tasks = api.get_tasks() + self.assertIn(uuid,tasks) + api.delete_task(uuid) + tasks = api.get_tasks() + self.assertNotIn(uuid,tasks) + +class MagicLinkTests(unittest.TestCase): + def test_state_changes(self): + api = magiclink.MagicLink() + taskdata = api.create_task(dict(requirements=[], description="UNITTEST EPHEMERAL TEST TASK", automate=False)) + uuid = taskdata['metadata']['uuid'] + tasks = api.get_tasks() + self.assertIn(uuid,tasks) + + avail = api.get_available_items(uuid) + itemname = avail[0]['name'] + + self.assertGreater(len(avail), 0) + self.assertEqual(api.get_item_state(uuid,itemname),constants.INCOMPLETE) + success = api.update_item_state(uuid, itemname, constants.INCOMPLETE, constants.IN_PROGRESS) + self.assertTrue(success) + self.assertEqual(api.get_item_state(uuid,itemname),constants.IN_PROGRESS) + + success = api.update_item_state(uuid, itemname, constants.IN_PROGRESS, constants.COMPLETE) + self.assertTrue(success) + self.assertEqual(api.get_item_state(uuid,itemname),constants.COMPLETE) + avail = api.get_available_items(uuid) + self.assertNotEqual(avail[0]['name'],itemname) + + api.delete_task(uuid) + tasks = api.get_tasks() + self.assertNotIn(uuid,tasks) + +if __name__ == "__main__": + unittest.main() diff --git a/util.py b/util.py new file mode 100644 index 0000000..912d6bb --- /dev/null +++ b/util.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python + +import config +import socket +import auto +import json +import time +import functools +import logging + +class TimeoutException(Exception): pass +class OrchestraError(Exception): pass +class OrchestraJSONError(OrchestraError): pass + +def timecall(func, *args, **argd): + start = time.time() + try: ret = func(*args,**argd) + finally: print time.time() - start + return ret + +def poll_until_not_none(func, delay=0.1, backoff=1, max_delay=None, timeout=None): + '''call a function regularly until it returns not None and then return the result + + the function is called every 'delay' seconds + after every unsuccessful call, the delay is multiplied by 'backoff', and limited to max_delay + + if timeout is set, a TimeoutException is raised if that much time has passed without result + the function will always be called at least once without delay, regardless of the value of timeout + if a timeout is set the function may be called immediately before timing out, even if the delay was longer + ''' + #if timeout is not None: + start = time.time() + while True: + beforecall = time.time()-start + ret = func() + if ret is not None: return ret + elapsed = time.time()-start + if timeout is not None and elapsed >= timeout: + raise TimeoutException() + sleepfor = delay - (elapsed-beforecall) + if timeout is not None and elapsed+sleepfor > timeout: + sleepfor = timeout-elapsed + if sleepfor > 0: + time.sleep(sleepfor) + delay *= backoff + if max_delay: + delay = min(delay, max_delay) + +class OrchestraClient(object): + '''Simple Orchestra Audience + + >>> oc = OrchestraClient() + >>> oc.submit_job('helloworld','one','orchestra.player.hostname.example') + 200034 + >>> oc.get_status(200034) + {u'Status': u'OK', u'Players': {u'orchestra.player.hostname.example': {u'Status': u'OK', u'Response': {}}}} + >>> oc.submit_job('madworld','one','orchestra.player.hostname.example') + 200035 + >>> oc.get_status(200035) + {u'Status': u'FAIL', u'Players': {u'orchestra.player.hostname.example': {u'Status': u'HOST_ERROR', u'Response': {}}}} + >>> oc.submit_job('echo','one','orchestra.player.hostname.example', {'foo': 12345, 'nyan': 54321, 'cat': 'dog'}) + 200038 + >>> oc.wait_for_completion(200038, timeout=30) + {u'Status': u'OK', u'Players': {u'orchestra.player.hostname.example': {u'Status': u'OK', u'Response': {u'echo': u'{"IFS": " \\t\\n", "ORC_nyan": "54321", "PWD": "/var/lib/service/player", "ORC_foo": "12345", "PATH": "/usr/bin:/usr/sbin:/bin:/sbin", "ORC_cat": "dog"}'}}}} + ''' + def _get_conductor_socket(self): + '''get a socket connected to the conductor + current implementation is a UNIX socket''' + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(config.orchestra_socket) + return sock + + def orchestra_request(self, **argd): + '''make a request to the Orchestra conductor + pre: all non-positional arguments passed form a well-formed orchestra request + ''' + sock = self._get_conductor_socket() + f = sock.makefile() + try: + json.dump(argd, f) + f.flush() + data = f.read() + response = json.loads(data) + except ValueError: + if data == '': + raise OrchestraError('Conductor burned socket. Probably invalid call') + raise OrchestraJSONError(("Couldn't decode as JSON",data)) + finally: + sock.close() + + if response[0] == 'OK': + # FIXME: This is not demarshalling any json coming back from individual players! + return response[1] + else: + raise OrchestraError(response[1]) + + def submit_job(self, score, scope, target, args={}): + '''submit a job to orchestra as per the Audience API + Any keys and values in args will be coerced into unicode objects + ''' + if isinstance(target, basestring): + target = [target] + args = dict((unicode(k),unicode(v)) for k,v in args.items()) + request = dict(Op='queue', Score=score, Scope=scope, Players=list(target), Params=args) + logging.debug('Submitting Orchestra job: %s', repr(request)) + return self.orchestra_request(**request) + + def get_status(self, jobid): + return self.orchestra_request(Op='status', Id=jobid) + + def completed_status_or_none(self, jobid): + '''return the job status if completed or failed, or None if still Pending''' + status = self.get_status(jobid) + return None if status['Status'] == 'PENDING' else status + + def wait_for_completion(self, jobid, timeout=None, delay=0.1, backoff=1.41414, max_poll_delay=2): + '''Block until an orchestra job leaves the Pending state + returns the status response of the job when it is available + raises TimeoutException iff timeout is not None and timeout seconds + have passed without the job leaving the Pending state + + Orchestra calls are async, so we just keep polling until we get + somewhere + ''' + pf = functools.partial(self.completed_status_or_none, jobid) + return poll_until_not_none(pf, delay=delay, backoff=backoff, timeout=timeout, max_delay=max_poll_delay) -- 2.30.2