--- /dev/null
+- First, get make-magic up and running. It's available from
+ https://github.com/anchor/make-magic
+
+- Install python2.6 or newer
+
+- Install 'requests' following from PyPi using easy_install or pip. e.g.:
+
+ pip install requests
+
+- Read the README to get started
--- /dev/null
+Copyright (c) 2011, Anchor Systems Pty Ltd
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of Anchor Systems Pty Ltd nor the
+ names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL ANCHOR SYSTEMS PTY LTD BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--- /dev/null
+mudpuppy is a Python based automation agent/client for make-magic
+
+It allows you to write independant 'modules' in Python code, to automate
+items in a make-magic task. When run, mudpuppy will poll a make-magic
+server for tasks that are outstanding, and will complete any items with
+which it has a module for. Once the module has run successfully, mudpuppy
+will tell make-magic that the item has been completed, and then will look
+for more work to do.
+
+There are some example modules in modules/test.py. The simplest case is:
+
+class TestModule(Automatia):
+ '''Test item automation model
+ Successfuly automates a specific item
+ '''
+ can_handle = ('TestModuleItem',)
+
+ def do_magic(self):
+ print "Hello world"
+
+which will automate an item with the name 'TestModuleItem' by printing
+"Hello world" to stdout, and then returning successfully (allowing the
+item to be marked as complete).
+
+In config.py you can define both the API URL of your make-magic server,
+as well as a list of modules to load. Only modules listed in the
+installed_items config item will by loaded by mudpuppy.
+
+There is also a helper base class that allows you to easily kick off
+an Orchestra job, wait for it to complete, and optionally update the
+make-magic task state from the key/value pairs returned by Orchestra.
+This makes it very easy to integrate Orchestra with make-magic/mudpuppy
+with very little work. Look in modules/test.py for some examples for
+doing this. See https://github.com/anchor/orchestra for more information
+about Orchestra.
+
+Although make-magic is designed to be able to be used with almost no python
+knowledge, mudpuppy is specifically designed to automate items with python
+code; Feel free to implement agents such as this in any other language you
+wish.
+
+For more information on make-magic, see https://github.com/anchor/make-magic
--- /dev/null
+#!/usr/bin/env python
+
+import config
+import modules
+import logging
+
+registered_modules = []
+
+class ModuleLoadException(Exception): pass
+
+def import_class(absolutename):
+ name = absolutename.split('.')
+ assert (len(name) >= 2)
+ modbase = ".".join(name[:-1])
+ classname = name[-1]
+ m = __import__(modbase, fromlist=[classname])
+ return getattr(m,classname)
+
+def register_module(module_name):
+ m = import_class(module_name)
+
+ if m in registered_modules:
+ logging.exception("Module already registered: "+str(module_name))
+ raise ModuleLoadException('Module already registered: %s' % (module_name))
+ registered_modules.append(m)
+
+def load_modules():
+ '''load in all modules and register them
+ returns the names of the modules loaded
+ '''
+ for module_name in config.installed_modules:
+ module_name = 'modules.'+module_name
+ logging.debug("Loading module "+module_name)
+ register_module(module_name)
+ return registered_modules
+
+def get_modules_for_item(item_state, task_metadata):
+ '''return the module classes to automate an item or empty list if one isn't found
+ This only returns the classes, and does not instansiate them. They should not be instansiated
+ until they are about to be run, as the constructor may throw an exception that must be
+ caught.
+ '''
+ return [mod for mod in registered_modules if mod.can_automate_item(item_state,task_metadata)]
+
+# Helper modules not for day to day use
+
+def scan_for_mudpuppy_modules(pkgname='modules'):
+ '''returns a list of Automatia subclasses in the modules directory
+ Warning: this actually imports the things
+ '''
+ classes = set()
+ import pkgutil
+ import modules.base
+ for pymodname in [name for _,name,_ in pkgutil.iter_modules([pkgname])]:
+ try:
+ if pymodname == 'test': continue
+ pymod = __import__(pkgname+'.'+pymodname)
+ pymod = getattr(pymod, pymodname)
+ for key in dir(pymod):
+ if key[:1] == '_': continue
+ obj = getattr(pymod, key)
+ if type(obj) == type and issubclass(obj, modules.base.Automatia):
+ if len(obj.can_handle) == 0: continue ## Abstract class
+ classes.add("%s.%s" % (pymodname,key))
+ except Exception, err:
+ logging.error("! Error importing %s.%s: %s" % (pkgname,str(pymodname).ljust(20), err))
+ return classes
+
+def get_uninstalled_mudpuppy_modules():
+ '''get a list of mudpuppy modules not listed in the config'''
+ allmods = scan_for_mudpuppy_modules()
+ ourmods = set(config.installed_modules)
+ return allmods - ourmods
--- /dev/null
+#!/usr/bin/env python
+
+mudpuppy_api_url = 'http://localhost:4554/'
+
+orchestra_socket = '/var/run/conductor.sock'
+default_orchestra_target = 'localhost' # see modules.orchestra
+
+installed_modules = (
+ 'test.TestAlwaysAutomates',
+ )
+
+try:
+ from localconfig import *
+except:
+ pass
--- /dev/null
+#!/usr/bin/env python
+
+'''constants for talking to make-magic'''
+
+# Item states
+#
+# These are the same as core.bits.Item.allowed_states in make-magic
+
+INCOMPLETE = 'INCOMPLETE'
+FAILED = 'FAILED'
+IN_PROGRESS = 'IN_PROGRESS'
+COMPLETE = 'COMPLETE'
+
+
+# Token name used for checking to see if we did a state change
+# This must be the same across all agents talking to a make-magic server
+#
+# This is the same as in make-magic's mclient.py stub shell client
+CHANGE_STATE_TOKEN = '_change_state_token'
--- /dev/null
+[loggers]
+keys=root
+
+[handlers]
+keys=fileHandler,stderrHandler
+
+[formatters]
+keys=simpleFormatter
+
+[logger_root]
+level=DEBUG
+#handlers=fileHandler
+handlers=stderrHandler
+
+[handler_fileHandler]
+class=FileHandler
+level=DEBUG
+formatter=simpleFormatter
+args=('/tmp/mudpuppy.log',)
+
+[handler_stderrHandler]
+class=StreamHandler
+level=DEBUG
+formatter=simpleFormatter
+args=(sys.stderr,)
+
+[formatter_simpleFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+datefmt=
--- /dev/null
+#!/usr/bin/env python
+
+'''Access the make-magic HTTP API
+
+contains both make-magic HTTP API access and the policy for dealing with the same
+'''
+import requests
+import json
+import random
+
+from constants import *
+import config
+
+class MagicError(Exception):
+ def __init__(self, error, description):
+ self.response,self.content = response,content
+ def __str__(self):
+ return "MudpuppyError: %s: %s" %(str(self.response), str(self.content))
+
+class MagicAPI(object):
+ '''Talk to the make-magic API over HTTP
+ '''
+ ## HTTP Specific
+ #
+ def json_http_request(self, method, url, decode_response = True, data=None):
+ '''Make a HTTP request, and demarshal the HTTP response
+
+ if POST is given, the data is marshalled to JSON put in the
+ HTTP request body
+ '''
+ assert url[:4] == 'http'
+ headers = {'User-Agent': 'mudpuppy MagicAPI'}
+ if method == 'POST':
+ json_data = json.dumps(data)
+ headers['Content-Type'] = 'application/json'
+ response = requests.request(method, url, headers=headers, data=json_data)
+ else:
+ response = requests.request(method, url)
+
+ if response.status_code == None:
+ # Didn't manage to get a HTTP response
+ response.raise_for_status()
+
+ if response.status_code != 200:
+ # Got an error, but hopefully make-magic gave us more information
+ try:
+ jsondata = json.loads(response.content)
+ raise MagicError(jsondata['error'], jsondata['message'])
+ except:
+ # Couldn't marshal. Raise a less interesting error.
+ response.raise_for_status()
+
+ # Yay! good response. Try and marshal to JSON and return
+ if decode_response:
+ return json.loads(response.content)
+ else:
+ return response.content
+
+ def full_url(self, relpath):
+ '''return the full URL from a relative API path'''
+ if config.mudpuppy_api_url[-1] == '/':
+ config.mudpuppy_api_url = config.mudpuppy_api_url[:-1]
+ return "%s/%s" % (config.mudpuppy_api_url,relpath)
+
+ def json_http_get(self, relpath, decode_response=True):
+ return self.json_http_request('GET', self.full_url(relpath), decode_response)
+ def json_http_post(self, relpath, data, decode_response=True):
+ return self.json_http_request('POST', self.full_url(relpath), decode_response, data)
+ def json_http_delete(self, relpath, decode_response=True):
+ return self.json_http_request('DELETE', self.full_url(relpath), decode_response)
+
+ ## API to expose
+ #
+ def get_tasks(self):
+ '''return a list of all task UUIDs'''
+ return self.json_http_get('task')
+ def get_task(self,uuid):
+ '''return all data for a task'''
+ return self.json_http_get('task/%s' % (str(uuid),))
+ def get_task_metadata(self,uuid):
+ '''return metadata for a task'''
+ return self.json_http_get('task/%s/metadata' % (str(uuid),))
+ def update_task_metadata(self,uuid, updatedict):
+ '''update metadata for a task'''
+ return self.json_http_post('task/%s/metadata' % (str(uuid),),updatedict)
+ def get_available_items(self,uuid):
+ '''return all items in a task that are currently ready to be automated'''
+ return self.json_http_get('task/%s/available' % (str(uuid),))
+ def update_item(self,uuid,itemname, updatedict):
+ '''update data for a specific item'''
+ return self.json_http_post('task/%s/%s' % (str(uuid),str(itemname)),updatedict)
+ def get_item(self,uuid,itemname):
+ '''return data for a specific item'''
+ return self.json_http_get('task/%s/%s' % (str(uuid),str(itemname)))
+ def get_item_state(self,uuid,itemname):
+ '''return item state for a specific item'''
+ return self.json_http_get('task/%s/%s/state' % (str(uuid),str(itemname)),decode_response=False)
+ def create_task(self,taskdatadict):
+ '''create a new task
+ mudpuppy shouldn't have to do this ever but it is here for
+ completeness and testing. Unless you want to automatically
+ create tasks from a task, in which case, power to you.
+ '''
+ return self.json_http_post('task/create', taskdatadict)
+ def delete_task(self,uuid):
+ '''delete a task
+ mudpuppy REALLY shoudn't be doing this, but is here for testing
+ and completeness. Unless you want to automatically delete
+ tasks from a task, in which case I'll be hiding behind this rock
+ '''
+ return self.json_http_delete('task/%s' % (str(uuid),))
+
+class MagicLink(MagicAPI):
+ '''wrap the make-magic API while adding some of the logic for dealing with it
+ '''
+ def update_item_state(self, uuid, item, old_state, new_state):
+ '''atomically update the item state, failing if we don't manage to
+
+ returns True iff the state was changed from old_state to new_state and this call made the change
+ '''
+ token = random.randint(1,2**48)
+ item_state_update = {"state": new_state, "onlyif": dict(state=old_state)}
+ item_state_update[CHANGE_STATE_TOKEN] = token
+
+ new_item = self.update_item(uuid, item, item_state_update)
+ return new_item.get('state') == new_state and new_item.get(CHANGE_STATE_TOKEN) == token
--- /dev/null
+class CannotAutomateException(Exception):
+ '''signal from an Automatia that we can't automate this item
+ This is NOT an error. It's just saying that the item cannot
+ be processed by this module.
+
+ This should ONLY be raised if the system state was not changed
+ by the module.
+ '''
+
+class Automatia(object):
+ '''Base class for anything that can automate an item'''
+ can_handle = []
+
+ def __init__(self, item_state, task_metadata):
+ self.item_state = item_state
+ self.task_metadata = task_metadata
+
+ module_name = property(lambda self: self.__class__.__name__)
+
+ @classmethod
+ def can_automate_item(cls, item_state, task_metadata):
+ '''return if this module can automate an item given item state and task metadata
+
+ default implementation to return True iff item name is in
+ cls.can_handle
+
+ This must be a class method as modules can count on
+ only being instantiated if they are going to be used
+ to attempt to automate something
+ '''
+ return item_state['name'] in cls.can_handle
+
+ def do_magic(self):
+ '''automate a build step'''
+ raise NotImplementedError(self.module_name+'.do_magic()')
--- /dev/null
+from modules.base import Automatia
+import config
+import util
+
+try: import json
+except: import simplejson as json
+
+class OrchestraAutomatia(Automatia):
+ """Abstract helper module to wrap an Orchestra score.
+
+ Use this helper to implement any item that can be directly
+ mapped to a single Orchestra score.
+
+ Override do_magic() if you want to do more than just wrap a single
+ score (almost multiple scores should almost certainly be instead made
+ multiple items in make-magic)
+
+ """
+ score_name = None
+ score_scope = 'one'
+ score_target = config.default_orchestra_target
+ score_args = {}
+ score_timeout = None
+
+ def do_magic(self):
+ """Execute an Orchestra score."""
+ if self.score_name == None:
+ raise NotImplementedError(self.module_name + '.score_name')
+
+ self.execute_score(
+ self.score_name,
+ self.score_scope,
+ self.score_target,
+ self.score_args,
+ self.score_timeout)
+
+ @staticmethod
+ def execute_score(name, scope='one',
+ target=config.default_orchestra_target, args={},
+ timeout=None):
+ """Execute an Orchestra score, block for completion, then return
+ the result as a (job_id, result) tuple.
+
+ Will raise ``util.OrchestraError`` on any Orchestra failure.
+
+ """
+ oc = util.OrchestraClient()
+ job_id = oc.submit_job(name, scope, target, args)
+ result = oc.wait_for_completion(job_id, timeout=timeout)
+
+ if result['Status'] == 'OK':
+ return (job_id, result, )
+ else:
+ raise util.OrchestraError(repr(result))
+
+class OrchestraMetadataAutomatia(OrchestraAutomatia):
+ '''Helper module to wrap an Orchestra score and update task metadata from the result
+ '''
+
+ # Mapping from orchestra response variables to metadata key names
+ orchestra_response_map = None
+
+ def get_variable_from_results(self, results, resultvarname):
+ '''look through orchestra results and find the value keyed on resultvarname
+
+ FIXME: THIS SHOULD BE IN util.py
+ pre: results status is OK and players have completed the score
+ '''
+ # response looks like:
+ # {u'Status': u'OK', u'Players': {u'orchestra.player.hostname.example': {u'Status': u'OK', u'Response': {u'echo': u'{"PATH": "/usr/bin:/usr/sbin:/bin:/sbin", "PWD": "/var/lib/service/player", "IFS": " \\t\\n", "ORC_foo": "bar", "ORC_fish": "heads"}'}}}}
+ assert results['Status'] == 'OK'
+ for player,result in results['Players'].items():
+ if result['Status'] != 'OK': continue
+ for score,playerresponse in result['Response'].items():
+ playerresponse = json.loads(playerresponse) # UGH. This should NOT be in here FIXME: Move unrolling to OrchestraClient
+ if playerresponse.has_key(resultvarname):
+ return playerresponse[resultvarname]
+ return KeyError("Variable '"+resultvarname+"' not returned by any Orchestra player")
+
+ def do_magic(self):
+ if self.orchestra_response_map == None:
+ raise NotImplementedError(str(self.module_name+'.orchestra_response_map'))
+ results = self.execute_score(self.score_args)
+ metadata_update = {}
+ for response_var,metadata_key in self.orchestra_response_map.items():
+ task_metadata_update[metadata_key] = self.get_variable_from_results(results, response_var)
+
+ return dict(task_metadata_update=metadata_update)
--- /dev/null
+'''modules for use in unit testing'''
+
+from modules.base import Automatia, CannotAutomateException
+from modules.orchestra import OrchestraAutomatia, OrchestraMetadataAutomatia
+
+class TestModule(Automatia):
+ '''Test item automation model
+ Successfuly automates a specific item
+ '''
+ can_handle = ('TestModuleItem',)
+
+ def do_magic(self):
+ return # Returning nothing is fine. The item will still succeed
+
+class TestModuleUnimplemented(Automatia):
+ '''Test item automation model
+
+ Raises NotImplementedError when mudpuppy tries to call do_magic
+ '''
+ can_handle = ('TestUnimplementedModuleItem',)
+
+class TestAlwaysAutomates(Automatia):
+ '''Test item that always automates anything it sees
+ also adds stuff to both the task metadata and the state metadata
+ '''
+ @classmethod
+ def can_automate_item(cls, item_state, task_metadata):
+ return True
+
+ def do_magic(self):
+ itemstuff = { 'guess what': ['TestAlwaysAutomates was here', {'fish': 'are cool'}], 'foo': 'bar' }
+ taskstuff = { 'TestAlwaysAutomates last touched': self.item_state['name'] }
+ return dict( task_metadata_update=taskstuff, item_metadata_update=itemstuff )
+
+class TestFakeIP(Automatia):
+ '''Test item automation model to add task metadata
+ Adds a dummy IP address to the task metadata
+ '''
+ can_handle = ('TestFakeIPitem',)
+
+ def do_magic(self):
+ return dict( task_metadata_update={'ip': '10.0.0.1'} )
+
+class TestCannotAutomate(Automatia):
+ can_handle = ('TestCannotAutomate item',)
+ def do_magic(self):
+ nyan = "NYAN NYAN NYAN NYAN NYAN NYAN NYAN NYAN NYAN"
+ raise CannotAutomateException(nyan)
+
+class TestOrchestraBad(OrchestraAutomatia):
+ can_handle = ('TestOrchestraBadItem',)
+
+class TestOrchestra(OrchestraAutomatia):
+ can_handle = ('TestOrchestraItem',)
+ score_name = 'helloworld'
+
+class TestOrchestraMetadata(OrchestraMetadataAutomatia):
+ can_handle = ('TestOrchestraMetadataItem',)
+
+ # echo score returns score args passed with ORC_ prepended to the key names
+ score_name = 'echo'
+
+ # get return variables from orchestra and put into into task metadata keys
+ orchestra_response_map = { 'ORC_fish': "fish", "ORC_foo": "taskfoo" }
+ score_args = { 'fish':'heads', 'foo':'bar' }
--- /dev/null
+#!/usr/bin/env python
+
+'''A very, very simple mudpuppy implementation
+'''
+
+import os
+import sys
+import auto
+import util
+import config
+import time
+import traceback
+import signal
+
+from constants import *
+from modules.base import CannotAutomateException
+from magiclink import MagicLink
+
+import logging
+import logging.config
+logging.config.fileConfig(os.path.join(os.path.dirname(os.path.abspath(__file__)),'logging.conf'))
+
+exit_gracefully = False
+
+def sighup_handler(signum, frame):
+ global exit_gracefully
+ exit_gracefully = True
+ print "got SIGHUP"
+signal.signal(signal.SIGHUP, sighup_handler)
+
+
+def automate_task(uuid):
+ '''find an item that we can do in a task and automate it
+
+ At the moment we return as soon as we automate a single item successfully
+
+ returns True if we managed to automate anything
+ '''
+
+ api = MagicLink()
+
+ automated_something = False
+
+ for item in api.get_available_items(uuid):
+ item_name = item['name']
+ if automated_something: break
+
+ # Needs to be at least as new as the available list
+ task_metadata = api.get_task_metadata(uuid)
+ for module in auto.get_modules_for_item(item, task_metadata):
+ if automated_something: break
+ # Lock
+ if not api.update_item_state(uuid, item_name, INCOMPLETE, IN_PROGRESS):
+ break # Couldn't lock item. Try the next
+ logging.info('Set %s/%s to IN_PROGRESS' %(uuid,item_name))
+
+ # Attempt to run
+ try:
+ builder = module(item, task_metadata)
+ result = builder.do_magic()
+ if result != None and result.has_key('task_metadata_update'):
+ api.update_task_metadata(uuid, result['task_metadata_update'])
+ logging.debug('updating task metadata for %s with: %s' %(uuid,result['task_metadata_update']))
+ if result != None and result.has_key('item_metadata_update'):
+ api.update_item(uuid, item_name, result['item_metadata_update'])
+ logging.debug('updating item metadata for %s/%s with: %s' %(uuid,item_name,result['item_metadata_update']))
+ except CannotAutomateException:
+ # Unlock and try the next builder
+ logging.debug('Module threw CANNOT_AUTOMATE. Continuing with other modules if there are there', exc_info=1)
+ logging.info('Setting %s/%s to INCOMPLETE' %(uuid,item_name))
+ worked = api.update_item_state(uuid, item_name, IN_PROGRESS, INCOMPLETE)
+ if not worked:
+ logging.error("Couldn't set item state from IN_PROGRESS to FAILED! Task may be in inconsistant state")
+ return False
+ continue
+ except Exception, e:
+ logging.error('Module threw an exception. Setting the item to FAILED', exc_info=1)
+ worked = api.update_item_state(uuid, item_name, IN_PROGRESS, FAILED)
+ if not worked:
+ logging.error("Couldn't set item state from IN_PROGRESS TO FAILED! Task may be in inconsistant state")
+ return False
+ automated_something = True
+ break
+
+ logging.info('Module finished. Setting %s/%s to COMPLETE' %(uuid,item['name']))
+ automated_something = True
+ worked = api.update_item_state(uuid, item_name, IN_PROGRESS, COMPLETE)
+ if not worked:
+ logging.error("Couldn't set item state from IN_PROGRESS TO COMPLETE! Task may be in inconsistant state")
+ return False
+
+ return automated_something
+
+def mudpuppy_main():
+ logging.config.fileConfig(os.path.join(os.path.dirname(os.path.abspath(__file__)),'logging.conf'))
+
+ api = MagicLink()
+
+ # Load in the modules that do the real work
+ auto.load_modules()
+
+ logging.info("Started. Waiting for things to do.")
+ while not exit_gracefully:
+ tasks = api.get_tasks()
+
+ automated_something = False
+ for uuid in tasks:
+ automated = automate_task(uuid)
+ automated_something |= automated
+
+ if not automated_something:
+ # Didn't manage to change the state of anything
+ # so wait at least a second before hammering
+ # the server again
+ time.sleep(1)
+ continue
+
+ logging.debug("Polling make-magic for serverbuilds for us to automate")
+ logging.info("Exiting gracefully after SIGHUP")
+
+
+if __name__ == '__main__':
+ mudpuppy_main()
--- /dev/null
+#!/usr/bin/env python
+
+import unittest2 as unittest
+import config
+import auto
+import sys
+import util
+from modules.base import CannotAutomateException
+
+import magiclink
+import constants
+
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+'''tests for mudpuppy'''
+
+class OrchestraTests(unittest.TestCase):
+ def test_orchestra_client(self):
+ oc = util.OrchestraClient()
+ jobid = oc.submit_job('madworld','one',config.default_orchestra_target)
+ status = oc.wait_for_completion(jobid)
+ self.assertEquals(status['Status'], 'FAIL')
+ jobid = oc.submit_job('helloworld','one',config.default_orchestra_target)
+ status = oc.wait_for_completion(jobid)
+ self.assertEquals(status['Status'], 'OK')
+
+class ModuleTests(unittest.TestCase):
+ def setUp(self):
+ reload(auto)
+ def tearDown(self):
+ reload(auto)
+ def test_1_load_modules(self):
+ config.installed_modules = ('test.TestModule', )
+ modules = auto.load_modules()
+ self.assertEquals(len(modules),1)
+
+ # Shouldn't allow duplication of loads
+ self.assertRaises(auto.ModuleLoadException, auto.load_modules)
+ def test_allocation_modules(self):
+ config.installed_modules = ('test.TestModule', )
+ modules = auto.load_modules()
+
+ _itemdata = {'name': 'FakeItem'}
+ bots = auto.get_modules_for_item(_itemdata,{})
+ self.assertEquals(bots, [])
+
+ _itemdata = {'name': 'TestModuleItem'}
+ bots = auto.get_modules_for_item(_itemdata,{})
+ for botclass in bots:
+ self.assertIsNotNone(botclass)
+ bot = botclass(_itemdata,{})
+ self.assertEquals(bot.module_name, 'TestModule')
+ def test_question_modules(self):
+ config.installed_modules = ('test.TestFakeIP', )
+ modules = auto.load_modules()
+ _item_state = {'name': 'TestFakeIPitem'}
+ bots = auto.get_modules_for_item(_item_state,{})
+ for botclass in bots:
+ bot = botclass(_item_state,{})
+ self.assertIsNotNone(bot)
+ self.assertEquals(bot.module_name, 'TestFakeIP')
+ self.assertEquals(bot.do_magic(), {'task_metadata_update': {'ip': '10.0.0.1'}})
+ def test_module_generated_cannot_automate(self):
+ config.installed_modules = ('test.TestCannotAutomate', )
+ modules = auto.load_modules()
+ _item_state = {'name': 'TestCannotAutomateItem'}
+ bots = auto.get_modules_for_item(_item_state,{})
+ for botclass in bots:
+ self.assertIsNotNone(botclass)
+ bot = botclass(_item_state, {})
+ self.assertRaises(CannotAutomateException, bot.do_magic)
+
+class MagicAPITests(unittest.TestCase):
+ def test_000_get_tasks(self):
+ api = magiclink.MagicAPI()
+ tasks = api.get_tasks()
+ self.assertIsInstance(tasks, list)
+ def test_create_delete_tasks(self):
+ api = magiclink.MagicAPI()
+ taskdata = api.create_task(dict(requirements=[], description="UNITTEST EPHEMERAL TEST TASK", automate=False))
+ uuid = taskdata['metadata']['uuid']
+ tasks = api.get_tasks()
+ self.assertIn(uuid,tasks)
+ api.delete_task(uuid)
+ tasks = api.get_tasks()
+ self.assertNotIn(uuid,tasks)
+
+class MagicLinkTests(unittest.TestCase):
+ def test_state_changes(self):
+ api = magiclink.MagicLink()
+ taskdata = api.create_task(dict(requirements=[], description="UNITTEST EPHEMERAL TEST TASK", automate=False))
+ uuid = taskdata['metadata']['uuid']
+ tasks = api.get_tasks()
+ self.assertIn(uuid,tasks)
+
+ avail = api.get_available_items(uuid)
+ itemname = avail[0]['name']
+
+ self.assertGreater(len(avail), 0)
+ self.assertEqual(api.get_item_state(uuid,itemname),constants.INCOMPLETE)
+ success = api.update_item_state(uuid, itemname, constants.INCOMPLETE, constants.IN_PROGRESS)
+ self.assertTrue(success)
+ self.assertEqual(api.get_item_state(uuid,itemname),constants.IN_PROGRESS)
+
+ success = api.update_item_state(uuid, itemname, constants.IN_PROGRESS, constants.COMPLETE)
+ self.assertTrue(success)
+ self.assertEqual(api.get_item_state(uuid,itemname),constants.COMPLETE)
+ avail = api.get_available_items(uuid)
+ self.assertNotEqual(avail[0]['name'],itemname)
+
+ api.delete_task(uuid)
+ tasks = api.get_tasks()
+ self.assertNotIn(uuid,tasks)
+
+if __name__ == "__main__":
+ unittest.main()
--- /dev/null
+#!/usr/bin/env python
+
+import config
+import socket
+import auto
+import json
+import time
+import functools
+import logging
+
+class TimeoutException(Exception): pass
+class OrchestraError(Exception): pass
+class OrchestraJSONError(OrchestraError): pass
+
+def timecall(func, *args, **argd):
+ start = time.time()
+ try: ret = func(*args,**argd)
+ finally: print time.time() - start
+ return ret
+
+def poll_until_not_none(func, delay=0.1, backoff=1, max_delay=None, timeout=None):
+ '''call a function regularly until it returns not None and then return the result
+
+ the function is called every 'delay' seconds
+ after every unsuccessful call, the delay is multiplied by 'backoff', and limited to max_delay
+
+ if timeout is set, a TimeoutException is raised if that much time has passed without result
+ the function will always be called at least once without delay, regardless of the value of timeout
+ if a timeout is set the function may be called immediately before timing out, even if the delay was longer
+ '''
+ #if timeout is not None:
+ start = time.time()
+ while True:
+ beforecall = time.time()-start
+ ret = func()
+ if ret is not None: return ret
+ elapsed = time.time()-start
+ if timeout is not None and elapsed >= timeout:
+ raise TimeoutException()
+ sleepfor = delay - (elapsed-beforecall)
+ if timeout is not None and elapsed+sleepfor > timeout:
+ sleepfor = timeout-elapsed
+ if sleepfor > 0:
+ time.sleep(sleepfor)
+ delay *= backoff
+ if max_delay:
+ delay = min(delay, max_delay)
+
+class OrchestraClient(object):
+ '''Simple Orchestra Audience
+
+ >>> oc = OrchestraClient()
+ >>> oc.submit_job('helloworld','one','orchestra.player.hostname.example')
+ 200034
+ >>> oc.get_status(200034)
+ {u'Status': u'OK', u'Players': {u'orchestra.player.hostname.example': {u'Status': u'OK', u'Response': {}}}}
+ >>> oc.submit_job('madworld','one','orchestra.player.hostname.example')
+ 200035
+ >>> oc.get_status(200035)
+ {u'Status': u'FAIL', u'Players': {u'orchestra.player.hostname.example': {u'Status': u'HOST_ERROR', u'Response': {}}}}
+ >>> oc.submit_job('echo','one','orchestra.player.hostname.example', {'foo': 12345, 'nyan': 54321, 'cat': 'dog'})
+ 200038
+ >>> oc.wait_for_completion(200038, timeout=30)
+ {u'Status': u'OK', u'Players': {u'orchestra.player.hostname.example': {u'Status': u'OK', u'Response': {u'echo': u'{"IFS": " \\t\\n", "ORC_nyan": "54321", "PWD": "/var/lib/service/player", "ORC_foo": "12345", "PATH": "/usr/bin:/usr/sbin:/bin:/sbin", "ORC_cat": "dog"}'}}}}
+ '''
+ def _get_conductor_socket(self):
+ '''get a socket connected to the conductor
+ current implementation is a UNIX socket'''
+ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ sock.connect(config.orchestra_socket)
+ return sock
+
+ def orchestra_request(self, **argd):
+ '''make a request to the Orchestra conductor
+ pre: all non-positional arguments passed form a well-formed orchestra request
+ '''
+ sock = self._get_conductor_socket()
+ f = sock.makefile()
+ try:
+ json.dump(argd, f)
+ f.flush()
+ data = f.read()
+ response = json.loads(data)
+ except ValueError:
+ if data == '':
+ raise OrchestraError('Conductor burned socket. Probably invalid call')
+ raise OrchestraJSONError(("Couldn't decode as JSON",data))
+ finally:
+ sock.close()
+
+ if response[0] == 'OK':
+ # FIXME: This is not demarshalling any json coming back from individual players!
+ return response[1]
+ else:
+ raise OrchestraError(response[1])
+
+ def submit_job(self, score, scope, target, args={}):
+ '''submit a job to orchestra as per the Audience API
+ Any keys and values in args will be coerced into unicode objects
+ '''
+ if isinstance(target, basestring):
+ target = [target]
+ args = dict((unicode(k),unicode(v)) for k,v in args.items())
+ request = dict(Op='queue', Score=score, Scope=scope, Players=list(target), Params=args)
+ logging.debug('Submitting Orchestra job: %s', repr(request))
+ return self.orchestra_request(**request)
+
+ def get_status(self, jobid):
+ return self.orchestra_request(Op='status', Id=jobid)
+
+ def completed_status_or_none(self, jobid):
+ '''return the job status if completed or failed, or None if still Pending'''
+ status = self.get_status(jobid)
+ return None if status['Status'] == 'PENDING' else status
+
+ def wait_for_completion(self, jobid, timeout=None, delay=0.1, backoff=1.41414, max_poll_delay=2):
+ '''Block until an orchestra job leaves the Pending state
+ returns the status response of the job when it is available
+ raises TimeoutException iff timeout is not None and timeout seconds
+ have passed without the job leaving the Pending state
+
+ Orchestra calls are async, so we just keep polling until we get
+ somewhere
+ '''
+ pf = functools.partial(self.completed_status_or_none, jobid)
+ return poll_until_not_none(pf, delay=delay, backoff=backoff, timeout=timeout, max_delay=max_poll_delay)