initialise repo

diff --git a/INSTALL b/INSTALL
new file mode 100644 (file)
index 0000000..50f54ab
--- /dev/null
+++ b/INSTALL
@@ -0,0 +1,10 @@
+- First, get make-magic up and running. It's available from
+  https://github.com/anchor/make-magic
+
+- Install python2.6 or newer
+
+- Install 'requests' following from PyPi using easy_install or pip. e.g.:
+
+       pip install requests
+
+- Read the README to get started

diff --git a/LICENSE b/LICENSE
new file mode 100644 (file)
index 0000000..7eca0f5
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,24 @@
+Copyright (c) 2011, Anchor Systems Pty Ltd
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of Anchor Systems Pty Ltd nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL ANCHOR SYSTEMS PTY LTD BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

diff --git a/README b/README
new file mode 100644 (file)
index 0000000..69b7468
--- /dev/null
+++ b/README
@@ -0,0 +1,42 @@
+mudpuppy is a Python based automation agent/client for make-magic
+
+It allows you to write independant 'modules' in Python code, to automate
+items in a make-magic task.  When run, mudpuppy will poll a make-magic
+server for tasks that are outstanding, and will complete any items with
+which it has a module for. Once the module has run successfully, mudpuppy
+will tell make-magic that the item has been completed, and then will look
+for more work to do.
+
+There are some example modules in modules/test.py. The simplest case is:
+
+class TestModule(Automatia):
+        '''Test item automation model
+        Successfuly automates a specific item
+        '''
+        can_handle = ('TestModuleItem',)
+
+        def do_magic(self):
+               print "Hello world"
+
+which will automate an item with the name 'TestModuleItem' by printing
+"Hello world" to stdout, and then returning successfully (allowing the
+item to be marked as complete).
+
+In config.py you can define both the API URL of your make-magic server, 
+as well as a list of modules to load. Only modules listed in the
+installed_items config item will by loaded by mudpuppy. 
+
+There is also a helper base class that allows you to easily kick off
+an Orchestra job, wait for it to complete, and optionally update the
+make-magic task state from the key/value pairs returned by Orchestra.
+This makes it very easy to integrate Orchestra with make-magic/mudpuppy
+with very little work. Look in modules/test.py for some examples for
+doing this.  See https://github.com/anchor/orchestra for more information
+about Orchestra.
+
+Although make-magic is designed to be able to be used with almost no python
+knowledge, mudpuppy is specifically designed to automate items with python
+code; Feel free to implement agents such as this in any other language you
+wish.
+
+For more information on make-magic, see https://github.com/anchor/make-magic

diff --git a/auto.py b/auto.py
new file mode 100644 (file)
index 0000000..207841a
--- /dev/null
+++ b/auto.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+import config
+import modules
+import logging
+
+registered_modules = []
+
+class ModuleLoadException(Exception): pass
+
+def import_class(absolutename):
+       name = absolutename.split('.')
+       assert (len(name) >= 2)
+       modbase = ".".join(name[:-1])
+       classname = name[-1]
+       m = __import__(modbase, fromlist=[classname])
+       return getattr(m,classname)
+
+def register_module(module_name):
+       m = import_class(module_name)
+
+       if m in registered_modules:
+               logging.exception("Module already registered: "+str(module_name))
+               raise ModuleLoadException('Module already registered: %s' % (module_name))
+       registered_modules.append(m)
+
+def load_modules():
+       '''load in all modules and register them
+       returns the names of the modules loaded
+       '''
+       for module_name in config.installed_modules:
+               module_name = 'modules.'+module_name
+               logging.debug("Loading module "+module_name)
+               register_module(module_name)
+       return registered_modules
+
+def get_modules_for_item(item_state, task_metadata):
+       '''return the module classes to automate an item or empty list if one isn't found
+       This only returns the classes, and does not instansiate them. They should not be instansiated
+       until they are about to be run, as the constructor may throw an exception that must be
+       caught.
+       '''
+       return [mod for mod in registered_modules if mod.can_automate_item(item_state,task_metadata)]
+
+# Helper modules not for day to day use
+
+def scan_for_mudpuppy_modules(pkgname='modules'):
+       '''returns a list of Automatia subclasses in the modules directory
+       Warning: this actually imports the things
+       '''
+       classes = set()
+       import pkgutil
+       import modules.base
+       for pymodname in [name for _,name,_ in pkgutil.iter_modules([pkgname])]:
+               try:
+                       if pymodname == 'test': continue
+                       pymod = __import__(pkgname+'.'+pymodname)
+                       pymod = getattr(pymod, pymodname)
+                       for key in dir(pymod):
+                               if key[:1] == '_': continue
+                               obj = getattr(pymod, key)
+                               if type(obj) == type and issubclass(obj, modules.base.Automatia):
+                                       if len(obj.can_handle) == 0: continue  ## Abstract class
+                                       classes.add("%s.%s" % (pymodname,key))
+               except Exception, err:
+                       logging.error("! Error importing %s.%s: %s" % (pkgname,str(pymodname).ljust(20), err))
+       return classes
+
+def get_uninstalled_mudpuppy_modules():
+       '''get a list of mudpuppy modules not listed in the config'''
+       allmods = scan_for_mudpuppy_modules()
+       ourmods = set(config.installed_modules)
+       return allmods - ourmods

diff --git a/config.py b/config.py
new file mode 100644 (file)
index 0000000..84a5cd5
--- /dev/null
+++ b/config.py
@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+
+mudpuppy_api_url = 'http://localhost:4554/'
+
+orchestra_socket = '/var/run/conductor.sock'
+default_orchestra_target = 'localhost' # see modules.orchestra
+
+installed_modules = (
+               'test.TestAlwaysAutomates',
+       )
+
+try:
+       from localconfig import *
+except:
+       pass

diff --git a/constants.py b/constants.py
new file mode 100644 (file)
index 0000000..942ee7e
--- /dev/null
+++ b/constants.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+
+'''constants for talking to make-magic'''
+
+# Item states
+#
+# These are the same as core.bits.Item.allowed_states in make-magic
+
+INCOMPLETE = 'INCOMPLETE'
+FAILED = 'FAILED'
+IN_PROGRESS = 'IN_PROGRESS'
+COMPLETE = 'COMPLETE'
+
+
+# Token name used for checking to see if we did a state change
+# This must be the same across all agents talking to a make-magic server
+#
+# This is the same as in make-magic's mclient.py stub shell client
+CHANGE_STATE_TOKEN = '_change_state_token'

diff --git a/logging.conf b/logging.conf
new file mode 100644 (file)
index 0000000..11b32af
--- /dev/null
+++ b/logging.conf
@@ -0,0 +1,29 @@
+[loggers]
+keys=root
+
+[handlers]
+keys=fileHandler,stderrHandler
+
+[formatters]
+keys=simpleFormatter
+
+[logger_root]
+level=DEBUG
+#handlers=fileHandler
+handlers=stderrHandler
+
+[handler_fileHandler]
+class=FileHandler
+level=DEBUG
+formatter=simpleFormatter
+args=('/tmp/mudpuppy.log',)
+
+[handler_stderrHandler]
+class=StreamHandler
+level=DEBUG
+formatter=simpleFormatter
+args=(sys.stderr,)
+
+[formatter_simpleFormatter] 
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s 
+datefmt= 

diff --git a/magiclink.py b/magiclink.py
new file mode 100644 (file)
index 0000000..d1f665d
--- /dev/null
+++ b/magiclink.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+
+'''Access the make-magic HTTP API
+
+contains both make-magic HTTP API access and the policy for dealing with the same
+'''
+import requests
+import json
+import random
+
+from constants import *
+import config
+
+class MagicError(Exception):
+        def __init__(self, error, description):
+                self.response,self.content = response,content
+        def __str__(self):
+               return "MudpuppyError: %s: %s" %(str(self.response), str(self.content))
+
+class MagicAPI(object):
+       '''Talk to the make-magic API over HTTP
+       '''
+       ## HTTP Specific
+       #
+       def json_http_request(self, method, url, decode_response = True, data=None):
+               '''Make a HTTP request, and demarshal the HTTP response
+
+               if POST is given, the data is marshalled to JSON put in the 
+               HTTP request body
+               '''
+               assert url[:4] == 'http'
+               headers = {'User-Agent': 'mudpuppy MagicAPI'}
+               if method == 'POST':
+                       json_data = json.dumps(data)
+                       headers['Content-Type'] = 'application/json'
+                       response = requests.request(method, url, headers=headers, data=json_data)
+               else:
+                       response = requests.request(method, url)
+
+               if response.status_code == None: 
+                       # Didn't manage to get a HTTP response
+                       response.raise_for_status()
+
+               if response.status_code != 200:
+                       # Got an error, but hopefully make-magic gave us more information
+                       try:
+                               jsondata = json.loads(response.content)
+                               raise MagicError(jsondata['error'], jsondata['message'])
+                       except: 
+                               # Couldn't marshal. Raise a less interesting error.
+                               response.raise_for_status()
+
+               # Yay! good response. Try and marshal to JSON and return
+               if decode_response:
+                       return json.loads(response.content)
+               else:
+                       return response.content
+
+       def full_url(self, relpath):
+               '''return the full URL from a relative API path'''
+               if config.mudpuppy_api_url[-1] == '/':
+                       config.mudpuppy_api_url = config.mudpuppy_api_url[:-1]
+               return "%s/%s" % (config.mudpuppy_api_url,relpath)
+
+       def json_http_get(self, relpath, decode_response=True):
+               return self.json_http_request('GET', self.full_url(relpath), decode_response)
+       def json_http_post(self, relpath, data, decode_response=True):
+               return self.json_http_request('POST', self.full_url(relpath), decode_response, data)
+       def json_http_delete(self, relpath, decode_response=True):
+               return self.json_http_request('DELETE', self.full_url(relpath), decode_response)
+
+       ## API to expose
+       #
+       def get_tasks(self):
+               '''return a list of all task UUIDs'''
+               return self.json_http_get('task')
+       def get_task(self,uuid):
+               '''return all data for a task'''
+               return self.json_http_get('task/%s' % (str(uuid),))
+       def get_task_metadata(self,uuid):
+               '''return metadata for a task'''
+               return self.json_http_get('task/%s/metadata' % (str(uuid),))
+       def update_task_metadata(self,uuid, updatedict):
+               '''update metadata for a task'''
+               return self.json_http_post('task/%s/metadata' % (str(uuid),),updatedict)
+       def get_available_items(self,uuid):
+               '''return all items in a task that are currently ready to be automated'''
+               return self.json_http_get('task/%s/available' % (str(uuid),))
+       def update_item(self,uuid,itemname, updatedict):
+               '''update data for a specific item'''
+               return self.json_http_post('task/%s/%s' % (str(uuid),str(itemname)),updatedict)
+       def get_item(self,uuid,itemname):
+               '''return data for a specific item'''
+               return self.json_http_get('task/%s/%s' % (str(uuid),str(itemname)))
+       def get_item_state(self,uuid,itemname):
+               '''return item state for a specific item'''
+               return self.json_http_get('task/%s/%s/state' % (str(uuid),str(itemname)),decode_response=False)
+       def create_task(self,taskdatadict):
+               '''create a new task
+               mudpuppy shouldn't have to do this ever but it is here for
+               completeness and testing.  Unless you want to automatically
+               create tasks from a task, in which case, power to you.
+               '''
+               return self.json_http_post('task/create', taskdatadict)
+       def delete_task(self,uuid):
+               '''delete a task
+               mudpuppy REALLY shoudn't be doing this, but is here for testing
+               and completeness. Unless you want to automatically delete
+               tasks from a task, in which case I'll be hiding behind this rock
+               '''
+               return self.json_http_delete('task/%s' % (str(uuid),))
+
+class MagicLink(MagicAPI):
+       '''wrap the make-magic API while adding some of the logic for dealing with it
+       '''
+       def update_item_state(self, uuid, item, old_state, new_state):
+               '''atomically update the item state, failing if we don't manage to
+
+               returns True iff the state was changed from old_state to new_state and this call made the change
+               '''
+               token = random.randint(1,2**48)
+               item_state_update = {"state": new_state, "onlyif": dict(state=old_state)}
+               item_state_update[CHANGE_STATE_TOKEN] = token
+
+               new_item = self.update_item(uuid, item, item_state_update)
+               return new_item.get('state') == new_state and new_item.get(CHANGE_STATE_TOKEN) == token

diff --git a/modules/__init__.py b/modules/__init__.py
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/modules/base.py b/modules/base.py
new file mode 100644 (file)
index 0000000..3de9285
--- /dev/null
+++ b/modules/base.py
@@ -0,0 +1,35 @@
+class CannotAutomateException(Exception):
+       '''signal from an Automatia that we can't automate this item
+       This is NOT an error. It's just saying that the item cannot
+       be processed by this module.
+
+       This should ONLY be raised if the system state was not changed
+       by the module.
+       '''
+
+class Automatia(object):
+       '''Base class for anything that can automate an item'''
+       can_handle = []
+
+       def __init__(self, item_state, task_metadata):
+               self.item_state = item_state
+               self.task_metadata = task_metadata
+
+       module_name = property(lambda self: self.__class__.__name__)
+
+       @classmethod
+       def can_automate_item(cls, item_state, task_metadata):
+               '''return if this module can automate an item given item state and task metadata
+
+               default implementation to return True iff item name is in
+               cls.can_handle
+
+               This must be a class method as modules can count on
+               only being instantiated if they are going to be used
+               to attempt to automate something
+               '''
+               return item_state['name'] in cls.can_handle
+
+       def do_magic(self):
+               '''automate a build step'''
+               raise NotImplementedError(self.module_name+'.do_magic()')

diff --git a/modules/orchestra.py b/modules/orchestra.py
new file mode 100644 (file)
index 0000000..1542f0f
--- /dev/null
+++ b/modules/orchestra.py
@@ -0,0 +1,88 @@
+from modules.base import Automatia
+import config
+import util
+
+try: import json
+except: import simplejson as json
+
+class OrchestraAutomatia(Automatia):
+       """Abstract helper module to wrap an Orchestra score.
+
+       Use this helper to implement any item that can be directly 
+       mapped to a single Orchestra score.
+
+       Override do_magic() if you want to do more than just wrap a single 
+       score (almost multiple scores should almost certainly be instead made
+       multiple items in make-magic)
+
+       """
+       score_name = None
+       score_scope = 'one'
+       score_target = config.default_orchestra_target
+       score_args = {}
+       score_timeout = None
+
+       def do_magic(self):
+               """Execute an Orchestra score."""
+               if self.score_name == None:
+                       raise NotImplementedError(self.module_name + '.score_name')
+
+               self.execute_score(
+                       self.score_name,
+                       self.score_scope,
+                       self.score_target,
+                       self.score_args,
+                       self.score_timeout)
+
+       @staticmethod
+       def execute_score(name, scope='one',
+                         target=config.default_orchestra_target, args={},
+                         timeout=None):
+               """Execute an Orchestra score, block for completion, then return 
+               the result as a (job_id, result) tuple.
+
+               Will raise ``util.OrchestraError`` on any Orchestra failure.
+
+               """
+               oc = util.OrchestraClient()
+               job_id = oc.submit_job(name, scope, target, args)
+               result = oc.wait_for_completion(job_id, timeout=timeout)
+
+               if result['Status'] == 'OK': 
+                       return (job_id, result, )
+               else: 
+                       raise util.OrchestraError(repr(result))
+
+class OrchestraMetadataAutomatia(OrchestraAutomatia):
+       '''Helper module to wrap an Orchestra score and update task metadata from the result
+       '''
+
+       # Mapping from orchestra response variables to metadata key names
+       orchestra_response_map = None
+
+       def get_variable_from_results(self, results, resultvarname):
+               '''look through orchestra results and find the value keyed on resultvarname
+
+               FIXME: THIS SHOULD BE IN util.py
+               pre: results status is OK and players have completed the score
+               '''
+               # response looks like:
+               # {u'Status': u'OK', u'Players': {u'orchestra.player.hostname.example': {u'Status': u'OK', u'Response': {u'echo': u'{"PATH": "/usr/bin:/usr/sbin:/bin:/sbin", "PWD": "/var/lib/service/player", "IFS": " \\t\\n", "ORC_foo": "bar", "ORC_fish": "heads"}'}}}} 
+               assert results['Status'] == 'OK'
+               for player,result in results['Players'].items():
+                       if result['Status'] != 'OK': continue
+                       for score,playerresponse in result['Response'].items():
+                               playerresponse = json.loads(playerresponse) # UGH. This should NOT be in here  FIXME: Move unrolling to OrchestraClient
+                               if playerresponse.has_key(resultvarname):
+                                       return playerresponse[resultvarname]
+               return KeyError("Variable '"+resultvarname+"' not returned by any Orchestra player")
+
+       def do_magic(self):
+               if self.orchestra_response_map == None: 
+                       raise NotImplementedError(str(self.module_name+'.orchestra_response_map'))
+               results = self.execute_score(self.score_args)
+               metadata_update = {}
+               for response_var,metadata_key in self.orchestra_response_map.items():
+                       task_metadata_update[metadata_key] = self.get_variable_from_results(results, response_var)
+
+               return dict(task_metadata_update=metadata_update)

diff --git a/modules/test.py b/modules/test.py
new file mode 100644 (file)
index 0000000..efed0a0
--- /dev/null
+++ b/modules/test.py
@@ -0,0 +1,65 @@
+'''modules for use in unit testing'''
+
+from modules.base import Automatia, CannotAutomateException
+from modules.orchestra import OrchestraAutomatia, OrchestraMetadataAutomatia
+
+class TestModule(Automatia):
+       '''Test item automation model
+       Successfuly automates a specific item
+       '''
+       can_handle = ('TestModuleItem',)
+
+       def do_magic(self):
+               return  # Returning nothing is fine. The item will still succeed
+
+class TestModuleUnimplemented(Automatia):
+       '''Test item automation model
+
+       Raises NotImplementedError when mudpuppy tries to call do_magic
+       '''
+       can_handle = ('TestUnimplementedModuleItem',)
+
+class TestAlwaysAutomates(Automatia):
+       '''Test item that always automates anything it sees
+       also adds stuff to both the task metadata and the state metadata
+       '''
+       @classmethod
+       def can_automate_item(cls, item_state, task_metadata):
+               return True
+
+       def do_magic(self):
+               itemstuff = { 'guess what': ['TestAlwaysAutomates was here', {'fish': 'are cool'}], 'foo': 'bar' }
+               taskstuff = { 'TestAlwaysAutomates last touched': self.item_state['name'] }
+               return dict( task_metadata_update=taskstuff, item_metadata_update=itemstuff )
+
+class TestFakeIP(Automatia):
+       '''Test item automation model to add task metadata
+       Adds a dummy IP address to the task metadata
+       '''
+       can_handle = ('TestFakeIPitem',)
+
+       def do_magic(self):
+               return dict( task_metadata_update={'ip': '10.0.0.1'} )
+
+class TestCannotAutomate(Automatia):
+       can_handle = ('TestCannotAutomate item',)
+       def do_magic(self):
+               nyan = "NYAN NYAN NYAN NYAN NYAN NYAN NYAN NYAN NYAN"
+               raise CannotAutomateException(nyan)
+
+class TestOrchestraBad(OrchestraAutomatia):
+       can_handle = ('TestOrchestraBadItem',)
+
+class TestOrchestra(OrchestraAutomatia):
+       can_handle = ('TestOrchestraItem',)
+       score_name = 'helloworld'
+
+class TestOrchestraMetadata(OrchestraMetadataAutomatia):
+       can_handle = ('TestOrchestraMetadataItem',)
+
+       # echo score returns score args passed with ORC_ prepended to the key names
+       score_name = 'echo' 
+
+       # get return variables from orchestra and put into into task metadata keys
+       orchestra_response_map = { 'ORC_fish': "fish", "ORC_foo": "taskfoo" }
+       score_args = { 'fish':'heads', 'foo':'bar' }

diff --git a/mudpuppy.py b/mudpuppy.py
new file mode 100755 (executable)
index 0000000..a1f272a
--- /dev/null
+++ b/mudpuppy.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+
+'''A very, very simple mudpuppy implementation
+'''
+
+import os
+import sys
+import auto
+import util
+import config
+import time
+import traceback
+import signal
+
+from constants import *
+from modules.base import CannotAutomateException
+from magiclink import MagicLink
+
+import logging
+import logging.config
+logging.config.fileConfig(os.path.join(os.path.dirname(os.path.abspath(__file__)),'logging.conf'))
+
+exit_gracefully = False
+
+def sighup_handler(signum, frame):
+       global exit_gracefully
+       exit_gracefully = True
+       print "got SIGHUP"
+signal.signal(signal.SIGHUP, sighup_handler)
+
+
+def automate_task(uuid):
+       '''find an item that we can do in a task and automate it
+
+       At the moment we return as soon as we automate a single item successfully
+
+       returns True if we managed to automate anything
+       '''     
+
+       api = MagicLink()
+
+       automated_something = False
+
+       for item in api.get_available_items(uuid):
+               item_name = item['name']
+               if automated_something: break
+
+               # Needs to be at least as new as the available list
+               task_metadata = api.get_task_metadata(uuid)
+               for module in auto.get_modules_for_item(item, task_metadata):
+                       if automated_something: break
+                       # Lock
+                       if not api.update_item_state(uuid, item_name, INCOMPLETE, IN_PROGRESS):
+                               break   # Couldn't lock item. Try the next
+                       logging.info('Set %s/%s to IN_PROGRESS' %(uuid,item_name))
+
+                       # Attempt to run
+                       try:
+                               builder = module(item, task_metadata)
+                               result = builder.do_magic()
+                               if result != None and result.has_key('task_metadata_update'):
+                                       api.update_task_metadata(uuid, result['task_metadata_update'])
+                                       logging.debug('updating task metadata for %s with: %s' %(uuid,result['task_metadata_update']))
+                               if result != None and result.has_key('item_metadata_update'):
+                                       api.update_item(uuid, item_name, result['item_metadata_update'])
+                                       logging.debug('updating item metadata for %s/%s with: %s' %(uuid,item_name,result['item_metadata_update']))
+                       except CannotAutomateException:
+                               # Unlock and try the next builder
+                               logging.debug('Module threw CANNOT_AUTOMATE. Continuing with other modules if there are there', exc_info=1)
+                               logging.info('Setting %s/%s to INCOMPLETE' %(uuid,item_name))
+                               worked = api.update_item_state(uuid, item_name, IN_PROGRESS, INCOMPLETE)
+                               if not worked:
+                                       logging.error("Couldn't set item state from IN_PROGRESS to FAILED! Task may be in inconsistant state")
+                                       return False
+                               continue
+                       except Exception, e:
+                               logging.error('Module threw an exception. Setting the item to FAILED', exc_info=1)
+                               worked = api.update_item_state(uuid, item_name, IN_PROGRESS, FAILED)
+                               if not worked:
+                                       logging.error("Couldn't set item state from IN_PROGRESS TO FAILED! Task may be in inconsistant state")
+                                       return False
+                               automated_something = True
+                               break
+
+                       logging.info('Module finished. Setting %s/%s to COMPLETE' %(uuid,item['name']))
+                       automated_something = True
+                       worked = api.update_item_state(uuid, item_name, IN_PROGRESS, COMPLETE)
+                       if not worked:
+                               logging.error("Couldn't set item state from IN_PROGRESS TO COMPLETE! Task may be in inconsistant state")
+                               return False
+
+       return automated_something
+       
+def mudpuppy_main():
+       logging.config.fileConfig(os.path.join(os.path.dirname(os.path.abspath(__file__)),'logging.conf'))
+
+       api = MagicLink()
+
+       # Load in the modules that do the real work
+       auto.load_modules()
+
+       logging.info("Started. Waiting for things to do.")
+       while not exit_gracefully:
+               tasks = api.get_tasks()
+
+               automated_something = False
+               for uuid in tasks:
+                       automated = automate_task(uuid)         
+                       automated_something |= automated
+
+               if not automated_something:
+                       # Didn't manage to change the state of anything
+                       # so wait at least a second before hammering
+                       # the server again
+                       time.sleep(1)
+                       continue
+
+               logging.debug("Polling make-magic for serverbuilds for us to automate")
+       logging.info("Exiting gracefully after SIGHUP")
+
+
+if __name__ == '__main__':
+       mudpuppy_main()

diff --git a/tests.py b/tests.py
new file mode 100755 (executable)
index 0000000..8d01ba8
--- /dev/null
+++ b/tests.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+
+import unittest2 as unittest
+import config
+import auto
+import sys
+import util
+from modules.base import CannotAutomateException
+
+import magiclink
+import constants
+
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+'''tests for mudpuppy'''
+
+class OrchestraTests(unittest.TestCase):
+       def test_orchestra_client(self):
+               oc = util.OrchestraClient()
+               jobid = oc.submit_job('madworld','one',config.default_orchestra_target)
+               status = oc.wait_for_completion(jobid)
+               self.assertEquals(status['Status'], 'FAIL')
+               jobid = oc.submit_job('helloworld','one',config.default_orchestra_target)
+               status = oc.wait_for_completion(jobid)
+               self.assertEquals(status['Status'], 'OK')
+
+class ModuleTests(unittest.TestCase):
+       def setUp(self):
+               reload(auto)
+       def tearDown(self):
+               reload(auto)
+       def test_1_load_modules(self):
+               config.installed_modules = ('test.TestModule', )
+               modules = auto.load_modules()
+               self.assertEquals(len(modules),1)
+
+               # Shouldn't allow duplication of loads
+               self.assertRaises(auto.ModuleLoadException, auto.load_modules)
+       def test_allocation_modules(self):
+               config.installed_modules = ('test.TestModule', )
+               modules = auto.load_modules()
+               
+               _itemdata = {'name': 'FakeItem'}
+               bots = auto.get_modules_for_item(_itemdata,{})
+               self.assertEquals(bots, [])
+
+               _itemdata = {'name': 'TestModuleItem'}
+               bots = auto.get_modules_for_item(_itemdata,{})
+               for botclass in bots:
+                       self.assertIsNotNone(botclass)
+                       bot = botclass(_itemdata,{})
+                       self.assertEquals(bot.module_name, 'TestModule')
+       def test_question_modules(self):
+               config.installed_modules = ('test.TestFakeIP', )
+               modules = auto.load_modules()
+               _item_state = {'name': 'TestFakeIPitem'}
+               bots = auto.get_modules_for_item(_item_state,{})
+               for botclass in bots:
+                       bot = botclass(_item_state,{})
+                       self.assertIsNotNone(bot)
+                       self.assertEquals(bot.module_name, 'TestFakeIP')
+                       self.assertEquals(bot.do_magic(), {'task_metadata_update': {'ip': '10.0.0.1'}})
+       def test_module_generated_cannot_automate(self):
+               config.installed_modules = ('test.TestCannotAutomate', )
+               modules = auto.load_modules()
+               _item_state = {'name': 'TestCannotAutomateItem'}
+               bots = auto.get_modules_for_item(_item_state,{})
+               for botclass in bots:
+                       self.assertIsNotNone(botclass)
+                       bot = botclass(_item_state, {})
+                       self.assertRaises(CannotAutomateException, bot.do_magic)
+
+class MagicAPITests(unittest.TestCase):
+       def test_000_get_tasks(self):
+               api = magiclink.MagicAPI()
+               tasks = api.get_tasks()
+               self.assertIsInstance(tasks, list)
+       def test_create_delete_tasks(self):
+               api = magiclink.MagicAPI()
+               taskdata = api.create_task(dict(requirements=[], description="UNITTEST EPHEMERAL TEST TASK", automate=False))
+               uuid = taskdata['metadata']['uuid']
+               tasks = api.get_tasks()
+               self.assertIn(uuid,tasks)
+               api.delete_task(uuid)
+               tasks = api.get_tasks()
+               self.assertNotIn(uuid,tasks)
+
+class MagicLinkTests(unittest.TestCase):
+       def test_state_changes(self):
+               api = magiclink.MagicLink()
+               taskdata = api.create_task(dict(requirements=[], description="UNITTEST EPHEMERAL TEST TASK", automate=False))
+               uuid = taskdata['metadata']['uuid']
+               tasks = api.get_tasks()
+               self.assertIn(uuid,tasks)
+
+               avail = api.get_available_items(uuid)
+               itemname = avail[0]['name']
+
+               self.assertGreater(len(avail), 0)
+               self.assertEqual(api.get_item_state(uuid,itemname),constants.INCOMPLETE)
+               success = api.update_item_state(uuid, itemname, constants.INCOMPLETE, constants.IN_PROGRESS)
+               self.assertTrue(success)
+               self.assertEqual(api.get_item_state(uuid,itemname),constants.IN_PROGRESS)
+
+               success = api.update_item_state(uuid, itemname, constants.IN_PROGRESS, constants.COMPLETE)
+               self.assertTrue(success)
+               self.assertEqual(api.get_item_state(uuid,itemname),constants.COMPLETE)
+               avail = api.get_available_items(uuid)
+               self.assertNotEqual(avail[0]['name'],itemname)
+
+               api.delete_task(uuid)
+               tasks = api.get_tasks()
+               self.assertNotIn(uuid,tasks)
+               
+if __name__ == "__main__":
+       unittest.main()

diff --git a/util.py b/util.py
new file mode 100644 (file)
index 0000000..912d6bb
--- /dev/null
+++ b/util.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+
+import config
+import socket
+import auto
+import json
+import time
+import functools
+import logging
+
+class TimeoutException(Exception): pass
+class OrchestraError(Exception): pass
+class OrchestraJSONError(OrchestraError): pass
+
+def timecall(func, *args, **argd):
+       start = time.time()
+       try: ret = func(*args,**argd)
+       finally: print time.time() - start
+       return ret
+
+def poll_until_not_none(func, delay=0.1, backoff=1, max_delay=None, timeout=None):
+       '''call a function regularly until it returns not None and then return the result
+
+       the function is called every 'delay' seconds
+       after every unsuccessful call, the delay is multiplied by 'backoff', and limited to max_delay
+
+       if timeout is set, a TimeoutException is raised if that much time has passed without result
+       the function will always be called at least once without delay, regardless of the value of timeout
+       if a timeout is set the function may be called immediately before timing out, even if the delay was longer
+       '''
+       #if timeout is not None:
+       start = time.time()
+       while True:
+               beforecall = time.time()-start
+               ret = func()
+               if ret is not None: return ret
+               elapsed = time.time()-start
+               if timeout is not None and elapsed >= timeout:
+                       raise TimeoutException()
+               sleepfor = delay - (elapsed-beforecall)
+               if timeout is not None and elapsed+sleepfor > timeout:
+                       sleepfor = timeout-elapsed
+               if sleepfor > 0:
+                       time.sleep(sleepfor)
+               delay *= backoff
+               if max_delay:
+                       delay = min(delay, max_delay)
+
+class OrchestraClient(object):
+       '''Simple Orchestra Audience
+
+       >>> oc = OrchestraClient()
+       >>> oc.submit_job('helloworld','one','orchestra.player.hostname.example')
+       200034
+       >>> oc.get_status(200034)
+       {u'Status': u'OK', u'Players': {u'orchestra.player.hostname.example': {u'Status': u'OK', u'Response': {}}}}
+       >>> oc.submit_job('madworld','one','orchestra.player.hostname.example')
+       200035
+       >>> oc.get_status(200035)
+       {u'Status': u'FAIL', u'Players': {u'orchestra.player.hostname.example': {u'Status': u'HOST_ERROR', u'Response': {}}}}
+       >>> oc.submit_job('echo','one','orchestra.player.hostname.example', {'foo': 12345, 'nyan': 54321, 'cat': 'dog'})
+       200038
+       >>> oc.wait_for_completion(200038, timeout=30)
+       {u'Status': u'OK', u'Players': {u'orchestra.player.hostname.example': {u'Status': u'OK', u'Response': {u'echo': u'{"IFS": " \\t\\n", "ORC_nyan": "54321", "PWD": "/var/lib/service/player", "ORC_foo": "12345", "PATH": "/usr/bin:/usr/sbin:/bin:/sbin", "ORC_cat": "dog"}'}}}}
+       '''
+       def _get_conductor_socket(self):
+               '''get a socket connected to the conductor
+               current implementation is a UNIX socket'''
+               sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+               sock.connect(config.orchestra_socket)
+               return sock
+
+       def orchestra_request(self, **argd):
+               '''make a request to the Orchestra conductor
+               pre: all non-positional arguments passed form a well-formed orchestra request
+               '''
+               sock = self._get_conductor_socket()
+               f = sock.makefile()
+               try:
+                       json.dump(argd, f)
+                       f.flush()
+                       data = f.read()
+                       response = json.loads(data)
+               except ValueError:
+                       if data == '':
+                               raise OrchestraError('Conductor burned socket. Probably invalid call')
+                       raise OrchestraJSONError(("Couldn't decode as JSON",data))
+               finally:
+                       sock.close()
+
+               if response[0] == 'OK':
+                       # FIXME: This is not demarshalling any json coming back from individual players!
+                       return response[1]      
+               else:
+                       raise OrchestraError(response[1])
+
+       def submit_job(self, score, scope, target, args={}):
+               '''submit a job to orchestra as per the Audience API
+               Any keys and values in args will be coerced into unicode objects
+               '''
+               if isinstance(target, basestring):
+                       target = [target]
+               args = dict((unicode(k),unicode(v)) for k,v in args.items())
+               request = dict(Op='queue', Score=score, Scope=scope, Players=list(target), Params=args)
+               logging.debug('Submitting Orchestra job: %s', repr(request))
+               return self.orchestra_request(**request)
+
+       def get_status(self, jobid):
+               return self.orchestra_request(Op='status', Id=jobid)
+
+       def completed_status_or_none(self, jobid):
+               '''return the job status if completed or failed, or None if still Pending'''
+               status = self.get_status(jobid)
+               return None if status['Status'] == 'PENDING' else status
+
+       def wait_for_completion(self, jobid, timeout=None, delay=0.1, backoff=1.41414, max_poll_delay=2):
+               '''Block until an orchestra job leaves the Pending state
+               returns the status response of the job when it is available
+               raises TimeoutException iff timeout is not None and timeout seconds 
+               have passed without the job leaving the Pending state
+
+               Orchestra calls are async, so we just keep polling until we get
+               somewhere
+               '''
+               pf = functools.partial(self.completed_status_or_none, jobid)
+               return poll_until_not_none(pf, delay=delay, backoff=backoff, timeout=timeout, max_delay=max_poll_delay)