CC-5990, CC-5991 - Python cleanup, removed need for /usr/lib/airtime

python_apps/media-monitor/mm2/media/metadata/__init__.py

@@ -0,0 +1 @@
+

python_apps/media-monitor/mm2/media/metadata/definitions.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+import process as md
+import re
+from os.path import normpath
+from ..monitor.pure import format_length, file_md5, is_airtime_recorded, \
+    no_extension_basename
+
+defs_loaded = False
+
+MAX_SIGNED_INT = 2**31-1
+
+def is_defs_loaded():
+    global defs_loaded
+    return defs_loaded
+
+def load_definitions():
+    with md.metadata('MDATA_KEY_DURATION') as t:
+        t.default(u'0.0')
+        t.depends('length')
+        t.translate(lambda k: format_length(k['length']))
+        
+    with md.metadata('MDATA_KEY_CUE_IN') as t:
+        t.default(u'0.0')
+        t.depends('cuein')
+        t.translate(lambda k: format_length(k['cuein']))
+
+    with md.metadata('MDATA_KEY_CUE_OUT') as t:
+        t.default(u'0.0')
+        t.depends('cueout')
+        t.translate(lambda k: format_length(k['cueout']))
+
+    with md.metadata('MDATA_KEY_MIME') as t:
+        t.default(u'')
+        t.depends('mime')
+        # Is this necessary?
+        t.translate(lambda k: k['mime'].replace('audio/vorbis','audio/ogg'))
+
+    with md.metadata('MDATA_KEY_BITRATE') as t:
+        t.default(u'')
+        t.depends('bitrate')
+        t.translate(lambda k: k['bitrate'])
+        t.max_value(MAX_SIGNED_INT)
+
+    with md.metadata('MDATA_KEY_SAMPLERATE') as t:
+        t.default(u'0')
+        t.depends('sample_rate')
+        t.translate(lambda k: k['sample_rate'])
+        t.max_value(MAX_SIGNED_INT)
+
+    with md.metadata('MDATA_KEY_FTYPE') as t:
+        t.depends('ftype') # i don't think this field even exists
+        t.default(u'audioclip')
+        t.translate(lambda k: k['ftype']) # but just in case
+
+    with md.metadata("MDATA_KEY_CREATOR") as t:
+        t.depends("artist")
+        # A little kludge to make sure that we have some value for when we parse
+        # MDATA_KEY_TITLE
+        t.default(u"")
+        t.max_length(512)
+
+    with md.metadata("MDATA_KEY_SOURCE") as t:
+        t.depends("album")
+        t.max_length(512)
+
+    with md.metadata("MDATA_KEY_GENRE") as t:
+        t.depends("genre")
+        t.max_length(64)
+
+    with md.metadata("MDATA_KEY_MOOD") as t:
+        t.depends("mood")
+        t.max_length(64)
+
+    with md.metadata("MDATA_KEY_TRACKNUMBER") as t:
+        t.depends("tracknumber")
+        t.max_value(MAX_SIGNED_INT)
+
+    with md.metadata("MDATA_KEY_BPM") as t:
+        t.depends("bpm")
+        t.max_value(MAX_SIGNED_INT)
+
+    with md.metadata("MDATA_KEY_LABEL") as t:
+        t.depends("organization")
+        t.max_length(512)
+
+    with md.metadata("MDATA_KEY_COMPOSER") as t:
+        t.depends("composer")
+        t.max_length(512)
+
+    with md.metadata("MDATA_KEY_ENCODER") as t:
+        t.depends("encodedby")
+        t.max_length(512)
+
+    with md.metadata("MDATA_KEY_CONDUCTOR") as t:
+        t.depends("conductor")
+        t.max_length(512)
+
+    with md.metadata("MDATA_KEY_YEAR") as t:
+        t.depends("date")
+        t.max_length(16)
+
+    with md.metadata("MDATA_KEY_URL") as t:
+        t.depends("website")
+
+    with md.metadata("MDATA_KEY_ISRC") as t:
+        t.depends("isrc")
+        t.max_length(512)
+
+    with md.metadata("MDATA_KEY_COPYRIGHT") as t:
+        t.depends("copyright")
+        t.max_length(512)
+
+    with md.metadata("MDATA_KEY_ORIGINAL_PATH") as t:
+        t.depends('path')
+        t.translate(lambda k: unicode(normpath(k['path'])))
+
+    with md.metadata("MDATA_KEY_MD5") as t:
+        t.depends('path')
+        t.optional(False)
+        t.translate(lambda k: file_md5(k['path'], max_length=100))
+
+    # owner is handled differently by (by events.py)
+
+    # MDATA_KEY_TITLE is the annoying special case b/c we sometimes read it
+    # from file name
+
+
+    # must handle 3 cases:
+    # 1. regular case (not recorded + title is present)
+    # 2. title is absent (read from file)
+    # 3. recorded file
+    def tr_title(k):
+        #unicode_unknown = u"unknown"
+        new_title = u""
+        if is_airtime_recorded(k) or k['title'] != u"":
+            new_title = k['title']
+        else:
+            default_title = no_extension_basename(k['path'])
+            default_title = re.sub(r'__\d+\.',u'.', default_title)
+
+            # format is: track_number-title-123kbps.mp3
+            m = re.match(".+?-(?P<title>.+)-(\d+kbps|unknown)$", default_title)
+            if m: new_title = m.group('title')
+            else: new_title = re.sub(r'-\d+kbps$', u'', default_title)
+
+        return new_title
+
+    with md.metadata('MDATA_KEY_TITLE') as t:
+        # Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is
+        # defaulted to "" from definitions above
+        t.depends('title','MDATA_KEY_CREATOR','path')
+        t.optional(False)
+        t.translate(tr_title)
+        t.max_length(512)
+
+    with md.metadata('MDATA_KEY_LABEL') as t:
+        t.depends('label')
+        t.max_length(512)

python_apps/media-monitor/mm2/media/metadata/process.py

@@ -0,0 +1,237 @@
+# -*- coding: utf-8 -*-
+from contextlib             import contextmanager
+from ..monitor.pure         import truncate_to_value, truncate_to_length, toposort
+from os.path                import normpath
+from ..monitor.exceptions   import BadSongFile
+from ..monitor.log          import Loggable
+from ..monitor              import pure as mmp
+from collections            import namedtuple
+import mutagen
+import subprocess
+import json
+import logging
+
+class FakeMutagen(dict):
+    """
+    Need this fake mutagen object so that airtime_special functions
+    return a proper default value instead of throwing an exceptions for
+    files that mutagen doesn't recognize
+    """
+    FakeInfo = namedtuple('FakeInfo','length bitrate')
+    def __init__(self,path):
+        self.path = path
+        self.mime = ['audio/wav']
+        self.info = FakeMutagen.FakeInfo(0.0, '')
+        dict.__init__(self)
+    def set_length(self,l):
+        old_bitrate = self.info.bitrate
+        self.info = FakeMutagen.FakeInfo(l, old_bitrate)
+
+
+class MetadataAbsent(Exception):
+    def __init__(self, name): self.name = name
+    def __str__(self): return "Could not obtain element '%s'" % self.name
+
+class MetadataElement(Loggable):
+
+    def __init__(self,name):
+        self.name = name
+        # "Sane" defaults
+        self.__deps          = set()
+        self.__normalizer    = lambda x: x
+        self.__optional      = True
+        self.__default       = None
+        self.__is_normalized = lambda _ : True
+        self.__max_length    = -1
+        self.__max_value     = -1
+        self.__translator    = None
+
+    def max_length(self,l):
+        self.__max_length = l
+
+    def max_value(self,v):
+        self.__max_value = v
+
+    def optional(self, setting):
+        self.__optional = setting
+
+    def is_optional(self):
+        return self.__optional
+
+    def depends(self, *deps):
+        self.__deps = set(deps)
+
+    def dependencies(self):
+        return self.__deps
+
+    def translate(self, f):
+        self.__translator = f
+
+    def is_normalized(self, f):
+        self.__is_normalized = f
+
+    def normalize(self, f):
+        self.__normalizer = f
+
+    def default(self,v):
+        self.__default = v
+
+    def get_default(self):
+        if hasattr(self.__default, '__call__'): return self.__default()
+        else: return self.__default
+
+    def has_default(self):
+        return self.__default is not None
+
+    def path(self):
+        return self.__path
+
+    def __slice_deps(self, d):
+        """
+        returns a dictionary of all the key value pairs in d that are also
+        present in self.__deps
+        """
+        return dict( (k,v) for k,v in d.iteritems() if k in self.__deps)
+
+    def __str__(self):
+        return "%s(%s)" % (self.name, ' '.join(list(self.__deps)))
+
+    def read_value(self, path, original, running={}):
+
+        # If value is present and normalized then we only check if it's
+        # normalized or not. We normalize if it's not normalized already
+
+        if self.name in original:
+            v = original[self.name]
+            if self.__is_normalized(v): return v
+            else: return self.__normalizer(v)
+
+        # We slice out only the dependencies that are required for the metadata
+        # element.
+        dep_slice_orig    = self.__slice_deps(original)
+        dep_slice_running = self.__slice_deps(running)
+        # TODO : remove this later
+        dep_slice_special = self.__slice_deps({'path' : path})
+        # We combine all required dependencies into a single dictionary
+        # that we will pass to the translator
+        full_deps         = dict( dep_slice_orig.items()
+                                + dep_slice_running.items()
+                                + dep_slice_special.items())
+
+        # check if any dependencies are absent
+        # note: there is no point checking the case that len(full_deps) >
+        # len(self.__deps) because we make sure to "slice out" any supefluous
+        # dependencies above.
+        if len(full_deps) != len(self.dependencies()) or \
+            len(self.dependencies()) == 0:
+            # If we have a default value then use that. Otherwise throw an
+            # exception
+            if self.has_default(): return self.get_default()
+            else: raise MetadataAbsent(self.name)
+
+        # We have all dependencies. Now for actual for parsing
+        def def_translate(dep):
+            def wrap(k):
+                e = [ x for x in dep ][0]
+                return k[e]
+            return wrap
+
+        # Only case where we can select a default translator
+        if self.__translator is None:
+            self.translate(def_translate(self.dependencies()))
+            if len(self.dependencies()) > 2: # dependencies include themselves
+                self.logger.info("Ignoring some dependencies in translate %s"
+                                 % self.name)
+                self.logger.info(self.dependencies())
+
+        r = self.__normalizer( self.__translator(full_deps) )
+        if self.__max_length != -1:
+            r = truncate_to_length(r, self.__max_length)
+        if self.__max_value != -1:
+            try: r = truncate_to_value(r, self.__max_value)
+            except ValueError, e: r = ''
+        return r
+
+def normalize_mutagen(path):
+    """
+    Consumes a path and reads the metadata using mutagen. normalizes some of
+    the metadata that isn't read through the mutagen hash
+    """
+    if not mmp.file_playable(path): raise BadSongFile(path)
+    try              : m = mutagen.File(path, easy=True)
+    except Exception : raise BadSongFile(path)
+    if m is None: m = FakeMutagen(path)
+    try:
+        if mmp.extension(path) == 'wav':
+            m.set_length(mmp.read_wave_duration(path))
+    except Exception: raise BadSongFile(path)
+    md = {}
+    for k,v in m.iteritems():
+        if type(v) is list:
+            if len(v) > 0: md[k] = v[0]
+        else: md[k] = v
+    # populate special metadata values
+    md['length']      = getattr(m.info, 'length', 0.0)
+    md['bitrate']     = getattr(m.info, 'bitrate', u'')
+    md['sample_rate'] = getattr(m.info, 'sample_rate', 0)
+    md['mime']        = m.mime[0] if len(m.mime) > 0 else u''
+    md['path']        = normpath(path)
+
+    # silence detect(set default cue in and out)
+    #try:
+        #command = ['silan', '-b', '-f', 'JSON', md['path']]
+        #proc = subprocess.Popen(command, stdout=subprocess.PIPE)
+        #out = proc.communicate()[0].strip('\r\n')
+
+        #info = json.loads(out)
+        #md['cuein'] = info['sound'][0][0]
+        #md['cueout'] = info['sound'][0][1]
+    #except Exception:
+        #self.logger.debug('silan is missing')
+
+    if 'title' not in md: md['title']  = u''
+    return md
+
+
+class OverwriteMetadataElement(Exception):
+    def __init__(self, m): self.m = m
+    def __str__(self): return "Trying to overwrite: %s" % self.m
+
+class MetadataReader(object):
+    def __init__(self):
+        self.clear()
+
+    def register_metadata(self,m):
+        if m in self.__mdata_name_map:
+            raise OverwriteMetadataElement(m)
+        self.__mdata_name_map[m.name] = m
+        d = dict( (name,m.dependencies()) for name,m in
+                self.__mdata_name_map.iteritems() )
+        new_list = list( toposort(d) )
+        self.__metadata = [ self.__mdata_name_map[name] for name in new_list
+                if name in self.__mdata_name_map]
+
+    def clear(self):
+        self.__mdata_name_map = {}
+        self.__metadata       = []
+
+    def read(self, path, muta_hash):
+        normalized_metadata = {}
+        for mdata in self.__metadata:
+            try:
+                normalized_metadata[mdata.name] = mdata.read_value(
+                        path, muta_hash, normalized_metadata)
+            except MetadataAbsent:
+                if not mdata.is_optional(): raise
+        return normalized_metadata
+
+    def read_mutagen(self, path):
+        return self.read(path, normalize_mutagen(path))
+
+global_reader = MetadataReader()
+
+@contextmanager
+def metadata(name):
+    t = MetadataElement(name)
+    yield t
+    global_reader.register_metadata(t)

python_apps/media-monitor/mm2/media/monitor/airtime.py

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+from kombu.messaging    import Exchange, Queue, Consumer
+from kombu.connection   import BrokerConnection
+from kombu.simple       import SimpleQueue
+from os.path            import normpath
+
+import json
+import os
+import copy
+import time
+
+from exceptions         import BadSongFile, InvalidMetadataElement, DirectoryIsNotListed
+from metadata           import Metadata
+from log                import Loggable
+from syncdb             import AirtimeDB
+from bootstrap          import Bootstrapper
+
+from ..saas.thread      import apc, user
+
+class AirtimeNotifier(Loggable):
+    """
+    AirtimeNotifier is responsible for interecepting RabbitMQ messages and
+    feeding them to the event_handler object it was initialized with. The only
+    thing it does to the messages is parse them from json
+    """
+    def __init__(self, cfg, message_receiver):
+        self.cfg = cfg
+        self.handler = message_receiver
+        while not self.init_rabbit_mq():
+            self.logger.error("Error connecting to RabbitMQ Server. Trying again in few seconds")
+            time.sleep(5)
+
+    def init_rabbit_mq(self):
+        try:
+            self.logger.info("Initializing RabbitMQ message consumer...")
+            schedule_exchange = Exchange("airtime-media-monitor", "direct",
+                    durable=True, auto_delete=True)
+            schedule_queue = Queue("media-monitor", exchange=schedule_exchange,
+                    key="filesystem")
+            self.connection = BrokerConnection(self.cfg["rabbitmq"]["host"],
+                    self.cfg["rabbitmq"]["user"], self.cfg["rabbitmq"]["password"],
+                    self.cfg["rabbitmq"]["vhost"])
+            channel  = self.connection.channel()
+
+            self.simple_queue = SimpleQueue(channel, schedule_queue)
+
+            self.logger.info("Initialized RabbitMQ consumer.")
+        except Exception as e:
+            self.logger.info("Failed to initialize RabbitMQ consumer")
+            self.logger.error(e)
+            return False
+
+        return True
+
+
+    def handle_message(self, message):
+        """
+        Messages received from RabbitMQ are handled here. These messages
+        instruct media-monitor of events such as a new directory being watched,
+        file metadata has been changed, or any other changes to the config of
+        media-monitor via the web UI.
+        """
+        self.logger.info("Received md from RabbitMQ: %s" % str(message))
+        m = json.loads(message)
+        # TODO : normalize any other keys that could be used to pass
+        # directories
+        if 'directory' in m: m['directory'] = normpath(m['directory'])
+        self.handler.message(m)
+
+class AirtimeMessageReceiver(Loggable):
+    def __init__(self, cfg, manager):
+        self.dispatch_table = {
+                'md_update'    : self.md_update,
+                'new_watch'    : self.new_watch,
+                'remove_watch' : self.remove_watch,
+                'rescan_watch' : self.rescan_watch,
+                'change_stor'  : self.change_storage,
+                'file_delete'  : self.file_delete,
+        }
+        self.cfg     = cfg
+        self.manager = manager
+
+    def message(self, msg):
+        """
+        This method is called by an AirtimeNotifier instance that
+        consumes the Rabbit MQ events that trigger this. The method
+        return true when the event was executed and false when it wasn't.
+        """
+        msg = copy.deepcopy(msg)
+        if msg['event_type'] in self.dispatch_table:
+            evt = msg['event_type']
+            del msg['event_type']
+            self.logger.info("Handling RabbitMQ message: '%s'" % evt)
+            self._execute_message(evt,msg)
+            return True
+        else:
+            self.logger.info("Received invalid message with 'event_type': '%s'"
+                    % msg['event_type'])
+            self.logger.info("Message details: %s" % str(msg))
+            return False
+    def _execute_message(self,evt,message):
+        self.dispatch_table[evt](message)
+
+    def __request_now_bootstrap(self, directory_id=None, directory=None,
+            all_files=True):
+        if (not directory_id) and (not directory):
+            raise ValueError("You must provide either directory_id or \
+                    directory")
+        sdb = AirtimeDB(apc())
+        if directory            : directory = os.path.normpath(directory)
+        if directory_id == None : directory_id = sdb.to_id(directory)
+        if directory    == None : directory = sdb.to_directory(directory_id)
+        try:
+            bs = Bootstrapper( sdb, self.manager.watch_signal() )
+            bs.flush_watch( directory=directory, last_ran=self.cfg.last_ran() )
+        except Exception as e:
+            self.fatal_exception("Exception bootstrapping: (dir,id)=(%s,%s)" %
+                                 (directory, directory_id), e)
+            raise DirectoryIsNotListed(directory, cause=e)
+
+    def md_update(self, msg):
+        self.logger.info("Updating metadata for: '%s'" %
+                msg['MDATA_KEY_FILEPATH'])
+        md_path = msg['MDATA_KEY_FILEPATH']
+        try: Metadata.write_unsafe(path=md_path, md=msg)
+        except BadSongFile as e:
+            self.logger.info("Cannot find metadata file: '%s'" % e.path)
+        except InvalidMetadataElement as e:
+            self.logger.info("Metadata instance not supported for this file '%s'" \
+                    % e.path)
+            self.logger.info(str(e))
+        except Exception as e:
+            # TODO : add md_path to problem path or something?
+            self.fatal_exception("Unknown error when writing metadata to: '%s'"
+                    % md_path, e)
+
+    def new_watch(self, msg, restart=False):
+        msg['directory'] = normpath(msg['directory'])
+        self.logger.info("Creating watch for directory: '%s'" %
+                msg['directory'])
+        if not os.path.exists(msg['directory']):
+            try: os.makedirs(msg['directory'])
+            except Exception as e:
+                self.fatal_exception("Failed to create watched dir '%s'" %
+                        msg['directory'],e)
+            else:
+                self.logger.info("Created new watch directory: '%s'" %
+                        msg['directory'])
+                self.new_watch(msg)
+        else:
+            self.__request_now_bootstrap( directory=msg['directory'],
+                    all_files=restart)
+            self.manager.add_watch_directory(msg['directory'])
+
+    def remove_watch(self, msg):
+        msg['directory'] = normpath(msg['directory'])
+        self.logger.info("Removing watch from directory: '%s'" %
+                msg['directory'])
+        self.manager.remove_watch_directory(msg['directory'])
+
+    def rescan_watch(self, msg):
+        self.logger.info("Trying to rescan watched directory: '%s'" %
+                         msg['directory'])
+        try:
+            # id is always an integer but in the dictionary the key is always a
+            # string
+            self.__request_now_bootstrap( unicode(msg['id']) )
+        except DirectoryIsNotListed as e:
+            self.fatal_exception("Bad rescan request", e)
+        except Exception as e:
+            self.fatal_exception("Bad rescan request. Unknown error.", e)
+        else:
+            self.logger.info("Successfully re-scanned: '%s'" % msg['directory'])
+
+    def change_storage(self, msg):
+        new_storage_directory = msg['directory']
+        self.manager.change_storage_root(new_storage_directory)
+        for to_bootstrap in [ self.manager.get_recorded_path(),
+                self.manager.get_imported_path() ]:
+            self.__request_now_bootstrap( directory=to_bootstrap )
+
+    def file_delete(self, msg):
+        # Deletes should be requested only from imported folder but we
+        # don't verify that. Security risk perhaps?
+        # we only delete if we are passed the special delete flag that is
+        # necessary with every "delete_file" request
+        if not msg['delete']:
+            self.logger.info("No clippy confirmation, ignoring event. \
+                    Out of curiousity we will print some details.")
+            self.logger.info(msg)
+            return
+        # TODO : Add validation that we are deleting a file that's under our
+        # surveillance. We don't to delete some random system file.
+        if os.path.exists(msg['filepath']):
+            try:
+                self.logger.info("Attempting to delete '%s'" %
+                        msg['filepath'])
+                # We use FileMediator to ignore any paths with
+                # msg['filepath'] so that we do not send a duplicate delete
+                # request that we'd normally get form pyinotify. But right
+                # now event contractor would take care of this sort of
+                # thing anyway so this might not be necessary after all
+                #user().file_mediator.ignore(msg['filepath'])
+                os.unlink(msg['filepath'])
+                # Verify deletion:
+                if not os.path.exists(msg['filepath']):
+                    self.logger.info("Successfully deleted: '%s'" %
+                            msg['filepath'])
+            except Exception as e:
+                self.fatal_exception("Failed to delete '%s'" % msg['filepath'],
+                        e)
+        else: # validation for filepath existence failed
+            self.logger.info("Attempting to delete file '%s' that does not \
+                    exist. Full request:" % msg['filepath'])
+            self.logger.info(msg)

python_apps/media-monitor/mm2/media/monitor/bootstrap.py

@@ -0,0 +1,63 @@
+import os
+from pydispatch       import dispatcher
+from events           import NewFile, DeleteFile, ModifyFile
+from log              import Loggable
+from ..saas.thread    import getsig
+import pure as mmp
+
+class Bootstrapper(Loggable):
+    """
+    Bootstrapper reads all the info in the filesystem flushes organize events
+    and watch events
+    """
+    def __init__(self,db,watch_signal):
+        """
+        db - AirtimeDB object; small layer over api client
+        last_ran - last time the program was ran.
+        watch_signal - the signals should send events for every file on.
+        """
+        self.db           = db
+        self.watch_signal = getsig(watch_signal)
+
+    def flush_all(self, last_ran):
+        """
+        bootstrap every single watched directory. only useful at startup note
+        that because of the way list_directories works we also flush the import
+        directory as well I think
+        """
+        for d in self.db.list_storable_paths(): self.flush_watch(d, last_ran)
+
+    def flush_watch(self, directory, last_ran, all_files=False):
+        """
+        flush a single watch/imported directory. useful when wanting to to
+        rescan, or add a watched/imported directory
+        """
+        songs = set([])
+        added = modded = deleted = 0
+        for f in mmp.walk_supported(directory, clean_empties=False):
+            songs.add(f)
+            # We decide whether to update a file's metadata by checking its
+            # system modification date. If it's above the value self.last_ran
+            # which is passed to us that means media monitor wasn't aware when
+            # this changes occured in the filesystem hence it will send the
+            # correct events to sync the database with the filesystem
+            if os.path.getmtime(f) > last_ran:
+                modded += 1
+                dispatcher.send(signal=self.watch_signal, sender=self,
+                        event=ModifyFile(f))
+        db_songs = set(( song for song in self.db.directory_get_files(directory,
+            all_files)
+            if mmp.sub_path(directory,song) ))
+        # Get all the files that are in the database but in the file
+        # system. These are the files marked for deletions
+        for to_delete in db_songs.difference(songs):
+            dispatcher.send(signal=self.watch_signal, sender=self,
+                            event=DeleteFile(to_delete))
+            deleted += 1
+        for to_add in songs.difference(db_songs):
+            dispatcher.send(signal=self.watch_signal, sender=self,
+                            event=NewFile(to_add))
+            added += 1
+        self.logger.info( "Flushed watch directory (%s). \
+                (added, modified, deleted) = (%d, %d, %d)"
+                % (directory, added, modded, deleted) )

python_apps/media-monitor/mm2/media/monitor/config.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+import os
+import copy
+from configobj import ConfigObj
+
+from exceptions import NoConfigFile, ConfigAccessViolation
+import pure as mmp
+
+class MMConfig(object):
+    def __init__(self, path):
+        if not os.path.exists(path): raise NoConfigFile(path)
+        self.cfg = ConfigObj(path)
+
+    def __getitem__(self, key):
+        """ We always return a copy of the config item to prevent
+        callers from doing any modifications through the returned
+        objects methods """
+        return copy.deepcopy(self.cfg[key])
+
+    def __setitem__(self, key, value):
+        """ We use this method not to allow anybody to mess around with
+        config file any settings made should be done through MMConfig's
+        instance methods """
+        raise ConfigAccessViolation(key)
+
+    def save(self): self.cfg.write()
+
+    def last_ran(self):
+        """ Returns the last time media monitor was ran by looking at
+        the time when the file at 'index_path' was modified """
+        return mmp.last_modified(self.cfg['media-monitor']['index_path'])
+

python_apps/media-monitor/mm2/media/monitor/eventcontractor.py

@@ -0,0 +1,40 @@
+from log    import Loggable
+from events import DeleteFile
+
+class EventContractor(Loggable):
+    def __init__(self):
+        self.store = {}
+    def event_registered(self, evt):
+        """
+        returns true if the event is registered which means that there is
+        another "unpacked" event somewhere out there with the same path
+        """
+        return evt.path in self.store
+
+    def get_old_event(self, evt):
+        """
+        get the previously registered event with the same path as 'evt'
+        """
+        return self.store[ evt.path ]
+
+    def register(self, evt):
+        if self.event_registered(evt):
+            ev_proxy = self.get_old_event(evt)
+            if ev_proxy.same_event(evt):
+                ev_proxy.merge_proxy(evt)
+                return False
+            # delete overrides any other event
+            elif evt.is_event(DeleteFile):
+                ev_proxy.merge_proxy(evt)
+                return False
+            else:
+                ev_proxy.run_hook()
+                ev_proxy.reset_hook()
+
+        self.store[ evt.path ] = evt
+        evt.set_pack_hook( lambda : self.__unregister(evt) )
+        return True
+
+    def __unregister(self, evt):
+        del self.store[evt.path]
+

python_apps/media-monitor/mm2/media/monitor/eventdrainer.py

@@ -0,0 +1,27 @@
+import socket
+import time
+from log     import Loggable
+from toucher import RepeatTimer
+from amqplib.client_0_8.exceptions import AMQPConnectionException
+
+class EventDrainer(Loggable):
+    """
+    Flushes events from RabbitMQ that are sent from airtime every
+    certain amount of time
+    """
+    def __init__(self, airtime_notifier, interval=1):
+        def cb():
+            try:
+                message = airtime_notifier.simple_queue.get(block=True)
+                airtime_notifier.handle_message(message.payload)
+                message.ack()
+            except (IOError, AttributeError, AMQPConnectionException), e:
+                self.logger.error('Exception: %s', e)
+                while not airtime_notifier.init_rabbit_mq():
+                    self.logger.error("Error connecting to RabbitMQ Server. \
+                                      Trying again in few seconds")
+                    time.sleep(5)
+
+        t        = RepeatTimer(interval, cb)
+        t.daemon = True
+        t.start()

python_apps/media-monitor/mm2/media/monitor/events.py

@@ -0,0 +1,261 @@
+# -*- coding: utf-8 -*-
+import os
+import abc
+import re
+import pure         as mmp
+from pure           import LazyProperty
+from metadata       import Metadata
+from log            import Loggable
+from exceptions     import BadSongFile
+from ..saas.thread  import getsig, user
+
+class PathChannel(object):
+    """ Simple struct to hold a 'signal' string and a related 'path'.
+    Basically used as a named tuple """
+    def __init__(self, signal, path):
+        self.signal = getsig(signal)
+        self.path   = path
+
+# TODO : Move this to it's file. Also possible unsingleton and use it as a
+# simple module just like m.m.owners
+class EventRegistry(object):
+    """ This class's main use is to keep track all events with a cookie
+    attribute. This is done mainly because some events must be 'morphed'
+    into other events because we later detect that they are move events
+    instead of delete events. """
+    def __init__(self):
+        self.registry = {}
+    def register(self,evt): self.registry[evt.cookie] = evt
+    def unregister(self,evt): del self.registry[evt.cookie]
+    def registered(self,evt): return evt.cookie in self.registry
+    def matching(self,evt):
+        event = self.registry[evt.cookie]
+        # Want to disallow accessing the same event twice
+        self.unregister(event)
+        return event
+
+class EventProxy(Loggable):
+    """ A container object for instances of BaseEvent (or it's
+    subclasses) used for event contractor """
+    def __init__(self, orig_evt):
+        self.orig_evt   = orig_evt
+        self.evt        = orig_evt
+        self.reset_hook()
+        if hasattr(orig_evt, 'path'): self.path = orig_evt.path
+
+    def set_pack_hook(self, l):
+        self._pack_hook = l
+
+    def reset_hook(self):
+        self._pack_hook = lambda : None
+
+    def run_hook(self):
+        self._pack_hook()
+
+    def safe_pack(self):
+        self.run_hook()
+        # make sure that cleanup hook is never called twice for the same event
+        self.reset_hook()
+        return self.evt.safe_pack()
+
+    def merge_proxy(self, proxy):
+        self.evt = proxy.evt
+
+    def is_event(self, real_event):
+        return isinstance(self.evt, real_event)
+
+    def same_event(self, proxy):
+        return self.evt.__class__ == proxy.evt.__class__
+
+
+class HasMetaData(object):
+    """ Any class that inherits from this class gains the metadata
+    attribute that loads metadata from the class's 'path' attribute.
+    This is done lazily so there is no performance penalty to inheriting
+    from this and subsequent calls to metadata are cached """
+    __metaclass__ = abc.ABCMeta
+    @LazyProperty
+    def metadata(self): return Metadata(self.path)
+
+class BaseEvent(Loggable):
+    __metaclass__ = abc.ABCMeta
+    def __init__(self, raw_event):
+        # TODO : clean up this idiotic hack
+        # we should use keyword constructors instead of this behaviour checking
+        # bs to initialize BaseEvent
+        if hasattr(raw_event,"pathname"):
+            self._raw_event = raw_event
+            self.path = os.path.normpath(raw_event.pathname)
+        else: self.path = raw_event
+        self.owner = user().owner.get_owner(self.path)
+        owner_re = re.search('stor/imported/(?P<owner>\d+)/', self.path)
+        if owner_re: 
+            self.logger.info("matched path: %s" % self.path)
+            self.owner = owner_re.group('owner')
+        else:
+            self.logger.info("did not match path: %s" % self.path)
+        self._pack_hook = lambda: None # no op
+        # into another event
+
+    # TODO : delete this method later
+    def reset_hook(self):
+        """ Resets the hook that is called after an event is packed.
+        Before resetting the hook we execute it to make sure that
+        whatever cleanup operations were queued are executed. """
+        self._pack_hook()
+        self._pack_hook = lambda: None
+
+    def exists(self): return os.path.exists(self.path)
+
+    @LazyProperty
+    def cookie(self): return getattr( self._raw_event, 'cookie', None )
+
+    def __str__(self):
+        return "Event(%s). Path(%s)" % ( self.path, self.__class__.__name__)
+
+    # TODO : delete this method later
+    def add_safe_pack_hook(self,k):
+        """ adds a callable object (function) that will be called after
+        the event has been "safe_packed" """
+        self._pack_hook = k
+
+    def proxify(self):
+        return EventProxy(self)
+
+    # As opposed to unsafe_pack...
+    def safe_pack(self):
+        """ returns exceptions instead of throwing them to be consistent
+        with events that must catch their own BadSongFile exceptions
+        since generate a set of exceptions instead of a single one """
+        try:
+            self._pack_hook()
+            ret = self.pack()
+            # Remove owner of this file only after packing. Otherwise packing
+            # will not serialize the owner correctly into the airtime request
+            user().owner.remove_file_owner(self.path)
+            return ret
+        except BadSongFile as e: return [e]
+        except Exception as e:
+            self.unexpected_exception(e)
+            return[e]
+
+    # nothing to see here, please move along
+    def morph_into(self, evt):
+        self.logger.info("Morphing %s into %s" % ( str(self), str(evt) ) )
+        self._raw_event   = evt._raw_event
+        self.path         = evt.path
+        self.__class__    = evt.__class__
+        # Clean up old hook and transfer the new events hook
+        self.reset_hook()
+        self.add_safe_pack_hook( evt._pack_hook )
+        return self
+
+    def assign_owner(self,req):
+        """ Packs self.owner to req if the owner is valid. I.e. it's not
+        -1. This method is used by various events that would like to
+        pass owner as a parameter. NewFile for example. """
+        if self.owner != -1: req['MDATA_KEY_OWNER_ID'] = self.owner
+
+class FakePyinotify(object):
+    """ sometimes we must create our own pyinotify like objects to
+    instantiate objects from the classes below whenever we want to turn
+    a single event into multiple events """
+    def __init__(self, path): self.pathname = path
+
+class OrganizeFile(BaseEvent, HasMetaData):
+    """ The only kind of event that does support the pack protocol. It's
+    used internally with mediamonitor to move files in the organize
+    directory. """
+    def __init__(self, *args, **kwargs):
+        super(OrganizeFile, self).__init__(*args, **kwargs)
+    def pack(self):
+        raise AttributeError("You can't send organize events to airtime!!!")
+
+class NewFile(BaseEvent, HasMetaData):
+    """ NewFile events are the only events that contain
+    MDATA_KEY_OWNER_ID metadata in them. """
+    def __init__(self, *args, **kwargs):
+        super(NewFile, self).__init__(*args, **kwargs)
+    def pack(self):
+        """ packs turns an event into a media monitor request """
+        req_dict = self.metadata.extract()
+        req_dict['mode'] = u'create'
+        req_dict['is_record'] = self.metadata.is_recorded()
+        self.assign_owner(req_dict)
+        req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path )
+        return [req_dict]
+
+class DeleteFile(BaseEvent):
+    """ DeleteFile event only contains the path to be deleted. No other
+    metadata can be or is included. (This is because this event is fired
+    after the deletion occurs). """
+    def __init__(self, *args, **kwargs):
+        super(DeleteFile, self).__init__(*args, **kwargs)
+    def pack(self):
+        req_dict = {}
+        req_dict['mode'] = u'delete'
+        req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path )
+        return [req_dict]
+
+class MoveFile(BaseEvent, HasMetaData):
+    """ Path argument should be the new path of the file that was moved """
+    def __init__(self, *args, **kwargs):
+        super(MoveFile, self).__init__(*args, **kwargs)
+    def old_path(self):
+        return self._raw_event.src_pathname
+    def pack(self):
+        req_dict                            = {}
+        req_dict['mode']                    = u'moved'
+        req_dict['MDATA_KEY_ORIGINAL_PATH'] = self.old_path()
+        req_dict['MDATA_KEY_FILEPATH']      = unicode( self.path )
+        req_dict['MDATA_KEY_MD5'] = self.metadata.extract()['MDATA_KEY_MD5']
+        return [req_dict]
+
+class ModifyFile(BaseEvent, HasMetaData):
+    def __init__(self, *args, **kwargs):
+        super(ModifyFile, self).__init__(*args, **kwargs)
+    def pack(self):
+        req_dict = self.metadata.extract()
+        req_dict['mode'] = u'modify'
+        # path to directory that is to be removed
+        req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path )
+        return [req_dict]
+
+def map_events(directory, constructor):
+    """ Walks 'directory' and creates an event using 'constructor'.
+    Returns a list of the constructed events. """
+    # -unknown-path should not appear in the path here but more testing
+    # might be necessary
+    for f in mmp.walk_supported(directory, clean_empties=False):
+        try:
+            for e in constructor( FakePyinotify(f) ).pack(): yield e
+        except BadSongFile as e: yield e
+
+class DeleteDir(BaseEvent):
+    """ A DeleteDir event unfolds itself into a list of DeleteFile
+    events for every file in the directory. """
+    def __init__(self, *args, **kwargs):
+        super(DeleteDir, self).__init__(*args, **kwargs)
+    def pack(self):
+        return map_events( self.path, DeleteFile )
+
+class MoveDir(BaseEvent):
+    """ A MoveDir event unfolds itself into a list of MoveFile events
+    for every file in the directory. """
+    def __init__(self, *args, **kwargs):
+        super(MoveDir, self).__init__(*args, **kwargs)
+    def pack(self):
+        return map_events( self.path, MoveFile )
+
+class DeleteDirWatch(BaseEvent):
+    """ Deleting a watched directory is different from deleting any
+    other directory. Hence we must have a separate event to handle this
+    case """
+    def __init__(self, *args, **kwargs):
+        super(DeleteDirWatch, self).__init__(*args, **kwargs)
+    def pack(self):
+        req_dict = {}
+        req_dict['mode']               = u'delete_dir'
+        req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path + "/" )
+        return [req_dict]
+

python_apps/media-monitor/mm2/media/monitor/exceptions.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+class BadSongFile(Exception):
+    def __init__(self, path): self.path = path
+    def __str__(self): return "Can't read %s" % self.path
+
+class NoConfigFile(Exception):
+    def __init__(self, path): self.path = path
+    def __str__(self):
+        return "Path '%s' for config file does not exit" % self.path
+
+class ConfigAccessViolation(Exception):
+    def __init__(self,key): self.key = key
+    def __str__(self): return "You must not access key '%s' directly" % self.key
+
+class FailedToSetLocale(Exception):
+    def __str__(self): return "Failed to set locale"
+
+class FailedToObtainLocale(Exception):
+    def __init__(self, path, cause):
+        self.path = path
+        self.cause = cause
+    def __str__(self): return "Failed to obtain locale from '%s'" % self.path
+
+class CouldNotCreateIndexFile(Exception):
+    """exception whenever index file cannot be created"""
+    def __init__(self, path, cause=None):
+        self.path = path
+        self.cause = cause
+    def __str__(self): return "Failed to create touch file '%s'" % self.path
+
+class DirectoryIsNotListed(Exception):
+    def __init__(self,dir_id,cause=None):
+        self.dir_id = dir_id
+        self.cause = cause
+    def __str__(self):
+        return "%d was not listed as a directory in the database" % self.dir_id
+
+class FailedToCreateDir(Exception):
+    def __init__(self,path, parent):
+        self.path = path
+        self.parent = parent
+    def __str__(self): return "Failed to create path '%s'" % self.path
+
+class NoDirectoryInAirtime(Exception):
+    def __init__(self,path, does_exist):
+        self.path = path
+        self.does_exist = does_exist
+    def __str__(self):
+        return "Directory '%s' does not exist in Airtime.\n \
+                However: %s do exist." % (self.path, self.does_exist)
+
+class InvalidMetadataElement(Exception):
+    def __init__(self, parent, key, path):
+        self.parent = parent
+        self.key    = key
+        self.path   = path
+    def __str__(self):
+        return "InvalidMetadataElement: (key,path) = (%s,%s)" \
+                % (self.key, self.path)
+

python_apps/media-monitor/mm2/media/monitor/handler.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+from pydispatch import dispatcher
+import abc
+
+from log import Loggable
+from ..saas.thread import getsig
+import pure as mmp
+
+# Defines the handle interface
+class Handles(object):
+    __metaclass__ = abc.ABCMeta
+    @abc.abstractmethod
+    def handle(self, sender, event, *args, **kwargs): pass
+
+# TODO : Investigate whether weak reffing in dispatcher.connect could possibly
+# cause a memory leak
+
+class ReportHandler(Handles):
+    """
+    A handler that can also report problem files when things go wrong
+    through the report_problem_file routine
+    """
+    __metaclass__ = abc.ABCMeta
+    def __init__(self, signal, weak=False):
+        self.signal = getsig(signal)
+        self.report_signal = getsig("badfile")
+        def dummy(sender, event): self.handle(sender,event)
+        dispatcher.connect(dummy, signal=self.signal, sender=dispatcher.Any,
+                weak=weak)
+
+    def report_problem_file(self, event, exception=None):
+        dispatcher.send(signal=self.report_signal, sender=self, event=event,
+                exception=exception)
+
+class ProblemFileHandler(Handles, Loggable):
+    """
+    Responsible for answering to events passed through the 'badfile'
+    signal. Moves the problem file passed to the designated directory.
+    """
+    def __init__(self, channel, **kwargs):
+        self.channel = channel
+        self.signal = getsig(self.channel.signal)
+        self.problem_dir = self.channel.path
+        def dummy(sender, event, exception):
+            self.handle(sender, event, exception)
+        dispatcher.connect(dummy, signal=self.signal, sender=dispatcher.Any,
+                weak=False)
+        mmp.create_dir( self.problem_dir )
+        self.logger.info("Initialized problem file handler. Problem dir: '%s'" %
+                self.problem_dir)
+
+    def handle(self, sender, event, exception=None):
+        # TODO : use the exception parameter for something
+        self.logger.info("Received problem file: '%s'. Supposed to move it to \
+                problem dir", event.path)
+        try: mmp.move_to_dir(dir_path=self.problem_dir, file_path=event.path)
+        except Exception as e:
+            self.logger.info("Could not move file: '%s' to problem dir: '%s'" %
+                    (event.path, self.problem_dir))
+            self.logger.info("Exception: %s" % str(e))

python_apps/media-monitor/mm2/media/monitor/listeners.py

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+import pyinotify
+from pydispatch import dispatcher
+from functools import wraps
+
+import pure as mmp
+from pure import IncludeOnly
+from events import OrganizeFile, NewFile, MoveFile, DeleteFile, \
+                                 DeleteDir, MoveDir,\
+                                 DeleteDirWatch
+from log import Loggable
+from ..saas.thread import getsig, user
+# Note: Because of the way classes that inherit from pyinotify.ProcessEvent
+# interact with constructors. you should only instantiate objects from them
+# using keyword arguments. For example:
+# OrganizeListener('watch_signal') <= wrong
+# OrganizeListener(signal='watch_signal') <= right
+
+class FileMediator(Loggable):
+    # TODO : this class is not actually used. remove all references to it 
+    # everywhere (including tests).
+    """ FileMediator is used an intermediate mechanism that filters out
+    certain events. """
+    def __init__(self)        : self.ignored_set = set([]) # for paths only
+    def is_ignored(self,path) : return path in self.ignored_set
+    def ignore(self, path)    : self.ignored_set.add(path)
+    def unignore(self, path)  : self.ignored_set.remove(path)
+
+def mediate_ignored(fn):
+    @wraps(fn)
+    def wrapped(self, event, *args,**kwargs):
+        event.pathname = unicode(event.pathname, "utf-8")
+        if user().file_mediator.is_ignored(event.pathname):
+            user().file_mediator.logger.info("Ignoring: '%s' (once)" % event.pathname)
+            user().file_mediator.unignore(event.pathname)
+        else: return fn(self, event, *args, **kwargs)
+    return wrapped
+
+class BaseListener(object):
+    def __str__(self):
+        return "Listener(%s), Signal(%s)" % \
+                (self.__class__.__name__, self.  signal)
+    def my_init(self, signal): self.signal = getsig(signal)
+
+class OrganizeListener(BaseListener, pyinotify.ProcessEvent, Loggable):
+    def process_IN_CLOSE_WRITE(self, event):
+        #self.logger.info("===> handling: '%s'" % str(event))
+        self.process_to_organize(event)
+
+    def process_IN_MOVED_TO(self, event):
+        #self.logger.info("===> handling: '%s'" % str(event))
+        self.process_to_organize(event)
+
+    def flush_events(self, path):
+        """
+        organize the whole directory at path. (pretty much by doing what
+        handle does to every file
+        """
+        flushed = 0
+        for f in mmp.walk_supported(path, clean_empties=True):
+            self.logger.info("Bootstrapping: File in 'organize' directory: \
+                    '%s'" % f)
+            if not mmp.file_locked(f):
+                dispatcher.send(signal=getsig(self.signal), sender=self,
+                        event=OrganizeFile(f))
+            flushed += 1
+        #self.logger.info("Flushed organized directory with %d files" % flushed)
+
+    @IncludeOnly(mmp.supported_extensions)
+    def process_to_organize(self, event):
+        dispatcher.send(signal=getsig(self.signal), sender=self,
+                event=OrganizeFile(event))
+
+class StoreWatchListener(BaseListener, Loggable, pyinotify.ProcessEvent):
+    def process_IN_CLOSE_WRITE(self, event):
+        self.process_create(event)
+    def process_IN_MOVED_TO(self, event):
+        if user().event_registry.registered(event):
+            # We need this trick because we don't how to "expand" dir events
+            # into file events until we know for sure if we deleted or moved
+            morph = MoveDir(event) if event.dir else MoveFile(event)
+            user().event_registry.matching(event).morph_into(morph)
+        else: self.process_create(event)
+    def process_IN_MOVED_FROM(self, event):
+        # Is either delete dir or delete file
+        evt = self.process_delete(event)
+        # evt can be none whenever event points that a file that would be
+        # ignored by @IncludeOnly
+        if hasattr(event,'cookie') and (evt != None):
+            user().event_registry.register(evt)
+    def process_IN_DELETE(self,event): self.process_delete(event)
+    def process_IN_MOVE_SELF(self, event):
+        if '-unknown-path' in event.pathname:
+            event.pathname = event.pathname.replace('-unknown-path','')
+            self.delete_watch_dir(event)
+
+    def delete_watch_dir(self, event):
+        e = DeleteDirWatch(event)
+        dispatcher.send(signal=getsig('watch_move'), sender=self, event=e)
+        dispatcher.send(signal=getsig(self.signal), sender=self, event=e)
+
+    @mediate_ignored
+    @IncludeOnly(mmp.supported_extensions)
+    def process_create(self, event):
+        evt = NewFile(event)
+        dispatcher.send(signal=getsig(self.signal), sender=self, event=evt)
+        return evt
+
+    @mediate_ignored
+    @IncludeOnly(mmp.supported_extensions)
+    def process_delete(self, event):
+        evt = None
+        if event.dir : evt = DeleteDir(event)
+        else         : evt = DeleteFile(event)
+        dispatcher.send(signal=getsig(self.signal), sender=self, event=evt)
+        return evt
+
+    @mediate_ignored
+    def process_delete_dir(self, event):
+        evt = DeleteDir(event)
+        dispatcher.send(signal=getsig(self.signal), sender=self, event=evt)
+        return evt
+
+    def flush_events(self, path):
+        """
+        walk over path and send a NewFile event for every file in this
+        directory.  Not to be confused with bootstrapping which is a more
+        careful process that involved figuring out what's in the database
+        first.
+        """
+        # Songs is a dictionary where every key is the watched the directory
+        # and the value is a set with all the files in that directory.
+        added = 0
+        for f in mmp.walk_supported(path, clean_empties=False):
+            added += 1
+            dispatcher.send( signal=getsig(self.signal), sender=self, event=NewFile(f) )
+        self.logger.info( "Flushed watch directory. added = %d" % added )
+

python_apps/media-monitor/mm2/media/monitor/log.py

@@ -0,0 +1,36 @@
+import logging
+import abc
+import traceback
+from pure import LazyProperty
+
+appname = 'root'
+
+def setup_logging(log_path):
+    """ Setup logging by writing log to 'log_path' """
+    #logger = logging.getLogger(appname)
+    logging.basicConfig(filename=log_path, level=logging.DEBUG)
+
+def get_logger():
+    """ in case we want to use the common logger from a procedural
+    interface """
+    return logging.getLogger()
+
+class Loggable(object):
+    """ Any class that wants to log can inherit from this class and
+    automatically get a logger attribute that can be used like:
+    self.logger.info(...) etc. """
+    __metaclass__ = abc.ABCMeta
+    @LazyProperty
+    def logger(self): return get_logger()
+
+    def unexpected_exception(self,e):
+        """ Default message for 'unexpected' exceptions """
+        self.fatal_exception("'Unexpected' exception has occured:", e)
+
+    def fatal_exception(self, message, e):
+        """ Prints an exception 'e' with 'message'. Also outputs the
+        traceback. """
+        self.logger.error( message )
+        self.logger.error( str(e) )
+        self.logger.error( traceback.format_exc() )
+

python_apps/media-monitor/mm2/media/monitor/manager.py

@@ -0,0 +1,236 @@
+import pyinotify
+import time
+import os
+from pydispatch     import dispatcher
+
+from os.path        import normpath
+from events         import PathChannel
+from log            import Loggable
+from listeners      import StoreWatchListener, OrganizeListener
+from handler        import ProblemFileHandler
+from organizer      import Organizer
+from ..saas.thread  import InstanceInheritingThread, getsig
+import pure         as mmp
+
+
+class ManagerTimeout(InstanceInheritingThread,Loggable):
+    """ The purpose of this class is to flush the organize directory
+    every 3 secnods. This used to be just a work around for cc-4235
+    but recently became a permanent solution because it's "cheap" and
+    reliable """
+    def __init__(self, manager, interval=1.5):
+        # TODO : interval should be read from config and passed here instead
+        # of just using the hard coded value
+        super(ManagerTimeout, self).__init__()
+        self.manager  = manager
+        self.interval = interval
+    def run(self):
+        while True:
+            time.sleep(self.interval)
+            self.manager.flush_organize()
+
+class Manager(Loggable):
+    # NOTE : this massive class is a source of many problems of mm and
+    # is in dire need of breaking up and refactoring.
+    """ An abstraction over media monitors core pyinotify functions.
+    These include adding watched,store, organize directories, etc.
+    Basically composes over WatchManager from pyinotify """
+    def __init__(self):
+        self.wm = pyinotify.WatchManager()
+        # These two instance variables are assumed to be constant
+        self.watch_channel    = getsig('watch')
+        self.organize_channel = getsig('organize')
+        self.watch_listener   = StoreWatchListener(signal = self.watch_channel)
+        self.__timeout_thread = ManagerTimeout(self)
+        self.__timeout_thread.daemon = True
+        self.__timeout_thread.start()
+        self.organize = {
+            'organize_path'      : None,
+            'imported_path'      : None,
+            'recorded_path'      : None,
+            'problem_files_path' : None,
+            'organizer'          : None,
+            'problem_handler'    : None,
+            'organize_listener'  : OrganizeListener(signal=
+                self.organize_channel),
+        }
+        def dummy(sender, event): self.watch_move( event.path, sender=sender )
+        dispatcher.connect(dummy, signal=getsig('watch_move'), 
+                sender=dispatcher.Any, weak=False)
+        def subwatch_add(sender, directory):
+            self.__add_watch(directory, self.watch_listener)
+        dispatcher.connect(subwatch_add, signal=getsig('add_subwatch'),
+                sender=dispatcher.Any, weak=False)
+        # A private mapping path => watch_descriptor
+        # we use the same dictionary for organize, watch, store wd events.
+        # this is a little hacky because we are unable to have multiple wd's
+        # on the same path.
+        self.__wd_path = {}
+        # The following set isn't really necessary anymore. Should be
+        # removed...
+        self.watched_directories = set([])
+
+    # This is the only event that we are unable to process "normally". I.e.
+    # through dedicated handler objects. Because we must have access to a
+    # manager instance. Hence we must slightly break encapsulation.
+    def watch_move(self, watch_dir, sender=None):
+        """ handle 'watch move' events directly sent from listener """
+        self.logger.info("Watch dir '%s' has been renamed (hence removed)" %
+                watch_dir)
+        self.remove_watch_directory(normpath(watch_dir))
+
+    def watch_signal(self):
+        """ Return the signal string our watch_listener is reading
+        events from """
+        return getsig(self.watch_listener.signal)
+
+    def __remove_watch(self,path):
+        """ Remove path from being watched (first will check if 'path'
+        is watched) """
+        # only delete if dir is actually being watched
+        if path in self.__wd_path:
+            wd = self.__wd_path[path]
+            self.wm.rm_watch(wd, rec=True)
+            del(self.__wd_path[path])
+
+    def __add_watch(self,path,listener):
+        """ Start watching 'path' using 'listener'. First will check if
+        directory is being watched before adding another watch """
+
+        self.logger.info("Attempting to add listener to path '%s'" % path)
+        self.logger.info( 'Listener: %s' % str(listener) )
+
+        if not self.has_watch(path):
+            wd = self.wm.add_watch(path, pyinotify.ALL_EVENTS, rec=True,
+                    auto_add=True, proc_fun=listener)
+            if wd: self.__wd_path[path] = wd.values()[0]
+
+    def __create_organizer(self, target_path, recorded_path):
+        """ creates an organizer at new destination path or modifies the
+        old one """
+        # TODO : find a proper fix for the following hack
+        # We avoid creating new instances of organize because of the way
+        # it interacts with pydispatch. We must be careful to never have
+        # more than one instance of OrganizeListener but this is not so
+        # easy. (The singleton hack in Organizer) doesn't work. This is
+        # the only thing that seems to work.
+        if self.organize['organizer']:
+            o               = self.organize['organizer']
+            o.channel       = self.organize_channel
+            o.target_path   = target_path
+            o.recorded_path = recorded_path
+        else:
+            self.organize['organizer'] = Organizer(channel=
+                    self.organize_channel, target_path=target_path,
+                    recorded_path=recorded_path)
+
+    def get_problem_files_path(self):
+        """ returns the path where problem files should go """
+        return self.organize['problem_files_path']
+
+    def set_problem_files_path(self, new_path):
+        """ Set the path where problem files should go """
+        self.organize['problem_files_path'] = new_path
+        self.organize['problem_handler'] = \
+            ProblemFileHandler( PathChannel(signal=getsig('badfile'),
+                path=new_path) )
+
+    def get_recorded_path(self):
+        """ returns the path of the recorded directory """
+        return self.organize['recorded_path']
+
+    def set_recorded_path(self, new_path):
+        self.__remove_watch(self.organize['recorded_path'])
+        self.organize['recorded_path'] = new_path
+        self.__create_organizer( self.organize['imported_path'], new_path)
+        self.__add_watch(new_path, self.watch_listener)
+
+    def get_organize_path(self):
+        """ returns the current path that is being watched for
+        organization """
+        return self.organize['organize_path']
+
+    def set_organize_path(self, new_path):
+        """ sets the organize path to be new_path. Under the current
+        scheme there is only one organize path but there is no reason
+        why more cannot be supported """
+        # if we are already organizing a particular directory we remove the
+        # watch from it first before organizing another directory
+        self.__remove_watch(self.organize['organize_path'])
+        self.organize['organize_path'] = new_path
+        # the OrganizeListener instance will walk path and dispatch an organize
+        # event for every file in that directory
+        self.organize['organize_listener'].flush_events(new_path)
+        #self.__add_watch(new_path, self.organize['organize_listener'])
+
+    def flush_organize(self):
+        path = self.organize['organize_path']
+        self.organize['organize_listener'].flush_events(path)
+
+    def get_imported_path(self):
+        return self.organize['imported_path']
+
+    def set_imported_path(self,new_path):
+        """ set the directory where organized files go to. """
+        self.__remove_watch(self.organize['imported_path'])
+        self.organize['imported_path'] = new_path
+        self.__create_organizer( new_path, self.organize['recorded_path'])
+        self.__add_watch(new_path, self.watch_listener)
+
+    def change_storage_root(self, store):
+        """ hooks up all the directories for you. Problem, recorded,
+        imported, organize. """
+        store_paths = mmp.expand_storage(store)
+        # First attempt to make sure that all paths exist before adding any
+        # watches
+        for path_type, path in store_paths.iteritems():
+            try: mmp.create_dir(path)
+            except mmp.FailedToCreateDir as e: self.unexpected_exception(e)
+
+        os.chmod(store_paths['organize'], 0775) 
+
+        self.set_problem_files_path(store_paths['problem_files'])
+        self.set_imported_path(store_paths['imported'])
+        self.set_recorded_path(store_paths['recorded'])
+        self.set_organize_path(store_paths['organize'])
+
+    def has_watch(self, path):
+        """ returns true if the path is being watched or not. Any kind
+        of watch: organize, store, watched. """
+        return path in self.__wd_path
+
+    def add_watch_directory(self, new_dir):
+        """ adds a directory to be "watched". "watched" directories are
+        those that are being monitored by media monitor for airtime in
+        this context and not directories pyinotify calls watched """
+        if self.has_watch(new_dir):
+            self.logger.info("Cannot add '%s' to watched directories. It's \
+                    already being watched" % new_dir)
+        else:
+            self.logger.info("Adding watched directory: '%s'" % new_dir)
+            self.__add_watch(new_dir, self.watch_listener)
+
+    def remove_watch_directory(self, watch_dir):
+        """ removes a directory from being "watched". Undoes
+        add_watch_directory """
+        if self.has_watch(watch_dir):
+            self.logger.info("Removing watched directory: '%s'", watch_dir)
+            self.__remove_watch(watch_dir)
+        else:
+            self.logger.info("'%s' is not being watched, hence cannot be \
+                    removed" % watch_dir)
+            self.logger.info("The directories we are watching now are:")
+            self.logger.info( self.__wd_path )
+
+    def loop(self):
+        """ block until we receive pyinotify events """
+        notifier = pyinotify.Notifier(self.wm)
+        notifier.coalesce_events()
+        notifier.loop()
+        #notifier = pyinotify.ThreadedNotifier(self.wm, read_freq=1)
+        #notifier.coalesce_events()
+        #notifier.start()
+        #return notifier
+        #import asyncore
+        #notifier = pyinotify.AsyncNotifier(self.wm)
+        #asyncore.loop()

python_apps/media-monitor/mm2/media/monitor/metadata.py

@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+import mutagen
+import os
+import copy
+from mutagen.easymp4    import EasyMP4KeyError
+from mutagen.easyid3    import EasyID3KeyError
+
+from exceptions         import BadSongFile, InvalidMetadataElement
+from log                import Loggable
+from pure               import format_length
+import pure             as mmp
+
+# emf related stuff
+from ..metadata.process import global_reader
+from ..metadata         import definitions as defs
+defs.load_definitions()
+
+"""
+list of supported easy tags in mutagen version 1.20
+['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry',
+'date', 'performer', 'musicbrainz_albumartistid', 'composer', 'encodedby',
+'tracknumber', 'musicbrainz_albumid', 'album', 'asin', 'musicbrainz_artistid',
+'mood', 'copyright', 'author', 'media', 'length', 'version', 'artistsort',
+'titlesort', 'discsubtitle', 'website', 'musicip_fingerprint', 'conductor',
+'compilation', 'barcode', 'performer:*', 'composersort', 'musicbrainz_discid',
+'musicbrainz_albumtype', 'genre', 'isrc', 'discnumber', 'musicbrainz_trmid',
+'replaygain_*_gain', 'musicip_puid', 'artist', 'title', 'bpm',
+'musicbrainz_trackid', 'arranger', 'albumsort', 'replaygain_*_peak',
+'organization']
+"""
+
+airtime2mutagen = {
+    "MDATA_KEY_TITLE"       : "title",
+    "MDATA_KEY_CREATOR"     : "artist",
+    "MDATA_KEY_SOURCE"      : "album",
+    "MDATA_KEY_GENRE"       : "genre",
+    "MDATA_KEY_MOOD"        : "mood",
+    "MDATA_KEY_TRACKNUMBER" : "tracknumber",
+    "MDATA_KEY_BPM"         : "bpm",
+    "MDATA_KEY_LABEL"       : "label",
+    "MDATA_KEY_COMPOSER"    : "composer",
+    "MDATA_KEY_ENCODER"     : "encodedby",
+    "MDATA_KEY_CONDUCTOR"   : "conductor",
+    "MDATA_KEY_YEAR"        : "date",
+    "MDATA_KEY_URL"         : "website",
+    "MDATA_KEY_ISRC"        : "isrc",
+    "MDATA_KEY_COPYRIGHT"   : "copyright",
+    "MDATA_KEY_CUE_IN"      : "cuein",
+    "MDATA_KEY_CUE_OUT"     : "cueout",
+}
+
+#doesn't make sense for us to write these values to a track's metadata
+mutagen_do_not_write = ["MDATA_KEY_CUE_IN", "MDATA_KEY_CUE_OUT"]
+
+# Some airtime attributes are special because they must use the mutagen object
+# itself to calculate the value that they need. The lambda associated with each
+# key should attempt to extract the corresponding value from the mutagen object
+# itself pass as 'm'. In the case when nothing can be extracted the lambda
+# should return some default value to be assigned anyway or None so that the
+# airtime metadata object will skip the attribute outright.
+
+airtime_special = {
+    "MDATA_KEY_DURATION" :
+        lambda m: format_length(getattr(m.info, u'length', 0.0)),
+    "MDATA_KEY_BITRATE" :
+        lambda m: getattr(m.info, "bitrate", ''),
+    "MDATA_KEY_SAMPLERATE" :
+        lambda m: getattr(m.info, u'sample_rate', 0),
+    "MDATA_KEY_MIME" :
+        lambda m: m.mime[0] if len(m.mime) > 0 else u'',
+}
+mutagen2airtime = dict( (v,k) for k,v in airtime2mutagen.iteritems()
+        if isinstance(v, str) )
+
+truncate_table = {
+        'MDATA_KEY_GENRE'     : 64,
+        'MDATA_KEY_TITLE'     : 512,
+        'MDATA_KEY_CREATOR'   : 512,
+        'MDATA_KEY_SOURCE'    : 512,
+        'MDATA_KEY_MOOD'      : 64,
+        'MDATA_KEY_LABEL'     : 512,
+        'MDATA_KEY_COMPOSER'  : 512,
+        'MDATA_KEY_ENCODER'   : 255,
+        'MDATA_KEY_CONDUCTOR' : 512,
+        'MDATA_KEY_YEAR'      : 16,
+        'MDATA_KEY_URL'       : 512,
+        'MDATA_KEY_ISRC'      : 512,
+        'MDATA_KEY_COPYRIGHT' : 512,
+}
+
+class Metadata(Loggable):
+    # TODO : refactor the way metadata is being handled. Right now things are a
+    # little bit messy. Some of the handling is in m.m.pure while the rest is
+    # here. Also interface is not very consistent
+
+    @staticmethod
+    def fix_title(path):
+        # If we have no title in path we will format it
+        # TODO : this is very hacky so make sure to fix it
+        m = mutagen.File(path, easy=True)
+        if u'title' not in m:
+            new_title = unicode( mmp.no_extension_basename(path) )
+            m[u'title'] = new_title
+            m.save()
+
+    @staticmethod
+    def write_unsafe(path,md):
+        """
+        Writes 'md' metadata into 'path' through mutagen. Converts all
+        dictionary values to strings because mutagen will not write anything
+        else
+        """
+        if not os.path.exists(path): raise BadSongFile(path)
+        song_file = mutagen.File(path, easy=True)
+        exceptions = [] # for bad keys
+        for airtime_k, airtime_v in md.iteritems():
+            if airtime_k in airtime2mutagen and \
+                    airtime_k not in mutagen_do_not_write:
+                # The unicode cast here is mostly for integers that need to be
+                # strings
+                if airtime_v is None: continue
+                try:
+                    song_file[ airtime2mutagen[airtime_k] ] = unicode(airtime_v)
+                except (EasyMP4KeyError, EasyID3KeyError) as e:
+                    exceptions.append(InvalidMetadataElement(e, airtime_k,
+                        path))
+        song_file.save()
+        # bubble dem up so that user knows that something is wrong
+        for e in exceptions: raise e
+
+    def __init__(self, fpath):
+        # Forcing the unicode through
+        try    : fpath = fpath.decode("utf-8")
+        except : pass
+        self.__metadata = global_reader.read_mutagen(fpath)
+
+    def is_recorded(self):
+        """
+        returns true if the file has been created by airtime through recording
+        """
+        return mmp.is_airtime_recorded( self.__metadata )
+
+    def extract(self):
+        """
+        returns a copy of the metadata that was loaded when object was
+        constructed
+        """
+        return copy.deepcopy(self.__metadata)
+
+    def utf8(self):
+        """
+        Returns a unicode aware representation of the data that is compatible
+        with what is spent to airtime
+        """
+        return mmp.convert_dict_value_to_utf8(self.extract())

python_apps/media-monitor/mm2/media/monitor/organizer.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+import pure         as mmp
+from handler        import ReportHandler
+from log            import Loggable
+from exceptions     import BadSongFile
+from events         import OrganizeFile
+from pydispatch     import dispatcher
+from os.path        import dirname
+from ..saas.thread  import getsig, user
+import os.path
+
+class Organizer(ReportHandler,Loggable):
+    """ Organizer is responsible to to listening to OrganizeListener
+    events and committing the appropriate changes to the filesystem.
+    It does not in any interact with WatchSyncer's even when the the
+    WatchSyncer is a "storage directory". The "storage" directory picks
+    up all of its events through pyinotify. (These events are fed to it
+    through StoreWatchListener) """
+
+    # Commented out making this class a singleton because it's just a band aid
+    # for the real issue. The real issue being making multiple Organizer
+    # instances with pydispatch
+
+    #_instance = None
+    #def __new__(cls, channel, target_path, recorded_path):
+        #if cls._instance:
+            #cls._instance.channel       = channel
+            #cls._instance.target_path   = target_path
+            #cls._instance.recorded_path = recorded_path
+        #else:
+            #cls._instance = super(Organizer, cls).__new__( cls, channel,
+                    #target_path, recorded_path)
+        #return cls._instance
+
+    def __init__(self, channel, target_path, recorded_path):
+        self.channel       = channel
+        self.target_path   = target_path
+        self.recorded_path = recorded_path
+        super(Organizer, self).__init__(signal=getsig(self.channel), weak=False)
+
+    def handle(self, sender, event):
+        """ Intercept events where a new file has been added to the
+        organize directory and place it in the correct path (starting
+        with self.target_path) """
+        # Only handle this event type
+        assert isinstance(event, OrganizeFile), \
+            "Organizer can only handle OrganizeFile events.Given '%s'" % event
+        try:
+            # We must select the target_path based on whether file was recorded
+            # by airtime or not.
+            # Do we need to "massage" the path using mmp.organized_path?
+            target_path = self.recorded_path if event.metadata.is_recorded() \
+                                             else self.target_path
+            # nasty hack do this properly
+            owner_id = mmp.owner_id(event.path)
+            if owner_id != -1:
+                target_path = os.path.join(target_path, unicode(owner_id))
+
+            mdata = event.metadata.extract()
+            new_path = mmp.organized_path(event.path, target_path, mdata)
+
+            # See hack in mmp.magic_move
+            def new_dir_watch(d):
+                # TODO : rewrite as return lambda : dispatcher.send(...
+                def cb():
+                    dispatcher.send(signal=getsig("add_subwatch"), sender=self,
+                            directory=d)
+                return cb
+
+            mmp.magic_move(event.path, new_path,
+                    after_dir_make=new_dir_watch(dirname(new_path)))
+
+            # The reason we need to go around saving the owner in this
+            # backwards way is because we are unable to encode the owner id
+            # into the file itself so that the StoreWatchListener listener can
+            # detect it from the file
+            user().owner.add_file_owner(new_path, owner_id )
+
+            self.logger.info('Organized: "%s" into "%s"' %
+                    (event.path, new_path))
+        except BadSongFile as e:
+            self.report_problem_file(event=event, exception=e)
+        # probably general error in mmp.magic.move...
+        except Exception as e:
+            self.unexpected_exception( e )
+            self.report_problem_file(event=event, exception=e)
+

python_apps/media-monitor/mm2/media/monitor/owners.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+from log import Loggable
+
+class Owner(Loggable):
+    def __init__(self):
+        # hash: 'filepath' => owner_id
+        self.owners = {}
+
+    def get_owner(self,f):
+        """ Get the owner id of the file 'f' """
+        o = self.owners[f] if f in self.owners else -1
+        self.logger.info("Received owner for %s. Owner: %s" % (f, o))
+        return o
+
+
+    def add_file_owner(self,f,owner):
+        """ Associate file f with owner. If owner is -1 then do we will not record
+        it because -1 means there is no owner. Returns True if f is being stored
+        after the function. False otherwise.  """
+        if owner == -1: return False
+        if f in self.owners:
+            if owner != self.owners[f]: # check for fishiness
+                self.logger.info("Warning ownership of file '%s' changed from '%d' to '%d'"
+                        % (f, self.owners[f], owner))
+            else: return True
+        self.owners[f] = owner
+        return True
+
+    def has_owner(self,f):
+        """ True if f is owned by somebody. False otherwise. """
+        return f in self.owners
+
+    def remove_file_owner(self,f):
+        """ Try and delete any association made with file f. Returns true if
+        the the association was actually deleted. False otherwise. """
+        if f in self.owners:
+            del self.owners[f]
+            return True
+        else: return False
+

python_apps/media-monitor/mm2/media/monitor/pure.py

@@ -0,0 +1,508 @@
+# -*- coding: utf-8 -*-
+import copy
+from subprocess import Popen, PIPE
+import subprocess
+import os
+import math
+import wave
+import contextlib
+import shutil, pipes
+import re
+import sys
+import stat
+import hashlib
+import locale
+import operator as op
+
+from os.path   import normpath
+from itertools import takewhile
+# you need to import reduce in python 3
+try: from functools import reduce
+except: pass
+from configobj import ConfigObj
+
+from exceptions import FailedToSetLocale, FailedToCreateDir
+
+supported_extensions = [u"mp3", u"ogg", u"oga", u"flac", u"wav",
+                        u'm4a', u'mp4', 'opus']
+
+unicode_unknown = u'unknown'
+
+path_md = ['MDATA_KEY_TITLE', 'MDATA_KEY_CREATOR', 'MDATA_KEY_SOURCE',
+            'MDATA_KEY_TRACKNUMBER', 'MDATA_KEY_BITRATE']
+
+class LazyProperty(object):
+    """
+    meant to be used for lazy evaluation of an object attribute.
+    property should represent non-mutable data, as it replaces itself.
+    """
+    def __init__(self,fget):
+        self.fget      = fget
+        self.func_name = fget.__name__
+
+    def __get__(self,obj,cls):
+        if obj is None: return None
+        value = self.fget(obj)
+        setattr(obj,self.func_name,value)
+        return value
+
+class IncludeOnly(object):
+    """
+    A little decorator to help listeners only be called on extensions
+    they support
+    NOTE: this decorator only works on methods and not functions. Maybe
+    fix this?
+    """
+    def __init__(self, *deco_args):
+        self.exts = set([])
+        for arg in deco_args:
+            if isinstance(arg,str): self.add(arg)
+            elif hasattr(arg, '__iter__'):
+                for x in arg: self.exts.add(x)
+    def __call__(self, func):
+        def _wrap(moi, event, *args, **kwargs):
+            ext = extension(event.pathname)
+            # Checking for emptiness b/c we don't want to skip direcotries
+            if (ext.lower() in self.exts) or event.dir:
+                return func(moi, event, *args, **kwargs)
+        return _wrap
+
+def partition(f, alist):
+    """
+    Partition is very similar to filter except that it also returns the
+    elements for which f return false but in a tuple.
+    >>> partition(lambda x : x > 3, [1,2,3,4,5,6])
+    ([4, 5, 6], [1, 2, 3])
+    """
+    return (filter(f, alist), filter(lambda x: not f(x), alist))
+
+def is_file_supported(path):
+    """
+    Checks if a file's path(filename) extension matches the kind that we
+    support note that this is case insensitive.
+    >>> is_file_supported("test.mp3")
+    True
+    >>> is_file_supported("/bs/path/test.mP3")
+    True
+    >>> is_file_supported("test.txt")
+    False
+    """
+    return extension(path).lower() in supported_extensions
+
+# TODO : In the future we would like a better way to find out whether a show
+# has been recorded
+def is_airtime_recorded(md):
+    """ Takes a metadata dictionary and returns True if it belongs to a
+    file that was recorded by Airtime. """
+    if not 'MDATA_KEY_CREATOR' in md: return False
+    return md['MDATA_KEY_CREATOR'] == u'Airtime Show Recorder'
+
+def read_wave_duration(path):
+    """ Read the length of .wav file (mutagen does not handle this) """
+    with contextlib.closing(wave.open(path,'r')) as f:
+        frames   = f.getnframes()
+        rate     = f.getframerate()
+        duration = frames/float(rate)
+        return duration
+
+def clean_empty_dirs(path):
+    """ walks path and deletes every empty directory it finds """
+    # TODO : test this function
+    if path.endswith('/'): clean_empty_dirs(path[0:-1])
+    else:
+        for root, dirs, _ in os.walk(path, topdown=False):
+            full_paths = ( os.path.join(root, d) for d in dirs )
+            for d in full_paths:
+                if os.path.exists(d):
+                    #Try block avoids a race condition where a file is added AFTER listdir
+                    #is run but before removedirs. (Dir is not empty and removedirs throws
+                    #an exception in that case then.)
+                    try: 
+                        if not os.listdir(d): os.rmdir(d)
+                    except OSError:
+                        pass
+
+def extension(path):
+    """
+    return extension of path, empty string otherwise. Prefer to return empty
+    string instead of None because of bad handling of "maybe" types in python.
+    I.e. interpreter won't enforce None checks on the programmer
+    >>> extension("testing.php")
+    'php'
+    >>> extension("a.b.c.d.php")
+    'php'
+    >>> extension('/no/extension')
+    ''
+    >>> extension('/path/extension.ml')
+    'ml'
+    """
+    ext = path.split(".")
+    if len(ext) < 2: return ""
+    else: return ext[-1]
+
+def no_extension_basename(path):
+    """
+    returns the extensionsless basename of a filepath
+    >>> no_extension_basename("/home/test.mp3")
+    u'test'
+    >>> no_extension_basename("/home/test")
+    u'test'
+    >>> no_extension_basename('blah.ml')
+    u'blah'
+    >>> no_extension_basename('a.b.c.d.mp3')
+    u'a.b.c.d'
+    """
+    base = unicode(os.path.basename(path))
+    if extension(base) == "": return base
+    else: return '.'.join(base.split(".")[0:-1])
+
+def walk_supported(directory, clean_empties=False):
+    """ A small generator wrapper around os.walk to only give us files
+    that support the extensions we are considering. When clean_empties
+    is True we recursively delete empty directories left over in
+    directory after the walk. """
+    if directory is None:
+        return
+
+    for root, dirs, files in os.walk(directory):
+        full_paths = ( os.path.join(root, name) for name in files
+                if is_file_supported(name) )
+        for fp in full_paths: yield fp
+    if clean_empties: clean_empty_dirs(directory)
+
+
+def file_locked(path):
+    #Capture stderr to avoid polluting py-interpreter.log
+    proc = Popen(["lsof", path], stdout=PIPE, stderr=PIPE)
+    out = proc.communicate()[0].strip('\r\n')
+    return bool(out)
+
+def magic_move(old, new, after_dir_make=lambda : None):
+    """ Moves path old to new and constructs the necessary to
+    directories for new along the way """
+    new_dir = os.path.dirname(new)
+    if not os.path.exists(new_dir): os.makedirs(new_dir)
+    # We need this crusty hack because anytime a directory is created we must
+    # re-add it with add_watch otherwise putting files in it will not trigger
+    # pyinotify events
+    after_dir_make()
+    shutil.move(old,new)
+
+def move_to_dir(dir_path,file_path):
+    """ moves a file at file_path into dir_path/basename(filename) """
+    bs = os.path.basename(file_path)
+    magic_move(file_path, os.path.join(dir_path, bs))
+
+def apply_rules_dict(d, rules):
+    """ Consumes a dictionary of rules that maps some keys to lambdas
+    which it applies to every matching element in d and returns a new
+    dictionary with the rules applied. If a rule returns none then it's
+    not applied """
+    new_d = copy.deepcopy(d)
+    for k, rule in rules.iteritems():
+        if k in d:
+            new_val = rule(d[k])
+            if new_val is not None: new_d[k] = new_val
+    return new_d
+
+def default_to_f(dictionary, keys, default, condition):
+    new_d = copy.deepcopy(dictionary)
+    for k in keys:
+        if condition(dictionary=new_d, key=k): new_d[k] = default
+    return new_d
+
+def default_to(dictionary, keys, default):
+    """ Checks if the list of keys 'keys' exists in 'dictionary'. If
+    not then it returns a new dictionary with all those missing keys
+    defaults to 'default' """
+    cnd = lambda dictionary, key: key not in dictionary
+    return default_to_f(dictionary, keys, default, cnd)
+
+def remove_whitespace(dictionary):
+    """ Remove values that empty whitespace in the dictionary """
+    nd = copy.deepcopy(dictionary)
+    bad_keys = []
+    for k,v in nd.iteritems():
+        if hasattr(v,'strip'):
+            stripped = v.strip()
+            # ghetto and maybe unnecessary
+            if stripped == '' or stripped == u'': bad_keys.append(k)
+    for bad_key in bad_keys: del nd[bad_key]
+    return nd
+
+def parse_int(s):
+    # TODO : this function isn't used anywhere yet but it may useful for emf
+    """
+    Tries very hard to get some sort of integer result from s. Defaults to 0
+    when it fails
+    >>> parse_int("123")
+    '123'
+    >>> parse_int("123saf")
+    '123'
+    >>> parse_int("asdf")
+    None
+    """
+    if s.isdigit(): return s
+    else:
+        try   : return str(reduce(op.add, takewhile(lambda x: x.isdigit(), s)))
+        except: return None
+
+
+def organized_path(old_path, root_path, orig_md):
+    """
+    old_path  - path where file is store at the moment <= maybe not necessary?
+    root_path - the parent directory where all organized files go
+    orig_md - original meta data of the file as given by mutagen AFTER being
+    normalized
+    return value: new file path
+    """
+    filepath = None
+    ext = extension(old_path)
+    def default_f(dictionary, key):
+        if key in dictionary: return len(str(dictionary[key])) == 0
+        else: return True
+    # We set some metadata elements to a default "unknown" value because we use
+    # these fields to create a path hence they cannot be empty Here "normal"
+    # means normalized only for organized path
+
+    # MDATA_KEY_BITRATE is in bytes/second i.e. (256000) we want to turn this
+    # into 254kbps
+
+    # Some metadata elements cannot be empty, hence we default them to some
+    # value just so that we can create a correct path
+    normal_md = default_to_f(orig_md, path_md, unicode_unknown, default_f)
+    try:
+        formatted = str(int(normal_md['MDATA_KEY_BITRATE']) / 1000)
+        normal_md['MDATA_KEY_BITRATE'] = formatted + 'kbps'
+    except:
+        normal_md['MDATA_KEY_BITRATE'] = unicode_unknown
+
+    if is_airtime_recorded(normal_md):
+        # normal_md['MDATA_KEY_TITLE'] = 'show_name-yyyy-mm-dd-hh:mm:ss'
+        r = "(?P<show>.+)-(?P<date>\d+-\d+-\d+)-(?P<time>\d+:\d+:\d+)$"
+        title_re    = re.match(r, normal_md['MDATA_KEY_TITLE'])
+        show_name   = title_re.group('show')
+        #date        = title_re.group('date')
+        yyyy, mm, dd = normal_md['MDATA_KEY_YEAR'].split('-',2)
+        fname_base  = '%s-%s-%s.%s' % \
+                (title_re.group('time'), show_name,
+                        normal_md['MDATA_KEY_BITRATE'], ext)
+        filepath = os.path.join(root_path, yyyy, mm, dd, fname_base)
+    elif len(normal_md['MDATA_KEY_TRACKNUMBER']) == 0:
+        fname = u'%s-%s.%s' % (normal_md['MDATA_KEY_TITLE'],
+                normal_md['MDATA_KEY_BITRATE'], ext)
+        path = os.path.join(root_path, normal_md['MDATA_KEY_CREATOR'],
+                            normal_md['MDATA_KEY_SOURCE'] )
+        filepath = os.path.join(path, fname)
+    else: # The "normal" case
+        fname = u'%s-%s-%s.%s' % (normal_md['MDATA_KEY_TRACKNUMBER'],
+                                  normal_md['MDATA_KEY_TITLE'],
+                                  normal_md['MDATA_KEY_BITRATE'], ext)
+        path = os.path.join(root_path, normal_md['MDATA_KEY_CREATOR'],
+                            normal_md['MDATA_KEY_SOURCE'])
+        filepath = os.path.join(path, fname)
+    return filepath
+
+# TODO : Get rid of this function and every one of its uses. We no longer use
+# the md5 signature of a song for anything
+def file_md5(path,max_length=100):
+    """ Get md5 of file path (if it exists). Use only max_length
+    characters to save time and memory. Pass max_length=-1 to read the
+    whole file (like in mm1) """
+    if os.path.exists(path):
+        with open(path, 'rb') as f:
+            m = hashlib.md5()
+            # If a file is shorter than "max_length" python will just return
+            # whatever it was able to read which is acceptable behaviour
+            m.update(f.read(max_length))
+            return m.hexdigest()
+    else: raise ValueError("'%s' must exist to find its md5" % path)
+
+def encode_to(obj, encoding='utf-8'):
+    # TODO : add documentation + unit tests for this function
+    if isinstance(obj, unicode): obj = obj.encode(encoding)
+    return obj
+
+def convert_dict_value_to_utf8(md):
+    """ formats a dictionary to send as a request to api client """
+    return dict([(item[0], encode_to(item[1], "utf-8")) for item in md.items()])
+
+def get_system_locale(locale_path='/etc/default/locale'):
+    """ Returns the configuration object for the system's default
+    locale. Normally requires root access. """
+    if os.path.exists(locale_path):
+        try:
+            config = ConfigObj(locale_path)
+            return config
+        except Exception as e: raise FailedToSetLocale(locale_path,cause=e)
+    else: raise ValueError("locale path '%s' does not exist. \
+            permissions issue?" % locale_path)
+
+def configure_locale(config):
+    """ sets the locale according to the system's locale. """
+    current_locale = locale.getlocale()
+    if current_locale[1] is None:
+        default_locale = locale.getdefaultlocale()
+        if default_locale[1] is None:
+            lang = config.get('LANG')
+            new_locale = lang
+        else: new_locale = default_locale
+        locale.setlocale(locale.LC_ALL, new_locale)
+    reload(sys)
+    sys.setdefaultencoding("UTF-8")
+    current_locale_encoding = locale.getlocale()[1].lower()
+    if current_locale_encoding not in ['utf-8', 'utf8']:
+        raise FailedToSetLocale()
+
+def fondle(path,times=None):
+    # TODO : write unit tests for this
+    """ touch a file to change the last modified date. Beware of calling
+    this function on the same file from multiple threads. """
+    with file(path, 'a'): os.utime(path, times)
+
+def last_modified(path):
+    """ return the time of the last time mm2 was ran. path refers to the
+    index file whose date modified attribute contains this information.
+    In the case when the file does not exist we set this time 0 so that
+    any files on the filesystem were modified after it """
+    if os.path.exists(path): return os.path.getmtime(path)
+    else: return 0
+
+def expand_storage(store):
+    """ A storage directory usually consists of 4 different
+    subdirectories. This function returns their paths """
+    store = os.path.normpath(store)
+    return {
+        'organize'      : os.path.join(store, 'organize'),
+        'recorded'      : os.path.join(store, 'recorded'),
+        'problem_files' : os.path.join(store, 'problem_files'),
+        'imported'      : os.path.join(store, 'imported'),
+    }
+
+def create_dir(path):
+    """ will try and make sure that path exists at all costs. raises an
+    exception if it fails at this task. """
+    if not os.path.exists(path):
+        try                   : os.makedirs(path)
+        except Exception as e : raise FailedToCreateDir(path, e)
+        else: # if no error occurs we still need to check that dir exists
+            if not os.path.exists: raise FailedToCreateDir(path)
+
+def sub_path(directory,f):
+    """
+    returns true if 'f' is in the tree of files under directory.
+    NOTE: does not look at any symlinks or anything like that, just looks at
+    the paths.
+    """
+    normalized = normpath(directory)
+    common     = os.path.commonprefix([ normalized, normpath(f) ])
+    return common == normalized
+
+def owner_id(original_path):
+    """ Given 'original_path' return the file name of the of
+    'identifier' file. return the id that is contained in it. If no file
+    is found or nothing is read then -1 is returned. File is deleted
+    after the number has been read """
+    fname = "%s.identifier" % original_path
+    owner_id = -1
+    try:
+        f = open(fname)
+        for line in f:
+            owner_id = int(line)
+            break
+        f.close()
+    except Exception: pass
+    else:
+        try: os.unlink(fname)
+        except Exception: raise
+    return owner_id
+
+def file_playable(pathname):
+    """ Returns True if 'pathname' is playable by liquidsoap. False
+    otherwise. """
+
+    #currently disabled because this confuses inotify....
+    return True
+    #remove all write permissions. This is due to stupid taglib library bug
+    #where all files are opened in write mode. The only way around this is to
+    #modify the file permissions
+    os.chmod(pathname, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
+
+    # when there is an single apostrophe inside of a string quoted by
+    # apostrophes, we can only escape it by replace that apostrophe with
+    # '\''. This breaks the string into two, and inserts an escaped
+    # single quote in between them.
+    command = ("airtime-liquidsoap -c 'output.dummy" + \
+        "(audio_to_stereo(single(\"%s\")))' > /dev/null 2>&1") % \
+        pathname.replace("'", "'\\''")
+
+    return_code = subprocess.call(command, shell=True)
+
+    #change/restore permissions to acceptable
+    os.chmod(pathname, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | \
+             stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
+    return (return_code == 0)
+
+def toposort(data):
+    """
+    Topological sort on 'data' where 'data' is of the form:
+        data = [
+            'one'   : set('two','three'),
+            'two'   : set('three'),
+            'three' : set()
+        ]
+    """
+    for k, v in data.items():
+        v.discard(k) # Ignore self dependencies
+    extra_items_in_deps = reduce(set.union, data.values()) - set(data.keys())
+    data.update(dict((item,set()) for item in extra_items_in_deps))
+    while True:
+        ordered = set(item for item,dep in data.items() if not dep)
+        if not ordered: break
+        for e in sorted(ordered): yield e
+        data = dict((item,(dep - ordered)) for item,dep in data.items()
+                if item not in ordered)
+    assert not data, "A cyclic dependency exists amongst %r" % data
+
+def truncate_to_length(item, length):
+    """ Truncates 'item' to 'length' """
+    if isinstance(item, int): item = str(item)
+    if isinstance(item, basestring):
+        if len(item) > length: return item[0:length]
+        else: return item
+
+def truncate_to_value(item, value):
+    """ Truncates 'item' to 'value' """
+    if isinstance(item, basestring): item = int(item)
+    if isinstance(item, int):
+        item = abs(item)
+        if item > value: item = value
+    return str(item)
+
+def format_length(mutagen_length):
+    if convert_format(mutagen_length):
+        """ Convert mutagen length to airtime length """
+        t = float(mutagen_length)
+        h = int(math.floor(t / 3600))
+        t = t % 3600
+        m = int(math.floor(t / 60))
+        s = t % 60
+        # will be ss.uuu
+        s = str('{0:f}'.format(s))
+        seconds = s.split(".")
+        s = seconds[0]
+        # have a maximum of 6 subseconds.
+        if len(seconds[1]) >= 6: ss = seconds[1][0:6]
+        else: ss = seconds[1][0:]
+        return "%s:%s:%s.%s" % (h, m, s, ss)
+
+def convert_format(value):
+    regCompiled = re.compile("^[0-9][0-9]:[0-9][0-9]:[0-9][0-9](\.\d+)?$")
+    if re.search(regCompiled, str(value)) is None:
+        return True
+    else:
+        return False
+
+if __name__ == '__main__':
+    import doctest
+    doctest.testmod()

python_apps/media-monitor/mm2/media/monitor/request.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+
+from exceptions import BadSongFile
+from log        import Loggable
+from ..saas.thread import apc, InstanceInheritingThread
+
+class ThreadedRequestSync(InstanceInheritingThread, Loggable):
+    def __init__(self, rs):
+        super(ThreadedRequestSync, self).__init__()
+        self.rs = rs
+        self.daemon = True
+        self.start()
+
+    def run(self):
+        self.rs.run_request()
+
+class RequestSync(Loggable):
+    """ This class is responsible for making the api call to send a
+    request to airtime. In the process it packs the requests and retries
+    for some number of times """
+    @classmethod
+    def create_with_api_client(cls, watcher, requests):
+        apiclient = apc()
+        self = cls(watcher, requests, apiclient)
+        return self
+
+    def __init__(self, watcher, requests, apiclient):
+        self.watcher   = watcher
+        self.requests  = requests
+        self.apiclient = apiclient
+
+    def run_request(self):
+        self.logger.info("Attempting request with %d items." %
+                len(self.requests))
+        packed_requests = []
+        for request_event in self.requests:
+            try:
+                for request in request_event.safe_pack():
+                    if isinstance(request, BadSongFile):
+                        self.logger.info("Bad song file: '%s'" % request.path)
+                    else: packed_requests.append(request)
+            except Exception as e:
+                self.unexpected_exception( e )
+                if hasattr(request_event, 'path'):
+                    self.logger.info("Possibly related to path: '%s'" %
+                            request_event.path)
+        try: self.apiclient.send_media_monitor_requests( packed_requests )
+        # most likely we did not get json response as we expected
+        except ValueError:
+            self.logger.info("ApiController.php probably crashed, we \
+                    diagnose this from the fact that it did not return \
+                    valid json")
+        except Exception as e: self.unexpected_exception(e)
+        else: self.logger.info("Request was successful")
+        self.watcher.flag_done() # poor man's condition variable
+

python_apps/media-monitor/mm2/media/monitor/syncdb.py

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+import os
+from log            import Loggable
+from exceptions     import NoDirectoryInAirtime
+from ..saas.thread  import user
+from os.path        import normpath, join
+import pure         as mmp
+
+class AirtimeDB(Loggable):
+    def __init__(self, apc, reload_now=True):
+        self.apc = apc
+        if reload_now: self.reload_directories()
+
+    def reload_directories(self):
+        """ this is the 'real' constructor, should be called if you ever
+        want the class reinitialized. there's not much point to doing
+        it yourself however, you should just create a new AirtimeDB
+        instance. """
+
+        saas = user().root_path
+
+        try:
+            # dirs_setup is a dict with keys:
+            # u'watched_dirs' and u'stor' which point to lists of corresponding
+            # dirs
+            dirs_setup = self.apc.setup_media_monitor()
+            dirs_setup[u'stor'] = normpath( join(saas, dirs_setup[u'stor'] ) )
+            dirs_setup[u'watched_dirs'] = map(lambda p: normpath(join(saas,p)),
+                dirs_setup[u'watched_dirs'])
+            dirs_with_id = dict([ (k,normpath(v)) for k,v in
+                self.apc.list_all_watched_dirs()['dirs'].iteritems() ])
+
+            self.id_to_dir = dirs_with_id
+            self.dir_to_id = dict([ (v,k) for k,v in dirs_with_id.iteritems() ])
+
+            self.base_storage = dirs_setup[u'stor']
+            self.storage_paths = mmp.expand_storage( self.base_storage )
+            self.base_id = self.dir_to_id[self.base_storage]
+
+            # hack to get around annoying schema of airtime db
+            self.dir_to_id[ self.recorded_path() ] = self.base_id
+            self.dir_to_id[ self.import_path() ] = self.base_id
+
+            # We don't know from the x_to_y dict which directory is watched or
+            # store...
+            self.watched_directories = set([ os.path.normpath(p) for p in
+                dirs_setup[u'watched_dirs'] ])
+        except Exception, e:
+            self.logger.info(str(e))
+
+
+    def to_id(self, directory):
+        """ directory path -> id """
+        return self.dir_to_id[ directory ]
+
+    def to_directory(self, dir_id):
+        """ id -> directory path """
+        return self.id_to_dir[ dir_id ]
+
+    def storage_path(self)  : return self.base_storage
+    def organize_path(self) : return self.storage_paths['organize']
+    def problem_path(self)  : return self.storage_paths['problem_files']
+    def import_path(self)   : return self.storage_paths['imported']
+    def recorded_path(self) : return self.storage_paths['recorded']
+
+    def list_watched(self):
+        """ returns all watched directories as a list """
+        return list(self.watched_directories)
+
+    def list_storable_paths(self):
+        """ returns a list of all the watched directories in the
+        datatabase. (Includes the imported directory and the recorded
+        directory) """
+        l = self.list_watched()
+        l.append(self.import_path())
+        l.append(self.recorded_path())
+        return l
+
+    def dir_id_get_files(self, dir_id, all_files=True):
+        """ Get all files in a directory with id dir_id """
+        base_dir = self.id_to_dir[ dir_id ]
+        return set(( join(base_dir,p) for p in
+            self.apc.list_all_db_files( dir_id, all_files ) ))
+
+    def directory_get_files(self, directory, all_files=True):
+        """ returns all the files(recursively) in a directory. a
+        directory is an "actual" directory path instead of its id. This
+        is super hacky because you create one request for the recorded
+        directory and one for the imported directory even though they're
+        the same dir in the database so you get files for both dirs in 1
+        request... """
+        normal_dir = os.path.normpath(unicode(directory))
+        if normal_dir not in self.dir_to_id:
+            raise NoDirectoryInAirtime( normal_dir, self.dir_to_id )
+        all_files = self.dir_id_get_files( self.dir_to_id[normal_dir],
+                all_files )
+        if normal_dir == self.recorded_path():
+            all_files = [ p for p in all_files if
+                    mmp.sub_path( self.recorded_path(), p ) ]
+        elif normal_dir == self.import_path():
+            all_files = [ p for p in all_files if
+                    mmp.sub_path( self.import_path(), p ) ]
+        elif normal_dir == self.storage_path():
+            self.logger.info("Warning, you're getting all files in '%s' which \
+                    includes imported + record" % normal_dir)
+        return set(all_files)
+
+

python_apps/media-monitor/mm2/media/monitor/toucher.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+import pure         as mmp
+import os
+from log            import Loggable
+from exceptions     import CouldNotCreateIndexFile
+from ..saas.thread  import InstanceInheritingThread
+
+class Toucher(Loggable):
+    """
+    Class responsible for touching a file at a certain path when called
+    """
+    def __init__(self,path):
+        self.path = path
+    def __call__(self):
+        try: mmp.fondle(self.path)
+        except Exception as e:
+            self.logger.info("Failed to touch file: '%s'. Logging exception." %
+                    self.path)
+            self.logger.info(str(e))
+
+import time
+
+class RepeatTimer(InstanceInheritingThread):
+    def __init__(self, interval, callable, *args, **kwargs):
+        super(RepeatTimer, self).__init__()
+        self.interval = interval
+        self.callable = callable
+        self.args = args
+        self.kwargs = kwargs
+    def run(self):
+        while True:
+            time.sleep(self.interval)
+            self.callable(*self.args, **self.kwargs)
+
+class ToucherThread(Loggable):
+    """ Creates a thread that touches a file 'path' every 'interval'
+    seconds """
+    def __init__(self, path, interval=5):
+        if not os.path.exists(path):
+            try:
+                # TODO : rewrite using with?
+                f = open(path,'w')
+                f.write('')
+                f.close()
+            except Exception as e:
+                raise CouldNotCreateIndexFile(path,e)
+        cb = Toucher(path)
+        t = RepeatTimer(interval, cb)
+        t.daemon = True # thread terminates once process is done
+        t.start()
+

python_apps/media-monitor/mm2/media/monitor/watchersyncer.py

@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+import time
+import copy
+
+from handler         import ReportHandler
+from log             import Loggable
+from exceptions      import BadSongFile
+from eventcontractor import EventContractor
+from events          import EventProxy
+from request         import ThreadedRequestSync, RequestSync
+from ..saas.thread   import InstanceInheritingThread, getsig
+
+class TimeoutWatcher(InstanceInheritingThread,Loggable):
+    """
+    The job of this thread is to keep an eye on WatchSyncer and force a
+    request whenever the requests go over time out
+    """
+    def __init__(self, watcher, timeout=5):
+        self.logger.info("Created timeout thread...")
+        super(TimeoutWatcher, self).__init__()
+        self.watcher = watcher
+        self.timeout = timeout
+
+    def run(self):
+        # We try to launch a new thread every self.timeout seconds
+        # so that the people do not have to wait for the queue to fill up
+        while True:
+            time.sleep(self.timeout)
+            # If there is any requests left we launch em. Note that this
+            # isn't strictly necessary since RequestSync threads already
+            # chain themselves
+            if self.watcher.requests_in_queue():
+                self.logger.info("We have %d requests waiting to be launched" %
+                        self.watcher.requests_left_count())
+                self.watcher.request_do()
+            # Same for events, this behaviour is mandatory however.
+            if self.watcher.events_in_queue():
+                self.logger.info("We have %d events that are unflushed" %
+                        self.watcher.events_left_count())
+                self.watcher.flush_events()
+
+class WatchSyncer(ReportHandler,Loggable):
+    def __init__(self, signal, chunking_number = 100, timeout=15):
+        self.timeout          = float(timeout)
+        self.chunking_number  = int(chunking_number)
+        self.request_running  = False
+        self.__current_thread = None
+        self.__requests       = []
+        self.contractor       = EventContractor()
+        self.__reset_queue()
+
+        tc = TimeoutWatcher(self, self.timeout)
+        tc.daemon = True
+        tc.start()
+        super(WatchSyncer, self).__init__(signal=getsig(signal))
+
+    def handle(self, sender, event):
+        """
+        We implement this abstract method from ReportHandler
+        """
+        if hasattr(event, 'pack'):
+            # We push this event into queue
+            self.logger.info("Received event '%s'. Path: '%s'" % \
+                    ( event.__class__.__name__,
+                      getattr(event,'path','No path exists') ))
+            try:
+                # If there is a strange bug anywhere in the code the next line
+                # should be a suspect
+                ev = EventProxy(event)
+                if self.contractor.register(ev): self.push_queue(ev)
+                #self.push_queue( event )
+            except BadSongFile as e:
+                self.fatal_exception("Received bas song file '%s'" % e.path, e)
+            except Exception as e: self.unexpected_exception(e)
+        else:
+            self.logger.info("Received event that does not implement packing.\
+                    Printing its representation:")
+            self.logger.info( repr(event) )
+
+    def requests_left_count(self):
+        """
+        returns the number of requests left in the queue. requests are
+        functions that create RequestSync threads
+        """
+        return len(self.__requests)
+    def events_left_count(self):
+        """
+        Returns the number of events left in the queue to create a request
+        """
+        return len(self.__queue)
+
+    def push_queue(self, elem):
+        """
+        Added 'elem' to the event queue and launch a request if we are
+        over the the chunking number
+        """
+        self.logger.info("Added event into queue")
+        if self.events_left_count() >= self.chunking_number:
+            self.push_request()
+            self.request_do() # Launch the request if nothing is running
+        self.__queue.append(elem)
+
+    def flush_events(self):
+        """
+        Force flush the current events held in the queue
+        """
+        self.logger.info("Force flushing events...")
+        self.push_request()
+        self.request_do()
+
+    def events_in_queue(self):
+        """
+        returns true if there are events in the queue that haven't been
+        processed yet
+        """
+        return len(self.__queue) > 0
+
+    def requests_in_queue(self):
+        """
+        Returns true if there are any requests in the queue. False otherwise.
+        """
+        return len(self.__requests) > 0
+
+    def flag_done(self):
+        """
+        called by request thread when it finishes operating
+        """
+        self.request_running = False
+        self.__current_thread = None
+        # This call might not be necessary but we would like to get the
+        # ball running with the requests as soon as possible
+        if self.requests_in_queue() > 0: self.request_do()
+
+    def request_do(self):
+        """
+        launches a request thread only if one is not running right now
+        """
+        if not self.request_running:
+            self.request_running = True
+            self.__requests.pop()()
+
+    def push_request(self):
+        """
+        Create a request from the current events in the queue and schedule it
+        """
+        self.logger.info("WatchSyncer : Unleashing request")
+        # want to do request asyncly and empty the queue
+        requests = copy.copy(self.__queue)
+        def launch_request():
+            # Need shallow copy here
+            t = ThreadedRequestSync( RequestSync.create_with_api_client(
+                watcher=self, requests=requests) )
+            self.__current_thread = t
+        self.__requests.append(launch_request)
+        self.__reset_queue()
+
+    def __reset_queue(self): self.__queue = []
+
+    def __del__(self):
+        #this destructor is completely untested and it's unclear whether
+        #it's even doing anything useful. consider removing it
+        if self.events_in_queue():
+            self.logger.warn("Terminating with events still in the queue...")
+        if self.requests_in_queue():
+            self.logger.warn("Terminating with http requests still pending...")
+

python_apps/media-monitor/mm2/media/saas/airtimeinstance.py

@@ -0,0 +1,73 @@
+import os
+from os.path import join, basename, dirname
+
+from ..monitor.exceptions import NoConfigFile
+from ..monitor.pure import LazyProperty
+from ..monitor.config import MMConfig
+from ..monitor.owners import Owner
+from ..monitor.events import EventRegistry
+from ..monitor.listeners import FileMediator
+from api_clients.api_client import AirtimeApiClient
+
+# poor man's phantom types...
+class SignalString(str): pass
+
+class AirtimeInstance(object):
+    """ AirtimeInstance is a class that abstracts away every airtime
+    instance by providing all the necessary objects required to interact
+    with the instance. ApiClient, configs, root_directory """
+
+    @classmethod
+    def root_make(cls, name, root):
+        cfg = {
+                'api_client' : join(root, 'etc/airtime/api_client.cfg'),
+                'media_monitor' : join(root, 'etc/airtime/airtime.conf'),
+        }
+        return cls(name, root, cfg)
+
+    def __init__(self,name, root_path, config_paths):
+        """ name is an internal name only """
+        for cfg in ['api_client','media_monitor']:
+            if cfg not in config_paths: raise NoConfigFile(config_paths)
+            elif not os.path.exists(config_paths[cfg]):
+                raise NoConfigFile(config_paths[cfg])
+        self.name         = name
+        self.config_paths = config_paths
+        self.root_path    = root_path
+
+    def signal(self, sig):
+        if isinstance(sig, SignalString): return sig
+        else: return SignalString("%s_%s" % (self.name, sig))
+
+    def touch_file_path(self):
+        """ Get the path of the touch file for every instance """
+        touch_base_path = self.mm_config['media-monitor']['index_path']
+        touch_base_name = basename(touch_base_path)
+        new_base_name   = self.name + touch_base_name
+        return join(dirname(touch_base_path), new_base_name)
+
+
+    def __str__(self):
+        return "%s,%s(%s)" % (self.name, self.root_path, self.config_paths)
+
+    @LazyProperty
+    def api_client(self):
+        return AirtimeApiClient(config_path=self.config_paths['api_client'])
+
+    @LazyProperty
+    def mm_config(self):
+        return MMConfig(self.config_paths['media_monitor'])
+
+    # I'm well aware that I'm using the service locator pattern
+    # instead of normal constructor injection as I should be.
+    # It's recommended to rewrite this using proper constructor injection
+
+    @LazyProperty
+    def owner(self): return Owner()
+
+    @LazyProperty
+    def event_registry(self): return EventRegistry()
+
+    @LazyProperty
+    def file_mediator(self): return FileMediator()
+

python_apps/media-monitor/mm2/media/saas/launcher.py

@@ -0,0 +1,133 @@
+import os, sys
+import logging
+import logging.config
+
+from ..monitor               import pure as mmp
+
+from ..monitor.exceptions    import FailedToObtainLocale, FailedToSetLocale
+from ..monitor.log           import get_logger, setup_logging
+from std_err_override            import LogWriter
+from ..saas.thread           import InstanceThread, user, apc, getsig
+from ..monitor.log           import Loggable
+from ..monitor.exceptions    import CouldNotCreateIndexFile
+from ..monitor.toucher       import ToucherThread
+from ..monitor.airtime       import AirtimeNotifier, AirtimeMessageReceiver
+from ..monitor.watchersyncer import WatchSyncer
+from ..monitor.eventdrainer  import EventDrainer
+from ..monitor.manager       import Manager
+from ..monitor.syncdb        import AirtimeDB
+from airtimeinstance  import AirtimeInstance
+
+class MM2(InstanceThread, Loggable):
+
+    def index_create(self, index_create_attempt=False):
+        config = user().mm_config
+        if not index_create_attempt:
+            if not os.path.exists(config['media-monitor']['index_path']):
+                self.logger.info("Attempting to create index file:...")
+                try:
+                    with open(config['media-monitor']['index_path'], 'w') as f: f.write(" ")
+                except Exception as e:
+                    self.logger.info("Failed to create index file with exception: %s" \
+                             % str(e))
+                else:
+                    self.logger.info("Created index file, reloading configuration:")
+                    self.index_create(index_create_attempt=True)
+        else:
+            self.logger.info("Already tried to create index. Will not try again ")
+
+        if not os.path.exists(config['media-monitor']['index_path']):
+            raise CouldNotCreateIndexFile(config['media-monitor']['index_path'])
+
+    def run(self):
+        self.index_create()
+        manager = Manager()
+        apiclient = apc()
+        config = user().mm_config
+        WatchSyncer(signal=getsig('watch'),
+                chunking_number=config['media-monitor']['chunking_number'],
+                timeout=config['media-monitor']['request_max_wait'])
+        airtime_receiver = AirtimeMessageReceiver(config,manager)
+        airtime_notifier = AirtimeNotifier(config, airtime_receiver)
+
+
+        adb = AirtimeDB(apiclient)
+        store = {
+                u'stor' : adb.storage_path(),
+                u'watched_dirs' : adb.list_watched(),
+        }
+
+        self.logger.info("initializing mm with directories: %s" % str(store))
+
+        self.logger.info(
+                "Initing with the following airtime response:%s" % str(store))
+
+        airtime_receiver.change_storage({ 'directory':store[u'stor'] })
+
+        for watch_dir in store[u'watched_dirs']:
+            if not os.path.exists(watch_dir):
+                # Create the watch_directory here
+                try: os.makedirs(watch_dir)
+                except Exception:
+                    self.logger.error("Could not create watch directory: '%s' \
+                            (given from the database)." % watch_dir)
+            if os.path.exists(watch_dir):
+                airtime_receiver.new_watch({ 'directory':watch_dir }, restart=True)
+            else: self.logger.info("Failed to add watch on %s" % str(watch_dir))
+
+        EventDrainer(airtime_notifier,
+                interval=float(config['media-monitor']['rmq_event_wait']))
+
+        # Launch the toucher that updates the last time when the script was
+        # ran every n seconds.
+        # TODO : verify that this does not interfere with bootstrapping because the
+        # toucher thread might update the last_ran variable too fast
+        ToucherThread(path=user().touch_file_path(),
+                interval=int(config['media-monitor']['touch_interval']))
+
+        success = False
+        while not success:
+            try:
+                apiclient.register_component('media-monitor')
+                success = True
+            except Exception, e:
+                self.logger.error(str(e))
+                import time
+                time.sleep(10)
+
+        manager.loop()
+
+def launch_instance(name, root, global_cfg):
+    cfg = {
+        'api_client'    : global_cfg,
+        'media_monitor' : global_cfg,
+    }
+    ai = AirtimeInstance(name, root, cfg)
+    MM2(ai).start()
+
+def setup_global(log):
+    """ setup unicode and other stuff """
+    log.info("Attempting to set the locale...")
+    try: mmp.configure_locale(mmp.get_system_locale())
+    except FailedToSetLocale as e:
+        log.info("Failed to set the locale...")
+        sys.exit(1)
+    except FailedToObtainLocale as e:
+        log.info("Failed to obtain the locale form the default path: \
+                '/etc/default/locale'")
+        sys.exit(1)
+    except Exception as e:
+        log.info("Failed to set the locale for unknown reason. \
+                Logging exception.")
+        log.info(str(e))
+
+def setup_logger(log_config, logpath):
+    logging.config.fileConfig(log_config)
+    #need to wait for Python 2.7 for this..
+    #logging.captureWarnings(True)
+    logger = logging.getLogger()
+    LogWriter.override_std_err(logger)
+    logfile = unicode(logpath)
+    setup_logging(logfile)
+    log = get_logger()
+    return log

python_apps/media-monitor/mm2/media/saas/thread.py

@@ -0,0 +1,28 @@
+import threading
+
+class UserlessThread(Exception):
+    def __str__(self):
+        return "Current thread: %s is not an instance of InstanceThread \
+                of InstanceInheritingThread" % str(threading.current_thread())
+
+class HasUser(object):
+    def user(self): return self._user
+    def assign_user(self): self._user = threading.current_thread().user()
+
+class InstanceThread(threading.Thread, HasUser):
+    def __init__(self,user, *args, **kwargs):
+        super(InstanceThread, self).__init__(*args, **kwargs)
+        self._user = user
+        
+class InstanceInheritingThread(threading.Thread, HasUser):
+    def __init__(self, *args, **kwargs):
+        self.assign_user()
+        super(InstanceInheritingThread, self).__init__(*args, **kwargs)
+
+def user():
+    try: return threading.current_thread().user()
+    except AttributeError: raise UserlessThread()
+
+def apc(): return user().api_client
+
+def getsig(s): return user().signal(s)