CC-5990, CC-5991 - Python cleanup, removed need for /usr/lib/airtime

python_apps/media-monitor/media_monitor/__main__.py

@@ -0,0 +1,11 @@
+import logging
+import time
+import sys
+import os
+import mm2.mm2 as mm2
+from std_err_override import LogWriter
+
+global_cfg = '/etc/airtime/airtime.conf'
+logging_cfg = os.path.join(os.path.dirname(__file__), 'logging.cfg')
+
+mm2.main( global_cfg, logging_cfg )

python_apps/media-monitor/media_monitor/airtime-media-monitor-init-d

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+### BEGIN INIT INFO
+# Provides:          airtime-media-monitor
+# Required-Start:    $local_fs $remote_fs $network $syslog $all
+# Required-Stop:     $local_fs $remote_fs $network $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Manage airtime-media-monitor daemon
+### END INIT INFO
+
+USERID=WEB_USER
+GROUPID=WEB_USER
+NAME=Airtime\ Media\ Monitor
+
+DAEMON=/usr/lib/airtime/media-monitor/airtime-media-monitor
+PIDFILE=/var/run/airtime/airtime-media-monitor.pid
+
+start () {
+        start-stop-daemon --start --background --quiet --chuid $USERID:$GROUPID \
+            --make-pidfile --pidfile $PIDFILE --startas $DAEMON
+}
+
+stop () {
+        # Send TERM after 5 seconds, wait at most 30 seconds.
+        start-stop-daemon --stop --oknodo --retry TERM/5/0/30 --quiet --pidfile $PIDFILE
+        rm -f $PIDFILE
+}
+
+case "${1:-''}" in
+  'start')
+            # start commands here
+            echo -n "Starting $NAME: "
+            start
+            echo "Done."
+        ;;
+  'stop')
+            # stop commands here
+            echo -n "Stopping $NAME: "
+            stop
+            echo "Done."
+        ;;
+  'restart')
+           # restart commands here
+           echo -n "Restarting $NAME: "
+           stop
+           start
+           echo "Done."
+        ;;
+  'status')
+           # status commands here
+           /usr/bin/airtime-check-system
+        ;;
+  *)      # no parameter specified
+        echo "Usage: $SELF start|stop|restart|status"
+        exit 1
+        ;;
+esac

python_apps/media-monitor/media_monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py

@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+
+import os
+import time
+import pyinotify
+import shutil
+
+class AirtimeMediaMonitorBootstrap():
+
+    """AirtimeMediaMonitorBootstrap constructor
+
+    Keyword Arguments:
+    logger      -- reference to the media-monitor logging facility
+    pe          -- reference to an instance of ProcessEvent
+    api_clients -- reference of api_clients to communicate with airtime-server
+    """
+    def __init__(self, logger, pe, api_client, mmc, wm, config):
+        self.logger = logger
+        self.pe = pe
+        self.api_client = api_client
+        self.mmc = mmc
+        self.wm = wm
+        self.config = config
+        # add /etc on watch list so we can detect mount
+        self.mount_file = "/etc"
+        self.curr_mtab_file = "/var/tmp/airtime/media-monitor/currMtab"
+        self.logger.info("Adding %s on watch list...", self.mount_file)
+        self.wm.add_watch(self.mount_file, pyinotify.ALL_EVENTS, rec=False, auto_add=False)
+
+        tmp_dir = os.path.dirname(self.curr_mtab_file)
+        if not os.path.exists(tmp_dir):
+            os.makedirs(tmp_dir)
+
+        # create currMtab file if it's the first time
+        if not os.path.exists(self.curr_mtab_file):
+            shutil.copy('/etc/mtab', self.curr_mtab_file)
+
+    """On bootup we want to scan all directories and look for files that
+    weren't there or files that changed before media-monitor process
+    went offline.
+    """
+    def scan(self):
+        directories = self.get_list_of_watched_dirs()
+        self.logger.info("watched directories found: %s", directories)
+
+        for id, dir in directories.iteritems():
+            self.logger.debug("%s, %s", id, dir)
+            self.sync_database_to_filesystem(id, dir)
+
+    """Gets a list of files that the Airtime database knows for a specific directory.
+    You need to provide the directory's row ID, which is obtained when calling
+    get_list_of_watched_dirs function.
+    dir_id -- row id of the directory in the cc_watched_dirs database table
+    """
+    def list_db_files(self, dir_id):
+        return self.api_client.list_all_db_files(dir_id)
+
+    """
+    returns the path and its corresponding database row idfor all watched directories. Also
+    returns the Stor directory, which can be identified by its row id (always has value of "1")
+
+    Return type is a dictionary similar to:
+    {"1":"/srv/airtime/stor/"}
+    """
+    def get_list_of_watched_dirs(self):
+        json = self.api_client.list_all_watched_dirs()
+
+        try:
+            return json["dirs"]
+        except KeyError as e:
+            self.logger.error("Could not find index 'dirs' in dictionary: %s", str(json))
+            self.logger.error(str(e))
+            return {}
+
+    """
+    This function takes in a path name provided by the database (and its corresponding row id)
+    and reads the list of files in the local file system. Its purpose is to discover which files
+    exist on the file system but not in the database and vice versa, as well as which files have
+    been modified since the database was last updated. In each case, this method will call an
+    appropiate method to ensure that the database actually represents the filesystem.
+    dir_id -- row id of the directory in the cc_watched_dirs database table
+    dir    -- pathname of the directory
+    """
+    def sync_database_to_filesystem(self, dir_id, dir):
+        # TODO: is this line even necessary?
+        dir = os.path.normpath(dir)+"/"
+        """
+        set to hold new and/or modified files. We use a set to make it ok if files are added
+        twice. This is because some of the tests for new files return result sets that are not
+        mutually exclusive from each other.
+        """
+        removed_files = set()
+
+
+        db_known_files_set = set()
+        files = self.list_db_files(dir_id)
+
+        for f in files:
+            db_known_files_set.add(f)
+
+        all_files = self.mmc.clean_dirty_file_paths( self.mmc.scan_dir_for_new_files(dir) )
+
+        all_files_set = set()
+        for file_path in all_files:
+            if self.config.problem_directory not in file_path:
+                all_files_set.add(file_path[len(dir):])
+
+        # if dir doesn't exists, update db
+        if not os.path.exists(dir):
+            self.pe.handle_stdout_files(dir)
+
+        if os.path.exists(self.mmc.timestamp_file):
+            """find files that have been modified since the last time media-monitor process started."""
+            time_diff_sec = time.time() - os.path.getmtime(self.mmc.timestamp_file)
+            command = self.mmc.find_command(directory=dir, extra_arguments=("-type f -readable -mmin -%d" % (time_diff_sec/60+1)))
+        else:
+            command = self.mmc.find_command(directory=dir, extra_arguments="-type f -readable")
+
+        self.logger.debug(command)
+        stdout = self.mmc.exec_command(command)
+
+        if stdout is None:
+            new_files = []
+        else:
+            new_files = stdout.splitlines()
+
+        new_and_modified_files = set()
+        for file_path in new_files:
+            if self.config.problem_directory not in file_path:
+                new_and_modified_files.add(file_path[len(dir):])
+
+        """
+        new_and_modified_files gives us a set of files that were either copied or modified
+        since the last time media-monitor was running. These files were collected based on
+        their modified timestamp. But this is not all that has changed in the directory. Files
+        could have been removed, or files could have been moved into this directory (moving does
+        not affect last modified timestamp). Lets get a list of files that are on the file-system
+        that the db has no record of, and vice-versa.
+        """
+        deleted_files_set = db_known_files_set - all_files_set
+        new_files_set = all_files_set - db_known_files_set
+        modified_files_set = new_and_modified_files - new_files_set
+
+        self.logger.info(u"Deleted files: \n%s\n\n", deleted_files_set)
+        self.logger.info(u"New files: \n%s\n\n", new_files_set)
+        self.logger.info(u"Modified files: \n%s\n\n", modified_files_set)
+
+        #"touch" file timestamp
+        try:
+            self.mmc.touch_index_file()
+        except Exception, e:
+            self.logger.warn(e)
+
+        for file_path in deleted_files_set:
+            self.logger.debug("deleted file")
+            full_file_path = os.path.join(dir, file_path)
+            self.logger.debug(full_file_path)
+            self.pe.handle_removed_file(False, full_file_path)
+
+
+        for file_set, debug_message, handle_attribute in [(new_files_set, "new file", "handle_created_file"),
+                                                          (modified_files_set, "modified file", "handle_modified_file")]:
+            for file_path in file_set:
+                self.logger.debug(debug_message)
+                full_file_path = os.path.join(dir, file_path)
+                self.logger.debug(full_file_path)
+                if os.path.exists(full_file_path):
+                    getattr(self.pe,handle_attribute)(False,full_file_path, os.path.basename(full_file_path))

python_apps/media-monitor/media_monitor/airtimefilemonitor/airtimemetadata.py

@@ -0,0 +1,268 @@
+# -*- coding: utf-8 -*-
+
+import replaygain
+
+import os
+import hashlib
+import mutagen
+import logging
+import math
+import traceback
+
+
+"""
+list of supported easy tags in mutagen version 1.20
+['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry',
+'date', 'performer', 'musicbrainz_albumartistid', 'composer', 'encodedby',
+'tracknumber', 'musicbrainz_albumid', 'album', 'asin', 'musicbrainz_artistid',
+'mood', 'copyright', 'author', 'media', 'length', 'version', 'artistsort',
+'titlesort', 'discsubtitle', 'website', 'musicip_fingerprint', 'conductor',
+'compilation', 'barcode', 'performer:*', 'composersort', 'musicbrainz_discid',
+'musicbrainz_albumtype', 'genre', 'isrc', 'discnumber', 'musicbrainz_trmid',
+'replaygain_*_gain', 'musicip_puid', 'artist', 'title', 'bpm', 'musicbrainz_trackid',
+'arranger', 'albumsort', 'replaygain_*_peak', 'organization']
+"""
+
+class AirtimeMetadata:
+
+    def __init__(self):
+
+        self.airtime2mutagen = {\
+        "MDATA_KEY_TITLE": "title", \
+        "MDATA_KEY_CREATOR": "artist", \
+        "MDATA_KEY_SOURCE": "album", \
+        "MDATA_KEY_GENRE": "genre", \
+        "MDATA_KEY_MOOD": "mood", \
+        "MDATA_KEY_TRACKNUMBER": "tracknumber", \
+        "MDATA_KEY_BPM": "bpm", \
+        "MDATA_KEY_LABEL": "organization", \
+        "MDATA_KEY_COMPOSER": "composer", \
+        "MDATA_KEY_ENCODER": "encodedby", \
+        "MDATA_KEY_CONDUCTOR": "conductor", \
+        "MDATA_KEY_YEAR": "date", \
+        "MDATA_KEY_URL": "website", \
+        "MDATA_KEY_ISRC": "isrc", \
+        "MDATA_KEY_COPYRIGHT": "copyright", \
+        }
+
+        self.mutagen2airtime = {\
+        "title": "MDATA_KEY_TITLE", \
+        "artist": "MDATA_KEY_CREATOR", \
+        "album": "MDATA_KEY_SOURCE", \
+        "genre": "MDATA_KEY_GENRE", \
+        "mood": "MDATA_KEY_MOOD", \
+        "tracknumber": "MDATA_KEY_TRACKNUMBER", \
+        "bpm": "MDATA_KEY_BPM", \
+        "organization": "MDATA_KEY_LABEL", \
+        "composer": "MDATA_KEY_COMPOSER", \
+        "encodedby": "MDATA_KEY_ENCODER", \
+        "conductor": "MDATA_KEY_CONDUCTOR", \
+        "date": "MDATA_KEY_YEAR", \
+        "website": "MDATA_KEY_URL", \
+        "isrc": "MDATA_KEY_ISRC", \
+        "copyright": "MDATA_KEY_COPYRIGHT", \
+        }
+
+        self.logger = logging.getLogger()
+
+    def get_md5(self, filepath):
+        """
+        Returns an md5 of the file located at filepath. Returns an empty string
+        if there was an error reading the file.
+        """
+        try:
+            f = open(filepath, 'rb')
+            m = hashlib.md5()
+            m.update(f.read())
+            md5 = m.hexdigest()
+        except Exception, e:
+            return ""
+
+        return md5
+
+    ## mutagen_length is in seconds with the format (d+).dd
+    ## return format hh:mm:ss.uuu
+    def format_length(self, mutagen_length):
+        t = float(mutagen_length)
+        h = int(math.floor(t / 3600))
+        t = t % 3600
+        m = int(math.floor(t / 60))
+
+        s = t % 60
+        # will be ss.uuu
+        s = str(s)
+        seconds = s.split(".")
+        s = seconds[0]
+
+        # have a maximum of 6 subseconds.
+        if len(seconds[1]) >= 6:
+            ss = seconds[1][0:6]
+        else:
+            ss = seconds[1][0:]
+
+        length = "%s:%s:%s.%s" % (h, m, s, ss)
+
+        return length
+
+    def save_md_to_file(self, m):
+        try:
+            airtime_file = mutagen.File(m['MDATA_KEY_FILEPATH'], easy=True)
+
+            for key in m:
+                if key in self.airtime2mutagen:
+                    value = m[key]
+
+                    if value is not None:
+                        value = unicode(value)
+                    else:
+                        value = unicode('');
+
+                        #if len(value) > 0:
+                    self.logger.debug("Saving key '%s' with value '%s' to file", key, value)
+                    airtime_file[self.airtime2mutagen[key]] = value
+
+            airtime_file.save()
+        except Exception, e:
+            self.logger.error('Trying to save md')
+            self.logger.error('Exception: %s', e)
+            self.logger.error('Filepath %s', m['MDATA_KEY_FILEPATH'])
+
+    def truncate_to_length(self, item, length):
+        if isinstance(item, int):
+            item = str(item)
+        if isinstance(item, basestring):
+            if len(item) > length:
+                return item[0:length]
+            else:
+                return item
+
+    def get_md_from_file(self, filepath):
+        """
+        Returns None if error retrieving metadata. Otherwise returns a dictionary
+        representing the file's metadata
+        """
+
+        self.logger.info("getting info from filepath %s", filepath)
+
+        md = {}
+
+        replay_gain_val = replaygain.calculate_replay_gain(filepath)
+        self.logger.info('ReplayGain calculated as %s for %s' % (replay_gain_val, filepath))
+        md['MDATA_KEY_REPLAYGAIN'] = replay_gain_val
+
+        try:
+
+            md5 = self.get_md5(filepath)
+            md['MDATA_KEY_MD5'] = md5
+
+            file_info = mutagen.File(filepath, easy=True)
+        except Exception, e:
+            self.logger.error("failed getting metadata from %s", filepath)
+            self.logger.error("Exception %s", e)
+            return None
+
+
+        #check if file has any metadata
+        if file_info is None:
+            return None
+
+        for key in file_info.keys() :
+            if key in self.mutagen2airtime:
+                val = file_info[key]
+                try:
+                    if val is not None and len(val) > 0 and val[0] is not None and len(val[0]) > 0:
+                        md[self.mutagen2airtime[key]] = val[0]
+                except Exception, e:
+                    self.logger.error('Exception: %s', e)
+                    self.logger.error("traceback: %s", traceback.format_exc())
+        if 'MDATA_KEY_TITLE' not in md:
+            #get rid of file extension from original name, name might have more than 1 '.' in it.
+            original_name = os.path.basename(filepath)
+            original_name = original_name.split(".")[0:-1]
+            original_name = ''.join(original_name)
+            md['MDATA_KEY_TITLE'] = original_name
+
+        #incase track number is in format u'4/11'
+        #need to also check that the tracknumber is even a tracknumber (cc-2582)
+        if 'MDATA_KEY_TRACKNUMBER' in md:
+            try:
+                md['MDATA_KEY_TRACKNUMBER'] = int(md['MDATA_KEY_TRACKNUMBER'])
+            except Exception, e:
+                pass
+
+            if isinstance(md['MDATA_KEY_TRACKNUMBER'], basestring):
+                try:
+                    md['MDATA_KEY_TRACKNUMBER'] = int(md['MDATA_KEY_TRACKNUMBER'].split("/")[0], 10)
+                except Exception, e:
+                    del md['MDATA_KEY_TRACKNUMBER']
+
+        #make sure bpm is valid, need to check more types of formats for this tag to assure correct parsing.
+        if 'MDATA_KEY_BPM' in md:
+            if isinstance(md['MDATA_KEY_BPM'], basestring):
+                try:
+                    md['MDATA_KEY_BPM'] = int(md['MDATA_KEY_BPM'])
+                except Exception, e:
+                    del md['MDATA_KEY_BPM']
+
+        #following metadata is truncated if needed to fit db requirements.
+        if 'MDATA_KEY_GENRE' in md:
+            md['MDATA_KEY_GENRE'] = self.truncate_to_length(md['MDATA_KEY_GENRE'], 64)
+
+        if 'MDATA_KEY_TITLE' in md:
+            md['MDATA_KEY_TITLE'] = self.truncate_to_length(md['MDATA_KEY_TITLE'], 512)
+
+        if 'MDATA_KEY_CREATOR' in md:
+            md['MDATA_KEY_CREATOR'] = self.truncate_to_length(md['MDATA_KEY_CREATOR'], 512)
+
+        if 'MDATA_KEY_SOURCE' in md:
+            md['MDATA_KEY_SOURCE'] = self.truncate_to_length(md['MDATA_KEY_SOURCE'], 512)
+
+        if 'MDATA_KEY_MOOD' in md:
+            md['MDATA_KEY_MOOD'] = self.truncate_to_length(md['MDATA_KEY_MOOD'], 64)
+
+        if 'MDATA_KEY_LABEL' in md:
+            md['MDATA_KEY_LABEL'] = self.truncate_to_length(md['MDATA_KEY_LABEL'], 512)
+
+        if 'MDATA_KEY_COMPOSER' in md:
+            md['MDATA_KEY_COMPOSER'] = self.truncate_to_length(md['MDATA_KEY_COMPOSER'], 512)
+
+        if 'MDATA_KEY_ENCODER' in md:
+            md['MDATA_KEY_ENCODER'] = self.truncate_to_length(md['MDATA_KEY_ENCODER'], 255)
+
+        if 'MDATA_KEY_CONDUCTOR' in md:
+            md['MDATA_KEY_CONDUCTOR'] = self.truncate_to_length(md['MDATA_KEY_CONDUCTOR'], 512)
+
+        if 'MDATA_KEY_YEAR' in md:
+            md['MDATA_KEY_YEAR'] = self.truncate_to_length(md['MDATA_KEY_YEAR'], 16)
+
+        if 'MDATA_KEY_URL' in md:
+            md['MDATA_KEY_URL'] = self.truncate_to_length(md['MDATA_KEY_URL'], 512)
+
+        if 'MDATA_KEY_ISRC' in md:
+            md['MDATA_KEY_ISRC'] = self.truncate_to_length(md['MDATA_KEY_ISRC'], 512)
+
+        if 'MDATA_KEY_COPYRIGHT' in md:
+            md['MDATA_KEY_COPYRIGHT'] = self.truncate_to_length(md['MDATA_KEY_COPYRIGHT'], 512)
+        #end of db truncation checks.
+
+        try:
+            md['MDATA_KEY_BITRATE'] = getattr(file_info.info, "bitrate", 0)
+            md['MDATA_KEY_SAMPLERATE'] = getattr(file_info.info, "sample_rate", 0)
+
+            md['MDATA_KEY_DURATION'] = self.format_length(getattr(file_info.info, "length", 0.0))
+
+            md['MDATA_KEY_MIME'] = ""
+            if len(file_info.mime) > 0:
+                md['MDATA_KEY_MIME'] = file_info.mime[0]
+        except Exception as e:
+            self.logger.warn(e)
+
+        if "mp3" in md['MDATA_KEY_MIME']:
+            md['MDATA_KEY_FTYPE'] = "audioclip"
+        elif "vorbis" in md['MDATA_KEY_MIME']:
+            md['MDATA_KEY_FTYPE'] = "audioclip"
+        else:
+            self.logger.error("File %s of mime type %s does not appear to be a valid vorbis or mp3 file." % (filepath, md['MDATA_KEY_MIME']))
+            return None
+
+        return md

python_apps/media-monitor/media_monitor/airtimefilemonitor/airtimenotifier.py

@@ -0,0 +1,213 @@
+# -*- coding: utf-8 -*-
+
+import json
+import time
+import os
+import logging
+import traceback
+
+# For RabbitMQ
+from kombu.connection import BrokerConnection
+from kombu.messaging import Exchange, Queue, Consumer
+
+import pyinotify
+from pyinotify import Notifier
+
+from airtimemetadata import AirtimeMetadata
+
+class AirtimeNotifier(Notifier):
+
+    def __init__(self, watch_manager, default_proc_fun=None, read_freq=0, threshold=0, timeout=None, airtime_config=None, api_client=None, bootstrap=None, mmc=None):
+        Notifier.__init__(self, watch_manager, default_proc_fun, read_freq, threshold, timeout)
+
+        self.logger = logging.getLogger()
+        self.config = airtime_config
+        self.api_client = api_client
+        self.bootstrap = bootstrap
+        self.md_manager = AirtimeMetadata()
+        self.import_processes = {}
+        self.watched_folders = []
+        self.mmc = mmc
+        self.wm = watch_manager
+        self.mask = pyinotify.ALL_EVENTS
+
+        while not self.init_rabbit_mq():
+            self.logger.error("Error connecting to RabbitMQ Server. Trying again in few seconds")
+            time.sleep(5)
+
+    def init_rabbit_mq(self):
+        """
+        This function will attempt to connect to RabbitMQ Server and if successful
+        return 'True'. Returns 'False' otherwise.
+        """
+ 
+        self.logger.info("Initializing RabbitMQ stuff")
+        try:
+            schedule_exchange = Exchange("airtime-media-monitor", "direct", durable=True, auto_delete=True)
+            schedule_queue = Queue("media-monitor", exchange=schedule_exchange, key="filesystem")
+            self.connection = BrokerConnection(self.config.cfg["rabbitmq"]["rabbitmq_host"], self.config.cfg["rabbitmq"]["rabbitmq_user"], self.config.cfg["rabbitmq"]["rabbitmq_password"], self.config.cfg["rabbitmq"]["rabbitmq_vhost"])
+            channel = self.connection.channel()
+            consumer = Consumer(channel, schedule_queue)
+            consumer.register_callback(self.handle_message)
+            consumer.consume()
+        except Exception, e:
+            self.logger.error(e)
+            return False
+
+        return True
+
+    def handle_message(self, body, message):
+        """
+        Messages received from RabbitMQ are handled here. These messages
+        instruct media-monitor of events such as a new directory being watched,
+        file metadata has been changed, or any other changes to the config of
+        media-monitor via the web UI.
+        """
+        # ACK the message to take it off the queue
+        message.ack()
+
+        self.logger.info("Received md from RabbitMQ: " + body)
+        m = json.loads(message.body)
+
+        if m['event_type'] == "md_update":
+            self.logger.info("AIRTIME NOTIFIER md update event")
+            self.md_manager.save_md_to_file(m)
+
+        elif m['event_type'] == "new_watch":
+            self.logger.info("AIRTIME NOTIFIER add watched folder event " + m['directory'])
+            self.walk_newly_watched_directory(m['directory'])
+
+            self.watch_directory(m['directory'])
+
+        elif m['event_type'] == "remove_watch":
+            watched_directory = m['directory']
+
+            mm = self.proc_fun()
+            wd = mm.wm.get_wd(watched_directory)
+            self.logger.info("Removing watch on: %s wd %s", watched_directory, wd)
+            mm.wm.rm_watch(wd, rec=True)
+
+        elif m['event_type'] == "rescan_watch":
+            self.bootstrap.sync_database_to_filesystem(str(m['id']), m['directory'])
+
+        elif m['event_type'] == "change_stor":
+            storage_directory = self.config.storage_directory
+            new_storage_directory = m['directory']
+            new_storage_directory_id = str(m['dir_id'])
+
+            mm = self.proc_fun()
+
+            wd = mm.wm.get_wd(storage_directory)
+            self.logger.info("Removing watch on: %s wd %s", storage_directory, wd)
+            mm.wm.rm_watch(wd, rec=True)
+
+            self.bootstrap.sync_database_to_filesystem(new_storage_directory_id, new_storage_directory)
+
+            self.config.storage_directory = os.path.normpath(new_storage_directory)
+            self.config.imported_directory = os.path.normpath(os.path.join(new_storage_directory, '/imported'))
+            self.config.organize_directory = os.path.normpath(os.path.join(new_storage_directory, '/organize'))
+
+            for directory in [self.config.storage_directory, self.config.imported_directory, self.config.organize_directory]:
+                self.mmc.ensure_is_dir(directory)
+                self.mmc.is_readable(directory, True)
+
+            self.watch_directory(new_storage_directory)
+        elif m['event_type'] == "file_delete":
+            filepath = m['filepath']
+
+            mm = self.proc_fun()
+            self.logger.info("Adding file to ignore: %s ", filepath)
+            mm.add_filepath_to_ignore(filepath)
+
+            if m['delete']:
+                self.logger.info("Deleting file: %s ", filepath)
+                try:
+                    os.unlink(filepath)
+                except Exception, e:
+                    self.logger.error('Exception: %s', e)
+                    self.logger.error("traceback: %s", traceback.format_exc())
+
+
+    def update_airtime(self, event):
+        """
+        Update airtime with information about files discovered in our
+        watched directories.
+        event: a dict() object with the following attributes:
+        -filepath
+        -mode
+        -data
+        -is_recorded_show
+        """
+        try:
+            self.logger.info("updating filepath: %s ", event['filepath'])
+            filepath = event['filepath']
+            mode = event['mode']
+
+            md = {}
+            md['MDATA_KEY_FILEPATH'] = os.path.normpath(filepath)
+
+            if 'data' in event:
+                file_md = event['data']
+                md.update(file_md)
+            else:
+                file_md = None
+
+            if (os.path.exists(filepath) and (mode == self.config.MODE_CREATE)):
+                if file_md is None:
+                    mutagen = self.md_manager.get_md_from_file(filepath)
+                    if mutagen is None:
+                        return
+                    md.update(mutagen)
+
+                if 'is_recorded_show' in event and event['is_recorded_show']:
+                    self.api_client.update_media_metadata(md, mode, True)
+                else:
+                    self.api_client.update_media_metadata(md, mode)
+
+            elif (os.path.exists(filepath) and (mode == self.config.MODE_MODIFY)):
+                mutagen = self.md_manager.get_md_from_file(filepath)
+                if mutagen is None:
+                    return
+                md.update(mutagen)
+                if 'is_recorded_show' in event and event['is_recorded_show']:
+                    self.api_client.update_media_metadata(md, mode, True)
+                else:
+                    self.api_client.update_media_metadata(md, mode)
+            elif (mode == self.config.MODE_MOVED):
+                md['MDATA_KEY_MD5'] = self.md_manager.get_md5(filepath)
+                if 'is_recorded_show' in event and event['is_recorded_show']:
+                    self.api_client.update_media_metadata(md, mode, True)
+                else:
+                    self.api_client.update_media_metadata(md, mode)
+            elif (mode == self.config.MODE_DELETE):
+                self.api_client.update_media_metadata(md, mode)
+
+            elif (mode == self.config.MODE_DELETE_DIR):
+                self.api_client.update_media_metadata(md, mode)
+
+        except Exception, e:
+            self.logger.error("failed updating filepath: %s ", event['filepath'])
+            self.logger.error('Exception: %s', e)
+            self.logger.error('Traceback: %s', traceback.format_exc())
+
+    #define which directories the pyinotify WatchManager should watch.
+    def watch_directory(self, directory):
+        return self.wm.add_watch(directory, self.mask, rec=True, auto_add=True)
+
+    def walk_newly_watched_directory(self, directory):
+
+        mm = self.proc_fun()
+
+        self.mmc.is_readable(directory, True)
+        for (path, dirs, files) in os.walk(directory):
+            for filename in files:
+                full_filepath = os.path.join(path, filename)
+
+                if self.mmc.is_audio_file(full_filepath):
+                    if self.mmc.is_readable(full_filepath, False):
+                        self.logger.info("importing %s", full_filepath)
+                        event = {'filepath': full_filepath, 'mode': self.config.MODE_CREATE, 'is_recorded_show': False}
+                        mm.multi_queue.put(event)
+                    else:
+                        self.logger.warn("file '%s' has does not have sufficient read permissions. Ignoring.", full_filepath)
+

python_apps/media-monitor/media_monitor/airtimefilemonitor/airtimeprocessevent.py

@@ -0,0 +1,431 @@
+# -*- coding: utf-8 -*-
+
+import socket
+import logging
+import time
+import os
+import shutil
+import difflib
+import traceback
+from subprocess import Popen, PIPE
+
+from pyinotify import ProcessEvent
+
+from airtimemetadata import AirtimeMetadata
+from airtimefilemonitor.mediaconfig import AirtimeMediaConfig
+
+from api_clients import api_client
+
+class AirtimeProcessEvent(ProcessEvent):
+
+    #TODO
+    def my_init(self, queue, airtime_config=None, wm=None, mmc=None, api_client=api_client):
+        """
+        Method automatically called from ProcessEvent.__init__(). Additional
+        keyworded arguments passed to ProcessEvent.__init__() are then
+        delegated to my_init().
+        """
+        self.logger = logging.getLogger()
+        self.config = airtime_config
+
+        #put the file path into this dict if we want to ignore certain
+        #events. For example, when deleting a file from the web ui, we
+        #are going to delete it from the db on the server side, so media-monitor
+        #doesn't need to contact the server and tell it to delete again.
+        self.ignore_event = set()
+
+        self.temp_files = {}
+        self.cookies_IN_MOVED_FROM = {}
+        self.file_events = []
+        self.multi_queue = queue
+        self.wm = wm
+        self.md_manager = AirtimeMetadata()
+        self.mmc = mmc
+        self.api_client = api_client
+        self.create_dict = {}
+        self.mount_file_dir = "/etc";
+        self.mount_file = "/etc/mtab";
+        self.curr_mtab_file = "/var/tmp/airtime/media-monitor/currMtab"
+        self.prev_mtab_file = "/var/tmp/airtime/media-monitor/prevMtab"
+
+    def add_filepath_to_ignore(self, filepath):
+        self.ignore_event.add(filepath)
+
+    def process_IN_MOVE_SELF(self, event):
+        self.logger.info("event: %s", event)
+        path = event.path
+        if event.dir:
+            if "-unknown-path" in path:
+                unknown_path = path
+                pos = path.find("-unknown-path")
+                path = path[0:pos] + "/"
+
+                list = self.api_client.list_all_watched_dirs()
+                # case where the dir that is being watched is moved to somewhere
+                if path in list[u'dirs'].values():
+                    self.logger.info("Requesting the airtime server to remove '%s'", path)
+                    res = self.api_client.remove_watched_dir(path)
+                    if(res is None):
+                        self.logger.info("Unable to connect to the Airtime server.")
+                    # sucess
+                    if(res['msg']['code'] == 0):
+                        self.logger.info("%s removed from watch folder list successfully.", path)
+                    else:
+                        self.logger.info("Removing the watch folder failed: %s", res['msg']['error'])
+                else:
+                    # subdir being moved
+                    # in this case, it has to remove watch manualy and also have to manually delete all records
+                    # on cc_files table
+                    wd = self.wm.get_wd(unknown_path)
+                    self.logger.info("Removing watch on: %s wd %s", unknown_path, wd)
+                    self.wm.rm_watch(wd, rec=True)
+                    self.file_events.append({'mode': self.config.MODE_DELETE_DIR, 'filepath': path})
+
+
+    def process_IN_DELETE_SELF(self, event):
+
+        #we only care about files that have been moved away from imported/ or organize/ dir
+        if event.path in self.config.problem_directory or event.path in self.config.organize_directory:
+            return
+
+        self.logger.info("event: %s", event)
+        path = event.path + '/'
+        if event.dir:
+            list = self.api_client.list_all_watched_dirs()
+            if path in list[u'dirs'].values():
+                self.logger.info("Requesting the airtime server to remove '%s'", path)
+                res = self.api_client.remove_watched_dir(path)
+                if(res is None):
+                    self.logger.info("Unable to connect to the Airtime server.")
+                # sucess
+                if(res['msg']['code'] == 0):
+                    self.logger.info("%s removed from watch folder list successfully.", path)
+                else:
+                    self.logger.info("Removing the watch folder failed: %s", res['msg']['error'])
+
+    def process_IN_CREATE(self, event):
+        if event.path in self.mount_file_dir:
+            return
+        self.logger.info("event: %s", event)
+        if not event.dir:
+            # record the timestamp of the time on IN_CREATE event
+            self.create_dict[event.pathname] = time.time()
+
+    #event.dir: True if the event was raised against a directory.
+    #event.name: filename
+    #event.pathname: pathname (str): Concatenation of 'path' and 'name'.
+    # we used to use IN_CREATE event, but the IN_CREATE event gets fired before the
+    # copy was done. Hence, IN_CLOSE_WRITE is the correct one to handle.
+    def process_IN_CLOSE_WRITE(self, event):
+        if event.path in self.mount_file_dir:
+            return
+        self.logger.info("event: %s", event)
+        self.logger.info("create_dict: %s", self.create_dict)
+
+        try:
+            del self.create_dict[event.pathname]
+            self.handle_created_file(event.dir, event.pathname, event.name)
+        except KeyError, e:
+            pass
+            #self.logger.warn("%s does not exist in create_dict", event.pathname)
+            #Uncomment the above warning when we fix CC-3830 for 2.1.1
+
+
+    def handle_created_file(self, dir, pathname, name):
+        if not dir:
+            self.logger.debug("PROCESS_IN_CLOSE_WRITE: %s, name: %s, pathname: %s ", dir, name, pathname)
+
+            if self.mmc.is_temp_file(name) :
+                #file created is a tmp file which will be modified and then moved back to the original filename.
+                #Easy Tag creates this when changing metadata of ogg files.
+                self.temp_files[pathname] = None
+            #file is being overwritten/replaced in GUI.
+            elif "goutputstream" in pathname:
+                self.temp_files[pathname] = None
+            elif self.mmc.is_audio_file(name):
+                if self.mmc.is_parent_directory(pathname, self.config.organize_directory):
+
+                    #file was created in /srv/airtime/stor/organize. Need to process and move
+                    #to /srv/airtime/stor/imported
+                    file_md = self.md_manager.get_md_from_file(pathname)
+                    playable = self.mmc.test_file_playability(pathname)
+
+                    if file_md and playable:
+                        self.mmc.organize_new_file(pathname, file_md)
+                    else:
+                        #move to problem_files
+                        self.mmc.move_to_problem_dir(pathname)
+
+                else:
+                    # only append to self.file_events if the file isn't going to be altered by organize_new_file(). If file is going
+                    # to be altered by organize_new_file(), then process_IN_MOVED_TO event will handle appending it to self.file_events
+                    is_recorded = self.mmc.is_parent_directory(pathname, self.config.recorded_directory)
+                    self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': pathname, 'is_recorded_show': is_recorded})
+
+
+    def process_IN_MODIFY(self, event):
+        # if IN_MODIFY is followed by IN_CREATE, it's not true modify event
+        if not event.pathname in self.create_dict:
+            self.logger.info("process_IN_MODIFY: %s", event)
+            self.handle_modified_file(event.dir, event.pathname, event.name)
+
+    def handle_modified_file(self, dir, pathname, name):
+        # if /etc/mtab is modified
+        if pathname in self.mount_file:
+            self.handle_mount_change()
+        # update timestamp on create_dict for the entry with pathname as the key
+        if pathname in self.create_dict:
+            self.create_dict[pathname] = time.time()
+        if not dir and not self.mmc.is_parent_directory(pathname, self.config.organize_directory):
+            self.logger.info("Modified: %s", pathname)
+            if self.mmc.is_audio_file(name):
+                is_recorded = self.mmc.is_parent_directory(pathname, self.config.recorded_directory)
+                self.file_events.append({'filepath': pathname, 'mode': self.config.MODE_MODIFY, 'is_recorded_show': is_recorded})
+
+    # if change is detected on /etc/mtab, we check what mount(file system) was added/removed
+    # and act accordingly
+    def handle_mount_change(self):
+        self.logger.info("Mount change detected, handling changes...");
+        # take snapshot of mtab file and update currMtab and prevMtab
+        # move currMtab to prevMtab and create new currMtab
+        shutil.move(self.curr_mtab_file, self.prev_mtab_file)
+        # create the file
+        shutil.copy(self.mount_file, self.curr_mtab_file)
+
+        d = difflib.Differ()
+        curr_fh = open(self.curr_mtab_file, 'r')
+        prev_fh = open(self.prev_mtab_file, 'r')
+
+        diff = list(d.compare(prev_fh.readlines(), curr_fh.readlines()))
+        added_mount_points = []
+        removed_mount_points = []
+
+        for dir in diff:
+            info = dir.split(' ')
+            if info[0] == '+':
+                added_mount_points.append(info[2])
+            elif info[0] == '-':
+                removed_mount_points.append(info[2])
+
+        self.logger.info("added: %s", added_mount_points)
+        self.logger.info("removed: %s", removed_mount_points)
+
+        # send current mount information to Airtime
+        self.api_client.update_file_system_mount(added_mount_points, removed_mount_points);
+
+    def handle_watched_dir_missing(self, dir):
+        self.api_client.handle_watched_dir_missing(dir);
+
+    #if a file is moved somewhere, this callback is run. With details about
+    #where the file is being moved from. The corresponding process_IN_MOVED_TO
+    #callback is only called if the destination of the file is also in a watched
+    #directory.
+    def process_IN_MOVED_FROM(self, event):
+
+        #we don't care about files that have been moved from problem_directory
+        if event.path in self.config.problem_directory:
+            return
+
+        self.logger.info("process_IN_MOVED_FROM: %s", event)
+        if not event.dir:
+            if event.pathname in self.temp_files:
+                self.temp_files[event.cookie] = event.pathname
+            elif not self.mmc.is_parent_directory(event.pathname, self.config.organize_directory):
+                #we don't care about moved_from events from the organize dir.
+                if self.mmc.is_audio_file(event.name):
+                    self.cookies_IN_MOVED_FROM[event.cookie] = (event, time.time())
+        else:
+            self.cookies_IN_MOVED_FROM[event.cookie] = (event, time.time())
+
+    def process_IN_MOVED_TO(self, event):
+        self.logger.info("process_IN_MOVED_TO: %s", event)
+        # if /etc/mtab is modified
+        filename = self.mount_file_dir + "/mtab"
+        if event.pathname in filename:
+            self.handle_mount_change()
+
+        if event.path in self.config.problem_directory:
+            return
+
+        if not event.dir:
+            if self.mmc.is_audio_file(event.name):
+                if event.cookie in self.temp_files:
+                    self.file_events.append({'filepath': event.pathname, 'mode': self.config.MODE_MODIFY})
+                    del self.temp_files[event.cookie]
+                elif event.cookie in self.cookies_IN_MOVED_FROM:
+                    #file's original location was also in a watched directory
+                    del self.cookies_IN_MOVED_FROM[event.cookie]
+                    if self.mmc.is_parent_directory(event.pathname, self.config.organize_directory):
+
+
+
+                        pathname = event.pathname
+                        #file was created in /srv/airtime/stor/organize. Need to process and move
+                        #to /srv/airtime/stor/imported
+                        file_md = self.md_manager.get_md_from_file(pathname)
+                        playable = self.mmc.test_file_playability(pathname)
+
+                        if file_md and playable:
+                            filepath = self.mmc.organize_new_file(pathname, file_md)
+                        else:
+                            #move to problem_files
+                            self.mmc.move_to_problem_dir(pathname)
+
+
+
+                    else:
+                        filepath = event.pathname
+
+                    if (filepath is not None):
+                        self.file_events.append({'filepath': filepath, 'mode': self.config.MODE_MOVED})
+                else:
+                    #file's original location is from outside an inotify watched dir.
+                    pathname = event.pathname
+                    if self.mmc.is_parent_directory(pathname, self.config.organize_directory):
+
+
+
+
+                        #file was created in /srv/airtime/stor/organize. Need to process and move
+                        #to /srv/airtime/stor/imported
+                        file_md = self.md_manager.get_md_from_file(pathname)
+                        playable = self.mmc.test_file_playability(pathname)
+
+                        if file_md and playable:
+                            self.mmc.organize_new_file(pathname, file_md)
+                        else:
+                            #move to problem_files
+                            self.mmc.move_to_problem_dir(pathname)
+
+
+
+
+                    else:
+                        #show moved from unwatched folder into a watched folder. Do not "organize".
+                        is_recorded = self.mmc.is_parent_directory(event.pathname, self.config.recorded_directory)
+                        self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': event.pathname, 'is_recorded_show': is_recorded})
+        else:
+            #When we move a directory into a watched_dir, we only get a notification that the dir was created,
+            #and no additional information about files that came along with that directory.
+            #need to scan the entire directory for files.
+
+            if event.cookie in self.cookies_IN_MOVED_FROM:
+                del self.cookies_IN_MOVED_FROM[event.cookie]
+                mode = self.config.MODE_MOVED
+            else:
+                mode = self.config.MODE_CREATE
+
+            files = self.mmc.scan_dir_for_new_files(event.pathname)
+            if self.mmc.is_parent_directory(event.pathname, self.config.organize_directory):
+                for pathname in files:
+
+
+
+                    #file was created in /srv/airtime/stor/organize. Need to process and move
+                    #to /srv/airtime/stor/imported
+                    file_md = self.md_manager.get_md_from_file(pathname)
+                    playable = self.mmc.test_file_playability(pathname)
+
+                    if file_md and playable:
+                        self.mmc.organize_new_file(pathname, file_md)
+                        #self.file_events.append({'mode': mode, 'filepath': filepath, 'is_recorded_show': False})
+                    else:
+                        #move to problem_files
+                        self.mmc.move_to_problem_dir(pathname)
+
+
+
+            else:
+                for file in files:
+                    self.file_events.append({'mode': mode, 'filepath': file, 'is_recorded_show': False})
+
+
+    def process_IN_DELETE(self, event):
+        if event.path in self.mount_file_dir:
+            return
+        self.logger.info("process_IN_DELETE: %s", event)
+        self.handle_removed_file(event.dir, event.pathname)
+
+    def handle_removed_file(self, dir, pathname):
+        self.logger.info("Deleting %s", pathname)
+        if not dir:
+            if self.mmc.is_audio_file(pathname):
+                if pathname in self.ignore_event:
+                    self.logger.info("pathname in ignore event")
+                    self.ignore_event.remove(pathname)
+                elif not self.mmc.is_parent_directory(pathname, self.config.organize_directory):
+                    self.logger.info("deleting a file not in organize")
+                    #we don't care if a file was deleted from the organize directory.
+                    self.file_events.append({'filepath': pathname, 'mode': self.config.MODE_DELETE})
+
+
+    def process_default(self, event):
+        pass
+
+    def notifier_loop_callback(self, notifier):
+        if len(self.file_events) > 0:
+            for event in self.file_events:
+                self.multi_queue.put(event)
+            self.mmc.touch_index_file()
+
+            self.file_events = []
+
+        #yield to worker thread
+        time.sleep(0)
+
+        #use items() because we are going to be modifying this
+        #dictionary while iterating over it.
+        for k, pair in self.cookies_IN_MOVED_FROM.items():
+            event = pair[0]
+            timestamp = pair[1]
+
+            timestamp_now = time.time()
+
+            if timestamp_now - timestamp > 5:
+                #in_moved_from event didn't have a corresponding
+                #in_moved_to event in the last 5 seconds.
+                #This means the file was moved to outside of the
+                #watched directories. Let's handle this by deleting
+                #it from the Airtime directory.
+                del self.cookies_IN_MOVED_FROM[k]
+                self.handle_removed_file(False, event.pathname)
+
+        # we don't want create_dict grow infinitely
+        # this part is like a garbage collector
+        for k, t in self.create_dict.items():
+            now = time.time()
+            if now - t > 5:
+                # check if file exist
+                # When whole directory is copied to the organized dir,
+                # inotify doesn't fire IN_CLOSE_WRITE, hench we need special way of
+                # handling those cases. We are manully calling handle_created_file
+                # function.
+                if os.path.exists(k):
+                    # check if file is open
+                    try:
+                        command = "lsof " + k
+                        #f = os.popen(command)
+                        f = Popen(command, shell=True, stdout=PIPE).stdout
+                    except Exception, e:
+                        self.logger.error('Exception: %s', e)
+                        self.logger.error("traceback: %s", traceback.format_exc())
+                        continue
+
+                    if not f.readlines():
+                        self.logger.info("Handling file: %s", k)
+                        self.handle_created_file(False, k, os.path.basename(k))
+                        del self.create_dict[k]
+                else:
+                        del self.create_dict[k]
+
+        #check for any events received from Airtime.
+        try:
+            notifier.connection.drain_events(timeout=0.1)
+        #avoid logging a bunch of timeout messages.
+        except socket.timeout:
+            pass
+        except Exception, e:
+            self.logger.error('Exception: %s', e)
+            self.logger.error("traceback: %s", traceback.format_exc())
+            time.sleep(3)
+

python_apps/media-monitor/media_monitor/airtimefilemonitor/mediaconfig.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+
+import sys
+
+from configobj import ConfigObj
+
+class AirtimeMediaConfig:
+
+    MODE_CREATE = "create"
+    MODE_MODIFY = "modify"
+    MODE_MOVED = "moved"
+    MODE_DELETE = "delete"
+    MODE_DELETE_DIR = "delete_dir"
+
+    def __init__(self, logger):
+
+        # loading config file
+        try:
+            config = ConfigObj('/etc/airtime/airtime.conf')
+            self.cfg = config
+        except Exception, e:
+            logger.info('Error loading config: ', e)
+            sys.exit(1)
+
+        self.storage_directory = None
+
+

python_apps/media-monitor/media_monitor/airtimefilemonitor/mediamonitorcommon.py

@@ -0,0 +1,341 @@
+# -*- coding: utf-8 -*-
+
+import os
+import grp
+import pwd
+import logging
+import stat
+import subprocess
+import traceback
+
+from subprocess import Popen, PIPE
+from airtimemetadata import AirtimeMetadata
+import pyinotify
+
+class MediaMonitorCommon:
+
+    timestamp_file = "/var/tmp/airtime/media-monitor/last_index"
+    supported_file_formats = ['mp3', 'ogg']
+
+    def __init__(self, airtime_config, wm=None):
+        self.logger = logging.getLogger()
+        self.config = airtime_config
+        self.md_manager = AirtimeMetadata()
+        self.wm = wm
+
+
+    def clean_dirty_file_paths(self, dirty_files):
+        """ clean dirty file paths by removing blanks and removing trailing/leading whitespace"""
+        return filter(lambda e: len(e) > 0, [ f.strip(" \n") for f in dirty_files ])
+
+    def find_command(self, directory, extra_arguments=""):
+        """ Builds a find command that respects supported_file_formats list
+        Note: Use single quotes to quote arguments """
+        ext_globs = [ "-iname '*.%s'" % ext for ext in self.supported_file_formats ]
+        find_glob = ' -o '.join(ext_globs)
+        return "find '%s' %s %s" % (directory, find_glob, extra_arguments)
+
+    def is_parent_directory(self, filepath, directory):
+        filepath = os.path.normpath(filepath)
+        directory = os.path.normpath(directory)
+        return (directory == filepath[0:len(directory)])
+
+    def is_temp_file(self, filename):
+        info = filename.split(".")
+        # if file doesn't have any extension, info[-2] throws exception
+        # Hence, checking length of info before we do anything
+        if(len(info) >= 2):
+            return info[-2].lower() in self.supported_file_formats
+        else:
+            return False
+
+    def is_audio_file(self, filename):
+        info = filename.split(".")
+        if len(info) < 2: return False # handle cases like filename="mp3"
+        return info[-1].lower() in self.supported_file_formats
+
+    #check if file is readable by "nobody"
+    def is_user_readable(self, filepath, euid='nobody', egid='nogroup'):
+        f = None
+        try:
+            uid = pwd.getpwnam(euid)[2]
+            gid = grp.getgrnam(egid)[2]
+            #drop root permissions and become "nobody"
+            os.setegid(gid)
+            os.seteuid(uid)
+            f = open(filepath)
+            readable = True
+        except IOError:
+            self.logger.warn("File does not have correct permissions: '%s'", filepath)
+            readable = False
+        except Exception, e:
+            self.logger.error("Unexpected exception thrown: %s", e)
+            readable = False
+            self.logger.error("traceback: %s", traceback.format_exc())
+        finally:
+            #reset effective user to root
+            if f: f.close()
+            os.seteuid(0)
+            os.setegid(0)
+        return readable
+
+    # the function only changes the permission if its not readable by www-data
+    def is_readable(self, item, is_dir):
+        try:
+            return self.is_user_readable(item, 'www-data', 'www-data')
+        except Exception:
+            self.logger.warn(u"Failed to check owner/group/permissions for %s", item)
+            return False
+
+    def make_file_readable(self, pathname, is_dir):
+        if is_dir:
+            #set to 755
+            os.chmod(pathname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
+        else:
+            #set to 644
+            os.chmod(pathname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
+
+    def make_readable(self, pathname):
+        """
+        Should only call this function if is_readable() returns False. This function
+        will attempt to make the file world readable by modifying the file's permission's
+        as well as the file's parent directory permissions. We should only call this function
+        on files in Airtime's stor directory, not watched directories!
+
+        Returns True if we were able to make the file world readable. False otherwise.
+        """
+        original_file = pathname
+        is_dir = False
+        try:
+            while not self.is_readable(original_file, is_dir):
+                #Not readable. Make appropriate permission changes.
+                self.make_file_readable(pathname, is_dir)
+
+                dirname = os.path.dirname(pathname)
+                if dirname == pathname:
+                    #most likey reason for this is that we've hit '/'. Avoid infinite loop by terminating loop
+                    raise Exception()
+                else:
+                    pathname = dirname
+                    is_dir = True
+        except Exception:
+            #something went wrong while we were trying to make world readable.
+            return False
+
+        return True
+
+    #checks if path is a directory, and if it doesnt exist, then creates it.
+    #Otherwise prints error to log file.
+    def ensure_is_dir(self, directory):
+        try:
+            omask = os.umask(0)
+            if not os.path.exists(directory):
+                os.makedirs(directory, 02777)
+                self.wm.add_watch(directory, pyinotify.ALL_EVENTS, rec=True, auto_add=True)
+            elif not os.path.isdir(directory):
+                #path exists but it is a file not a directory!
+                self.logger.error(u"path %s exists, but it is not a directory!!!", directory)
+        finally:
+            os.umask(omask)
+
+    #moves file from source to dest but also recursively removes the
+    #the source file's parent directories if they are now empty.
+    def move_file(self, source, dest):
+        try:
+            omask = os.umask(0)
+            os.rename(source, dest)
+        except Exception, e:
+            self.logger.error("failed to move file. %s", e)
+            self.logger.error("traceback: %s", traceback.format_exc())
+        finally:
+            os.umask(omask)
+
+        dir = os.path.dirname(source)
+        self.cleanup_empty_dirs(dir)
+
+    #keep moving up the file hierarchy and deleting parent
+    #directories until we hit a non-empty directory, or we
+    #hit the organize dir.
+    def cleanup_empty_dirs(self, dir):
+        if os.path.normpath(dir) != self.config.organize_directory:
+            if len(os.listdir(dir)) == 0:
+                try:
+                    os.rmdir(dir)
+                    self.cleanup_empty_dirs(os.path.dirname(dir))
+                except Exception:
+                    #non-critical exception because we probably tried to delete a non-empty dir.
+                    #Don't need to log this, let's just "return"
+                    pass
+
+
+
+    #checks if path exists already in stor. If the path exists and the md5s are the
+    #same just overwrite.
+    def create_unique_filename(self, filepath, old_filepath):
+
+        try:
+            if(os.path.exists(filepath)):
+                self.logger.info("Path %s exists", filepath)
+
+                self.logger.info("Checking if md5s are the same.")
+                md5_fp = self.md_manager.get_md5(filepath)
+                md5_ofp = self.md_manager.get_md5(old_filepath)
+
+                if(md5_fp == md5_ofp):
+                    self.logger.info("Md5s are the same, moving to same filepath.")
+                    return filepath
+
+                self.logger.info("Md5s aren't the same, appending to filepath.")
+                file_dir = os.path.dirname(filepath)
+                filename = os.path.basename(filepath).split(".")[0]
+                #will be in the format .ext
+                file_ext = os.path.splitext(filepath)[1]
+                i = 1;
+                while(True):
+                    new_filepath = '%s/%s(%s)%s' % (file_dir, filename, i, file_ext)
+                    self.logger.error("Trying %s", new_filepath)
+
+                    if(os.path.exists(new_filepath)):
+                        i = i + 1;
+                    else:
+                        filepath = new_filepath
+                        break
+
+        except Exception, e:
+            self.logger.error("Exception %s", e)
+
+        return filepath
+
+    #create path in /srv/airtime/stor/imported/[song-metadata]
+    def create_file_path(self, original_path, orig_md):
+
+        storage_directory = self.config.storage_directory
+        try:
+            #will be in the format .ext
+            file_ext = os.path.splitext(original_path)[1].lower()
+            path_md = ['MDATA_KEY_TITLE', 'MDATA_KEY_CREATOR', 'MDATA_KEY_SOURCE', 'MDATA_KEY_TRACKNUMBER', 'MDATA_KEY_BITRATE']
+
+            md = {}
+            for m in path_md:
+                if m not in orig_md:
+                    md[m] = u'unknown'
+                else:
+                    #get rid of any "/" which will interfere with the filepath.
+                    if isinstance(orig_md[m], basestring):
+                        md[m] = orig_md[m].replace("/", "-")
+                    else:
+                        md[m] = orig_md[m]
+
+            if 'MDATA_KEY_TRACKNUMBER' in orig_md:
+                #make sure all track numbers are at least 2 digits long in the filepath.
+                md['MDATA_KEY_TRACKNUMBER'] = "%02d" % (int(md['MDATA_KEY_TRACKNUMBER']))
+
+            #format bitrate as 128kbps
+            md['MDATA_KEY_BITRATE'] = str(md['MDATA_KEY_BITRATE'] / 1000) + "kbps"
+
+            filepath = None
+            #file is recorded by Airtime
+            #/srv/airtime/stor/recorded/year/month/year-month-day-time-showname-bitrate.ext
+            if(md['MDATA_KEY_CREATOR'] == u"Airtime Show Recorder"):
+                #yyyy-mm-dd-hh-MM-ss
+                y = orig_md['MDATA_KEY_YEAR'].split("-")
+                filepath = u'%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "recorded", y[0], y[1], orig_md['MDATA_KEY_YEAR'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext)
+
+                #"Show-Title-2011-03-28-17:15:00"
+                title = md['MDATA_KEY_TITLE'].split("-")
+                show_hour = title[0]
+                show_min = title[1]
+                show_sec = title[2]
+                show_name = '-'.join(title[3:])
+
+                new_md = {}
+                new_md['MDATA_KEY_FILEPATH'] = os.path.normpath(original_path)
+                new_md['MDATA_KEY_TITLE'] = '%s-%s-%s:%s:%s' % (show_name, orig_md['MDATA_KEY_YEAR'], show_hour, show_min, show_sec)
+                self.md_manager.save_md_to_file(new_md)
+
+            elif(md['MDATA_KEY_TRACKNUMBER'] == u'unknown'):
+                filepath = u'%s/%s/%s/%s/%s-%s%s' % (storage_directory, "imported", md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext)
+            else:
+                filepath = u'%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "imported", md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TRACKNUMBER'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext)
+
+            filepath = self.create_unique_filename(filepath, original_path)
+            self.logger.info('Unique filepath: %s', filepath)
+            self.ensure_is_dir(os.path.dirname(filepath))
+
+        except Exception, e:
+            self.logger.error('Exception: %s', e)
+            self.logger.error("traceback: %s", traceback.format_exc())
+
+        return filepath
+
+    def exec_command(self, command):
+        p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)
+        stdout, stderr = p.communicate()
+        if p.returncode != 0:
+            self.logger.warn("command \n%s\n return with a non-zero return value", command)
+            self.logger.error(stderr)
+
+        try:
+            """
+            File name charset encoding is UTF-8.
+            """
+            stdout = stdout.decode("UTF-8")
+        except Exception:
+            stdout = None
+            self.logger.error("Could not decode %s using UTF-8" % stdout)
+
+        return stdout
+
+    def scan_dir_for_new_files(self, dir):
+        command = self.find_command(directory=dir, extra_arguments="-type f -readable")
+        self.logger.debug(command)
+        stdout = self.exec_command(command)
+
+        if stdout is None:
+            return []
+        else:
+            return stdout.splitlines()
+
+    def touch_index_file(self):
+        dirname = os.path.dirname(self.timestamp_file)
+        try:
+            if not os.path.exists(dirname):
+                os.makedirs(dirname)
+            open(self.timestamp_file, "w")
+        except Exception, e:
+            self.logger.error('Exception: %s', e)
+            self.logger.error("traceback: %s", traceback.format_exc())
+
+    def organize_new_file(self, pathname, file_md):
+        self.logger.info("Organizing new file: %s", pathname)
+
+        filepath = self.create_file_path(pathname, file_md)
+
+        self.logger.debug(u"Moving from %s to %s", pathname, filepath)
+        self.move_file(pathname, filepath)
+        self.make_readable(filepath)
+        return filepath
+
+    def test_file_playability(self, pathname):
+        #when there is an single apostrophe inside of a string quoted by apostrophes, we can only escape it by replace that apostrophe
+        #with '\''. This breaks the string into two, and inserts an escaped single quote in between them.
+        #We run the command as pypo because otherwise the target file is opened with write permissions, and this causes an inotify ON_CLOSE_WRITE event
+        #to be fired :/
+        command = "sudo -u pypo airtime-liquidsoap -c 'output.dummy(audio_to_stereo(single(\"%s\")))' > /dev/null 2>&1" % pathname.replace("'", "'\\''")
+        return_code = subprocess.call(command, shell=True)
+        if return_code != 0:
+            #print pathname for py-interpreter.log
+            print pathname
+        return (return_code == 0)
+
+    def move_to_problem_dir(self, source):
+        dest = os.path.join(self.config.problem_directory, os.path.basename(source))
+        try:
+            omask = os.umask(0)
+            os.rename(source, dest)
+        except Exception, e:
+            self.logger.error("failed to move file. %s", e)
+            self.logger.error("traceback: %s", traceback.format_exc())
+        finally:
+            os.umask(omask)
+

python_apps/media-monitor/media_monitor/airtimefilemonitor/replaygain.py

@@ -0,0 +1,142 @@
+from subprocess import Popen, PIPE
+import re
+import os
+import sys
+import shutil
+import tempfile
+import logging
+
+
+logger = logging.getLogger()
+
+def get_process_output(command):
+    """
+    Run subprocess and return stdout
+    """
+    #logger.debug(command)
+    p = Popen(command, shell=True, stdout=PIPE)
+    return p.communicate()[0].strip()
+
+def run_process(command):
+    """
+    Run subprocess and return "return code"
+    """
+    p = Popen(command, shell=True)
+    return os.waitpid(p.pid, 0)[1]
+
+def get_mime_type(file_path):
+    """
+    Attempts to get the mime type but will return prematurely if the process
+    takes longer than 5 seconds. Note that this function should only be called
+    for files which do not have a mp3/ogg/flac extension.
+    """
+
+    return get_process_output("timeout 5 file -b --mime-type %s" % file_path)
+
+def duplicate_file(file_path):
+    """
+    Makes a duplicate of the file and returns the path of this duplicate file.
+    """
+    fsrc = open(file_path, 'r')
+    fdst = tempfile.NamedTemporaryFile(delete=False)
+
+    #logger.info("Copying %s to %s" % (file_path, fdst.name))
+
+    shutil.copyfileobj(fsrc, fdst)
+
+    fsrc.close()
+    fdst.close()
+
+    return fdst.name
+
+def get_file_type(file_path):
+    file_type = None
+    if re.search(r'mp3$', file_path, re.IGNORECASE):
+        file_type = 'mp3'
+    elif re.search(r'og(g|a)$', file_path, re.IGNORECASE):
+        file_type = 'vorbis'
+    elif re.search(r'flac$', file_path, re.IGNORECASE):
+        file_type = 'flac'
+    elif re.search(r'(mp4|m4a)$', file_path, re.IGNORECASE):
+        file_type = 'mp4'
+    else:
+        mime_type = get_mime_type(file_path) == "audio/mpeg"
+        if 'mpeg' in mime_type:
+            file_type = 'mp3'
+        elif 'ogg' in mime_type or "oga" in mime_type:
+            file_type = 'vorbis'
+        elif 'flac' in mime_type:
+            file_type = 'flac'
+        elif 'mp4' in mime_type or "m4a" in mime_type:
+            file_type = 'mp4'
+
+    return file_type
+
+
+def calculate_replay_gain(file_path):
+    """
+    This function accepts files of type mp3/ogg/flac and returns a calculated
+    ReplayGain value in dB.  If the value cannot be calculated for some reason,
+    then we default to 0 (Unity Gain).
+
+    http://wiki.hydrogenaudio.org/index.php?title=ReplayGain_1.0_specification
+    """
+
+    try:
+        """
+        Making a duplicate is required because the ReplayGain extraction
+        utilities we use make unwanted modifications to the file.
+        """
+
+        search = None
+        temp_file_path = duplicate_file(file_path)
+
+        file_type = get_file_type(file_path)
+
+        if file_type:
+            if file_type == 'mp3':
+                if run_process("which mp3gain > /dev/null") == 0:
+                    out = get_process_output('mp3gain -q "%s" 2> /dev/null' % temp_file_path)
+                    search = re.search(r'Recommended "Track" dB change: (.*)', out)
+                else:
+                    logger.warn("mp3gain not found")
+            elif file_type == 'vorbis':
+                if run_process("which vorbisgain > /dev/null  && which ogginfo > /dev/null") == 0:
+                    run_process('vorbisgain -q -f "%s" 2>/dev/null >/dev/null' % temp_file_path)
+                    out = get_process_output('ogginfo "%s"' % temp_file_path)
+                    search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out)
+                else:
+                    logger.warn("vorbisgain/ogginfo not found")
+            elif file_type == 'flac':
+                if run_process("which metaflac > /dev/null") == 0:
+                    out = get_process_output('metaflac --show-tag=REPLAYGAIN_TRACK_GAIN "%s"' % temp_file_path)
+                    search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out)
+                else:
+                    logger.warn("metaflac not found")
+            elif file_type == 'mp4':
+                if run_process("which aacgain > /dev/null") == 0:
+                    out = get_process_output('aacgain -q "%s" 2> /dev/null' % temp_file_path)
+                    search = re.search(r'Recommended "Track" dB change: (.*)', out)
+                else:
+                    logger.warn("aacgain not found")
+            else:
+                pass
+
+        #no longer need the temp, file simply remove it.
+        os.remove(temp_file_path)
+    except Exception, e:
+        logger.error(str(e))
+
+    replay_gain = 0
+    if search:
+        matches = search.groups()
+        if len(matches) == 1:
+            replay_gain = matches[0]
+
+    return replay_gain
+
+
+# Example of running from command line:
+# python replay_gain.py /path/to/filename.mp3
+if __name__ == "__main__":
+    print calculate_replay_gain(sys.argv[1])

python_apps/media-monitor/media_monitor/airtimefilemonitor/workerprocess.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+
+import traceback
+import os
+
+class MediaMonitorWorkerProcess:
+
+    def __init__(self, config, mmc):
+        self.config = config
+        self.mmc = mmc
+
+    #this function is run in its own process, and continuously
+    #checks the queue for any new file events.
+    def process_file_events(self, queue, notifier):
+        while True:
+            try:
+                event = queue.get()
+                notifier.logger.info("received event %s", event)
+                notifier.update_airtime(event)
+            except Exception, e:
+                notifier.logger.error(e)
+                notifier.logger.error("traceback: %s", traceback.format_exc())

python_apps/media-monitor/media_monitor/logging.cfg

@@ -0,0 +1,32 @@
+[loggers]
+keys= root,notifier,metadata
+
+[handlers]
+keys=fileOutHandler
+
+[formatters]
+keys=simpleFormatter
+
+[logger_root]
+level=DEBUG
+handlers=fileOutHandler
+
+[logger_notifier]
+level=DEBUG
+handlers=fileOutHandler
+qualname=notifier
+
+[logger_metadata]
+level=DEBUG
+handlers=fileOutHandler
+qualname=metadata
+
+[handler_fileOutHandler]
+class=logging.handlers.RotatingFileHandler
+level=DEBUG
+formatter=simpleFormatter
+args=("/var/log/airtime/media-monitor/media-monitor.log", 'a', 10000000, 5,)
+
+[formatter_simpleFormatter]
+format=%(asctime)s %(levelname)s - [%(threadName)s] [%(filename)s : %(funcName)s()] : LINE %(lineno)d - %(message)s
+datefmt=

python_apps/media-monitor/media_monitor/mm1.py

@@ -0,0 +1,142 @@
+# -*- coding: utf-8 -*-
+
+import time
+import logging.config
+import sys
+import os
+import traceback
+import locale
+
+from configobj import ConfigObj
+
+from api_clients import api_client as apc
+from std_err_override import LogWriter
+
+from multiprocessing import Queue as mpQueue
+
+from threading import Thread
+
+from pyinotify import WatchManager
+
+from airtimefilemonitor.airtimenotifier import AirtimeNotifier
+from airtimefilemonitor.mediamonitorcommon import MediaMonitorCommon
+from airtimefilemonitor.airtimeprocessevent import AirtimeProcessEvent
+from airtimefilemonitor.mediaconfig import AirtimeMediaConfig
+from airtimefilemonitor.workerprocess import MediaMonitorWorkerProcess
+from airtimefilemonitor.airtimemediamonitorbootstrap import AirtimeMediaMonitorBootstrap
+
+def configure_locale():
+    logger.debug("Before %s", locale.nl_langinfo(locale.CODESET))
+    current_locale = locale.getlocale()
+
+    if current_locale[1] is None:
+        logger.debug("No locale currently set. Attempting to get default locale.")
+        default_locale = locale.getdefaultlocale()
+
+        if default_locale[1] is None:
+            logger.debug("No default locale exists. Let's try loading from /etc/default/locale")
+            if os.path.exists("/etc/default/locale"):
+                config = ConfigObj('/etc/default/locale')
+                lang = config.get('LANG')
+                new_locale = lang
+            else:
+                logger.error("/etc/default/locale could not be found! Please run 'sudo update-locale' from command-line.")
+                sys.exit(1)
+        else:
+            new_locale = default_locale
+
+        logger.info("New locale set to: %s", locale.setlocale(locale.LC_ALL, new_locale))
+
+
+
+    reload(sys)
+    sys.setdefaultencoding("UTF-8")
+    current_locale_encoding = locale.getlocale()[1].lower()
+    logger.debug("sys default encoding %s", sys.getdefaultencoding())
+    logger.debug("After %s", locale.nl_langinfo(locale.CODESET))
+
+    if current_locale_encoding not in ['utf-8', 'utf8']:
+        logger.error("Need a UTF-8 locale. Currently '%s'. Exiting..." % current_locale_encoding)
+        sys.exit(1)
+
+# configure logging
+try:
+    logging.config.fileConfig("%s/logging.cfg" % os.path.dirname(os.path.realpath(__file__)))
+
+    #need to wait for Python 2.7 for this..
+    #logging.captureWarnings(True)
+
+    logger = logging.getLogger()
+    LogWriter.override_std_err(logger)
+
+except Exception, e:
+    print 'Error configuring logging: ', e
+    sys.exit(1)
+
+logger.info("\n\n*** Media Monitor bootup ***\n\n")
+
+
+try:
+    configure_locale()
+
+    config = AirtimeMediaConfig(logger)
+    api_client = apc.AirtimeApiClient()
+    api_client.register_component("media-monitor")
+
+    logger.info("Setting up monitor")
+    response = None
+    while response is None:
+        response = api_client.setup_media_monitor()
+        time.sleep(5)
+
+    storage_directory = response["stor"]
+    watched_dirs = response["watched_dirs"]
+    logger.info("Storage Directory is: %s", storage_directory)
+    config.storage_directory = os.path.normpath(storage_directory)
+    config.imported_directory = os.path.normpath(os.path.join(storage_directory, 'imported'))
+    config.organize_directory = os.path.normpath(os.path.join(storage_directory, 'organize'))
+    config.recorded_directory = os.path.normpath(os.path.join(storage_directory, 'recorded'))
+    config.problem_directory = os.path.normpath(os.path.join(storage_directory, 'problem_files'))
+
+    dirs = [config.imported_directory, config.organize_directory, config.recorded_directory, config.problem_directory]
+    for d in dirs:
+        if not os.path.exists(d):
+            os.makedirs(d, 02775)
+
+    multi_queue = mpQueue()
+    logger.info("Initializing event processor")
+
+    wm = WatchManager()
+    mmc = MediaMonitorCommon(config, wm=wm)
+    pe = AirtimeProcessEvent(queue=multi_queue, airtime_config=config, wm=wm, mmc=mmc, api_client=api_client)
+
+    bootstrap = AirtimeMediaMonitorBootstrap(logger, pe, api_client, mmc, wm, config)
+    bootstrap.scan()
+
+    notifier = AirtimeNotifier(wm, pe, read_freq=0, timeout=0, airtime_config=config, api_client=api_client, bootstrap=bootstrap, mmc=mmc)
+    notifier.coalesce_events()
+
+    #create 5 worker threads
+    wp = MediaMonitorWorkerProcess(config, mmc)
+    for i in range(5):
+        threadName = "Thread #%d" % i
+        t = Thread(target=wp.process_file_events, name=threadName, args=(multi_queue, notifier))
+        t.start()
+
+    wdd = notifier.watch_directory(storage_directory)
+    logger.info("Added watch to %s", storage_directory)
+    logger.info("wdd result %s", wdd[storage_directory])
+
+    for dir in watched_dirs:
+        wdd = notifier.watch_directory(dir)
+        logger.info("Added watch to %s", dir)
+        logger.info("wdd result %s", wdd[dir])
+
+    notifier.loop(callback=pe.notifier_loop_callback)
+
+except KeyboardInterrupt:
+    notifier.stop()
+    logger.info("Keyboard Interrupt")
+except Exception, e:
+    logger.error('Exception: %s', e)
+    logger.error("traceback: %s", traceback.format_exc())