Merge branch 'cc-5709-airtime-analyzer' into cc-5709-airtime-analyzer-saas

Conflicts:
	airtime_mvc/public/index.php

* Reverted some SaaS-only thing Martin did a year ago. Looks benign but
  only one way to find out...

python_apps/airtime_analyzer/MANIFEST.in

@@ -0,0 +1 @@
+include README.rst

python_apps/airtime_analyzer/README.rst

@@ -0,0 +1,73 @@
+
+Ghetto temporary installation instructions
+==========
+
+    $ sudo python setup.py install
+
+You will need to allow the "airtime" RabbitMQ user to access all exchanges and queues within the /airtime vhost:
+
+    sudo rabbitmqctl set_permissions -p /airtime airtime .\* .\* .\* 
+
+
+Usage
+==========
+
+This program must run as a user with permissions to write to your Airtime music library
+directory. For standard Airtime installations, run it as the www-data user:
+
+    $ sudo -u www-data airtime_analyzer --daemon
+   
+Or during development, add the --debug flag for more verbose output:
+
+    $ sudo -u www-data airtime_analyzer --debug
+
+To print usage instructions, run:
+
+    $ airtime_analyzer --help
+
+This application can be run as a daemon by running:
+
+    $ airtime_analyzer -d
+
+
+
+Developers
+==========
+
+For development, you want to install airtime_analyzer system-wide but with everything symlinked back to the source 
+directory for convenience. This is super easy to do, just run:
+    
+    $ sudo python setup.py develop
+
+To send an test message to airtime_analyzer, you can use the message_sender.php script in the tools directory.
+For example, run:
+
+    $ php tools/message_sender.php '{ "tmp_file_path" : "foo.mp3", "final_directory" : ".", "callback_url" : "http://airtime.localhost/rest/media/1", "api_key" : "YOUR_API_KEY" }'
+
+    $ php tools/message_sender.php '{"tmp_file_path":"foo.mp3", "import_directory":"/srv/airtime/stor/imported/1","original_filename":"foo.mp3","callback_url": "http://airtime.localhost/rest/media/1", "api_key":"YOUR_API_KEY"}'
+
+Logging
+=========
+
+By default, logs are saved to:
+
+    /var/log/airtime/airtime_analyzer.log
+
+This application takes care of rotating logs for you.
+
+
+Unit Tests
+==========
+
+To run the unit tests, execute:
+
+    $ nosetests
+
+If you care about seeing console output (stdout), like when you're debugging or developing
+a test, run:
+
+    $ nosetests -s
+
+To run the unit tests and generate a code coverage report, run:
+
+    $ nosetests --with-coverage --cover-package=airtime_analyzer

python_apps/airtime_analyzer/airtime_analyzer/__init__.py

@@ -0,0 +1 @@
+

python_apps/airtime_analyzer/airtime_analyzer/airtime_analyzer.py

@@ -0,0 +1,79 @@
+import ConfigParser
+import logging
+import logging.handlers
+import sys
+from metadata_analyzer import MetadataAnalyzer
+from replaygain_analyzer import ReplayGainAnalyzer
+from message_listener import MessageListener
+
+
+class AirtimeAnalyzerServer:
+
+    # Constants 
+    _CONFIG_PATH = '/etc/airtime/airtime.conf'
+    _LOG_PATH = "/var/log/airtime/airtime_analyzer.log"
+   
+    # Variables
+    _log_level = logging.INFO
+
+    def __init__(self, debug=False):
+
+        # Configure logging
+        self.setup_logging(debug)
+
+        # Read our config file
+        rabbitmq_config = self.read_config_file()
+
+        # Start listening for RabbitMQ messages telling us about newly
+        # uploaded files.
+        self._msg_listener = MessageListener(rabbitmq_config)
+    
+
+    def setup_logging(self, debug):
+   
+        if debug:
+            self._log_level = logging.DEBUG
+        else:
+            #Disable most pika/rabbitmq logging:
+            pika_logger = logging.getLogger('pika')
+            pika_logger.setLevel(logging.CRITICAL)
+        
+        #self.log = logging.getLogger(__name__)
+
+        # Set up logging
+        logFormatter = logging.Formatter("%(asctime)s [%(module)s] [%(levelname)-5.5s]  %(message)s")
+        rootLogger = logging.getLogger()
+        rootLogger.setLevel(self._log_level)
+
+        fileHandler = logging.handlers.RotatingFileHandler(filename=self._LOG_PATH, maxBytes=1024*1024*30,
+                                                  backupCount=8)
+        fileHandler.setFormatter(logFormatter)
+        rootLogger.addHandler(fileHandler)
+
+        consoleHandler = logging.StreamHandler()
+        consoleHandler.setFormatter(logFormatter)
+        rootLogger.addHandler(consoleHandler)
+
+
+    def read_config_file(self):
+        config = ConfigParser.SafeConfigParser()
+        config_path = AirtimeAnalyzerServer._CONFIG_PATH 
+        try:
+            config.readfp(open(config_path))
+        except IOError as e:
+            print "Failed to open config file at " + config_path + ": " + e.strerror 
+            exit(-1)
+        except Exception:
+            print e.strerror 
+            exit(-1)
+
+        return config
+        
+
+''' When being run from the command line, analyze a file passed
+    as an argument. '''
+if __name__ == "__main__":
+    import sys
+    analyzers = AnalyzerPipeline()
+
+

python_apps/airtime_analyzer/airtime_analyzer/analyzer.py

@@ -0,0 +1,12 @@
+
+class Analyzer:
+
+    @staticmethod
+    def analyze(filename, metadata):
+        raise NotImplementedError
+
+'''
+class AnalyzerError(Exception):
+    def __init__(self):
+        super.__init__(self)
+'''

python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py

@@ -0,0 +1,37 @@
+import logging
+import multiprocessing 
+from metadata_analyzer import MetadataAnalyzer
+from filemover_analyzer import FileMoverAnalyzer
+
+class AnalyzerPipeline:
+
+    # Take message dictionary and perform the necessary analysis.
+    @staticmethod
+    def run_analysis(queue, audio_file_path, import_directory, original_filename):
+
+        if not isinstance(queue, multiprocessing.queues.Queue):
+            raise TypeError("queue must be a multiprocessing.Queue()")
+        if not isinstance(audio_file_path, unicode):
+            raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__ + " instead.")
+        if not isinstance(import_directory, unicode):
+            raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__ + " instead.")
+        if not isinstance(original_filename, unicode):
+            raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__ + " instead.")
+
+        #print ReplayGainAnalyzer.analyze("foo.mp3")
+
+        # Analyze the audio file we were told to analyze:
+        # First, we extract the ID3 tags and other metadata:
+        metadata = dict()
+        metadata = MetadataAnalyzer.analyze(audio_file_path, metadata)
+        metadata = FileMoverAnalyzer.move(audio_file_path, import_directory, original_filename, metadata)
+        metadata["import_status"] = 0 # imported
+
+        # Note that the queue we're putting the results into is our interprocess communication 
+        # back to the main process.
+
+        #Pass all the file metadata back to the main analyzer process, which then passes
+        #it back to the Airtime web application.
+        queue.put(metadata)
+
+

python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py

@@ -0,0 +1,79 @@
+import logging
+import os
+import time
+import shutil
+import os, errno
+import time
+import uuid 
+
+from analyzer import Analyzer
+
+class FileMoverAnalyzer(Analyzer):
+    
+    @staticmethod
+    def analyze(audio_file_path, metadata):
+        raise Exception("Use FileMoverAnalyzer.move() instead.")
+    
+    @staticmethod
+    def move(audio_file_path, import_directory, original_filename, metadata):
+        if not isinstance(audio_file_path, unicode):
+            raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__)
+        if not isinstance(import_directory, unicode):
+            raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__)
+        if not isinstance(original_filename, unicode):
+            raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__)
+        if not isinstance(metadata, dict):
+            raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__)
+        
+        #Import the file over to it's final location.
+        # TODO: Also, handle the case where the move fails and write some code
+        # to possibly move the file to problem_files.
+       
+        final_file_path = import_directory
+        if metadata.has_key("artist_name"):
+            final_file_path += "/" + metadata["artist_name"]
+        if metadata.has_key("album_title"):
+            final_file_path += "/" + metadata["album_title"]
+        final_file_path += "/" + original_filename
+
+        #Ensure any redundant slashes are stripped
+        final_file_path = os.path.normpath(final_file_path)
+
+        #If a file with the same name already exists in the "import" directory, then
+        #we add a unique string to the end of this one. We never overwrite a file on import
+        #because if we did that, it would mean Airtime's database would have 
+        #the wrong information for the file we just overwrote (eg. the song length would be wrong!)
+        #If the final file path is the same as the file we've been told to import (which
+        #you often do when you're debugging), then don't move the file at all.
+        if os.path.exists(final_file_path):
+            if os.path.samefile(audio_file_path, final_file_path):
+                metadata["full_path"] = final_file_path
+                return metadata
+            base_file_path, file_extension = os.path.splitext(final_file_path)
+            final_file_path = "%s_%s%s" % (base_file_path, time.strftime("%m-%d-%Y-%H-%M-%S", time.localtime()), file_extension)
+
+        #If THAT path exists, append a UUID instead:
+        while os.path.exists(final_file_path):
+            base_file_path, file_extension = os.path.splitext(final_file_path)
+            final_file_path = "%s_%s%s" % (base_file_path, str(uuid.uuid4()), file_extension)
+
+        #Ensure the full path to the file exists
+        mkdir_p(os.path.dirname(final_file_path))
+        
+        #Move the file into its final destination directory 
+        logging.debug("Moving %s to %s" % (audio_file_path, final_file_path))
+        shutil.move(audio_file_path, final_file_path)
+    
+        metadata["full_path"] = final_file_path
+        return metadata
+    
+def mkdir_p(path):
+    if path == "":
+        return
+    try:
+        os.makedirs(path)
+    except OSError as exc: # Python >2.5
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else: raise
+

python_apps/airtime_analyzer/airtime_analyzer/message_listener.py

@@ -0,0 +1,144 @@
+import sys
+import pika
+import json
+import time
+import logging 
+import multiprocessing 
+from analyzer_pipeline import AnalyzerPipeline
+from status_reporter import StatusReporter
+
+EXCHANGE = "airtime-uploads"
+EXCHANGE_TYPE = "topic"
+ROUTING_KEY = "" #"airtime.analyzer.tasks"
+QUEUE = "airtime-uploads"
+
+
+''' TODO: Document me
+    - round robin messaging
+    - acking
+    - why we use the multiprocess architecture
+    - in general, how it works and why it works this way
+'''
+class MessageListener:
+
+    def __init__(self, config):
+
+        # Read the RabbitMQ connection settings from the config file
+        # The exceptions throw here by default give good error messages. 
+        RMQ_CONFIG_SECTION = "rabbitmq"
+        self._host = config.get(RMQ_CONFIG_SECTION, 'host')
+        self._port = config.getint(RMQ_CONFIG_SECTION, 'port')
+        self._username = config.get(RMQ_CONFIG_SECTION, 'user')
+        self._password = config.get(RMQ_CONFIG_SECTION, 'password')
+        self._vhost = config.get(RMQ_CONFIG_SECTION, 'vhost')
+
+        while True:
+            try:
+                self.connect_to_messaging_server()
+                self.wait_for_messages()
+            except KeyboardInterrupt:
+                self.disconnect_from_messaging_server()
+                break
+            except pika.exceptions.AMQPError as e:
+                logging.error("Connection to message queue failed. ")
+                logging.error(e)
+                logging.info("Retrying in 5 seconds...")
+                time.sleep(5)
+
+        self._connection.close()
+
+
+    def connect_to_messaging_server(self):
+
+        self._connection = pika.BlockingConnection(pika.ConnectionParameters(host=self._host, 
+            port=self._port, virtual_host=self._vhost, 
+            credentials=pika.credentials.PlainCredentials(self._username, self._password)))
+        self._channel = self._connection.channel()
+        self._channel.exchange_declare(exchange=EXCHANGE, type=EXCHANGE_TYPE, durable=True)
+        result = self._channel.queue_declare(queue=QUEUE, durable=True)
+
+        self._channel.queue_bind(exchange=EXCHANGE, queue=QUEUE, routing_key=ROUTING_KEY)
+         
+        logging.info(" Listening for messages...")
+        self._channel.basic_consume(MessageListener.msg_received_callback, 
+                                    queue=QUEUE, no_ack=False)
+
+    def wait_for_messages(self):
+        self._channel.start_consuming()
+
+    def disconnect_from_messaging_server(self):
+        self._channel.stop_consuming()
+
+
+    # consume callback function
+    @staticmethod
+    def msg_received_callback(channel, method_frame, header_frame, body):
+        logging.info(" - Received '%s' on routing_key '%s'" % (body, method_frame.routing_key))
+
+        # Spin up a worker process. We use the multiprocessing module and multiprocessing.Queue 
+        # to pass objects between the processes so that if the analyzer process crashes, it does not
+        # take down the rest of the daemon and we NACK that message so that it doesn't get 
+        # propagated to other airtime_analyzer daemons (eg. running on other servers). 
+        # We avoid cascading failure this way.
+        try:
+            msg_dict = json.loads(body)
+            audio_file_path = msg_dict["tmp_file_path"]
+            #final_file_path = msg_dict["final_file_path"]
+            import_directory = msg_dict["import_directory"]
+            original_filename = msg_dict["original_filename"]
+            callback_url    = msg_dict["callback_url"]
+            api_key         = msg_dict["api_key"]
+            
+            audio_metadata = MessageListener.spawn_analyzer_process(audio_file_path, import_directory, original_filename)
+            StatusReporter.report_success_to_callback_url(callback_url, api_key, audio_metadata)
+
+        except KeyError as e:
+            # A field in msg_dict that we needed was missing (eg. audio_file_path)
+            logging.exception("A mandatory airtime_analyzer message field was missing from the message.")
+            # See the huge comment about NACK below.
+            channel.basic_nack(delivery_tag=method_frame.delivery_tag, multiple=False,
+                               requeue=False) #Important that it doesn't requeue the message
+        
+        except Exception as e:
+            logging.exception(e)
+            #If ANY exception happens while processing a file, we're going to NACK to the 
+            #messaging server and tell it to remove the message from the queue. 
+            #(NACK is a negative acknowledgement. We could use ACK instead, but this might come
+            # in handy in the future.)
+            #Exceptions in this context are unexpected, unhandled errors. We try to recover
+            #from as many errors as possble in AnalyzerPipeline, but we're safeguarding ourselves
+            #here from any catastrophic or genuinely unexpected errors:
+            channel.basic_nack(delivery_tag=method_frame.delivery_tag, multiple=False,
+                               requeue=False) #Important that it doesn't requeue the message
+
+            # TODO: Report this as a failed upload to the File Upload REST API.
+            #
+            # TODO: If the JSON was invalid or the web server is down, 
+            #       then don't report that failure to the REST API
+            #TODO: Catch exceptions from this HTTP request too:
+            StatusReporter.report_failure_to_callback_url(callback_url, api_key, import_status=2,
+                                                          reason=u'An error occurred while importing this file')
+            
+
+        else:
+            # ACK at the very end, after the message has been successfully processed.
+            # If we don't ack, then RabbitMQ will redeliver the message in the future.
+            channel.basic_ack(delivery_tag=method_frame.delivery_tag)
+    
+    @staticmethod
+    def spawn_analyzer_process(audio_file_path, import_directory, original_filename):
+
+        q = multiprocessing.Queue()
+        p = multiprocessing.Process(target=AnalyzerPipeline.run_analysis, 
+                        args=(q, audio_file_path, import_directory, original_filename))
+        p.start()
+        p.join()
+        if p.exitcode == 0:
+            results = q.get()
+            logging.info("Main process received results from child: ")
+            logging.info(results)
+        else:
+            raise Exception("Analyzer process terminated unexpectedly.")
+
+        return results
+

python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py

@@ -0,0 +1,142 @@
+import time
+import datetime
+import mutagen
+import magic # For MIME type detection
+from analyzer import Analyzer
+
+class MetadataAnalyzer(Analyzer):
+
+    @staticmethod
+    def analyze(filename, metadata):
+        if not isinstance(filename, unicode):
+            raise TypeError("filename must be unicode. Was of type " + type(filename).__name__)
+        if not isinstance(metadata, dict):
+            raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__)
+
+        #Extract metadata from an audio file using mutagen
+        audio_file = mutagen.File(filename, easy=True)
+
+        #Grab other file information that isn't encoded in a tag, but instead usually
+        #in the file header. Mutagen breaks that out into a separate "info" object:
+        info = audio_file.info
+        metadata["sample_rate"] = info.sample_rate
+        metadata["length_seconds"] = info.length 
+        #Converting the length in seconds (float) to a formatted time string
+        track_length = datetime.timedelta(seconds=info.length)
+        metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length)
+        metadata["bit_rate"] = info.bitrate
+        #metadata["channels"] = info.channels
+      
+        #Use the python-magic module to get the MIME type.
+        mime_magic = magic.Magic(mime=True)
+        metadata["mime"] = mime_magic.from_file(filename)
+   
+        if isinstance(info, mutagen.mp3.MPEGInfo):
+            print "mode is: " + str(info.mode)
+
+        #Try to get the number of channels if mutagen can...
+        try:
+            #Special handling for getting the # of channels from MP3s. It's in the "mode" field
+            #which is 0=Stereo, 1=Joint Stereo, 2=Dual Channel, 3=Mono. Part of the ID3 spec...
+            if metadata["mime"] == "audio/mpeg":
+                if info.mode == 3:
+                    metadata["channels"] = 1
+                else:
+                    metadata["channels"] = 2
+            else:
+                metadata["channels"] = info.channels
+        except (AttributeError, KeyError):
+            #If mutagen can't figure out the number of channels, we'll just leave it out...
+            pass
+        
+        #Try to extract the number of tracks on the album if we can (the "track total")
+        try:
+            track_number = audio_file["tracknumber"]
+            if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh 
+                track_number = track_number[0]
+            track_number_tokens = track_number.split(u'/')
+            track_number = track_number_tokens[0]
+            metadata["track_number"] = track_number
+            track_total = track_number_tokens[1]
+            metadata["track_total"] = track_total
+        except (AttributeError, KeyError, IndexError):
+            #If we couldn't figure out the track_number or track_total, just ignore it...
+            pass
+
+        #We normalize the mutagen tags slightly here, so in case mutagen changes,
+        #we find the 
+        mutagen_to_airtime_mapping = {
+            'title':        'track_title',
+            'artist':       'artist_name',
+            'album':        'album_title',
+            'bpm':          'bpm',
+            'composer':     'composer',
+            'conductor':    'conductor',
+            'copyright':    'copyright',
+            'comment':      'comment',
+            'encoded_by':   'encoder',
+            'genre':        'genre',
+            'isrc':         'isrc',
+            'label':        'label',
+            'length':       'length',
+            'language':     'language',
+            'last_modified':'last_modified',
+            'mood':         'mood',
+            'replay_gain':  'replaygain',
+            #'tracknumber':  'track_number',
+            #'track_total':  'track_total',
+            'website':      'website',
+            'date':         'year',
+            #'mime_type':    'mime',
+        }
+
+        for mutagen_tag, airtime_tag in mutagen_to_airtime_mapping.iteritems():
+            try:
+                metadata[airtime_tag] = audio_file[mutagen_tag]
+
+                # Some tags are returned as lists because there could be multiple values.
+                # This is unusual so we're going to always just take the first item in the list.
+                if isinstance(metadata[airtime_tag], list):
+                    metadata[airtime_tag] = metadata[airtime_tag][0]
+
+            except KeyError:
+                continue 
+
+        #Airtime <= 2.5.x nonsense:
+        metadata["ftype"] = "audioclip"
+        #Other fields we'll want to set for Airtime:
+        metadata["cueout"] = metadata["length"] 
+        metadata["hidden"] = False
+
+        return metadata
+
+
+
+'''
+For reference, the Airtime metadata fields are:
+            title
+            artist ("Creator" in Airtime)
+            album
+            bit rate
+            BPM
+            composer
+            conductor
+            copyright
+            cue in
+            cue out
+            encoded by
+            genre
+            ISRC
+            label
+            language
+            last modified
+            length
+            mime
+            mood
+            owner
+            replay gain
+            sample rate
+            track number
+            website
+            year
+'''

python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py

@@ -0,0 +1,12 @@
+from analyzer import Analyzer
+
+''' TODO: everything '''
+class ReplayGainAnalyzer(Analyzer):
+    
+    def __init__(self):
+        pass
+    
+    @staticmethod
+    def analyze(filename):
+        pass
+    

python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py

@@ -0,0 +1,44 @@
+import requests
+import json
+import logging
+
+class StatusReporter():
+
+    _HTTP_REQUEST_TIMEOUT = 30
+
+    # Report the extracted metadata and status of the successfully imported file 
+    # to the callback URL (which should be the Airtime File Upload API)
+    @classmethod
+    def report_success_to_callback_url(self, callback_url, api_key, audio_metadata):
+
+        # encode the audio metadata as json and post it back to the callback_url
+        put_payload = json.dumps(audio_metadata)
+        logging.debug("sending http put with payload: " + put_payload)
+        r = requests.put(callback_url, data=put_payload, 
+                         auth=requests.auth.HTTPBasicAuth(api_key, ''),
+                         timeout=StatusReporter._HTTP_REQUEST_TIMEOUT)
+        logging.debug("HTTP request returned status: " + str(r.status_code))
+        logging.debug(r.text) # log the response body
+
+        #todo: queue up failed requests and try them again later.
+        r.raise_for_status() # raise an exception if there was an http error code returned
+
+    @classmethod
+    def report_failure_to_callback_url(self, callback_url, api_key, import_status, reason):
+        # TODO: Make import_status is an int?
+      
+        logging.debug("Reporting import failure to Airtime REST API...")
+        audio_metadata = dict()
+        audio_metadata["import_status"] = import_status
+        audio_metadata["comment"] = reason  # hack attack
+        put_payload = json.dumps(audio_metadata)
+        logging.debug("sending http put with payload: " + put_payload)
+        r = requests.put(callback_url, data=put_payload, 
+                         auth=requests.auth.HTTPBasicAuth(api_key, ''),
+                         timeout=StatusReporter._HTTP_REQUEST_TIMEOUT)
+        logging.debug("HTTP request returned status: " + str(r.status_code))
+        logging.debug(r.text) # log the response body
+
+        #TODO: queue up failed requests and try them again later.
+        r.raise_for_status() # raise an exception if there was an http error code returned
+

python_apps/airtime_analyzer/bin/airtime_analyzer

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+
+import daemon
+import argparse
+import os
+import airtime_analyzer.airtime_analyzer as aa
+
+VERSION = "1.0"
+
+print "Airtime Analyzer " + VERSION
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-d", "--daemon", help="run as a daemon", action="store_true")
+parser.add_argument("--debug", help="log full debugging output", action="store_true")
+args = parser.parse_args()
+
+'''Ensure media_monitor isn't running before we start, because it'll move newly uploaded
+   files into the library on us and screw up the operation of airtime_analyzer.
+   media_monitor is deprecated.
+'''
+def check_if_media_monitor_is_running():
+    pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
+
+    for pid in pids:
+        try:
+            process_name = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read()
+            if 'media_monitor.py' in process_name:
+                print "Error: This process conflicts with media_monitor, and media_monitor is running."
+                print "       Please terminate the running media_monitor.py process and try again."
+                exit(1)
+        except IOError: # proc has already terminated
+            continue
+
+check_if_media_monitor_is_running()
+
+if args.daemon:
+    with daemon.DaemonContext():
+        analyzer = aa.AirtimeAnalyzerServer(debug=args.debug)
+else: 
+    # Run without daemonizing
+    analyzer = aa.AirtimeAnalyzerServer(debug=args.debug)
+

python_apps/airtime_analyzer/setup.py

@@ -0,0 +1,22 @@
+from setuptools import setup
+
+setup(name='airtime_analyzer',
+      version='0.1',
+      description='Airtime Analyzer Worker and File Importer',
+      url='http://github.com/sourcefabric/Airtime',
+      author='Albert Santoni',
+      author_email='albert.santoni@sourcefabric.org',
+      license='MIT',
+      packages=['airtime_analyzer'],
+      scripts=['bin/airtime_analyzer'],
+      install_requires=[
+          'mutagen',
+          'python-magic',
+          'pika',
+          'nose',
+          'coverage',
+          'mock',
+          'python-daemon',
+          'requests',
+      ],
+      zip_safe=False)

python_apps/airtime_analyzer/tests/airtime_analyzer_tests.py

@@ -0,0 +1,12 @@
+from nose.tools import *
+import airtime_analyzer
+
+def setup():
+    pass
+
+def teardown():
+    pass
+
+def test_basic():
+    pass
+

python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py

@@ -0,0 +1,51 @@
+from nose.tools import *
+import os
+import shutil
+import multiprocessing
+import Queue
+import datetime
+from airtime_analyzer.analyzer_pipeline import AnalyzerPipeline
+
+DEFAULT_AUDIO_FILE = u'tests/test_data/44100Hz-16bit-mono.mp3'
+DEFAULT_IMPORT_DEST = u'Test Artist/Test Album/44100Hz-16bit-mono.mp3'
+
+def setup():
+    pass
+
+def teardown():
+    #Move the file back
+    shutil.move(DEFAULT_IMPORT_DEST, DEFAULT_AUDIO_FILE)
+    assert os.path.exists(DEFAULT_AUDIO_FILE)
+
+def test_basic():
+    filename = os.path.basename(DEFAULT_AUDIO_FILE)
+    q = multiprocessing.Queue()
+    #This actually imports the file into the "./Test Artist" directory.
+    AnalyzerPipeline.run_analysis(q, DEFAULT_AUDIO_FILE, u'.', filename)
+    results = q.get()
+    assert results['track_title'] == u'Test Title'
+    assert results['artist_name'] == u'Test Artist'
+    assert results['album_title'] == u'Test Album'
+    assert results['year'] == u'1999'
+    assert results['genre'] == u'Test Genre'
+    assert results['mime'] == 'audio/mpeg' # Not unicode because MIMEs aren't.
+    assert results['length_seconds'] == 3.90925
+    assert results["length"] == str(datetime.timedelta(seconds=results["length_seconds"]))
+    assert os.path.exists(DEFAULT_IMPORT_DEST)
+
+@raises(TypeError)
+def test_wrong_type_queue_param():
+    AnalyzerPipeline.run_analysis(Queue.Queue(), u'', u'', u'')
+
+@raises(TypeError)
+def test_wrong_type_string_param2():
+    AnalyzerPipeline.run_analysis(multiprocessing.queues.Queue(), '', u'', u'')
+
+@raises(TypeError)
+def test_wrong_type_string_param3():
+    AnalyzerPipeline.run_analysis(multiprocessing.queues.Queue(), u'', '', u'')
+
+@raises(TypeError)
+def test_wrong_type_string_param4():
+    AnalyzerPipeline.run_analysis(multiprocessing.queues.Queue(), u'', u'', '')
+

python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py

@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+import datetime
+import mutagen
+import mock
+from nose.tools import *
+from airtime_analyzer.metadata_analyzer import MetadataAnalyzer 
+
+def setup():
+    pass
+
+def teardown():
+    pass
+
+def check_default_metadata(metadata):
+    assert metadata['track_title'] == u'Test Title'
+    assert metadata['artist_name'] == u'Test Artist'
+    assert metadata['album_title'] == u'Test Album'
+    assert metadata['year'] == u'1999'
+    assert metadata['genre'] == u'Test Genre'
+    assert metadata['track_number'] == u'1'
+    assert metadata["length"] == str(datetime.timedelta(seconds=metadata["length_seconds"]))
+
+def test_mp3_mono():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.mp3', dict())
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 1
+    assert metadata['bit_rate'] == 64000
+    assert metadata['length_seconds'] == 3.90925 
+    assert metadata['mime'] == 'audio/mpeg' # Not unicode because MIMEs aren't.
+    assert metadata['track_total'] == u'10' # MP3s can have a track_total
+    #Mutagen doesn't extract comments from mp3s it seems
+
+def test_mp3_jointstereo():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-jointstereo.mp3', dict())
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 2
+    assert metadata['bit_rate'] == 128000
+    assert metadata['length_seconds'] == 3.90075 
+    assert metadata['mime'] == 'audio/mpeg'
+    assert metadata['track_total'] == u'10' # MP3s can have a track_total
+
+def test_mp3_simplestereo():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-simplestereo.mp3', dict())
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 2
+    assert metadata['bit_rate'] == 128000
+    assert metadata['length_seconds'] == 3.90075 
+    assert metadata['mime'] == 'audio/mpeg'
+    assert metadata['track_total'] == u'10' # MP3s can have a track_total
+
+def test_mp3_dualmono():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-dualmono.mp3', dict())
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 2
+    assert metadata['bit_rate'] == 128000
+    assert metadata['length_seconds'] == 3.90075 
+    assert metadata['mime'] == 'audio/mpeg'
+    assert metadata['track_total'] == u'10' # MP3s can have a track_total
+
+
+def test_ogg_mono():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.ogg', dict())
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 1
+    assert metadata['bit_rate'] == 80000
+    assert metadata['length_seconds'] == 3.8394104308390022
+    assert metadata['mime'] == 'application/ogg'
+    assert metadata['comment'] == u'Test Comment'
+
+def test_ogg_stereo():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.ogg', dict())
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 2
+    assert metadata['bit_rate'] == 112000
+    assert metadata['length_seconds'] == 3.8394104308390022
+    assert metadata['mime'] == 'application/ogg'
+    assert metadata['comment'] == u'Test Comment'
+
+''' faac and avconv can't seem to create a proper mono AAC file... ugh
+def test_aac_mono():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.m4a')
+    print "Mono AAC metadata:"
+    print metadata
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 1
+    assert metadata['bit_rate'] == 80000
+    assert metadata['length_seconds'] == 3.8394104308390022
+    assert metadata['mime'] == 'video/mp4'
+    assert metadata['comment'] == u'Test Comment'
+'''
+
+def test_aac_stereo():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.m4a', dict())
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 2
+    assert metadata['bit_rate'] == 102619
+    assert metadata['length_seconds'] == 3.8626303854875284 
+    assert metadata['mime'] == 'video/mp4'
+    assert metadata['comment'] == u'Test Comment'
+
+def test_mp3_utf8():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict())
+    # Using a bunch of different UTF-8 codepages here. Test data is from:
+    #   http://winrus.com/utf8-jap.htm
+    assert metadata['track_title'] == u'ｱｲｳｴｵｶｷｸｹｺｻｼｽｾｿﾀﾁﾂﾃ'
+    assert metadata['artist_name'] == u'てすと'
+    assert metadata['album_title'] == u'Ä ä Ü ü ß'
+    assert metadata['year'] == u'1999'
+    assert metadata['genre'] == u'Я Б Г Д Ж Й'
+    assert metadata['track_number'] == u'1'
+    assert metadata['channels'] == 2
+    assert metadata['bit_rate'] == 128000
+    assert metadata['length_seconds'] == 3.90075 
+    assert metadata['mime'] == 'audio/mpeg'
+    assert metadata['track_total'] == u'10' # MP3s can have a track_total
+
+# Make sure the parameter checking works
+@raises(TypeError)
+def test_move_wrong_string_param1():
+    not_unicode = 'asdfasdf'
+    MetadataAnalyzer.analyze(not_unicode, dict())
+
+@raises(TypeError)
+def test_move_wrong_metadata_dict():
+    not_a_dict = list()
+    MetadataAnalyzer.analyze(u'asdfasdf', not_a_dict)
+
+# Test an mp3 file where the number of channels is invalid or missing:
+def test_mp3_bad_channels():
+    filename = u'tests/test_data/44100Hz-16bit-mono.mp3'
+    ''' 
+        It'd be a pain in the ass to construct a real MP3 with an invalid number
+        of channels by hand because that value is stored in every MP3 frame in the file
+    '''
+    print "testing bad channels..."
+    audio_file = mutagen.File(filename, easy=True)
+    audio_file.info.mode = 1777
+    with mock.patch('airtime_analyzer.metadata_analyzer.mutagen') as mock_mutagen:
+        mock_mutagen.File.return_value = audio_file
+        #mock_mutagen.side_effect = lambda *args, **kw: audio_file #File(*args, **kw)
+
+    metadata = MetadataAnalyzer.analyze(filename, dict())
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 1
+    assert metadata['bit_rate'] == 64000
+    assert metadata['length_seconds'] == 3.90925 
+    assert metadata['mime'] == 'audio/mpeg' # Not unicode because MIMEs aren't.
+    assert metadata['track_total'] == u'10' # MP3s can have a track_total
+    #Mutagen doesn't extract comments from mp3s it seems

python_apps/airtime_analyzer/tools/ftp-upload-hook.sh

@@ -0,0 +1,5 @@
+#! /bin/bash
+
+path=$1
+filename="${path##*/}"
+curl http://localhost/rest/media -u 3188BDIMPJROQP89Z0OX: -X POST -F "file=@$path" -F "name=$filename"

python_apps/airtime_analyzer/tools/message_sender.php

@@ -0,0 +1,47 @@
+<?
+require_once('php-amqplib/amqp.inc');
+
+//use PhpAmqpLibConnectionAMQPConnection;
+//use PhpAmqpLibMessageAMQPMessage;
+
+define('HOST', '127.0.0.1');
+define('PORT', '5672');
+define('USER', 'airtime');
+define('PASS', 'QEFKX5GMKT4YNMOAL9R8');
+define('VHOST', '/airtime');//'/airtime');
+
+$exchange = "airtime-uploads";
+$exchangeType = "topic";
+$queue = "airtime-uploads";
+$routingKey = ""; //"airtime.analyzer.tasks";
+
+if ($argc <= 1)
+{
+    echo("Usage: " . $argv[0] . " message\n");
+    exit();
+}
+
+$message = $argv[1]; 
+ 
+$connection = new AMQPConnection(HOST, PORT, USER, PASS, VHOST);
+if (!isset($connection))
+{
+    echo "Failed to connect to the RabbitMQ server.";
+    return;
+}
+
+$channel = $connection->channel();
+ 
+// declare/create the queue
+$channel->queue_declare($queue, false, true, false, false);
+  
+// declare/create the exchange as a topic exchange.
+$channel->exchange_declare($exchange, $exchangeType, false, true, false);
+
+$msg = new AMQPMessage($message, array("content_type" => "text/plain"));
+
+$channel->basic_publish($msg, $exchange, $routingKey);
+print "Sent $message ($routingKey)\n";
+$channel->close();
+$connection->close();
+

python_apps/airtime_analyzer/tools/php-amqplib

@@ -0,0 +1 @@
+../../../airtime_mvc/library/php-amqplib