CC-5709: Airtime Analyzer

* Added MetadataAnalyzer unit tests and test data
* Improved debug logging and squashed pika logging
* Implemented file moving
* Extract the track number/total
* Fixed mapping of mutagen to Airtime fields in a few spots. The mapping
  matches the DB column names now.
* Fixed the bin/airtime_analyzer binary
* Started work on PluploadController to make it work with the new File
  API

airtime_mvc/application/controllers/PluploadController.php

@@ -35,6 +35,20 @@ class PluploadController extends Zend_Controller_Action
         $this->_helper->json->sendJson(array("jsonrpc" => "2.0", "tempfilepath" => $tempFileName));
     }
 
+    public function uploadFinishedAction()
+    {
+        $upload_dir = ini_get("upload_tmp_dir") . DIRECTORY_SEPARATOR . "plupload";
+        $filename = $this->_getParam('name');
+        $tempname = $this->_getParam('tempname');
+        $result = Application_Model_StoredFile::importUploadedFile($upload_dir, $filename, $tempname);
+        if (!is_null($result))
+            $this->_helper->json->sendJson(array("jsonrpc" => "2.0", "error" => $result));
+        
+        $this->_helper->json->sendJson(array("jsonrpc" => "2.0"));
+        
+    }
+    /* FIXME: I renamed this guy to uploadFinishedAction and am just starting to rewrite it to use the new File API.
+     *        -- Albert March 10, 2014
     public function copyfileAction()
     {
         $upload_dir = ini_get("upload_tmp_dir") . DIRECTORY_SEPARATOR . "plupload";
@@ -46,5 +60,5 @@ class PluploadController extends Zend_Controller_Action
            $this->_helper->json->sendJson(array("jsonrpc" => "2.0", "error" => $result));
 
         $this->_helper->json->sendJson(array("jsonrpc" => "2.0"));
-    }
+    }*/
 }

python_apps/airtime_analyzer/README.rst

@@ -53,4 +53,9 @@ To run the unit tests, execute:
 
     $ nosetests
 
+If you care about seeing console output (stdout), like when you're debugging or developing
+a test, run:
+
+    $ nosetests -s
+
 

python_apps/airtime_analyzer/airtime_analyzer/__init__.py

@@ -1,2 +1 @@
-from airtime_analyzer import AirtimeAnalyzerServer
 

python_apps/airtime_analyzer/airtime_analyzer/airtime_analyzer.py

@@ -33,6 +33,11 @@ class AirtimeAnalyzerServer:
    
         if debug:
             self._log_level = logging.DEBUG
+        else:
+            #Disable most pika/rabbitmq logging:
+            pika_logger = logging.getLogger('pika')
+            pika_logger.setLevel(logging.CRITICAL)
+        
         #self.log = logging.getLogger(__name__)
 
         # Set up logging

python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py

@@ -1,5 +1,7 @@
 import logging
 import multiprocessing 
+import shutil
+import os
 from metadata_analyzer import MetadataAnalyzer
 
 class AnalyzerPipeline:
@@ -15,9 +17,9 @@ class AnalyzerPipeline:
         if not isinstance(queue, multiprocessing.queues.Queue):
             raise TypeError("queue must be a multiprocessing.Queue()")
         if not isinstance(audio_file_path, unicode):
-            raise TypeError("audio_file_path must be a string. Was of type " + type(audio_file_path).__name__ + " instead.")
+            raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__ + " instead.")
         if not isinstance(final_directory, unicode):
-            raise TypeError("final_directory must be a string. Was of type " + type(final_directory).__name__ + " instead.")
+            raise TypeError("final_directory must be unicode. Was of type " + type(final_directory).__name__ + " instead.")
 
 
         # Analyze the audio file we were told to analyze:
@@ -29,3 +31,9 @@ class AnalyzerPipeline:
 
         #print ReplayGainAnalyzer.analyze("foo.mp3")
 
+        final_audio_file_path = final_directory + os.sep + os.path.basename(audio_file_path)
+        if os.path.exists(final_audio_file_path) and not os.path.samefile(audio_file_path, final_audio_file_path):
+            os.remove(final_audio_file_path)
+
+        shutil.move(audio_file_path, final_audio_file_path)
+

python_apps/airtime_analyzer/airtime_analyzer/message_listener.py

@@ -91,6 +91,7 @@ class MessageListener:
             StatusReporter.report_success_to_callback_url(callback_url, api_key, audio_metadata)
 
         except KeyError as e:
+            # A field in msg_dict that we needed was missing (eg. audio_file_path)
             logging.exception("A mandatory airtime_analyzer message field was missing from the message.")
             # See the huge comment about NACK below.
             channel.basic_nack(delivery_tag=method_frame.delivery_tag, multiple=False,

python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py

@@ -40,6 +40,20 @@ class MetadataAnalyzer(Analyzer):
         except (AttributeError, KeyError):
             #If mutagen can't figure out the number of channels, we'll just leave it out...
             pass
+        
+        #Try to extract the number of tracks on the album if we can (the "track total")
+        try:
+            track_number = audio_file["tracknumber"]
+            if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh 
+                track_number = track_number[0]
+            track_number_tokens = track_number.split(u'/')
+            track_number = track_number_tokens[0]
+            metadata["track_number"] = track_number
+            track_total = track_number_tokens[1]
+            metadata["track_total"] = track_total
+        except (AttributeError, KeyError, IndexError):
+            #If we couldn't figure out the track_number or track_total, just ignore it...
+            pass
 
         #We normalize the mutagen tags slightly here, so in case mutagen changes,
         #we find the 
@@ -51,6 +65,7 @@ class MetadataAnalyzer(Analyzer):
             'composer':     'composer',
             'conductor':    'conductor',
             'copyright':    'copyright',
+            'comment':      'comment',
             'encoded_by':   'encoder',
             'genre':        'genre',
             'isrc':         'isrc',
@@ -59,10 +74,10 @@ class MetadataAnalyzer(Analyzer):
             'last_modified':'last_modified',
             'mood':         'mood',
             'replay_gain':  'replaygain',
-            'track_number': 'track_number',
-            'track_total':  'track_total',
+            #'tracknumber':  'track_number',
+            #'track_total':  'track_total',
             'website':      'website',
-            'year':         'year',
+            'date':         'year',
             'mime_type':    'mime',
         }
 

python_apps/airtime_analyzer/bin/airtime_analyzer

@@ -2,7 +2,7 @@
 
 import daemon
 import argparse
-import airtime_analyzer as aa
+import airtime_analyzer.airtime_analyzer as aa
 
 VERSION = "1.0"
 

python_apps/airtime_analyzer/tests/airtime_analyzer_tests.py

@@ -2,11 +2,11 @@ from nose.tools import *
 import airtime_analyzer
 
 def setup():
-    print "SETUP!"
+    pass
 
 def teardown():
-    print "TEAR DOWN!"
+    pass
 
 def test_basic():
-    print "I RAN!"
+    pass
 

python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py

@@ -0,0 +1,24 @@
+from nose.tools import *
+import multiprocessing
+from airtime_analyzer.analyzer_pipeline import AnalyzerPipeline
+
+DEFAULT_AUDIO_FILE = u'tests/test_data/44100Hz-16bit-mono.mp3'
+
+def setup():
+    pass
+
+def teardown():
+    pass
+
+def test_basic():
+    q = multiprocessing.Queue()
+    AnalyzerPipeline.run_analysis(q, DEFAULT_AUDIO_FILE, u'.')
+    results = q.get()
+    assert results['track_title'] == u'Test Title'
+    assert results['artist_name'] == u'Test Artist'
+    assert results['album_title'] == u'Test Album'
+    assert results['year'] == u'1999'
+    assert results['genre'] == u'Test Genre'
+    assert results['mime_type'] == 'audio/mpeg' # Not unicode because MIMEs aren't.
+    assert results['length_seconds'] == 3.90925 
+

python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+from nose.tools import *
+from airtime_analyzer.metadata_analyzer import MetadataAnalyzer 
+
+def setup():
+    pass
+
+def teardown():
+    pass
+
+def check_default_metadata(metadata):
+    assert metadata['track_title'] == u'Test Title'
+    assert metadata['artist_name'] == u'Test Artist'
+    assert metadata['album_title'] == u'Test Album'
+    assert metadata['year'] == u'1999'
+    assert metadata['genre'] == u'Test Genre'
+    assert metadata['track_number'] == u'1'
+
+def test_mp3_mono():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.mp3')
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 1
+    assert metadata['bit_rate'] == 64000
+    assert metadata['length_seconds'] == 3.90925 
+    assert metadata['mime_type'] == 'audio/mpeg' # Not unicode because MIMEs aren't.
+    assert metadata['track_total'] == u'10' # MP3s can have a track_total
+    #Mutagen doesn't extract comments from mp3s it seems
+
+def test_mp3_jointstereo():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-jointstereo.mp3')
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 2
+    assert metadata['bit_rate'] == 128000
+    assert metadata['length_seconds'] == 3.90075 
+    assert metadata['mime_type'] == 'audio/mpeg'
+    assert metadata['track_total'] == u'10' # MP3s can have a track_total
+
+def test_mp3_simplestereo():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-simplestereo.mp3')
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 2
+    assert metadata['bit_rate'] == 128000
+    assert metadata['length_seconds'] == 3.90075 
+    assert metadata['mime_type'] == 'audio/mpeg'
+    assert metadata['track_total'] == u'10' # MP3s can have a track_total
+
+def test_mp3_dualmono():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-dualmono.mp3')
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 2
+    assert metadata['bit_rate'] == 128000
+    assert metadata['length_seconds'] == 3.90075 
+    assert metadata['mime_type'] == 'audio/mpeg'
+    assert metadata['track_total'] == u'10' # MP3s can have a track_total
+
+
+def test_ogg_mono():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.ogg')
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 1
+    assert metadata['bit_rate'] == 80000
+    assert metadata['length_seconds'] == 3.8394104308390022
+    assert metadata['mime_type'] == 'application/ogg'
+    assert metadata['comment'] == u'Test Comment'
+
+def test_ogg_stereo():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.ogg')
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 2
+    assert metadata['bit_rate'] == 112000
+    assert metadata['length_seconds'] == 3.8394104308390022
+    assert metadata['mime_type'] == 'application/ogg'
+    assert metadata['comment'] == u'Test Comment'
+
+''' faac and avconv can't seem to create a proper mono AAC file... ugh
+def test_aac_mono():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.m4a')
+    print "Mono AAC metadata:"
+    print metadata
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 1
+    assert metadata['bit_rate'] == 80000
+    assert metadata['length_seconds'] == 3.8394104308390022
+    assert metadata['mime_type'] == 'video/mp4'
+    assert metadata['comment'] == u'Test Comment'
+'''
+
+def test_aac_stereo():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.m4a')
+    check_default_metadata(metadata)
+    assert metadata['channels'] == 2
+    assert metadata['bit_rate'] == 102619
+    assert metadata['length_seconds'] == 3.8626303854875284 
+    assert metadata['mime_type'] == 'video/mp4'
+    assert metadata['comment'] == u'Test Comment'
+
+def test_mp3_utf8():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-utf8.mp3')
+    # Using a bunch of different UTF-8 codepages here. Test data is from:
+    #   http://winrus.com/utf8-jap.htm
+    assert metadata['track_title'] == u'ｱｲｳｴｵｶｷｸｹｺｻｼｽｾｿﾀﾁﾂﾃ'
+    assert metadata['artist_name'] == u'てすと'
+    assert metadata['album_title'] == u'Ä ä Ü ü ß'
+    assert metadata['year'] == u'1999'
+    assert metadata['genre'] == u'Я Б Г Д Ж Й'
+    assert metadata['track_number'] == u'1'
+    assert metadata['channels'] == 2
+    assert metadata['bit_rate'] == 128000
+    assert metadata['length_seconds'] == 3.90075 
+    assert metadata['mime_type'] == 'audio/mpeg'
+    assert metadata['track_total'] == u'10' # MP3s can have a track_total
+