From b0670f5f478b94bdaca1acf6259050d940afcf25 Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Wed, 16 Apr 2014 13:05:02 -0400 Subject: [PATCH 1/3] CC-5709: Airtime Analyzer * Log tracebacks from child process --- .../airtime_analyzer/analyzer_pipeline.py | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py b/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py index 0ba5fb22e..3a1465b7b 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py @@ -30,27 +30,33 @@ class AnalyzerPipeline: temporary randomly generated name, which is why we want to know what the original name was. """ - if not isinstance(queue, multiprocessing.queues.Queue): - raise TypeError("queue must be a multiprocessing.Queue()") - if not isinstance(audio_file_path, unicode): - raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__ + " instead.") - if not isinstance(import_directory, unicode): - raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__ + " instead.") - if not isinstance(original_filename, unicode): - raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__ + " instead.") - - # Analyze the audio file we were told to analyze: - # First, we extract the ID3 tags and other metadata: - metadata = dict() - metadata = MetadataAnalyzer.analyze(audio_file_path, metadata) - metadata = FileMoverAnalyzer.move(audio_file_path, import_directory, original_filename, metadata) - metadata["import_status"] = 0 # imported - - # Note that the queue we're putting the results into is our interprocess communication - # back to the main process. - - # Pass all the file metadata back to the main analyzer process, which then passes - # it back to the Airtime web application. - queue.put(metadata) + try: + if not isinstance(queue, multiprocessing.queues.Queue): + raise TypeError("queue must be a multiprocessing.Queue()") + if not isinstance(audio_file_path, unicode): + raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__ + " instead.") + if not isinstance(import_directory, unicode): + raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__ + " instead.") + if not isinstance(original_filename, unicode): + raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__ + " instead.") + + + # Analyze the audio file we were told to analyze: + # First, we extract the ID3 tags and other metadata: + metadata = dict() + metadata = MetadataAnalyzer.analyze(audio_file_path, metadata) + metadata = FileMoverAnalyzer.move(audio_file_path, import_directory, original_filename, metadata) + metadata["import_status"] = 0 # imported + + # Note that the queue we're putting the results into is our interprocess communication + # back to the main process. + + # Pass all the file metadata back to the main analyzer process, which then passes + # it back to the Airtime web application. + queue.put(metadata) + except Exception as e: + # Ensures the traceback for this child process gets written to our log files: + logging.exception(e) + raise e From 9a3c88f00974d4c2ba1174ece48b73ddfaa1adf7 Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Wed, 16 Apr 2014 15:33:02 -0400 Subject: [PATCH 2/3] CC-5709: Airtime Analyzer * Explicitly set the locale in upstart conf to avoid UTF-8/unicode errors from python. Fixes CC-5794. --- .../airtime_analyzer/install/upstart/airtime_analyzer.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python_apps/airtime_analyzer/install/upstart/airtime_analyzer.conf b/python_apps/airtime_analyzer/install/upstart/airtime_analyzer.conf index 03bba725c..6fccc21c9 100644 --- a/python_apps/airtime_analyzer/install/upstart/airtime_analyzer.conf +++ b/python_apps/airtime_analyzer/install/upstart/airtime_analyzer.conf @@ -6,5 +6,8 @@ stop on runlevel [!2345] respawn +env LANG='en_US.UTF-8' +env LC_ALL='en_US.UTF-8' + exec su www-data -c "airtime_analyzer" From cdce4ca10457ef0961d9b3f868200142392a855a Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Wed, 16 Apr 2014 16:42:37 -0400 Subject: [PATCH 3/3] CC-5709: Airtime Analyzer * Fixed the unit tests * Improved length testing because it seems to fluctuate a bit with your mutagen version (great, eh?) --- .../airtime_analyzer/metadata_analyzer.py | 3 --- .../tests/analyzer_pipeline_tests.py | 18 +++++++------- .../tests/metadata_analyzer_tests.py | 24 +++++++++++-------- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py index 1e29830d6..71462c45d 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py @@ -49,9 +49,6 @@ class MetadataAnalyzer(Analyzer): mime_magic = magic.Magic(mime=True) metadata["mime"] = mime_magic.from_file(filename) - if isinstance(info, mutagen.mp3.MPEGInfo): - print "mode is: " + str(info.mode) - #Try to get the number of channels if mutagen can... try: #Special handling for getting the # of channels from MP3s. It's in the "mode" field diff --git a/python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py b/python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py index ba7b6f202..cafa07b47 100644 --- a/python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py +++ b/python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py @@ -22,15 +22,15 @@ def test_basic(): q = multiprocessing.Queue() #This actually imports the file into the "./Test Artist" directory. AnalyzerPipeline.run_analysis(q, DEFAULT_AUDIO_FILE, u'.', filename) - results = q.get() - assert results['track_title'] == u'Test Title' - assert results['artist_name'] == u'Test Artist' - assert results['album_title'] == u'Test Album' - assert results['year'] == u'1999' - assert results['genre'] == u'Test Genre' - assert results['mime'] == 'audio/mpeg' # Not unicode because MIMEs aren't. - assert results['length_seconds'] == 3.90925 - assert results["length"] == str(datetime.timedelta(seconds=results["length_seconds"])) + metadata = q.get() + assert metadata['track_title'] == u'Test Title' + assert metadata['artist_name'] == u'Test Artist' + assert metadata['album_title'] == u'Test Album' + assert metadata['year'] == u'1999' + assert metadata['genre'] == u'Test Genre' + assert metadata['mime'] == 'audio/mpeg' # Not unicode because MIMEs aren't. + assert abs(metadata['length_seconds'] - 3.9) < 0.1 + assert metadata["length"] == str(datetime.timedelta(seconds=metadata["length_seconds"])) assert os.path.exists(DEFAULT_IMPORT_DEST) @raises(TypeError) diff --git a/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py b/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py index 8baad0c54..6e59ae15a 100644 --- a/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py +++ b/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py @@ -25,7 +25,7 @@ def test_mp3_mono(): check_default_metadata(metadata) assert metadata['channels'] == 1 assert metadata['bit_rate'] == 64000 - assert metadata['length_seconds'] == 3.90925 + assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mpeg' # Not unicode because MIMEs aren't. assert metadata['track_total'] == u'10' # MP3s can have a track_total #Mutagen doesn't extract comments from mp3s it seems @@ -35,7 +35,7 @@ def test_mp3_jointstereo(): check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 128000 - assert metadata['length_seconds'] == 3.90075 + assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mpeg' assert metadata['track_total'] == u'10' # MP3s can have a track_total @@ -44,7 +44,7 @@ def test_mp3_simplestereo(): check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 128000 - assert metadata['length_seconds'] == 3.90075 + assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mpeg' assert metadata['track_total'] == u'10' # MP3s can have a track_total @@ -53,7 +53,7 @@ def test_mp3_dualmono(): check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 128000 - assert metadata['length_seconds'] == 3.90075 + assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mpeg' assert metadata['track_total'] == u'10' # MP3s can have a track_total @@ -63,7 +63,7 @@ def test_ogg_mono(): check_default_metadata(metadata) assert metadata['channels'] == 1 assert metadata['bit_rate'] == 80000 - assert metadata['length_seconds'] == 3.8394104308390022 + assert abs(metadata['length_seconds'] - 3.8) < 0.1 assert metadata['mime'] == 'application/ogg' assert metadata['comment'] == u'Test Comment' @@ -72,7 +72,7 @@ def test_ogg_stereo(): check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 112000 - assert metadata['length_seconds'] == 3.8394104308390022 + assert abs(metadata['length_seconds'] - 3.8) < 0.1 assert metadata['mime'] == 'application/ogg' assert metadata['comment'] == u'Test Comment' @@ -84,7 +84,7 @@ def test_aac_mono(): check_default_metadata(metadata) assert metadata['channels'] == 1 assert metadata['bit_rate'] == 80000 - assert metadata['length_seconds'] == 3.8394104308390022 + assert abs(metadata['length_seconds'] - 3.8) < 0.1 assert metadata['mime'] == 'video/mp4' assert metadata['comment'] == u'Test Comment' ''' @@ -94,7 +94,7 @@ def test_aac_stereo(): check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 102619 - assert metadata['length_seconds'] == 3.8626303854875284 + assert abs(metadata['length_seconds'] - 3.8) < 0.1 assert metadata['mime'] == 'video/mp4' assert metadata['comment'] == u'Test Comment' @@ -110,7 +110,7 @@ def test_mp3_utf8(): assert metadata['track_number'] == u'1' assert metadata['channels'] == 2 assert metadata['bit_rate'] == 128000 - assert metadata['length_seconds'] == 3.90075 + assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mpeg' assert metadata['track_total'] == u'10' # MP3s can have a track_total @@ -143,7 +143,11 @@ def test_mp3_bad_channels(): check_default_metadata(metadata) assert metadata['channels'] == 1 assert metadata['bit_rate'] == 64000 - assert metadata['length_seconds'] == 3.90925 + print metadata['length_seconds'] + assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mpeg' # Not unicode because MIMEs aren't. assert metadata['track_total'] == u'10' # MP3s can have a track_total #Mutagen doesn't extract comments from mp3s it seems + +def test_unparsable_file(): + MetadataAnalyzer.analyze(u'README.rst', dict())