Silan cue-in/out analysis now moved to airtime_analyzer

* Added a new CuePointAnalyzer using silan (will remove silan stuff from pypo later)
* Makes silan analysis slightly more reliable for certain short files.
* Fixes CC-5961: Audio duration cutoff for WAVE files
* Added unit tests for the new analyzer and improved code coverage slightly
* Added unit tests for WAVE metadata extraction and an invalid WMA file

python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py

@@ -5,6 +5,7 @@ import threading
 import multiprocessing 
 from metadata_analyzer import MetadataAnalyzer
 from filemover_analyzer import FileMoverAnalyzer
+from cuepoint_analyzer import CuePointAnalyzer
 
 class AnalyzerPipeline:
     """ Analyzes and imports an audio file into the Airtime library. 
@@ -51,6 +52,7 @@ class AnalyzerPipeline:
             # First, we extract the ID3 tags and other metadata:
             metadata = dict()
             metadata = MetadataAnalyzer.analyze(audio_file_path, metadata)
+            metadata = CuePointAnalyzer.analyze(audio_file_path, metadata)
             metadata = FileMoverAnalyzer.move(audio_file_path, import_directory, original_filename, metadata)
             metadata["import_status"] = 0 # Successfully imported
 

python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py

@@ -0,0 +1,48 @@
+import subprocess
+import logging
+import traceback
+import json
+import datetime
+from analyzer import Analyzer
+
+
+class CuePointAnalyzer(Analyzer):
+    ''' This class extracts the cue-in time, cue-out time, and length of a track using silan. '''
+
+    SILAN_EXECUTABLE = 'silan'
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def analyze(filename, metadata):
+        ''' Extracts the cue-in and cue-out times along and sets the file duration based on that.
+            The cue points are there to skip the silence at the start and end of a track, and are determined
+            using "silan", which analyzes the loudness in a track.
+        :param filename: The full path to the file to analyzer
+        :param metadata: A metadata dictionary where the results will be put
+        :return: The metadata dictionary
+        '''
+        ''' The silan -F 0.99 parameter tweaks the highpass filter. The default is 0.98, but at that setting,
+            the unit test on the short m4a file fails. With the new setting, it gets the correct cue-in time and
+            all the unit tests pass.
+        '''
+        command = [CuePointAnalyzer.SILAN_EXECUTABLE, '-b', '-F', '0.99', '-f', 'JSON', filename]
+        try:
+            results_json = subprocess.check_output(command)
+            silan_results = json.loads(results_json)
+            metadata['length_seconds'] = float(silan_results['file duration'])
+            # Conver the length into a formatted time string
+            track_length = datetime.timedelta(seconds=metadata['length_seconds'])
+            metadata["length"] = str(track_length)
+            metadata['cuein'] = silan_results['sound'][0][0]
+            metadata['cueout'] = silan_results['sound'][0][1]
+
+        except OSError as e: # silan was not found
+            logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have silan installed?"))
+        except subprocess.CalledProcessError as e: # silan returned an error code
+            logging.warn("%s %s %s", e.cmd, e.message, e.returncode)
+        except Exception as e:
+            logging.warn(e)
+
+        return metadata

python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py

@@ -149,7 +149,7 @@ class MetadataAnalyzer(Analyzer):
             metadata["mime"] = magic.from_file(filename, mime=True)
             metadata["channels"] = reader.getnchannels()
             metadata["sample_rate"] = reader.getframerate()
-            length_seconds = float(reader.getnframes()) / float(metadata["channels"] * metadata["sample_rate"])
+            length_seconds = float(reader.getnframes()) / float(metadata["sample_rate"])
             #Converting the length in seconds (float) to a formatted time string
             track_length = datetime.timedelta(seconds=length_seconds)
             metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length)

python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py

@@ -1,14 +1,41 @@
+import subprocess
 from analyzer import Analyzer
 
-''' TODO: ReplayGain is currently calculated by pypo but it should
+
-          be done here in the analyzer.
-'''
 class ReplayGainAnalyzer(Analyzer):
-    
+    ''' This class extracts the cue-in time, cue-out time, and length of a track using silan. '''
+
+    BG1770GAIN_EXECUTABLE = 'bg1770gain'
+
     def __init__(self):
         pass
-    
+
     @staticmethod
-    def analyze(filename):
+    def analyze(filename, metadata):
-        pass
+        ''' Extracts the Replaygain loudness normalization factor of a track.
-    
+        :param filename: The full path to the file to analyzer
+        :param metadata: A metadata dictionary where the results will be put
+        :return: The metadata dictionary
+        '''
+        ''' The -d 00:01:00 flag means it will let the decoding run for a maximum of 1 minute. This is a safeguard
+            in case the libavcodec decoder gets stuck in an infinite loop.
+        '''
+        command = [ReplayGainAnalyzer.BG1770GAIN_EXECUTABLE, '--replaygain', '-d', '00:01:00', '-f', 'JSON', filename]
+        try:
+            results_json = subprocess.check_output(command)
+            silan_results = json.loads(results_json)
+            metadata['length_seconds'] = float(silan_results['file duration'])
+            # Conver the length into a formatted time string
+            track_length = datetime.timedelta(seconds=metadata['length_seconds'])
+            metadata["length"] = str(track_length)
+            metadata['cuein'] = silan_results['sound'][0][0]
+            metadata['cueout'] = silan_results['sound'][0][1]
+
+        except OSError as e: # silan was not found
+            logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have silan installed?"))
+        except subprocess.CalledProcessError as e: # silan returned an error code
+            logging.warn("%s %s %s", e.cmd, e.message, e.returncode)
+        except Exception as e:
+            logging.warn(e)
+
+        return metadata

python_apps/airtime_analyzer/tests/cuepoint_analyzer_tests.py

@@ -0,0 +1,76 @@
+from nose.tools import *
+from airtime_analyzer.cuepoint_analyzer import CuePointAnalyzer
+
+def test_constructor():
+    cpa = CuePointAnalyzer()
+
+
+def check_default_metadata(metadata):
+    ''' Check that the values extract by Silan/CuePointAnalyzer on our test audio files match what we expect.
+    :param metadata: a metadata dictionary
+    :return: Nothing
+    '''
+    # We give silan some leeway here by specifying a tolerance
+    tolerance_seconds = 0.1
+    length_seconds = 3.9
+    assert abs(metadata['length_seconds'] - length_seconds) < tolerance_seconds
+    assert abs(metadata['cuein']) < tolerance_seconds
+    assert abs(metadata['cueout'] - length_seconds) < tolerance_seconds
+
+def test_missing_silan():
+    old_silan = CuePointAnalyzer.SILAN_EXECUTABLE
+    CuePointAnalyzer.SILAN_EXECUTABLE = 'foosdaf'
+    metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict())
+    CuePointAnalyzer.SILAN_EXECUTABLE = old_silan # Need to put this back
+
+def test_invalid_filepath():
+    metadata = CuePointAnalyzer.analyze(u'non-existent-file', dict())
+
+
+def test_mp3_utf8():
+    metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict())
+    check_default_metadata(metadata)
+
+def test_mp3_dualmono():
+    metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-dualmono.mp3', dict())
+    check_default_metadata(metadata)
+
+def test_mp3_jointstereo():
+    metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-jointstereo.mp3', dict())
+    check_default_metadata(metadata)
+
+def test_mp3_simplestereo():
+    metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-simplestereo.mp3', dict())
+    check_default_metadata(metadata)
+
+def test_mp3_stereo():
+    metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.mp3', dict())
+    check_default_metadata(metadata)
+
+def test_mp3_mono():
+    metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.mp3', dict())
+    check_default_metadata(metadata)
+
+def test_ogg_stereo():
+    metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.ogg', dict())
+    check_default_metadata(metadata)
+
+def test_invalid_wma():
+    metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-invalid.wma', dict())
+
+def test_m4a_stereo():
+    metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.m4a', dict())
+    check_default_metadata(metadata)
+
+def test_wav_stereo():
+    metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.wav', dict())
+    check_default_metadata(metadata)
+
+    # FFMPEG / libav detect the AAC file as slightly shorter...
+'''
+    tolerance_seconds = 0.2
+    length_seconds = 3.8
+    assert abs(metadata['length_seconds'] - length_seconds) < tolerance_seconds
+    assert abs(metadata['cuein']) < tolerance_seconds
+    assert abs(metadata['cueout'] - length_seconds) < tolerance_seconds
+'''

python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py

@@ -114,6 +114,18 @@ def test_mp3_utf8():
     assert metadata['mime'] == 'audio/mp3'
     assert metadata['track_total'] == u'10' # MP3s can have a track_total
 
+def test_invalid_wma():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-invalid.wma', dict())
+    assert metadata['mime'] == 'audio/x-ms-wma'
+
+def test_wav_stereo():
+    metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.wav', dict())
+    assert metadata['mime'] == 'audio/x-wav'
+    assert abs(metadata['length_seconds'] - 3.9) < 0.1
+    assert metadata['channels'] == 2
+    assert metadata['sample_rate'] == 44100
+
+
 # Make sure the parameter checking works
 @raises(TypeError)
 def test_move_wrong_string_param1():
@@ -132,7 +144,6 @@ def test_mp3_bad_channels():
         It'd be a pain in the ass to construct a real MP3 with an invalid number
         of channels by hand because that value is stored in every MP3 frame in the file
     '''
-    print "testing bad channels..."
     audio_file = mutagen.File(filename, easy=True)
     audio_file.info.mode = 1777
     with mock.patch('airtime_analyzer.metadata_analyzer.mutagen') as mock_mutagen:
@@ -143,7 +154,6 @@ def test_mp3_bad_channels():
     check_default_metadata(metadata)
     assert metadata['channels'] == 1
     assert metadata['bit_rate'] == 64000
-    print metadata['length_seconds']
     assert abs(metadata['length_seconds'] - 3.9) < 0.1
     assert metadata['mime'] == 'audio/mp3' # Not unicode because MIMEs aren't.
     assert metadata['track_total'] == u'10' # MP3s can have a track_total