From 68c401b2964c686a1a955191bdbdbc2e6b379bf7 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Mon, 13 Aug 2012 11:04:21 -0400 Subject: [PATCH 1/9] airtime-check-system not using user define port in all cases --- utils/airtime-check-system.php | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/utils/airtime-check-system.php b/utils/airtime-check-system.php index 22ff7362c..fa244f3b3 100644 --- a/utils/airtime-check-system.php +++ b/utils/airtime-check-system.php @@ -17,7 +17,7 @@ if (substr($sapi_type, 0, 3) == 'cli') { $status = AirtimeCheck::GetStatus($baseUrl, $base_port, $apiKey); - AirtimeCheck::PrintStatus($baseUrl, $status); + AirtimeCheck::PrintStatus($baseUrl, $base_port, $status); } class AirtimeCheck { @@ -92,8 +92,9 @@ class AirtimeCheck { return $os_string." ".$machine; } - public static function GetServerType($p_baseUrl){ - $headerInfo = get_headers("http://$p_baseUrl",1); + public static function GetServerType($p_baseUrl, $p_basePort) + { + $headerInfo = get_headers("http://$p_baseUrl:$p_basePort",1); if (!isset($headerInfo['Server'][0])) return self::UNKNOWN; @@ -120,7 +121,7 @@ class AirtimeCheck { return $data; } - public static function PrintStatus($p_baseUrl, $p_status){ + public static function PrintStatus($p_baseUrl, $p_basePort, $p_status){ if ($p_status === false){ self::output_status("AIRTIME_SERVER_RESPONDING", "FAILED"); @@ -150,7 +151,7 @@ class AirtimeCheck { } self::output_status("OS", self::CheckOsTypeVersion()); self::output_status("CPU", self::GetCpuInfo()); - self::output_status("WEB_SERVER", self::GetServerType($p_baseUrl)); + self::output_status("WEB_SERVER", self::GetServerType($p_baseUrl, $p_basePort)); if (isset($data->services)) { $services = $data->services; From 831a1b192315ef7a957849cd4a03eab702a8c7ed Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Thu, 16 Aug 2012 23:49:37 -0400 Subject: [PATCH 2/9] CC-4242: MP3 streams have the artist name duplicated -fixed --- python_apps/pypo/liquidsoap_scripts/ls_lib.liq | 6 +++--- python_apps/pypo/liquidsoap_scripts/ls_script.liq | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python_apps/pypo/liquidsoap_scripts/ls_lib.liq b/python_apps/pypo/liquidsoap_scripts/ls_lib.liq index 83ab39322..78854f07d 100644 --- a/python_apps/pypo/liquidsoap_scripts/ls_lib.liq +++ b/python_apps/pypo/liquidsoap_scripts/ls_lib.liq @@ -7,11 +7,11 @@ end def append_title(m) = log("Using stream_format #{!stream_metadata_type}") if !stream_metadata_type == 1 then - [("artist","#{!show_name} - #{m['artist']}")] + [("title", "#{!show_name} - #{m['artist']}")] elsif !stream_metadata_type == 2 then - [("artist",!station_name), ("title", !show_name)] + [("title", "#{!station_name} - #{!show_name}")] else - [] + [("title", "#{m['artist']} - #{m['title']}")] end end diff --git a/python_apps/pypo/liquidsoap_scripts/ls_script.liq b/python_apps/pypo/liquidsoap_scripts/ls_script.liq index 482a60a5e..dabd164a7 100644 --- a/python_apps/pypo/liquidsoap_scripts/ls_script.liq +++ b/python_apps/pypo/liquidsoap_scripts/ls_script.liq @@ -29,7 +29,7 @@ just_switched = ref false %include "ls_lib.liq" queue = on_metadata(notify, queue) -queue = map_metadata(append_title, queue) +queue = map_metadata(update=false, append_title, queue) # the crossfade function controls fade in/out queue = crossfade(queue) ignore(output.dummy(queue, fallible=true)) From 93d21b510c99a4c5433d8f0e34877dded2e48f2b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 20 Aug 2012 18:01:42 -0400 Subject: [PATCH 3/9] MM2: Fixed wrong use of split (misunderstanding of the maxsplit optional argument) --- python_apps/media-monitor2/media/monitor/eventcontractor.py | 4 +++- python_apps/media-monitor2/media/monitor/pure.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/eventcontractor.py b/python_apps/media-monitor2/media/monitor/eventcontractor.py index 77a5045fb..7e444b3ea 100644 --- a/python_apps/media-monitor2/media/monitor/eventcontractor.py +++ b/python_apps/media-monitor2/media/monitor/eventcontractor.py @@ -59,4 +59,6 @@ class EventContractor(Loggable): try: del self.store[evt.path] except Exception as e: self.unexpected_exception(e) - self.logger.info("Unregistering. Left: '%d'" % len(self.store.keys())) + # the next line is commented out because it clutters up logging real + # bad + #self.logger.info("Unregistering. Left: '%d'" % len(self.store.keys())) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 3f0b8bbe4..3b97ffe6f 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -247,7 +247,7 @@ def normalized_metadata(md, original_path): # Note: I don't know why I'm doing a unicode string comparison here # that part is copied from the original code if is_airtime_recorded(new_md): - hour,minute,second,name = md['MDATA_KEY_TITLE'].split("-",3) + hour,minute,second,name = new_md['MDATA_KEY_TITLE'].split("-",3) # We assume that MDATA_KEY_YEAR is always given for airtime recorded # shows new_md['MDATA_KEY_TITLE'] = u'%s-%s-%s:%s:%s' % \ @@ -280,7 +280,7 @@ def organized_path(old_path, root_path, orig_md): fname = u'%s-%s-%s.%s' % ( normal_md['MDATA_KEY_YEAR'], normal_md['MDATA_KEY_TITLE'], normal_md['MDATA_KEY_BITRATE'], ext ) - yyyy, mm, _ = normal_md['MDATA_KEY_YEAR'].split('-',3) + yyyy, mm, _ = normal_md['MDATA_KEY_YEAR'].split('-',2) path = os.path.join(root_path, yyyy, mm) filepath = os.path.join(path,fname) elif len(normal_md['MDATA_KEY_TRACKNUMBER']) == 0: From bb1e67a83c34eed3ebe57ccaef7a415c85beac3a Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 21 Aug 2012 10:46:48 -0400 Subject: [PATCH 4/9] small refactoring of to os.path.join instead of manual path concatenation. --- python_apps/pypo/recorder.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python_apps/pypo/recorder.py b/python_apps/pypo/recorder.py index 00d51c010..de48afd1c 100644 --- a/python_apps/pypo/recorder.py +++ b/python_apps/pypo/recorder.py @@ -73,16 +73,18 @@ class ShowRecorder(Thread): else: filetype = "ogg"; - filepath = "%s%s.%s" % (config["base_recorded_files"], filename, filetype) + joined_path = os.path.join(config["base_recorded_files"], filename) + filepath = "%s.%s" % (joined_path, filetype) br = config["record_bitrate"] sr = config["record_samplerate"] - c = config["record_channels"] + c = config["record_channels"] ss = config["record_sample_size"] #-f:16,2,44100 #-b:256 - command = "ecasound -f:%s,%s,%s -i alsa -o %s,%s000 -t:%s" % (ss, c, sr, filepath, br, length) + command = "ecasound -f:%s,%s,%s -i alsa -o %s,%s000 -t:%s" % \ + (ss, c, sr, filepath, br, length) args = command.split(" ") self.logger.info("starting record") From 142eb6b66ed7b3ecb9c41a49e6352e20423fcd84 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 21 Aug 2012 11:59:29 -0400 Subject: [PATCH 5/9] cc-4241: Updated tests to test for this bug automatically. --- python_apps/media-monitor2/tests/test_pure.py | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/python_apps/media-monitor2/tests/test_pure.py b/python_apps/media-monitor2/tests/test_pure.py index 611c42f60..5868b96bb 100644 --- a/python_apps/media-monitor2/tests/test_pure.py +++ b/python_apps/media-monitor2/tests/test_pure.py @@ -2,6 +2,7 @@ import unittest import os import media.monitor.pure as mmp +from media.monitor.metadata import Metadata class TestMMP(unittest.TestCase): def setUp(self): @@ -34,23 +35,28 @@ class TestMMP(unittest.TestCase): for k in def_keys: self.assertEqual( sd[k], 'DEF' ) def test_normalized_metadata(self): - normal = mmp.normalized_metadata(self.md1,"") - self.assertTrue(hasattr(normal['MDATA_KEY_CREATOR'],'startswith')) - self.assertTrue('MDATA_KEY_CREATOR' in normal) - self.assertTrue('MDATA_KEY_SOURCE' in normal) + # Recorded show test first + orig = Metadata.airtime_dict({ + 'date': [u'2012-08-21'], + 'tracknumber': [u'2'], + 'title': [u'11-29-00-record'], + 'artist': [u'Airtime Show Recorder'] + }) + orga = Metadata.airtime_dict({ + 'date': [u'2012-08-21'], + 'tracknumber': [u'2'], + 'artist': [u'Airtime Show Recorder'], + 'title': [u'record-2012-08-21-11:29:00'] + }) + orga['MDATA_KEY_FTYPE'] = u'audioclip' + old_path = "/home/rudi/recorded/2012-08-21-11:29:00.ogg" + normalized = mmp.normalized_metadata(orig, old_path) + self.assertEqual( orga, normalized ) - def test_organized_path(self): - o_path = '/home/rudi/throwaway/ACDC_-_Back_In_Black-sample-64kbps.ogg' - normal = mmp.normalized_metadata(self.md1,o_path) - og = mmp.organized_path(o_path, - '/home/rudi/throwaway/fucking_around/watch/', - normal) - real_path1 = \ - u'/home/rudi/throwaway/fucking_around/watch/unknown/unknown/ACDC_-_Back_In_Black-sample-64kbps-64kbps.ogg' - self.assertTrue( 'unknown' in og, True ) - self.assertEqual( og, real_path1 ) # TODO : fix this failure - # for recorded it should be something like this - # ./recorded/2012/07/2012-07-09-17-55-00-Untitled Show-256kbps.ogg + organized_base_name = "2012-08-21-11-29-00-record-256kbps.ogg" + base = "/srv/airtime/stor/" + organized_path = mmp.organized_path(old_path,base, normalized) + self.assertEqual(os.path.basename(organized_path), organized_base_name) def test_file_md5(self): p = os.path.realpath(__file__) From 2a497bf941ef68ee9c7ee9c712d198b9f82e97fb Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Tue, 21 Aug 2012 12:06:07 -0400 Subject: [PATCH 6/9] remove symlink, not actual binary --- python_apps/pypo/install/pypo-initialize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python_apps/pypo/install/pypo-initialize.py b/python_apps/pypo/install/pypo-initialize.py index 357aa49c6..01fe004bb 100644 --- a/python_apps/pypo/install/pypo-initialize.py +++ b/python_apps/pypo/install/pypo-initialize.py @@ -85,16 +85,16 @@ try: p = Popen("which liquidsoap", shell=True, stdout=PIPE) liq_path = p.communicate()[0].strip() + symlink_path = "/usr/bin/airtime-liquidsoap" if p.returncode == 0: try: - os.unlink(liq_path) + os.remove(symlink_path) except Exception: #liq_path DNE, which is OK. pass - - os.symlink(liq_path, "/usr/bin/airtime-liquidsoap") + os.symlink(liq_path, symlink_path) else: print " * Liquidsoap binary not found!" sys.exit(1) From bbf5a8641511ca888b151114896d1cd20ada0f32 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 21 Aug 2012 12:34:29 -0400 Subject: [PATCH 7/9] MM2: refactoring of metadata handling --- .../media-monitor2/media/monitor/metadata.py | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index a8c982b9a..5f222c4b4 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -105,10 +105,38 @@ class Metadata(Loggable): # little bit messy. Some of the handling is in m.m.pure while the rest is # here. Also interface is not very consistent + @staticmethod + def airtime_dict(d): + """ + Converts mutagen dictionary 'd' into airtime dictionary + """ + temp_dict = {} + for m_key, m_val in d.iteritems(): + # TODO : some files have multiple fields for the same metadata. + # genre is one example. In that case mutagen will return a list + # of values + assign_val = m_val[0] if isinstance(m_val, list) else m_val + temp_dict[ m_key ] = assign_val + airtime_dictionary = {} + for muta_k, muta_v in temp_dict.iteritems(): + # We must check if we can actually translate the mutagen key into + # an airtime key before doing the conversion + if muta_k in mutagen2airtime: + airtime_key = mutagen2airtime[muta_k] + # Apply truncation in the case where airtime_key is in our + # truncation table + muta_v = \ + truncate_to_length(muta_v, truncate_table[airtime_key])\ + if airtime_key in truncate_table else muta_v + airtime_dictionary[ airtime_key ] = muta_v + return airtime_dictionary + @staticmethod def write_unsafe(path,md): """ - Writes 'md' metadata into 'path' through mutagen + Writes 'md' metadata into 'path' through mutagen. Converts all + dictionary values to strings because mutagen will not write anything + else """ if not os.path.exists(path): raise BadSongFile(path) song_file = mutagen.File(path, easy=True) @@ -128,37 +156,13 @@ class Metadata(Loggable): self.path = fpath # TODO : Simplify the way all of these rules are handled right not it's # extremely unclear and needs to be refactored. - metadata = {} - # Load only the metadata avilable in mutagen into metdata - for k,v in full_mutagen.iteritems(): - # Special handling of attributes here - if isinstance(v, list): - # TODO : some files have multiple fields for the same metadata. - # genre is one example. In that case mutagen will return a list - # of values - metadata[k] = v[0] - #if len(v) == 1: metadata[k] = v[0] - #else: raise Exception("Unknown mutagen %s:%s" % (k,str(v))) - else: metadata[k] = v - self.__metadata = {} - # Start populating a dictionary of airtime metadata in __metadata - for muta_k, muta_v in metadata.iteritems(): - # We must check if we can actually translate the mutagen key into - # an airtime key before doing the conversion - if muta_k in mutagen2airtime: - airtime_key = mutagen2airtime[muta_k] - # Apply truncation in the case where airtime_key is in our - # truncation table - muta_v = \ - truncate_to_length(muta_v, truncate_table[airtime_key])\ - if airtime_key in truncate_table else muta_v - self.__metadata[ airtime_key ] = muta_v + self.__metadata = Metadata.airtime_dict(full_mutagen) # Now we extra the special values that are calculated from the mutagen # object itself: for special_key,f in airtime_special.iteritems(): new_val = f(full_mutagen) if new_val is not None: - self.__metadata[special_key] = f(full_mutagen) + self.__metadata[special_key] = new_val # Finally, we "normalize" all the metadata here: self.__metadata = mmp.normalized_metadata(self.__metadata, fpath) # Now we must load the md5: From 01060f560c791c505452e7e5c271003aff2a7326 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 21 Aug 2012 12:49:57 -0400 Subject: [PATCH 8/9] cc-4241: corrected unit test. --- python_apps/media-monitor2/tests/test_pure.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/tests/test_pure.py b/python_apps/media-monitor2/tests/test_pure.py index 5868b96bb..e822f3cf5 100644 --- a/python_apps/media-monitor2/tests/test_pure.py +++ b/python_apps/media-monitor2/tests/test_pure.py @@ -48,9 +48,13 @@ class TestMMP(unittest.TestCase): 'artist': [u'Airtime Show Recorder'], 'title': [u'record-2012-08-21-11:29:00'] }) - orga['MDATA_KEY_FTYPE'] = u'audioclip' + orga['MDATA_KEY_FTYPE'] = u'audioclip' + orig['MDATA_KEY_BITRATE'] = u'256000' + orga['MDATA_KEY_BITRATE'] = u'256kbps' + old_path = "/home/rudi/recorded/2012-08-21-11:29:00.ogg" normalized = mmp.normalized_metadata(orig, old_path) + print(normalized) self.assertEqual( orga, normalized ) organized_base_name = "2012-08-21-11-29-00-record-256kbps.ogg" From 8dc4bcb7d331c9bf77d2abadce84b55b76f74d1c Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 21 Aug 2012 12:50:29 -0400 Subject: [PATCH 9/9] cc-4241: fixed this bug. and maybe 4232 too. --- .../media-monitor2/media/monitor/pure.py | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 3b97ffe6f..4d768b0f8 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -2,6 +2,7 @@ import copy import os import shutil +import re import sys import hashlib import locale @@ -228,34 +229,33 @@ def normalized_metadata(md, original_path): # Specific rules that are applied in a per attribute basis format_rules = { 'MDATA_KEY_TRACKNUMBER' : parse_int, - 'MDATA_KEY_BITRATE' : lambda x: str(int(x) / 1000) + "kbps", 'MDATA_KEY_FILEPATH' : lambda x: os.path.normpath(x), 'MDATA_KEY_MIME' : lambda x: x.replace('-','/'), 'MDATA_KEY_BPM' : lambda x: x[0:8], } - new_md = remove_whitespace(new_md) + new_md = remove_whitespace(new_md) # remove whitespace fields + # Format all the fields in format_rules new_md = apply_rules_dict(new_md, format_rules) - new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_TITLE'], - default=no_extension_basename(original_path)) - new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_CREATOR', - 'MDATA_KEY_SOURCE'], default=u'') + # set filetype to audioclip by default new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_FTYPE'], default=u'audioclip') - # In the case where the creator is 'Airtime Show Recorder' we would like to - # format the MDATA_KEY_TITLE slightly differently - # Note: I don't know why I'm doing a unicode string comparison here - # that part is copied from the original code + + # Try to parse bpm but delete the whole key if that fails + if 'MDATA_KEY_BPM' in new_md: + new_md['MDATA_KEY_BPM'] = parse_int(new_md['MDATA_KEY_BPM']) + if new_md['MDATA_KEY_BPM'] is None: + del new_md['MDATA_KEY_BPM'] + if is_airtime_recorded(new_md): hour,minute,second,name = new_md['MDATA_KEY_TITLE'].split("-",3) - # We assume that MDATA_KEY_YEAR is always given for airtime recorded - # shows new_md['MDATA_KEY_TITLE'] = u'%s-%s-%s:%s:%s' % \ (name, new_md['MDATA_KEY_YEAR'], hour, minute, second) - # IMPORTANT: in the original code. MDATA_KEY_FILEPATH would also - # be set to the original path of the file for airtime recorded shows - # (before it was "organized"). We will skip this procedure for now - # because it's not clear why it was done + else: + # Read title from filename if it does not exist + new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_TITLE'], + default=no_extension_basename(original_path)) + return new_md def organized_path(old_path, root_path, orig_md): @@ -275,14 +275,21 @@ def organized_path(old_path, root_path, orig_md): else: return True # We set some metadata elements to a default "unknown" value because we use # these fields to create a path hence they cannot be empty + # Here "normal" means normalized only for organized path normal_md = default_to_f(orig_md, path_md, unicode_unknown, default_f) + if normal_md['MDATA_KEY_BITRATE']: + formatted = str(int(normal_md['MDATA_KEY_BITRATE']) / 1000) + normal_md['MDATA_KEY_BITRATE'] = formatted + 'kbps' + else: normal_md['MDATA_KEY_BITRATE'] = unicode_unknown if is_airtime_recorded(normal_md): - fname = u'%s-%s-%s.%s' % ( normal_md['MDATA_KEY_YEAR'], - normal_md['MDATA_KEY_TITLE'], - normal_md['MDATA_KEY_BITRATE'], ext ) + title_re = re.match("(?P\w+)-(?P\d+-\d+-\d+-\d+:\d+:\d+)$", + normal_md['MDATA_KEY_TITLE']) + show_name, = title_re.group('show'), + date = title_re.group('date').replace(':','-') yyyy, mm, _ = normal_md['MDATA_KEY_YEAR'].split('-',2) - path = os.path.join(root_path, yyyy, mm) - filepath = os.path.join(path,fname) + fname_base = '%s-%s-%s.%s' % \ + (date, show_name, normal_md['MDATA_KEY_BITRATE'], ext) + filepath = os.path.join(root_path, yyyy, mm, fname_base) elif len(normal_md['MDATA_KEY_TRACKNUMBER']) == 0: fname = u'%s-%s.%s' % (normal_md['MDATA_KEY_TITLE'], normal_md['MDATA_KEY_BITRATE'], ext)