From cad6a7e7b0783e24cddbb12db40cff9fa0c07efb Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 9 Oct 2012 12:25:40 -0400 Subject: [PATCH 01/59] added perliminary stuff for emf --- python_apps/media-monitor2/media/monitor/metadata.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index bd42e434c..7b7d6d184 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -185,6 +185,15 @@ class Metadata(Loggable): # TODO : Simplify the way all of these rules are handled right now it's # extremely unclear and needs to be refactored. #if full_mutagen is None: raise BadSongFile(fpath) + try: # emf stuff for testing: + import media.metadata.process as md + import pprint.pformat as pf + if full_mutagen: + normalized = md.global_reader.read('fpath', full_mutagen) + self.logger.info(pf(normalized)) + except Exception as e: + self.logger.unexpected_exception(e) + if full_mutagen is None: full_mutagen = FakeMutagen(fpath) self.__metadata = Metadata.airtime_dict(full_mutagen) # Now we extra the special values that are calculated from the mutagen @@ -209,6 +218,7 @@ class Metadata(Loggable): # from the file? self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100) + def is_recorded(self): """ returns true if the file has been created by airtime through recording From a88b7255ff9ec2d223bacc3cfaa380471e8db26b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 9 Oct 2012 12:36:52 -0400 Subject: [PATCH 02/59] removed automatic loading of emf definitions --- .../media/metadata/definitions.py | 171 +++++++++--------- .../media-monitor2/media/monitor/metadata.py | 12 +- 2 files changed, 97 insertions(+), 86 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index 51ef6fd14..cfeb0d2fb 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -3,109 +3,116 @@ import media.monitor.process as md from os.path import normpath from media.monitor.pure import format_length, file_md5 -with md.metadata('MDATA_KEY_DURATION') as t: - t.default(u'0.0') - t.depends('length') - t.translate(lambda k: format_length(k['length'])) +defs_loaded = False -with md.metadata('MDATA_KEY_MIME') as t: - t.default(u'') - t.depends('mime') - t.translate(lambda k: k['mime'].replace('-','/')) +def is_defs_loaded(): + global defs_loaded + return defs_loaded -with md.metadata('MDATA_KEY_BITRATE') as t: - t.default(u'') - t.depends('bitrate') - t.translate(lambda k: k['bitrate']) +def load_definitions(): + with md.metadata('MDATA_KEY_DURATION') as t: + t.default(u'0.0') + t.depends('length') + t.translate(lambda k: format_length(k['length'])) -with md.metadata('MDATA_KEY_SAMPLERATE') as t: - t.default(u'0') - t.depends('sample_rate') - t.translate(lambda k: k['sample_rate']) + with md.metadata('MDATA_KEY_MIME') as t: + t.default(u'') + t.depends('mime') + t.translate(lambda k: k['mime'].replace('-','/')) -with md.metadata('MDATA_KEY_FTYPE'): - t.depends('ftype') # i don't think this field even exists - t.default(u'audioclip') - t.translate(lambda k: k['ftype']) # but just in case + with md.metadata('MDATA_KEY_BITRATE') as t: + t.default(u'') + t.depends('bitrate') + t.translate(lambda k: k['bitrate']) -with md.metadata("MDATA_KEY_CREATOR") as t: - t.depends("artist") - # A little kludge to make sure that we have some value for when we parse - # MDATA_KEY_TITLE - t.default(u"") - t.max_length(512) + with md.metadata('MDATA_KEY_SAMPLERATE') as t: + t.default(u'0') + t.depends('sample_rate') + t.translate(lambda k: k['sample_rate']) -with md.metadata("MDATA_KEY_SOURCE") as t: - t.depends("album") - t.max_length(512) + with md.metadata('MDATA_KEY_FTYPE'): + t.depends('ftype') # i don't think this field even exists + t.default(u'audioclip') + t.translate(lambda k: k['ftype']) # but just in case -with md.metadata("MDATA_KEY_GENRE") as t: - t.depends("genre") - t.max_length(64) + with md.metadata("MDATA_KEY_CREATOR") as t: + t.depends("artist") + # A little kludge to make sure that we have some value for when we parse + # MDATA_KEY_TITLE + t.default(u"") + t.max_length(512) -with md.metadata("MDATA_KEY_MOOD") as t: - t.depends("mood") - t.max_length(64) + with md.metadata("MDATA_KEY_SOURCE") as t: + t.depends("album") + t.max_length(512) -with md.metadata("MDATA_KEY_TRACKNUMBER") as t: - t.depends("tracknumber") + with md.metadata("MDATA_KEY_GENRE") as t: + t.depends("genre") + t.max_length(64) -with md.metadata("MDATA_KEY_BPM") as t: - t.depends("bpm") - t.max_length(8) + with md.metadata("MDATA_KEY_MOOD") as t: + t.depends("mood") + t.max_length(64) -with md.metadata("MDATA_KEY_LABEL") as t: - t.depends("organization") - t.max_length(512) + with md.metadata("MDATA_KEY_TRACKNUMBER") as t: + t.depends("tracknumber") -with md.metadata("MDATA_KEY_COMPOSER") as t: - t.depends("composer") - t.max_length(512) + with md.metadata("MDATA_KEY_BPM") as t: + t.depends("bpm") + t.max_length(8) -with md.metadata("MDATA_KEY_ENCODER") as t: - t.depends("encodedby") - t.max_length(512) + with md.metadata("MDATA_KEY_LABEL") as t: + t.depends("organization") + t.max_length(512) -with md.metadata("MDATA_KEY_CONDUCTOR") as t: - t.depends("conductor") - t.max_length(512) + with md.metadata("MDATA_KEY_COMPOSER") as t: + t.depends("composer") + t.max_length(512) -with md.metadata("MDATA_KEY_YEAR") as t: - t.depends("date") - t.max_length(16) + with md.metadata("MDATA_KEY_ENCODER") as t: + t.depends("encodedby") + t.max_length(512) -with md.metadata("MDATA_KEY_URL") as t: - t.depends("website") + with md.metadata("MDATA_KEY_CONDUCTOR") as t: + t.depends("conductor") + t.max_length(512) -with md.metadata("MDATA_KEY_ISRC") as t: - t.depends("isrc") - t.max_length(512) + with md.metadata("MDATA_KEY_YEAR") as t: + t.depends("date") + t.max_length(16) -with md.metadata("MDATA_KEY_COPYRIGHT") as t: - t.depends("copyright") - t.max_length(512) + with md.metadata("MDATA_KEY_URL") as t: + t.depends("website") -with md.metadata("MDATA_KEY_FILEPATH") as t: - t.depends('path') - t.translate(lambda k: normpath(k['path'])) + with md.metadata("MDATA_KEY_ISRC") as t: + t.depends("isrc") + t.max_length(512) -with md.metadata("MDATA_KEY_MD5") as t: - t.depends('path') - t.optional(False) - t.translate(lambda k: file_md5(k['path'], max_length=100)) + with md.metadata("MDATA_KEY_COPYRIGHT") as t: + t.depends("copyright") + t.max_length(512) -# owner is handled differently by (by events.py) + with md.metadata("MDATA_KEY_FILEPATH") as t: + t.depends('path') + t.translate(lambda k: normpath(k['path'])) -with md.metadata('MDATA_KEY_ORIGINAL_PATH') as t: - t.depends('original_path') + with md.metadata("MDATA_KEY_MD5") as t: + t.depends('path') + t.optional(False) + t.translate(lambda k: file_md5(k['path'], max_length=100)) -# MDATA_KEY_TITLE is the annoying special case -with md.metadata('MDATA_KEY_TITLE') as t: - # Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is - # defaulted to "" from definitions above - t.depends('title','MDATA_KEY_CREATOR') - t.max_length(512) + # owner is handled differently by (by events.py) -with md.metadata('MDATA_KEY_LABEL') as t: - t.depends('label') - t.max_length(512) + with md.metadata('MDATA_KEY_ORIGINAL_PATH') as t: + t.depends('original_path') + + # MDATA_KEY_TITLE is the annoying special case + with md.metadata('MDATA_KEY_TITLE') as t: + # Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is + # defaulted to "" from definitions above + t.depends('title','MDATA_KEY_CREATOR') + t.max_length(512) + + with md.metadata('MDATA_KEY_LABEL') as t: + t.depends('label') + t.max_length(512) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 7b7d6d184..3e7fa893d 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -11,6 +11,12 @@ from media.monitor.log import Loggable from media.monitor.pure import format_length, truncate_to_length import media.monitor.pure as mmp +# emf related stuff +from media.metadata.process import global_reader +import media.metadata.definitions as defs +from pprint import pformat +defs.load_definitions() + """ list of supported easy tags in mutagen version 1.20 ['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry', @@ -186,11 +192,9 @@ class Metadata(Loggable): # extremely unclear and needs to be refactored. #if full_mutagen is None: raise BadSongFile(fpath) try: # emf stuff for testing: - import media.metadata.process as md - import pprint.pformat as pf if full_mutagen: - normalized = md.global_reader.read('fpath', full_mutagen) - self.logger.info(pf(normalized)) + normalized = global_reader.read('fpath', full_mutagen) + self.logger.info(pformat(normalized)) except Exception as e: self.logger.unexpected_exception(e) From 10c4a71866bbb0e9c0992f93b79515077a3dcb0e Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 9 Oct 2012 14:46:01 -0400 Subject: [PATCH 03/59] Added default translator for mdata elements that don't specify it --- .../media/metadata/definitions.py | 2 +- .../media-monitor2/media/metadata/process.py | 19 ++++++++++++++++++- .../media-monitor2/media/monitor/metadata.py | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index cfeb0d2fb..68ada6034 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -import media.monitor.process as md +import media.metadata.process as md from os.path import normpath from media.monitor.pure import format_length, file_md5 diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index a4c361468..6cebc0269 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from contextlib import contextmanager from media.monitor.pure import truncate_to_length, toposort +from media.monitor.log import Loggable import mutagen @@ -8,7 +9,12 @@ class MetadataAbsent(Exception): def __init__(self, name): self.name = name def __str__(self): return "Could not obtain element '%s'" % self.name -class MetadataElement(object): +class MetadataElement(Loggable): + + def __default_translator(k): + e = [ x for x in self.dependencies() ][0] + return k[e] + def __init__(self,name): self.name = name # "Sane" defaults @@ -19,6 +25,8 @@ class MetadataElement(object): self.__is_normalized = lambda _ : True self.__max_length = -1 + + def max_length(self,l): self.__max_length = l @@ -82,6 +90,15 @@ class MetadataElement(object): if self.has_default(): return self.get_default() else: raise MetadataAbsent(self.name) # We have all dependencies. Now for actual for parsing + + # Only case where we can select a default translator + if not self.__translator: + if len(self.dependencies()) == 1: + self.translate(MetadataElement.__default_translator) + else: + self.logger.info("Could not set more than 1 translator with \ + more than 1 dependancies") + r = self.__normalizer( self.__translator(full_deps) ) if self.__max_length != -1: r = truncate_to_length(r, self.__max_length) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 3e7fa893d..6cce20e34 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -196,7 +196,7 @@ class Metadata(Loggable): normalized = global_reader.read('fpath', full_mutagen) self.logger.info(pformat(normalized)) except Exception as e: - self.logger.unexpected_exception(e) + self.unexpected_exception(e) if full_mutagen is None: full_mutagen = FakeMutagen(fpath) self.__metadata = Metadata.airtime_dict(full_mutagen) From b7b11feae00dd1ef6b12a8fcf6bfe3d79e7f882e Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 9 Oct 2012 18:02:03 -0400 Subject: [PATCH 04/59] emf improvements --- .../media/metadata/definitions.py | 11 +++---- .../media-monitor2/media/metadata/process.py | 30 +++++++++++-------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index 68ada6034..f3399e24c 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -96,17 +96,18 @@ def load_definitions(): t.depends('path') t.translate(lambda k: normpath(k['path'])) - with md.metadata("MDATA_KEY_MD5") as t: - t.depends('path') - t.optional(False) - t.translate(lambda k: file_md5(k['path'], max_length=100)) + #with md.metadata("MDATA_KEY_MD5") as t: + #t.depends('path') + #t.optional(False) + #t.translate(lambda k: file_md5(k['path'], max_length=100)) # owner is handled differently by (by events.py) with md.metadata('MDATA_KEY_ORIGINAL_PATH') as t: t.depends('original_path') - # MDATA_KEY_TITLE is the annoying special case + # MDATA_KEY_TITLE is the annoying special case b/c we sometimes read it + # from file name with md.metadata('MDATA_KEY_TITLE') as t: # Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is # defaulted to "" from definitions above diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index 6cebc0269..88a8b5cc6 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -11,10 +11,6 @@ class MetadataAbsent(Exception): class MetadataElement(Loggable): - def __default_translator(k): - e = [ x for x in self.dependencies() ][0] - return k[e] - def __init__(self,name): self.name = name # "Sane" defaults @@ -24,8 +20,7 @@ class MetadataElement(Loggable): self.__default = None self.__is_normalized = lambda _ : True self.__max_length = -1 - - + self.__translator = None def max_length(self,l): self.__max_length = l @@ -71,6 +66,7 @@ class MetadataElement(Loggable): return "%s(%s)" % (self.name, ' '.join(list(self.__deps))) def read_value(self, path, original, running={}): + self.logger.info("Trying to read: %s" % str(original)) # If value is present and normalized then we don't touch it if self.name in original: v = original[self.name] @@ -83,21 +79,29 @@ class MetadataElement(Loggable): full_deps = dict( dep_slice_orig.items() + dep_slice_running.items() ) + full_deps['path'] = path + # check if any dependencies are absent - if len(full_deps) != len(self.__deps) or len(self.__deps) == 0: + if len(full_deps) < len(self.__deps) or len(self.__deps) == 0: # If we have a default value then use that. Otherwise throw an # exception if self.has_default(): return self.get_default() else: raise MetadataAbsent(self.name) # We have all dependencies. Now for actual for parsing + def def_translate(dep): + def wrap(k): + e = [ x for x in dep ][0] + return k[e] + return wrap + # Only case where we can select a default translator - if not self.__translator: - if len(self.dependencies()) == 1: - self.translate(MetadataElement.__default_translator) - else: - self.logger.info("Could not set more than 1 translator with \ - more than 1 dependancies") + if self.__translator is None: + self.translate(def_translate(self.dependencies())) + if len(self.dependencies()) > 2: # dependencies include themselves + self.logger.info("Ignoring some dependencies in translate %s" + % self.name) + self.logger.info(self.dependencies()) r = self.__normalizer( self.__translator(full_deps) ) if self.__max_length != -1: From 55567d1de03d7f993b33b93794a4fb4d93356a4a Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 10 Oct 2012 14:40:46 -0400 Subject: [PATCH 05/59] Aded diff_dict function to help locate differences between emf and old metadata parsing. --- .../media-monitor2/media/monitor/pure.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 877cbdc72..81b9cba49 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -68,6 +68,23 @@ class IncludeOnly(object): return _wrap + +def diff_dict(d1, d2, width=30): + """ + returns a formatted diff of 2 dictionaries + """ + out = "" + all_keys = d1.keys() + d2.keys() + for k in all_keys: + v1, v2 = d1.get(k), d2.get(k) + + # default values + if v1 is None: v1 = "N/A" + if v2 is None: v2 = "N/A" + + if d1[k] != d2[k]: + out += "%s%s%s" % (k, d1[k], d2[k]) + def partition(f, alist): """ Partition is very similar to filter except that it also returns the From 31b2a29392120de85fab4d05f3e1aaa7e76c838c Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 10 Oct 2012 14:41:12 -0400 Subject: [PATCH 06/59] Added docstirng for __slice_deps --- .../media-monitor2/media/metadata/process.py | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index 88a8b5cc6..d8be46bdf 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -60,35 +60,49 @@ class MetadataElement(Loggable): return self.__path def __slice_deps(self, d): + """ + returns a dictionary of all the key value pairs in d that are also + present in self.__deps + """ return dict( (k,v) for k,v in d.iteritems() if k in self.__deps) def __str__(self): return "%s(%s)" % (self.name, ' '.join(list(self.__deps))) def read_value(self, path, original, running={}): - self.logger.info("Trying to read: %s" % str(original)) - # If value is present and normalized then we don't touch it + + # If value is present and normalized then we only check if it's + # normalized or not. We normalize if it's not normalized already + if self.name in original: v = original[self.name] if self.__is_normalized(v): return v else: return self.__normalizer(v) - # A dictionary slice with all the dependencies and their values + # We slice out only the dependencies that are required for the metadata + # element. dep_slice_orig = self.__slice_deps(original) dep_slice_running = self.__slice_deps(running) + # TODO : remove this later + dep_slice_special = self.__slice_deps({'path' : path}) + # We combine all required dependencies into a single dictionary + # that we will pass to the translator full_deps = dict( dep_slice_orig.items() - + dep_slice_running.items() ) - - full_deps['path'] = path + + dep_slice_running.items() + + dep_slice_special.items()) # check if any dependencies are absent - if len(full_deps) < len(self.__deps) or len(self.__deps) == 0: + # note: there is no point checking the case that len(full_deps) > + # len(self.__deps) because we make sure to "slice out" any supefluous + # dependencies above. + if len(full_deps) != len(self.dependencies()) or \ + len(self.dependencies()) == 0: # If we have a default value then use that. Otherwise throw an # exception if self.has_default(): return self.get_default() else: raise MetadataAbsent(self.name) - # We have all dependencies. Now for actual for parsing + # We have all dependencies. Now for actual for parsing def def_translate(dep): def wrap(k): e = [ x for x in dep ][0] From 4bbc1fa95bf9afe8a47d2eaec2b68b88567d4c67 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 10 Oct 2012 14:41:39 -0400 Subject: [PATCH 07/59] Added comparison for emf and non emf metadata values. --- .../media-monitor2/media/monitor/metadata.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 6cce20e34..2b6c07a83 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -191,12 +191,6 @@ class Metadata(Loggable): # TODO : Simplify the way all of these rules are handled right now it's # extremely unclear and needs to be refactored. #if full_mutagen is None: raise BadSongFile(fpath) - try: # emf stuff for testing: - if full_mutagen: - normalized = global_reader.read('fpath', full_mutagen) - self.logger.info(pformat(normalized)) - except Exception as e: - self.unexpected_exception(e) if full_mutagen is None: full_mutagen = FakeMutagen(fpath) self.__metadata = Metadata.airtime_dict(full_mutagen) @@ -222,6 +216,17 @@ class Metadata(Loggable): # from the file? self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100) + try: # emf stuff for testing: + if full_mutagen: + normalized = global_reader.read_mutagen(fpath) + self.logger.info("EMF--------------------") + self.logger.info(pformat(normalized)) + self.logger.info("OLD--------------------") + self.logger.info(pformat(self.__metadata)) + self.logger.info("-----------------------") + + except Exception as e: + self.unexpected_exception(e) def is_recorded(self): """ From a2792b01acd7e13720b7c4674543585f67f94db5 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 10 Oct 2012 14:42:07 -0400 Subject: [PATCH 08/59] Got rid of emf bug where a list of 0 elements might be accessed. --- python_apps/media-monitor2/media/metadata/process.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index d8be46bdf..51a3c732d 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -130,10 +130,11 @@ def normalize_mutagen(path): m = mutagen.File(path, easy=True) md = {} for k,v in m.iteritems(): - if type(v) is list: md[k] = v[0] + if type(v) is list: + if len(v) > 0: md[k] = v[0] else: md[k] = v # populate special metadata values - md['length'] = getattr(m.info, u'length', 0.0) + md['length'] = getattr(m.info, 'length', 0.0) md['bitrate'] = getattr(m.info, 'bitrate', u'') md['sample_rate'] = getattr(m.info, 'sample_rate', 0) md['mime'] = m.mime[0] if len(m.mime) > 0 else u'' From 445ec23b8e0efe4fcfdce0834ceaaaf8ddfad006 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 10 Oct 2012 14:42:22 -0400 Subject: [PATCH 09/59] Added utility method for reading mutagen info through emf. --- python_apps/media-monitor2/media/metadata/process.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index 51a3c732d..1b62646a4 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -167,6 +167,9 @@ class MetadataReader(object): if not mdata.is_optional(): raise return normalized_metadata + def read_mutagen(self, path): + return self.read(path, normalize_mutagen(path)) + global_reader = MetadataReader() @contextmanager From 4242ed38be7c5a002f5f2f5af93bb269da9a3b5d Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 11:14:53 -0400 Subject: [PATCH 10/59] Added more testing for emf --- python_apps/media-monitor2/tests/test_emf.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 python_apps/media-monitor2/tests/test_emf.py diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py new file mode 100644 index 000000000..69fc566db --- /dev/null +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +import unittest + +from media.metadata.process import global_reader +import media.metadata.definitions as defs +defs.load_definitions() + +class TestMMP(unittest.TestCase): + def test_sanity(self): + m = global_reader.read_mutagen("/home/rudi/music/Nightingale.mp3") + self.assertTrue( len(m) > 0 ) From 09a8dfefc1d5f88ec4a4edfd50a458e1040ef6a4 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 13:55:03 -0400 Subject: [PATCH 11/59] Fixed metadata definitions in emf --- .../media-monitor2/media/metadata/definitions.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index f3399e24c..f0288bc61 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -30,7 +30,7 @@ def load_definitions(): t.depends('sample_rate') t.translate(lambda k: k['sample_rate']) - with md.metadata('MDATA_KEY_FTYPE'): + with md.metadata('MDATA_KEY_FTYPE') as t: t.depends('ftype') # i don't think this field even exists t.default(u'audioclip') t.translate(lambda k: k['ftype']) # but just in case @@ -92,9 +92,9 @@ def load_definitions(): t.depends("copyright") t.max_length(512) - with md.metadata("MDATA_KEY_FILEPATH") as t: + with md.metadata("MDATA_KEY_ORIGINAL_PATH") as t: t.depends('path') - t.translate(lambda k: normpath(k['path'])) + t.translate(lambda k: unicode(normpath(k['path']))) #with md.metadata("MDATA_KEY_MD5") as t: #t.depends('path') @@ -103,15 +103,13 @@ def load_definitions(): # owner is handled differently by (by events.py) - with md.metadata('MDATA_KEY_ORIGINAL_PATH') as t: - t.depends('original_path') - # MDATA_KEY_TITLE is the annoying special case b/c we sometimes read it # from file name with md.metadata('MDATA_KEY_TITLE') as t: # Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is # defaulted to "" from definitions above t.depends('title','MDATA_KEY_CREATOR') + t.translate(lambda k: k['title']) t.max_length(512) with md.metadata('MDATA_KEY_LABEL') as t: From 027153b88224376fddfc2a665f067dc249309baa Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 13:56:54 -0400 Subject: [PATCH 12/59] Fixed emf handling of bad definitions --- python_apps/media-monitor2/media/metadata/process.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index 1b62646a4..ca24feeb0 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -74,6 +74,7 @@ class MetadataElement(Loggable): # If value is present and normalized then we only check if it's # normalized or not. We normalize if it's not normalized already + if self.name in original: v = original[self.name] if self.__is_normalized(v): return v @@ -141,11 +142,18 @@ def normalize_mutagen(path): md['path'] = path return md + +class OverwriteMetadataElement(Exception): + def __init__(self, m): self.m = m + def __str__(self): return "Trying to overwrite: %s" % self.m + class MetadataReader(object): def __init__(self): self.clear() def register_metadata(self,m): + if m in self.__mdata_name_map: + raise OverwriteMetadataElement(m) self.__mdata_name_map[m.name] = m d = dict( (name,m.dependencies()) for name,m in self.__mdata_name_map.iteritems() ) From 591d2d741f814ed63cc4d7ce89bebf73fb23ce17 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 13:57:37 -0400 Subject: [PATCH 13/59] Added original path element in metadata --- python_apps/media-monitor2/media/monitor/pure.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 81b9cba49..1bb4231ae 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -312,6 +312,7 @@ def normalized_metadata(md, original_path): # TODO : wtf is this for again? new_md['MDATA_KEY_TITLE'] = re.sub(r'-?%s-?' % unicode_unknown, u'', new_md['MDATA_KEY_TITLE']) + new_md['MDATA_KEY_ORIGINAL_PATH'] = original_path return new_md def organized_path(old_path, root_path, orig_md): From 7db61cc4b4b1089f8de5746e8d63b360540cec78 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 13:58:05 -0400 Subject: [PATCH 14/59] fixed tests --- .../media-monitor2/media/monitor/metadata.py | 20 +++++++++---------- python_apps/media-monitor2/tests/test_emf.py | 14 ++++++++++++- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 2b6c07a83..30511c50d 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -216,17 +216,17 @@ class Metadata(Loggable): # from the file? self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100) - try: # emf stuff for testing: - if full_mutagen: - normalized = global_reader.read_mutagen(fpath) - self.logger.info("EMF--------------------") - self.logger.info(pformat(normalized)) - self.logger.info("OLD--------------------") - self.logger.info(pformat(self.__metadata)) - self.logger.info("-----------------------") + #try: # emf stuff for testing: + #if full_mutagen: + #normalized = global_reader.read_mutagen(fpath) + #self.logger.info("EMF--------------------") + #self.logger.info(pformat(normalized)) + #self.logger.info("OLD--------------------") + #self.logger.info(pformat(self.__metadata)) + #self.logger.info("-----------------------") - except Exception as e: - self.unexpected_exception(e) + #except Exception as e: + #self.unexpected_exception(e) def is_recorded(self): """ diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py index 69fc566db..355f93dd9 100644 --- a/python_apps/media-monitor2/tests/test_emf.py +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -1,11 +1,23 @@ # -*- coding: utf-8 -*- import unittest +from pprint import pprint as pp from media.metadata.process import global_reader +from media.monitor.metadata import Metadata + import media.metadata.definitions as defs defs.load_definitions() class TestMMP(unittest.TestCase): def test_sanity(self): - m = global_reader.read_mutagen("/home/rudi/music/Nightingale.mp3") + path = "/home/rudi/music/Nightingale.mp3" + m = global_reader.read_mutagen(path) self.assertTrue( len(m) > 0 ) + n = Metadata(path) + self.assertEqual(n.extract(), m) + pp(n.extract()) + print("--------------") + pp(m) + + +if __name__ == '__main__': unittest.main() From 482e2475ca6d49ca042921b06698e7259b164f7b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 14:24:33 -0400 Subject: [PATCH 15/59] cleaned up test --- python_apps/media-monitor2/media/metadata/definitions.py | 8 ++++---- python_apps/media-monitor2/tests/test_emf.py | 3 --- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index f0288bc61..97b3f6841 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -96,10 +96,10 @@ def load_definitions(): t.depends('path') t.translate(lambda k: unicode(normpath(k['path']))) - #with md.metadata("MDATA_KEY_MD5") as t: - #t.depends('path') - #t.optional(False) - #t.translate(lambda k: file_md5(k['path'], max_length=100)) + with md.metadata("MDATA_KEY_MD5") as t: + t.depends('path') + t.optional(False) + t.translate(lambda k: file_md5(k['path'], max_length=100)) # owner is handled differently by (by events.py) diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py index 355f93dd9..b4f9ef03b 100644 --- a/python_apps/media-monitor2/tests/test_emf.py +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -15,9 +15,6 @@ class TestMMP(unittest.TestCase): self.assertTrue( len(m) > 0 ) n = Metadata(path) self.assertEqual(n.extract(), m) - pp(n.extract()) - print("--------------") - pp(m) if __name__ == '__main__': unittest.main() From d00c74fdbe8f6779553f66ecbdb4c44279a52f36 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 14:32:56 -0400 Subject: [PATCH 16/59] Renamed test --- python_apps/media-monitor2/tests/test_emf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py index b4f9ef03b..545c059fd 100644 --- a/python_apps/media-monitor2/tests/test_emf.py +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -9,12 +9,11 @@ import media.metadata.definitions as defs defs.load_definitions() class TestMMP(unittest.TestCase): - def test_sanity(self): + def test_old_metadata(self): path = "/home/rudi/music/Nightingale.mp3" m = global_reader.read_mutagen(path) self.assertTrue( len(m) > 0 ) n = Metadata(path) self.assertEqual(n.extract(), m) - if __name__ == '__main__': unittest.main() From cefc5c99d93dd23fbdd3f0301e3573ea866db20e Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 15:39:35 -0400 Subject: [PATCH 17/59] Added special handling for title in emf --- .../media/metadata/definitions.py | 29 +++++++++++++++++-- .../media-monitor2/media/metadata/process.py | 1 + python_apps/media-monitor2/tests/test_emf.py | 6 ++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index 97b3f6841..a4f93848e 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- import media.metadata.process as md +import re from os.path import normpath -from media.monitor.pure import format_length, file_md5 +from media.monitor.pure import format_length, file_md5, is_airtime_recorded, \ + no_extension_basename defs_loaded = False @@ -105,11 +107,32 @@ def load_definitions(): # MDATA_KEY_TITLE is the annoying special case b/c we sometimes read it # from file name + + + # must handle 3 cases: + # 1. regular case (not recorded + title is present) + # 2. title is absent (read from file) + # 3. recorded file + def tr_title(k): + unicode_unknown = u"unknown" + new_title = u"" + if is_airtime_recorded(k) or k['title'] != u"": + new_title = k['title'] + else: + default_title = no_extension_basename(k['path']) + default_title = re.sub(r'__\d+\.',u'.', default_title) + if re.match(".+-%s-.+$" % unicode_unknown, default_title): + default_title = u'' + new_title = default_title + new_title = re.sub(r'-\d+kbps$', u'', new_title) + return new_title + with md.metadata('MDATA_KEY_TITLE') as t: # Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is # defaulted to "" from definitions above - t.depends('title','MDATA_KEY_CREATOR') - t.translate(lambda k: k['title']) + t.depends('title','MDATA_KEY_CREATOR','path') + t.optional(False) + t.translate(tr_title) t.max_length(512) with md.metadata('MDATA_KEY_LABEL') as t: diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index ca24feeb0..cde28cfaa 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -140,6 +140,7 @@ def normalize_mutagen(path): md['sample_rate'] = getattr(m.info, 'sample_rate', 0) md['mime'] = m.mime[0] if len(m.mime) > 0 else u'' md['path'] = path + if 'title' not in md: md['title'] = u'' return md diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py index 545c059fd..3114e2f41 100644 --- a/python_apps/media-monitor2/tests/test_emf.py +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -16,4 +16,10 @@ class TestMMP(unittest.TestCase): n = Metadata(path) self.assertEqual(n.extract(), m) + def test_recorded(self): + recorded_file = "./15:15:00-Untitled Show-256kbps.ogg" + m = global_reader.read_mutagen(recorded_file) + pp(m) + + if __name__ == '__main__': unittest.main() From ff8969efb997c7aefbbcd06593587ce7e3b03073 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 15:54:38 -0400 Subject: [PATCH 18/59] Improved handling for paths and added tests for it --- .../media-monitor2/media/metadata/definitions.py | 3 ++- .../media-monitor2/media/metadata/process.py | 3 ++- python_apps/media-monitor2/media/monitor/pure.py | 2 +- python_apps/media-monitor2/tests/test_emf.py | 13 ++++++++++--- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index a4f93848e..ee4175033 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -20,7 +20,8 @@ def load_definitions(): with md.metadata('MDATA_KEY_MIME') as t: t.default(u'') t.depends('mime') - t.translate(lambda k: k['mime'].replace('-','/')) + # Is this necessary? + t.translate(lambda k: k['mime'].replace('audio/vorbis','audio/ogg')) with md.metadata('MDATA_KEY_BITRATE') as t: t.default(u'') diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index cde28cfaa..85a462919 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from contextlib import contextmanager from media.monitor.pure import truncate_to_length, toposort +from os.path import normpath from media.monitor.log import Loggable import mutagen @@ -139,7 +140,7 @@ def normalize_mutagen(path): md['bitrate'] = getattr(m.info, 'bitrate', u'') md['sample_rate'] = getattr(m.info, 'sample_rate', 0) md['mime'] = m.mime[0] if len(m.mime) > 0 else u'' - md['path'] = path + md['path'] = normpath(path) if 'title' not in md: md['title'] = u'' return md diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 1bb4231ae..fd1c55a13 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -312,7 +312,7 @@ def normalized_metadata(md, original_path): # TODO : wtf is this for again? new_md['MDATA_KEY_TITLE'] = re.sub(r'-?%s-?' % unicode_unknown, u'', new_md['MDATA_KEY_TITLE']) - new_md['MDATA_KEY_ORIGINAL_PATH'] = original_path + new_md['MDATA_KEY_ORIGINAL_PATH'] = normpath(original_path) return new_md def organized_path(old_path, root_path, orig_md): diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py index 3114e2f41..0213fa918 100644 --- a/python_apps/media-monitor2/tests/test_emf.py +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- import unittest -from pprint import pprint as pp +#from pprint import pprint as pp from media.metadata.process import global_reader from media.monitor.metadata import Metadata @@ -9,6 +9,13 @@ import media.metadata.definitions as defs defs.load_definitions() class TestMMP(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + + def metadatas(self,f): + return global_reader.read_mutagen(f), Metadata(f).extract() + def test_old_metadata(self): path = "/home/rudi/music/Nightingale.mp3" m = global_reader.read_mutagen(path) @@ -18,8 +25,8 @@ class TestMMP(unittest.TestCase): def test_recorded(self): recorded_file = "./15:15:00-Untitled Show-256kbps.ogg" - m = global_reader.read_mutagen(recorded_file) - pp(m) + emf, old = self.metadatas(recorded_file) + self.assertEqual(emf, old) if __name__ == '__main__': unittest.main() From 5a593112b38209dc44c3cd43629eb5c0bdcf132b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 12:20:08 -0400 Subject: [PATCH 19/59] Use emf instead of omf --- python_apps/media-monitor2/media/monitor/metadata.py | 4 +++- python_apps/media-monitor2/tests/test_emf.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 30511c50d..2bd059323 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -14,7 +14,6 @@ import media.monitor.pure as mmp # emf related stuff from media.metadata.process import global_reader import media.metadata.definitions as defs -from pprint import pformat defs.load_definitions() """ @@ -173,6 +172,9 @@ class Metadata(Loggable): for e in exceptions: raise e def __init__(self, fpath): + self.__metadata = global_reader.read_mutagen(fpath) + + def __init__2(self, fpath): # Forcing the unicode through try : fpath = fpath.decode("utf-8") except : pass diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py index 0213fa918..140c0aa8b 100644 --- a/python_apps/media-monitor2/tests/test_emf.py +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -28,5 +28,4 @@ class TestMMP(unittest.TestCase): emf, old = self.metadatas(recorded_file) self.assertEqual(emf, old) - if __name__ == '__main__': unittest.main() From 7255241e6cbe293445d5ca098c557efd1aff3b1b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 12:27:58 -0400 Subject: [PATCH 20/59] Removed old commented code --- python_apps/media-monitor2/media/monitor/metadata.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 2bd059323..199bd7420 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -218,18 +218,6 @@ class Metadata(Loggable): # from the file? self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100) - #try: # emf stuff for testing: - #if full_mutagen: - #normalized = global_reader.read_mutagen(fpath) - #self.logger.info("EMF--------------------") - #self.logger.info(pformat(normalized)) - #self.logger.info("OLD--------------------") - #self.logger.info(pformat(self.__metadata)) - #self.logger.info("-----------------------") - - #except Exception as e: - #self.unexpected_exception(e) - def is_recorded(self): """ returns true if the file has been created by airtime through recording From 0638d2147365e394a342f7daed475f134ee0cd03 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 14:48:16 -0400 Subject: [PATCH 21/59] Refactored code --- .../media-monitor2/media/metadata/process.py | 28 ++++++++++++++++++- .../media-monitor2/media/monitor/metadata.py | 5 ++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index 85a462919..b500d029d 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -2,9 +2,28 @@ from contextlib import contextmanager from media.monitor.pure import truncate_to_length, toposort from os.path import normpath +from media.monitor.exceptions import BadSongFile from media.monitor.log import Loggable +import media.monitor.pure as mmp +from collections import namedtuple import mutagen +class FakeMutagen(dict): + """ + Need this fake mutagen object so that airtime_special functions + return a proper default value instead of throwing an exceptions for + files that mutagen doesn't recognize + """ + FakeInfo = namedtuple('FakeInfo','length bitrate') + def __init__(self,path): + self.path = path + self.mime = ['audio/wav'] + self.info = FakeMutagen.FakeInfo(0.0, '') + dict.__init__(self) + def set_length(self,l): + old_bitrate = self.info.bitrate + self.info = FakeMutagen.FakeInfo(l, old_bitrate) + class MetadataAbsent(Exception): def __init__(self, name): self.name = name @@ -129,7 +148,14 @@ def normalize_mutagen(path): Consumes a path and reads the metadata using mutagen. normalizes some of the metadata that isn't read through the mutagen hash """ - m = mutagen.File(path, easy=True) + if not mmp.file_playable(path): raise BadSongFile(path) + try : m = mutagen.File(path, easy=True) + except Exception : raise BadSongFile(path) + if m is None: m = FakeMutagen(path) + try: + if mmp.extension(path) == 'wav': + m.set_length(mmp.read_wave_duration(path)) + except Exception: raise BadSongFile(path) md = {} for k,v in m.iteritems(): if type(v) is list: diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 199bd7420..a496af61a 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -48,6 +48,8 @@ airtime2mutagen = { "MDATA_KEY_COPYRIGHT" : "copyright", } + +# TODO :Remove FakeMutagen class. Moved to media.metadata.process class FakeMutagen(dict): """ Need this fake mutagen object so that airtime_special functions @@ -172,6 +174,9 @@ class Metadata(Loggable): for e in exceptions: raise e def __init__(self, fpath): + # Forcing the unicode through + try : fpath = fpath.decode("utf-8") + except : pass self.__metadata = global_reader.read_mutagen(fpath) def __init__2(self, fpath): From 9a1948707c8ea2d0c0aa769ae0f5d048bff32032 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 14:51:51 -0400 Subject: [PATCH 22/59] Removed duplicate code --- .../media-monitor2/media/monitor/metadata.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index a496af61a..30bab7419 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -49,23 +49,6 @@ airtime2mutagen = { } -# TODO :Remove FakeMutagen class. Moved to media.metadata.process -class FakeMutagen(dict): - """ - Need this fake mutagen object so that airtime_special functions - return a proper default value instead of throwing an exceptions for - files that mutagen doesn't recognize - """ - FakeInfo = namedtuple('FakeInfo','length bitrate') - def __init__(self,path): - self.path = path - self.mime = ['audio/wav'] - self.info = FakeMutagen.FakeInfo(0.0, '') - dict.__init__(self) - def set_length(self,l): - old_bitrate = self.info.bitrate - self.info = FakeMutagen.FakeInfo(l, old_bitrate) - # Some airtime attributes are special because they must use the mutagen object # itself to calculate the value that they need. The lambda associated with each # key should attempt to extract the corresponding value from the mutagen object From 6a8d902a61e9dacea8ccecef1bbe74eb46c5448d Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 14:52:38 -0400 Subject: [PATCH 23/59] Removed omf --- .../media-monitor2/media/monitor/metadata.py | 45 ------------------- 1 file changed, 45 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 30bab7419..9754c1b0e 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -2,7 +2,6 @@ import mutagen import os import copy -from collections import namedtuple from mutagen.easymp4 import EasyMP4KeyError from mutagen.easyid3 import EasyID3KeyError @@ -162,50 +161,6 @@ class Metadata(Loggable): except : pass self.__metadata = global_reader.read_mutagen(fpath) - def __init__2(self, fpath): - # Forcing the unicode through - try : fpath = fpath.decode("utf-8") - except : pass - - if not mmp.file_playable(fpath): raise BadSongFile(fpath) - - try : full_mutagen = mutagen.File(fpath, easy=True) - except Exception : raise BadSongFile(fpath) - - self.path = fpath - if not os.path.exists(self.path): - self.logger.info("Attempting to read metadata of file \ - that does not exist. Setting metadata to {}") - self.__metadata = {} - return - # TODO : Simplify the way all of these rules are handled right now it's - # extremely unclear and needs to be refactored. - #if full_mutagen is None: raise BadSongFile(fpath) - - if full_mutagen is None: full_mutagen = FakeMutagen(fpath) - self.__metadata = Metadata.airtime_dict(full_mutagen) - # Now we extra the special values that are calculated from the mutagen - # object itself: - - if mmp.extension(fpath) == 'wav': - full_mutagen.set_length(mmp.read_wave_duration(fpath)) - - for special_key,f in airtime_special.iteritems(): - try: - new_val = f(full_mutagen) - if new_val is not None: - self.__metadata[special_key] = new_val - except Exception as e: - self.logger.info("Could not get special key %s for %s" % - (special_key, fpath)) - self.logger.info(str(e)) - # Finally, we "normalize" all the metadata here: - self.__metadata = mmp.normalized_metadata(self.__metadata, fpath) - # Now we must load the md5: - # TODO : perhaps we shouldn't hard code how many bytes we're reading - # from the file? - self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100) - def is_recorded(self): """ returns true if the file has been created by airtime through recording From 66b5bf304f0c692d3d59955162c6ccc3152db6ed Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 14:58:03 -0400 Subject: [PATCH 24/59] changed test strings --- python_apps/media-monitor2/tests/test_api_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/tests/test_api_client.py b/python_apps/media-monitor2/tests/test_api_client.py index 26f1a25ef..18595f860 100644 --- a/python_apps/media-monitor2/tests/test_api_client.py +++ b/python_apps/media-monitor2/tests/test_api_client.py @@ -19,8 +19,8 @@ class TestApiClient(unittest.TestCase): self.apc.register_component("api-client-tester") # All of the following requests should error out in some way self.bad_requests = [ - { 'mode' : 'dang it', 'is_record' : 0 }, - { 'mode' : 'damn frank', 'is_record' : 1 }, + { 'mode' : 'foo', 'is_record' : 0 }, + { 'mode' : 'bar', 'is_record' : 1 }, { 'no_mode' : 'at_all' }, ] def test_bad_requests(self): From dc361c11367f5bc5ab4f4d42107f9ce41a29124b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 14:59:06 -0400 Subject: [PATCH 25/59] Aligned constants --- airtime_mvc/application/configs/constants.php | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/airtime_mvc/application/configs/constants.php b/airtime_mvc/application/configs/constants.php index 1d1ed55e4..eb5cb3989 100644 --- a/airtime_mvc/application/configs/constants.php +++ b/airtime_mvc/application/configs/constants.php @@ -1,40 +1,40 @@ Date: Tue, 23 Oct 2012 12:06:30 -0400 Subject: [PATCH 26/59] Removed unused method --- .../media-monitor2/media/monitor/metadata.py | 36 ++----------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 9754c1b0e..5bbbafaeb 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -7,7 +7,7 @@ from mutagen.easyid3 import EasyID3KeyError from media.monitor.exceptions import BadSongFile, InvalidMetadataElement from media.monitor.log import Loggable -from media.monitor.pure import format_length, truncate_to_length +from media.monitor.pure import format_length import media.monitor.pure as mmp # emf related stuff @@ -89,6 +89,7 @@ class Metadata(Loggable): # little bit messy. Some of the handling is in m.m.pure while the rest is # here. Also interface is not very consistent + # TODO : what is this shit? maybe get rid of it? @staticmethod def fix_title(path): # If we have no title in path we will format it @@ -99,39 +100,6 @@ class Metadata(Loggable): m[u'title'] = new_title m.save() - @staticmethod - def airtime_dict(d): - """ - Converts mutagen dictionary 'd' into airtime dictionary - """ - temp_dict = {} - for m_key, m_val in d.iteritems(): - # TODO : some files have multiple fields for the same metadata. - # genre is one example. In that case mutagen will return a list - # of values - - if isinstance(m_val, list): - # TODO : does it make more sense to just skip the element in - # this case? - if len(m_val) == 0: assign_val = '' - else: assign_val = m_val[0] - else: assign_val = m_val - - temp_dict[ m_key ] = assign_val - airtime_dictionary = {} - for muta_k, muta_v in temp_dict.iteritems(): - # We must check if we can actually translate the mutagen key into - # an airtime key before doing the conversion - if muta_k in mutagen2airtime: - airtime_key = mutagen2airtime[muta_k] - # Apply truncation in the case where airtime_key is in our - # truncation table - muta_v = \ - truncate_to_length(muta_v, truncate_table[airtime_key])\ - if airtime_key in truncate_table else muta_v - airtime_dictionary[ airtime_key ] = muta_v - return airtime_dictionary - @staticmethod def write_unsafe(path,md): """ From 54e6bec16af71eb9ba26832e2acd1144a20d1f59 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 23 Oct 2012 15:00:09 -0400 Subject: [PATCH 27/59] Reshuffled/removed some unit tests to get rid of omf --- .../media-monitor2/tests/test_metadata.py | 6 -- python_apps/media-monitor2/tests/test_pure.py | 70 ++----------------- 2 files changed, 7 insertions(+), 69 deletions(-) diff --git a/python_apps/media-monitor2/tests/test_metadata.py b/python_apps/media-monitor2/tests/test_metadata.py index 6f24240b0..4b05a9fce 100644 --- a/python_apps/media-monitor2/tests/test_metadata.py +++ b/python_apps/media-monitor2/tests/test_metadata.py @@ -42,10 +42,4 @@ class TestMetadata(unittest.TestCase): x1 = 123456 print("Formatting '%s' to '%s'" % (x1, mmm.format_length(x1))) - def test_truncate_to_length(self): - s1 = "testing with non string literal" - s2 = u"testing with unicode literal" - self.assertEqual( len(mmm.truncate_to_length(s1, 5)), 5) - self.assertEqual( len(mmm.truncate_to_length(s2, 8)), 8) - if __name__ == '__main__': unittest.main() diff --git a/python_apps/media-monitor2/tests/test_pure.py b/python_apps/media-monitor2/tests/test_pure.py index 64d09dc62..6bc5d4906 100644 --- a/python_apps/media-monitor2/tests/test_pure.py +++ b/python_apps/media-monitor2/tests/test_pure.py @@ -2,7 +2,6 @@ import unittest import os import media.monitor.pure as mmp -from media.monitor.metadata import Metadata class TestMMP(unittest.TestCase): def setUp(self): @@ -34,68 +33,6 @@ class TestMMP(unittest.TestCase): sd = mmp.default_to(dictionary=sd, keys=def_keys, default='DEF') for k in def_keys: self.assertEqual( sd[k], 'DEF' ) - def test_normalized_metadata(self): - #Recorded show test first - orig = Metadata.airtime_dict({ - 'date' : [u'2012-08-21'], - 'tracknumber' : [u'2'], - 'title' : [u'record-2012-08-21-11:29:00'], - 'artist' : [u'Airtime Show Recorder'] - }) - orga = Metadata.airtime_dict({ - 'date' : [u'2012-08-21'], - 'tracknumber' : [u'2'], - 'artist' : [u'Airtime Show Recorder'], - 'title' : [u'record-2012-08-21-11:29:00'] - }) - orga['MDATA_KEY_FTYPE'] = u'audioclip' - orig['MDATA_KEY_BITRATE'] = u'256000' - orga['MDATA_KEY_BITRATE'] = u'256000' - old_path = "/home/rudi/recorded/2012-08-21-11:29:00.ogg" - normalized = mmp.normalized_metadata(orig, old_path) - normalized['MDATA_KEY_BITRATE'] = u'256000' - - self.assertEqual( orga, normalized ) - - organized_base_name = "11:29:00-record-256kbps.ogg" - base = "/srv/airtime/stor/" - organized_path = mmp.organized_path(old_path,base, normalized) - self.assertEqual(os.path.basename(organized_path), organized_base_name) - - def test_normalized_metadata2(self): - """ - cc-4305 - """ - orig = Metadata.airtime_dict({ - 'date' : [u'2012-08-27'], - 'tracknumber' : [u'3'], - 'title' : [u'18-11-00-Untitled Show'], - 'artist' : [u'Airtime Show Recorder'] - }) - old_path = "/home/rudi/recorded/doesnt_really_matter.ogg" - normalized = mmp.normalized_metadata(orig, old_path) - normalized['MDATA_KEY_BITRATE'] = u'256000' - opath = mmp.organized_path(old_path, "/srv/airtime/stor/", - normalized) - # TODO : add a better test than this... - self.assertTrue( len(opath) > 0 ) - - def test_normalized_metadata3(self): - """ - Test the case where the metadata is empty - """ - orig = Metadata.airtime_dict({}) - paths_unknown_title = [ - ("/testin/unknown-unknown-unknown.mp3",""), - ("/testin/01-unknown-123kbps.mp3",""), - ("/testin/02-unknown-140kbps.mp3",""), - ("/testin/unknown-unknown-123kbps.mp3",""), - ("/testin/unknown-bibimbop-unknown.mp3","bibimbop"), - ] - for p,res in paths_unknown_title: - normalized = mmp.normalized_metadata(orig, p) - self.assertEqual( normalized['MDATA_KEY_TITLE'], res) - def test_file_md5(self): p = os.path.realpath(__file__) m1 = mmp.file_md5(p) @@ -116,6 +53,13 @@ class TestMMP(unittest.TestCase): self.assertEqual( mmp.parse_int("123asf"), "123" ) self.assertEqual( mmp.parse_int("asdf"), None ) + def test_truncate_to_length(self): + s1 = "testing with non string literal" + s2 = u"testing with unicode literal" + self.assertEqual( len(mmp.truncate_to_length(s1, 5)), 5) + self.assertEqual( len(mmp.truncate_to_length(s2, 8)), 8) + + def test_owner_id(self): start_path = "testing.mp3" id_path = "testing.mp3.identifier" From 1359bb402e2af565bc94a9c83a443c291ac02fc5 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 23 Oct 2012 15:03:37 -0400 Subject: [PATCH 28/59] Removed useless comment --- python_apps/media-monitor2/media/monitor/pure.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index fd1c55a13..fa14089d7 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -22,7 +22,6 @@ from configobj import ConfigObj from media.monitor.exceptions import FailedToSetLocale, FailedToCreateDir -#supported_extensions = [u"mp3", u"ogg", u"oga"] supported_extensions = [u"mp3", u"ogg", u"oga", u"flac", u"wav", u'm4a', u'mp4'] From 29c96d2efe9f29c799d5f38eb7224beeb7e500b3 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 23 Oct 2012 15:24:27 -0400 Subject: [PATCH 29/59] Removed md5 related test --- python_apps/media-monitor2/tests/test_metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python_apps/media-monitor2/tests/test_metadata.py b/python_apps/media-monitor2/tests/test_metadata.py index 4b05a9fce..7a32b61a8 100644 --- a/python_apps/media-monitor2/tests/test_metadata.py +++ b/python_apps/media-monitor2/tests/test_metadata.py @@ -26,7 +26,6 @@ class TestMetadata(unittest.TestCase): i += 1 print("Sample metadata: '%s'" % md) self.assertTrue( len( md.keys() ) > 0 ) - self.assertTrue( 'MDATA_KEY_MD5' in md ) utf8 = md_full.utf8() for k,v in md.iteritems(): if hasattr(utf8[k], 'decode'): From 9169893e4d0deaebb19d5790d068367dcb4a683a Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 23 Oct 2012 16:44:11 -0400 Subject: [PATCH 30/59] Alignment of method definitions --- python_apps/media-monitor2/media/monitor/syncdb.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/syncdb.py b/python_apps/media-monitor2/media/monitor/syncdb.py index 0c14fb038..cc8abc294 100644 --- a/python_apps/media-monitor2/media/monitor/syncdb.py +++ b/python_apps/media-monitor2/media/monitor/syncdb.py @@ -53,11 +53,11 @@ class AirtimeDB(Loggable): """ return self.id_to_dir[ dir_id ] - def storage_path(self): return self.base_storage - def organize_path(self): return self.storage_paths['organize'] - def problem_path(self): return self.storage_paths['problem_files'] - def import_path(self): return self.storage_paths['imported'] - def recorded_path(self): return self.storage_paths['recorded'] + def storage_path(self) : return self.base_storage + def organize_path(self) : return self.storage_paths['organize'] + def problem_path(self) : return self.storage_paths['problem_files'] + def import_path(self) : return self.storage_paths['imported'] + def recorded_path(self) : return self.storage_paths['recorded'] def list_watched(self): """ From f718070d3cda1165f338888216ec76617e06364a Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 23 Oct 2012 17:00:22 -0400 Subject: [PATCH 31/59] Uncoupled the api_client instance from the RequestSync object --- .../media-monitor2/media/monitor/watchersyncer.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index e7a2cf9c0..191a0f8a1 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -18,10 +18,18 @@ class RequestSync(threading.Thread,Loggable): to airtime. In the process it packs the requests and retries for some number of times """ - def __init__(self, watcher, requests): + + @classmethod + def create_with_api_client(cls, watcher, requests): + apiclient = ac.AirtimeApiClient.create_right_config() + self = cls(watcher, requests, apiclient) + return self + + def __init__(self, watcher, requests, apiclient): threading.Thread.__init__(self) self.watcher = watcher self.requests = requests + self.apiclient = apiclient self.retries = 1 self.request_wait = 0.3 @@ -209,7 +217,8 @@ class WatchSyncer(ReportHandler,Loggable): requests = copy.copy(self.__queue) def launch_request(): # Need shallow copy here - t = RequestSync(watcher=self, requests=requests) + t = RequestSync.create_with_api_client(watcher=self, + requests=requests) t.start() self.__current_thread = t self.__requests.append(launch_request) From 86b262ed8f89b0fd0b07b4140a1803457bdecc5c Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 11:08:21 -0400 Subject: [PATCH 32/59] uncoupled threading from making a request --- .../media/monitor/watchersyncer.py | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index 191a0f8a1..6ed9ba594 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -6,13 +6,23 @@ import copy from media.monitor.handler import ReportHandler from media.monitor.log import Loggable from media.monitor.exceptions import BadSongFile -from media.monitor.pure import LazyProperty from media.monitor.eventcontractor import EventContractor from media.monitor.events import EventProxy import api_clients.api_client as ac -class RequestSync(threading.Thread,Loggable): + +class ThreadedRequestSync(threading.Thread, Loggable): + def __init__(self, rs): + threading.Thread.__init__(self) + self.rs = rs + self.daemon = True + self.start() + + def run(self): + self.rs.run_request() + +class RequestSync(Loggable): """ This class is responsible for making the api call to send a request to airtime. In the process it packs the requests and retries for @@ -26,18 +36,13 @@ class RequestSync(threading.Thread,Loggable): return self def __init__(self, watcher, requests, apiclient): - threading.Thread.__init__(self) self.watcher = watcher self.requests = requests self.apiclient = apiclient self.retries = 1 self.request_wait = 0.3 - @LazyProperty - def apiclient(self): - return ac.AirtimeApiClient.create_right_config() - - def run(self): + def run_request(self): self.logger.info("Attempting request with %d items." % len(self.requests)) # Note that we must attach the appropriate mode to every @@ -59,6 +64,8 @@ class RequestSync(threading.Thread,Loggable): request_event.path) def make_req(): self.apiclient.send_media_monitor_requests( packed_requests ) + # TODO : none of the shit below is necessary. get rid of it + # eventually for try_index in range(0,self.retries): try: make_req() # most likely we did not get json response as we expected @@ -76,7 +83,7 @@ class RequestSync(threading.Thread,Loggable): break else: self.logger.info("Failed to send request after '%d' tries..." % self.retries) - self.watcher.flag_done() + self.watcher.flag_done() # poor man's condition variable class TimeoutWatcher(threading.Thread,Loggable): """ @@ -217,9 +224,8 @@ class WatchSyncer(ReportHandler,Loggable): requests = copy.copy(self.__queue) def launch_request(): # Need shallow copy here - t = RequestSync.create_with_api_client(watcher=self, - requests=requests) - t.start() + t = ThreadedRequestSync( RequestSync.create_with_api_client( + watcher=self, requests=requests) ) self.__current_thread = t self.__requests.append(launch_request) self.__reset_queue() From 289d5575ece18521d4a21747a1634860a0d5e03e Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 11:18:41 -0400 Subject: [PATCH 33/59] Cleaned the shit out of RequestSync. Removed old comments and useless code --- .../media/monitor/watchersyncer.py | 40 +++++-------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index 6ed9ba594..6ab6509bf 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -28,7 +28,6 @@ class RequestSync(Loggable): to airtime. In the process it packs the requests and retries for some number of times """ - @classmethod def create_with_api_client(cls, watcher, requests): apiclient = ac.AirtimeApiClient.create_right_config() @@ -39,17 +38,10 @@ class RequestSync(Loggable): self.watcher = watcher self.requests = requests self.apiclient = apiclient - self.retries = 1 - self.request_wait = 0.3 def run_request(self): self.logger.info("Attempting request with %d items." % len(self.requests)) - # Note that we must attach the appropriate mode to every - # response. Also Not forget to attach the 'is_record' to any - # requests that are related to recorded shows - # TODO : recorded shows aren't flagged right - # Is this retry shit even necessary? Consider getting rid of this. packed_requests = [] for request_event in self.requests: try: @@ -62,27 +54,17 @@ class RequestSync(Loggable): if hasattr(request_event, 'path'): self.logger.info("Possibly related to path: '%s'" % request_event.path) - def make_req(): - self.apiclient.send_media_monitor_requests( packed_requests ) - # TODO : none of the shit below is necessary. get rid of it - # eventually - for try_index in range(0,self.retries): - try: make_req() - # most likely we did not get json response as we expected - except ValueError: - self.logger.info("ApiController.php probably crashed, we \ - diagnose this from the fact that it did not return \ - valid json") - self.logger.info("Trying again after %f seconds" % - self.request_wait) - time.sleep( self.request_wait ) - except Exception as e: self.unexpected_exception(e) - else: - self.logger.info("Request worked on the '%d' try" % - (try_index + 1)) - break - else: self.logger.info("Failed to send request after '%d' tries..." % - self.retries) + try: self.apiclient.send_media_monitor_requests( packed_requests ) + # most likely we did not get json response as we expected + except ValueError: + self.logger.info("ApiController.php probably crashed, we \ + diagnose this from the fact that it did not return \ + valid json") + self.logger.info("Trying again after %f seconds" % + self.request_wait) + except Exception as e: self.unexpected_exception(e) + else: + self.logger.info("Request was successful") self.watcher.flag_done() # poor man's condition variable class TimeoutWatcher(threading.Thread,Loggable): From a6ab91333f278375286e3be0bb635565b781970e Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 11:24:23 -0400 Subject: [PATCH 34/59] formatting --- python_apps/media-monitor2/media/monitor/watchersyncer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index 6ab6509bf..d7567a535 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -63,8 +63,7 @@ class RequestSync(Loggable): self.logger.info("Trying again after %f seconds" % self.request_wait) except Exception as e: self.unexpected_exception(e) - else: - self.logger.info("Request was successful") + else: self.logger.info("Request was successful") self.watcher.flag_done() # poor man's condition variable class TimeoutWatcher(threading.Thread,Loggable): From 38cbd7d7e42ac7020954e9b36223a4c4cc4b11bd Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 11:52:47 -0400 Subject: [PATCH 35/59] added emf unit tests --- .../media-monitor2/tests/test_metadata_def.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 python_apps/media-monitor2/tests/test_metadata_def.py diff --git a/python_apps/media-monitor2/tests/test_metadata_def.py b/python_apps/media-monitor2/tests/test_metadata_def.py new file mode 100644 index 000000000..e666ef68a --- /dev/null +++ b/python_apps/media-monitor2/tests/test_metadata_def.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +import unittest + +import media.metadata.process as md + +class TestMetadataDef(unittest.TestCase): + def test_simple(self): + + with md.metadata('MDATA_TESTING') as t: + t.optional(True) + t.depends('ONE','TWO') + t.default('unknown') + t.translate(lambda kw: kw['ONE'] + kw['TWO']) + + h = { 'ONE' : "testing", 'TWO' : "123" } + result = md.global_reader.read('test_path',h) + self.assertTrue( 'MDATA_TESTING' in result ) + self.assertEqual( result['MDATA_TESTING'], 'testing123' ) + h1 = { 'ONE' : 'big testing', 'two' : 'nothing' } + result1 = md.global_reader.read('bs path', h1) + self.assertEqual( result1['MDATA_TESTING'], 'unknown' ) + + def test_topo(self): + with md.metadata('MDATA_TESTING') as t: + t.depends('shen','sheni') + t.default('megitzda') + t.translate(lambda kw: kw['shen'] + kw['sheni']) + + with md.metadata('shen') as t: + t.default('vaxo') + + with md.metadata('sheni') as t: + t.default('gio') + + with md.metadata('vaxo') as t: + t.depends('shevetsi') + + v = md.global_reader.read('bs mang', {}) + self.assertEqual(v['MDATA_TESTING'], 'vaxogio') + self.assertTrue( 'vaxo' not in v ) + + md.global_reader.clear() + +if __name__ == '__main__': unittest.main() From 0422127689ef4a073090d766aae66fefea04a434 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 12:54:28 -0400 Subject: [PATCH 36/59] Added unit tests for requestsync --- .../media-monitor2/tests/test_requestsync.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 python_apps/media-monitor2/tests/test_requestsync.py diff --git a/python_apps/media-monitor2/tests/test_requestsync.py b/python_apps/media-monitor2/tests/test_requestsync.py new file mode 100644 index 000000000..a26b86b46 --- /dev/null +++ b/python_apps/media-monitor2/tests/test_requestsync.py @@ -0,0 +1,48 @@ +import unittest +from mock import MagicMock + +from media.monitor.watchersyncer import RequestSync + +class TestRequestSync(unittest.TestCase): + + def apc_mock(self): + fake_apc = MagicMock() + fake_apc.send_media_monitor_requests = MagicMock() + return fake_apc + + def watcher_mock(self): + fake_watcher = MagicMock() + fake_watcher.flag_done = MagicMock() + return fake_watcher + + def request_mock(self): + fake_request = MagicMock() + fake_request.safe_pack = MagicMock(return_value=[]) + return fake_request + + def test_send_media_monitor(self): + fake_apc = self.apc_mock() + fake_requests = [ self.request_mock() for x in range(1,5) ] + fake_watcher = self.watcher_mock() + rs = RequestSync(fake_watcher, fake_requests, fake_apc) + rs.run_request() + self.assertEquals(fake_apc.send_media_monitor_requests.call_count, 1) + + def test_flag_done(self): + fake_apc = self.apc_mock() + fake_requests = [ self.request_mock() for x in range(1,5) ] + fake_watcher = self.watcher_mock() + rs = RequestSync(fake_watcher, fake_requests, fake_apc) + rs.run_request() + self.assertEquals(fake_watcher.flag_done.call_count, 1) + + def test_safe_pack(self): + fake_apc = self.apc_mock() + fake_requests = [ self.request_mock() for x in range(1,5) ] + fake_watcher = self.watcher_mock() + rs = RequestSync(fake_watcher, fake_requests, fake_apc) + rs.run_request() + for req in fake_requests: + self.assertEquals(req.safe_pack.call_count, 1) + +if __name__ == '__main__': unittest.main() From 212e3bd30e614de05adbaa7d6f1b0ce69a8490a0 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:07:51 -0400 Subject: [PATCH 37/59] created a module for requests (refactorings) --- .../media-monitor2/media/monitor/request.py | 62 +++++++++++++++++++ .../media/monitor/watchersyncer.py | 58 +---------------- .../media-monitor2/tests/test_requestsync.py | 2 +- 3 files changed, 64 insertions(+), 58 deletions(-) create mode 100644 python_apps/media-monitor2/media/monitor/request.py diff --git a/python_apps/media-monitor2/media/monitor/request.py b/python_apps/media-monitor2/media/monitor/request.py new file mode 100644 index 000000000..934290a05 --- /dev/null +++ b/python_apps/media-monitor2/media/monitor/request.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +import threading + +from media.monitor.exceptions import BadSongFile +from media.monitor.log import Loggable +import api_clients.api_client as ac + +class ThreadedRequestSync(threading.Thread, Loggable): + def __init__(self, rs): + threading.Thread.__init__(self) + self.rs = rs + self.daemon = True + self.start() + + def run(self): + self.rs.run_request() + +class RequestSync(Loggable): + """ + This class is responsible for making the api call to send a request + to airtime. In the process it packs the requests and retries for + some number of times + """ + @classmethod + def create_with_api_client(cls, watcher, requests): + apiclient = ac.AirtimeApiClient.create_right_config() + self = cls(watcher, requests, apiclient) + return self + + def __init__(self, watcher, requests, apiclient): + self.watcher = watcher + self.requests = requests + self.apiclient = apiclient + + def run_request(self): + self.logger.info("Attempting request with %d items." % + len(self.requests)) + packed_requests = [] + for request_event in self.requests: + try: + for request in request_event.safe_pack(): + if isinstance(request, BadSongFile): + self.logger.info("Bad song file: '%s'" % request.path) + else: packed_requests.append(request) + except Exception as e: + self.unexpected_exception( e ) + if hasattr(request_event, 'path'): + self.logger.info("Possibly related to path: '%s'" % + request_event.path) + try: self.apiclient.send_media_monitor_requests( packed_requests ) + # most likely we did not get json response as we expected + except ValueError: + self.logger.info("ApiController.php probably crashed, we \ + diagnose this from the fact that it did not return \ + valid json") + self.logger.info("Trying again after %f seconds" % + self.request_wait) + except Exception as e: self.unexpected_exception(e) + else: self.logger.info("Request was successful") + self.watcher.flag_done() # poor man's condition variable + diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index d7567a535..9f83c6526 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -8,63 +8,7 @@ from media.monitor.log import Loggable from media.monitor.exceptions import BadSongFile from media.monitor.eventcontractor import EventContractor from media.monitor.events import EventProxy - -import api_clients.api_client as ac - - -class ThreadedRequestSync(threading.Thread, Loggable): - def __init__(self, rs): - threading.Thread.__init__(self) - self.rs = rs - self.daemon = True - self.start() - - def run(self): - self.rs.run_request() - -class RequestSync(Loggable): - """ - This class is responsible for making the api call to send a request - to airtime. In the process it packs the requests and retries for - some number of times - """ - @classmethod - def create_with_api_client(cls, watcher, requests): - apiclient = ac.AirtimeApiClient.create_right_config() - self = cls(watcher, requests, apiclient) - return self - - def __init__(self, watcher, requests, apiclient): - self.watcher = watcher - self.requests = requests - self.apiclient = apiclient - - def run_request(self): - self.logger.info("Attempting request with %d items." % - len(self.requests)) - packed_requests = [] - for request_event in self.requests: - try: - for request in request_event.safe_pack(): - if isinstance(request, BadSongFile): - self.logger.info("Bad song file: '%s'" % request.path) - else: packed_requests.append(request) - except Exception as e: - self.unexpected_exception( e ) - if hasattr(request_event, 'path'): - self.logger.info("Possibly related to path: '%s'" % - request_event.path) - try: self.apiclient.send_media_monitor_requests( packed_requests ) - # most likely we did not get json response as we expected - except ValueError: - self.logger.info("ApiController.php probably crashed, we \ - diagnose this from the fact that it did not return \ - valid json") - self.logger.info("Trying again after %f seconds" % - self.request_wait) - except Exception as e: self.unexpected_exception(e) - else: self.logger.info("Request was successful") - self.watcher.flag_done() # poor man's condition variable +from media.monitor.request import ThreadedRequestSync, RequestSync class TimeoutWatcher(threading.Thread,Loggable): """ diff --git a/python_apps/media-monitor2/tests/test_requestsync.py b/python_apps/media-monitor2/tests/test_requestsync.py index a26b86b46..2570bc34e 100644 --- a/python_apps/media-monitor2/tests/test_requestsync.py +++ b/python_apps/media-monitor2/tests/test_requestsync.py @@ -1,7 +1,7 @@ import unittest from mock import MagicMock -from media.monitor.watchersyncer import RequestSync +from media.monitor.request import RequestSync class TestRequestSync(unittest.TestCase): From fb28c3a6c5c7dc9ad3780d12f219f449c3756cc2 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:32:27 -0400 Subject: [PATCH 38/59] Formatting --- python_apps/media-monitor2/media/monitor/watchersyncer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index 9f83c6526..b008b10d0 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -71,8 +71,7 @@ class WatchSyncer(ReportHandler,Loggable): #self.push_queue( event ) except BadSongFile as e: self.fatal_exception("Received bas song file '%s'" % e.path, e) - except Exception as e: - self.unexpected_exception(e) + except Exception as e: self.unexpected_exception(e) else: self.logger.info("Received event that does not implement packing.\ Printing its representation:") From a42210e321f6a3e563ee655d8e6197756c620f38 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:38:11 -0400 Subject: [PATCH 39/59] added comment for destructor --- python_apps/media-monitor2/media/monitor/watchersyncer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index b008b10d0..d2df9ed3a 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -157,7 +157,8 @@ class WatchSyncer(ReportHandler,Loggable): def __reset_queue(self): self.__queue = [] def __del__(self): - # Ideally we would like to do a little more to ensure safe shutdown + #this destructor is completely untested and it's unclear whether + #it's even doing anything useful. consider removing it if self.events_in_queue(): self.logger.warn("Terminating with events still in the queue...") if self.requests_in_queue(): From a0f83c4db0600d99890fcf0b81f6d437aec02817 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:40:13 -0400 Subject: [PATCH 40/59] removed big unused method --- .../media-monitor2/media/monitor/pure.py | 48 ------------------- 1 file changed, 48 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index fa14089d7..462daf07a 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -265,54 +265,6 @@ def parse_int(s): try : return str(reduce(op.add, takewhile(lambda x: x.isdigit(), s))) except: return None -def normalized_metadata(md, original_path): - """ - consumes a dictionary of metadata and returns a new dictionary with the - formatted meta data. We also consume original_path because we must set - MDATA_KEY_CREATOR based on in it sometimes - """ - new_md = copy.deepcopy(md) - # replace all slashes with dashes - #for k,v in new_md.iteritems(): new_md[k] = unicode(v).replace('/','-') - # Specific rules that are applied in a per attribute basis - format_rules = { - 'MDATA_KEY_TRACKNUMBER' : parse_int, - 'MDATA_KEY_FILEPATH' : lambda x: os.path.normpath(x), - 'MDATA_KEY_BPM' : lambda x: x[0:8], - 'MDATA_KEY_MIME' : lambda x: x.replace('audio/vorbis','audio/ogg'), - # Whenever 0 is reported we change it to empty - #'MDATA_KEY_BITRATE' : lambda x: '' if str(x) == '0' else x - } - - new_md = remove_whitespace(new_md) # remove whitespace fields - # Format all the fields in format_rules - new_md = apply_rules_dict(new_md, format_rules) - # set filetype to audioclip by default - new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_FTYPE'], - default=u'audioclip') - - # Try to parse bpm but delete the whole key if that fails - if 'MDATA_KEY_BPM' in new_md: - new_md['MDATA_KEY_BPM'] = parse_int(new_md['MDATA_KEY_BPM']) - if new_md['MDATA_KEY_BPM'] is None: - del new_md['MDATA_KEY_BPM'] - - if not is_airtime_recorded(new_md): - # Read title from filename if it does not exist - default_title = no_extension_basename(original_path) - default_title = re.sub(r'__\d+\.',u'.', default_title) - if re.match(".+-%s-.+$" % unicode_unknown, default_title): - default_title = u'' - new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_TITLE'], - default=default_title) - new_md['MDATA_KEY_TITLE'] = re.sub(r'-\d+kbps$', u'', - new_md['MDATA_KEY_TITLE']) - - # TODO : wtf is this for again? - new_md['MDATA_KEY_TITLE'] = re.sub(r'-?%s-?' % unicode_unknown, u'', - new_md['MDATA_KEY_TITLE']) - new_md['MDATA_KEY_ORIGINAL_PATH'] = normpath(original_path) - return new_md def organized_path(old_path, root_path, orig_md): """ From 958b489fca148833214a4e698b434bfba191ebc0 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:43:15 -0400 Subject: [PATCH 41/59] formatted comments and removed unused function --- .../media-monitor2/media/monitor/pure.py | 31 +++---------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 462daf07a..1c747fdf0 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -66,24 +66,6 @@ class IncludeOnly(object): return func(moi, event, *args, **kwargs) return _wrap - - -def diff_dict(d1, d2, width=30): - """ - returns a formatted diff of 2 dictionaries - """ - out = "" - all_keys = d1.keys() + d2.keys() - for k in all_keys: - v1, v2 = d1.get(k), d2.get(k) - - # default values - if v1 is None: v1 = "N/A" - if v2 is None: v2 = "N/A" - - if d1[k] != d2[k]: - out += "%s%s%s" % (k, d1[k], d2[k]) - def partition(f, alist): """ Partition is very similar to filter except that it also returns the @@ -452,9 +434,8 @@ def owner_id(original_path): return owner_id def file_playable(pathname): - """ - Returns True if 'pathname' is playable by liquidsoap. False otherwise. - """ + """ Returns True if 'pathname' is playable by liquidsoap. False + otherwise. """ # when there is an single apostrophe inside of a string quoted by # apostrophes, we can only escape it by replace that apostrophe with # '\''. This breaks the string into two, and inserts an escaped @@ -490,18 +471,14 @@ def toposort(data): assert not data, "A cyclic dependency exists amongst %r" % data def truncate_to_length(item, length): - """ - Truncates 'item' to 'length' - """ + """ Truncates 'item' to 'length' """ if isinstance(item, int): item = str(item) if isinstance(item, basestring): if len(item) > length: return item[0:length] else: return item def format_length(mutagen_length): - """ - Convert mutagen length to airtime length - """ + """ Convert mutagen length to airtime length """ t = float(mutagen_length) h = int(math.floor(t / 3600)) t = t % 3600 From 5ed86e26665b8f090b22c89f7559ce1704f01147 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:44:51 -0400 Subject: [PATCH 42/59] added docstring for wave duration --- python_apps/media-monitor2/media/monitor/pure.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 1c747fdf0..1d8abf8f3 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -91,14 +91,13 @@ def is_file_supported(path): # TODO : In the future we would like a better way to find out whether a show # has been recorded def is_airtime_recorded(md): - """ - Takes a metadata dictionary and returns True if it belongs to a file that - was recorded by Airtime. - """ + """ Takes a metadata dictionary and returns True if it belongs to a + file that was recorded by Airtime. """ if not 'MDATA_KEY_CREATOR' in md: return False return md['MDATA_KEY_CREATOR'] == u'Airtime Show Recorder' def read_wave_duration(path): + """ Read the length of .wav file (mutagen does not handle this) """ with contextlib.closing(wave.open(path,'r')) as f: frames = f.getnframes() rate = f.getframerate() @@ -106,9 +105,7 @@ def read_wave_duration(path): return duration def clean_empty_dirs(path): - """ - walks path and deletes every empty directory it finds - """ + """ walks path and deletes every empty directory it finds """ # TODO : test this function if path.endswith('/'): clean_empty_dirs(path[0:-1]) else: From cccbfa2c3465df9b45d260a0863af6be1723e04b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:47:23 -0400 Subject: [PATCH 43/59] Formatted docstrings --- .../media-monitor2/media/monitor/pure.py | 66 +++++++------------ 1 file changed, 24 insertions(+), 42 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 1d8abf8f3..ad8ce336b 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -150,11 +150,10 @@ def no_extension_basename(path): else: return '.'.join(base.split(".")[0:-1]) def walk_supported(directory, clean_empties=False): - """ - A small generator wrapper around os.walk to only give us files that support - the extensions we are considering. When clean_empties is True we - recursively delete empty directories left over in directory after the walk. - """ + """ A small generator wrapper around os.walk to only give us files + that support the extensions we are considering. When clean_empties + is True we recursively delete empty directories left over in + directory after the walk. """ for root, dirs, files in os.walk(directory): full_paths = ( os.path.join(root, name) for name in files if is_file_supported(name) ) @@ -168,10 +167,8 @@ def file_locked(path): return bool(f.readlines()) def magic_move(old, new, after_dir_make=lambda : None): - """ - Moves path old to new and constructs the necessary to directories for new - along the way - """ + """ Moves path old to new and constructs the necessary to + directories for new along the way """ new_dir = os.path.dirname(new) if not os.path.exists(new_dir): os.makedirs(new_dir) # We need this crusty hack because anytime a directory is created we must @@ -181,18 +178,15 @@ def magic_move(old, new, after_dir_make=lambda : None): shutil.move(old,new) def move_to_dir(dir_path,file_path): - """ - moves a file at file_path into dir_path/basename(filename) - """ + """ moves a file at file_path into dir_path/basename(filename) """ bs = os.path.basename(file_path) magic_move(file_path, os.path.join(dir_path, bs)) def apply_rules_dict(d, rules): - """ - Consumes a dictionary of rules that maps some keys to lambdas which it - applies to every matching element in d and returns a new dictionary with - the rules applied. If a rule returns none then it's not applied - """ + """ Consumes a dictionary of rules that maps some keys to lambdas + which it applies to every matching element in d and returns a new + dictionary with the rules applied. If a rule returns none then it's + not applied """ new_d = copy.deepcopy(d) for k, rule in rules.iteritems(): if k in d: @@ -207,17 +201,14 @@ def default_to_f(dictionary, keys, default, condition): return new_d def default_to(dictionary, keys, default): - """ - Checks if the list of keys 'keys' exists in 'dictionary'. If not then it - returns a new dictionary with all those missing keys defaults to 'default' - """ + """ Checks if the list of keys 'keys' exists in 'dictionary'. If + not then it returns a new dictionary with all those missing keys + defaults to 'default' """ cnd = lambda dictionary, key: key not in dictionary return default_to_f(dictionary, keys, default, cnd) def remove_whitespace(dictionary): - """ - Remove values that empty whitespace in the dictionary - """ + """ Remove values that empty whitespace in the dictionary """ nd = copy.deepcopy(dictionary) bad_keys = [] for k,v in nd.iteritems(): @@ -303,10 +294,9 @@ def organized_path(old_path, root_path, orig_md): # TODO : Get rid of this function and every one of its uses. We no longer use # the md5 signature of a song for anything def file_md5(path,max_length=100): - """ - Get md5 of file path (if it exists). Use only max_length characters to save - time and memory. Pass max_length=-1 to read the whole file (like in mm1) - """ + """ Get md5 of file path (if it exists). Use only max_length + characters to save time and memory. Pass max_length=-1 to read the + whole file (like in mm1) """ if os.path.exists(path): with open(path, 'rb') as f: m = hashlib.md5() @@ -322,16 +312,12 @@ def encode_to(obj, encoding='utf-8'): return obj def convert_dict_value_to_utf8(md): - """ - formats a dictionary to send as a request to api client - """ + """ formats a dictionary to send as a request to api client """ return dict([(item[0], encode_to(item[1], "utf-8")) for item in md.items()]) def get_system_locale(locale_path='/etc/default/locale'): - """ - Returns the configuration object for the system's default locale. Normally - requires root access. - """ + """ Returns the configuration object for the system's default + locale. Normally requires root access. """ if os.path.exists(locale_path): try: config = ConfigObj(locale_path) @@ -341,9 +327,7 @@ def get_system_locale(locale_path='/etc/default/locale'): permissions issue?" % locale_path) def configure_locale(config): - """ - sets the locale according to the system's locale. - """ + """ sets the locale according to the system's locale. """ current_locale = locale.getlocale() if current_locale[1] is None: default_locale = locale.getdefaultlocale() @@ -360,10 +344,8 @@ def configure_locale(config): def fondle(path,times=None): # TODO : write unit tests for this - """ - touch a file to change the last modified date. Beware of calling this - function on the same file from multiple threads. - """ + """ touch a file to change the last modified date. Beware of calling + this function on the same file from multiple threads. """ with file(path, 'a'): os.utime(path, times) def last_modified(path): From 41b1b357eb8e9fc130a282df2d381d8c9a9fe3b0 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:48:01 -0400 Subject: [PATCH 44/59] formatted more docstrings --- .../media-monitor2/media/monitor/pure.py | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index ad8ce336b..45f4cc66e 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -349,20 +349,16 @@ def fondle(path,times=None): with file(path, 'a'): os.utime(path, times) def last_modified(path): - """ - return the time of the last time mm2 was ran. path refers to the index file - whose date modified attribute contains this information. In the case when - the file does not exist we set this time 0 so that any files on the - filesystem were modified after it - """ + """ return the time of the last time mm2 was ran. path refers to the + index file whose date modified attribute contains this information. + In the case when the file does not exist we set this time 0 so that + any files on the filesystem were modified after it """ if os.path.exists(path): return os.path.getmtime(path) else: return 0 def expand_storage(store): - """ - A storage directory usually consists of 4 different subdirectories. This - function returns their paths - """ + """ A storage directory usually consists of 4 different + subdirectories. This function returns their paths """ store = os.path.normpath(store) return { 'organize' : os.path.join(store, 'organize'), @@ -372,10 +368,8 @@ def expand_storage(store): } def create_dir(path): - """ - will try and make sure that path exists at all costs. raises an exception - if it fails at this task. - """ + """ will try and make sure that path exists at all costs. raises an + exception if it fails at this task. """ if not os.path.exists(path): try : os.makedirs(path) except Exception as e : raise FailedToCreateDir(path, e) @@ -393,11 +387,10 @@ def sub_path(directory,f): return common == normalized def owner_id(original_path): - """ - Given 'original_path' return the file name of the of 'identifier' file. - return the id that is contained in it. If no file is found or nothing is - read then -1 is returned. File is deleted after the number has been read - """ + """ Given 'original_path' return the file name of the of + 'identifier' file. return the id that is contained in it. If no file + is found or nothing is read then -1 is returned. File is deleted + after the number has been read """ fname = "%s.identifier" % original_path owner_id = -1 try: From b2650b9a21dc80639915903e3a549e29d746bc0d Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:49:01 -0400 Subject: [PATCH 45/59] Added comment --- python_apps/media-monitor2/media/monitor/pure.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 45f4cc66e..a879f449f 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -220,6 +220,7 @@ def remove_whitespace(dictionary): return nd def parse_int(s): + # TODO : this function isn't used anywhere yet but it may useful for emf """ Tries very hard to get some sort of integer result from s. Defaults to 0 when it fails From 0f1e843017662cc909e1d12d76f27f9d81303b69 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:49:49 -0400 Subject: [PATCH 46/59] Formatted docstring --- python_apps/media-monitor2/media/monitor/log.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/log.py b/python_apps/media-monitor2/media/monitor/log.py index 5d632bbf4..cf4efb79a 100644 --- a/python_apps/media-monitor2/media/monitor/log.py +++ b/python_apps/media-monitor2/media/monitor/log.py @@ -6,23 +6,19 @@ from media.monitor.pure import LazyProperty appname = 'root' def setup_logging(log_path): - """ - Setup logging by writing log to 'log_path' - """ + """ Setup logging by writing log to 'log_path' """ #logger = logging.getLogger(appname) logging.basicConfig(filename=log_path, level=logging.DEBUG) def get_logger(): - """ - in case we want to use the common logger from a procedural interface - """ + """ in case we want to use the common logger from a procedural + interface """ return logging.getLogger() class Loggable(object): - """ - Any class that wants to log can inherit from this class and automatically - get a logger attribute that can be used like: self.logger.info(...) etc. - """ + """ Any class that wants to log can inherit from this class and + automatically get a logger attribute that can be used like: + self.logger.info(...) etc. """ __metaclass__ = abc.ABCMeta @LazyProperty def logger(self): return get_logger() From 2e54fd64d3ab0312a98840ce7cc8fdb134437ac9 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:50:08 -0400 Subject: [PATCH 47/59] Formatted more docstrings --- python_apps/media-monitor2/media/monitor/log.py | 9 +++------ python_apps/media-monitor2/media/monitor/request.py | 8 +++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/log.py b/python_apps/media-monitor2/media/monitor/log.py index cf4efb79a..7e75c719d 100644 --- a/python_apps/media-monitor2/media/monitor/log.py +++ b/python_apps/media-monitor2/media/monitor/log.py @@ -24,15 +24,12 @@ class Loggable(object): def logger(self): return get_logger() def unexpected_exception(self,e): - """ - Default message for 'unexpected' exceptions - """ + """ Default message for 'unexpected' exceptions """ self.fatal_exception("'Unexpected' exception has occured:", e) def fatal_exception(self, message, e): - """ - Prints an exception 'e' with 'message'. Also outputs the traceback. - """ + """ Prints an exception 'e' with 'message'. Also outputs the + traceback. """ self.logger.error( message ) self.logger.error( str(e) ) self.logger.error( traceback.format_exc() ) diff --git a/python_apps/media-monitor2/media/monitor/request.py b/python_apps/media-monitor2/media/monitor/request.py index 934290a05..a2f3cdc8f 100644 --- a/python_apps/media-monitor2/media/monitor/request.py +++ b/python_apps/media-monitor2/media/monitor/request.py @@ -17,11 +17,9 @@ class ThreadedRequestSync(threading.Thread, Loggable): self.rs.run_request() class RequestSync(Loggable): - """ - This class is responsible for making the api call to send a request - to airtime. In the process it packs the requests and retries for - some number of times - """ + """ This class is responsible for making the api call to send a + request to airtime. In the process it packs the requests and retries + for some number of times """ @classmethod def create_with_api_client(cls, watcher, requests): apiclient = ac.AirtimeApiClient.create_right_config() From 10a6a4d541e47cee48f552e4c04a7586f20cac40 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 15:03:59 -0400 Subject: [PATCH 48/59] formatted docstrings. fixed decorator with functools --- .../media-monitor2/media/monitor/listeners.py | 2 ++ .../media-monitor2/media/monitor/organizer.py | 22 ++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/listeners.py b/python_apps/media-monitor2/media/monitor/listeners.py index 4c860a97e..b33a5c1a9 100644 --- a/python_apps/media-monitor2/media/monitor/listeners.py +++ b/python_apps/media-monitor2/media/monitor/listeners.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import pyinotify from pydispatch import dispatcher +from functools import wraps import media.monitor.pure as mmp from media.monitor.pure import IncludeOnly @@ -31,6 +32,7 @@ class FileMediator(object): def unignore(path): FileMediator.ignored_set.remove(path) def mediate_ignored(fn): + @wraps(fn) def wrapped(self, event, *args,**kwargs): event.pathname = unicode(event.pathname, "utf-8") if FileMediator.is_ignored(event.pathname): diff --git a/python_apps/media-monitor2/media/monitor/organizer.py b/python_apps/media-monitor2/media/monitor/organizer.py index ea6851356..ce1849b90 100644 --- a/python_apps/media-monitor2/media/monitor/organizer.py +++ b/python_apps/media-monitor2/media/monitor/organizer.py @@ -10,14 +10,12 @@ from os.path import dirname import os.path class Organizer(ReportHandler,Loggable): - """ - Organizer is responsible to to listening to OrganizeListener events - and committing the appropriate changes to the filesystem. It does - not in any interact with WatchSyncer's even when the the WatchSyncer - is a "storage directory". The "storage" directory picks up all of - its events through pyinotify. (These events are fed to it through - StoreWatchListener) - """ + """ Organizer is responsible to to listening to OrganizeListener + events and committing the appropriate changes to the filesystem. + It does not in any interact with WatchSyncer's even when the the + WatchSyncer is a "storage directory". The "storage" directory picks + up all of its events through pyinotify. (These events are fed to it + through StoreWatchListener) """ # Commented out making this class a singleton because it's just a band aid # for the real issue. The real issue being making multiple Organizer @@ -41,11 +39,9 @@ class Organizer(ReportHandler,Loggable): super(Organizer, self).__init__(signal=self.channel, weak=False) def handle(self, sender, event): - """ - Intercept events where a new file has been added to the organize - directory and place it in the correct path (starting with - self.target_path) - """ + """ Intercept events where a new file has been added to the + organize directory and place it in the correct path (starting + with self.target_path) """ # Only handle this event type assert isinstance(event, OrganizeFile), \ "Organizer can only handle OrganizeFile events.Given '%s'" % event From 0135fbe9dd457c0c6762bec91999e70d6dde5859 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 16:20:15 -0400 Subject: [PATCH 49/59] Fixed code formatting --- python_apps/media-monitor2/mm2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 17731cad8..ea1178a2f 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -59,7 +59,8 @@ def main(global_config, api_client_config, log_config, try: with open(config['index_path'], 'w') as f: f.write(" ") except Exception as e: - log.info("Failed to create index file with exception: %s" % str(e)) + log.info("Failed to create index file with exception: %s" \ + % str(e)) else: log.info("Created index file, reloading configuration:") main( global_config, api_client_config, log_config, From b8a5c6ce955ffd7339249dad7c8e7f64a92a1c21 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 26 Oct 2012 15:44:58 -0400 Subject: [PATCH 50/59] cc-4634: Fixed --- python_apps/media-monitor2/media/monitor/metadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 5bbbafaeb..d5dba3b51 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -114,6 +114,7 @@ class Metadata(Loggable): if airtime_k in airtime2mutagen: # The unicode cast here is mostly for integers that need to be # strings + if airtime_v is None: continue try: song_file[ airtime2mutagen[airtime_k] ] = unicode(airtime_v) except (EasyMP4KeyError, EasyID3KeyError) as e: From 7c48683492772523cac454aad89f45d148a1bf16 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 26 Oct 2012 16:02:32 -0400 Subject: [PATCH 51/59] cc-4635: Fix --- python_apps/media-monitor2/media/metadata/definitions.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index ee4175033..c53d091f3 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -122,8 +122,16 @@ def load_definitions(): else: default_title = no_extension_basename(k['path']) default_title = re.sub(r'__\d+\.',u'.', default_title) + + # format is: track_number-title-123kbps.mp3 + m = re.match(".+-(?P.+)-\d+kbps", default_title) + if m: + if m.group('title') == unicode_unknown: new_title = '' + else: new_title = m.group('title') + if re.match(".+-%s-.+$" % unicode_unknown, default_title): default_title = u'' + new_title = default_title new_title = re.sub(r'-\d+kbps$', u'', new_title) return new_title From 1ee228558ac3dffcf95fb889af65800f776ebe25 Mon Sep 17 00:00:00 2001 From: James <james@sourcefabric-DX4840.(none)> Date: Fri, 26 Oct 2012 16:24:17 -0400 Subject: [PATCH 52/59] CC-4574: Audio preview fails with certain characters in metadata - fixed --- airtime_mvc/public/js/airtime/common/common.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtime_mvc/public/js/airtime/common/common.js b/airtime_mvc/public/js/airtime/common/common.js index 3125d8d55..19de0d0a9 100644 --- a/airtime_mvc/public/js/airtime/common/common.js +++ b/airtime_mvc/public/js/airtime/common/common.js @@ -54,7 +54,7 @@ function open_audio_preview(type, id, audioFileTitle, audioFileArtist) { audioFileTitle = audioFileTitle.substring(0,index); } - openPreviewWindow('audiopreview/audio-preview/audioFileID/'+id+'/audioFileArtist/'+encodeURIComponent(audioFileArtist)+'/audioFileTitle/'+encodeURIComponent(audioFileTitle)+'/type/'+type); + openPreviewWindow('audiopreview/audio-preview/audioFileID/'+id+'/audioFileArtist/'+encodeURIComponent(encodeURIComponent(audioFileArtist))+'/audioFileTitle/'+encodeURIComponent(encodeURIComponent(audioFileTitle))+'/type/'+type); _preview_window.focus(); } From e667a721adfb8350e092d9dd05b66df9bb72b160 Mon Sep 17 00:00:00 2001 From: James <james@sourcefabric-DX4840.(none)> Date: Fri, 26 Oct 2012 16:26:48 -0400 Subject: [PATCH 53/59] - comment for some code --- airtime_mvc/public/js/airtime/common/common.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airtime_mvc/public/js/airtime/common/common.js b/airtime_mvc/public/js/airtime/common/common.js index 19de0d0a9..0d9883b66 100644 --- a/airtime_mvc/public/js/airtime/common/common.js +++ b/airtime_mvc/public/js/airtime/common/common.js @@ -54,6 +54,9 @@ function open_audio_preview(type, id, audioFileTitle, audioFileArtist) { audioFileTitle = audioFileTitle.substring(0,index); } + // The reason that we need to encode artist and title string is that + // sometime they contain '/' or '\' and apache reject %2f or %5f + // so the work around is to encode it twice. openPreviewWindow('audiopreview/audio-preview/audioFileID/'+id+'/audioFileArtist/'+encodeURIComponent(encodeURIComponent(audioFileArtist))+'/audioFileTitle/'+encodeURIComponent(encodeURIComponent(audioFileTitle))+'/type/'+type); _preview_window.focus(); From 7cd8541696372bb15e83d64f1c46686d72b7fa80 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Fri, 26 Oct 2012 16:41:29 -0400 Subject: [PATCH 54/59] cc-4635: Fix --- python_apps/media-monitor2/media/metadata/definitions.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index c53d091f3..0e7282b20 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -124,16 +124,12 @@ def load_definitions(): default_title = re.sub(r'__\d+\.',u'.', default_title) # format is: track_number-title-123kbps.mp3 - m = re.match(".+-(?P<title>.+)-\d+kbps", default_title) + m = re.match(".+-(?P<title>.+)-\d+kbps$", default_title) if m: if m.group('title') == unicode_unknown: new_title = '' else: new_title = m.group('title') + else: new_title = re.sub(r'-\d+kbps$', u'', new_title) - if re.match(".+-%s-.+$" % unicode_unknown, default_title): - default_title = u'' - - new_title = default_title - new_title = re.sub(r'-\d+kbps$', u'', new_title) return new_title with md.metadata('MDATA_KEY_TITLE') as t: From e10f81aec85b77a327aa7938fea1c90340103fb8 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Fri, 26 Oct 2012 16:48:03 -0400 Subject: [PATCH 55/59] cc-4635: Fix --- python_apps/media-monitor2/media/metadata/definitions.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index 0e7282b20..a6d8221df 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -115,7 +115,7 @@ def load_definitions(): # 2. title is absent (read from file) # 3. recorded file def tr_title(k): - unicode_unknown = u"unknown" + #unicode_unknown = u"unknown" new_title = u"" if is_airtime_recorded(k) or k['title'] != u"": new_title = k['title'] @@ -125,9 +125,7 @@ def load_definitions(): # format is: track_number-title-123kbps.mp3 m = re.match(".+-(?P<title>.+)-\d+kbps$", default_title) - if m: - if m.group('title') == unicode_unknown: new_title = '' - else: new_title = m.group('title') + if m: new_title = m.group('title') else: new_title = re.sub(r'-\d+kbps$', u'', new_title) return new_title From 72928bb5e65f1a7ab68212a2f71b2cd15b05ca92 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Fri, 26 Oct 2012 16:59:32 -0400 Subject: [PATCH 56/59] cc-4635: Fix --- python_apps/media-monitor2/media/metadata/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index a6d8221df..31d28b563 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -126,7 +126,7 @@ def load_definitions(): # format is: track_number-title-123kbps.mp3 m = re.match(".+-(?P<title>.+)-\d+kbps$", default_title) if m: new_title = m.group('title') - else: new_title = re.sub(r'-\d+kbps$', u'', new_title) + else: new_title = re.sub(r'-\d+kbps$', u'', default_title) return new_title From c4f0491ff643ca8b17038118c41be32123e15255 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Fri, 26 Oct 2012 17:04:42 -0400 Subject: [PATCH 57/59] cc-4635: Fix --- python_apps/media-monitor2/media/metadata/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index 31d28b563..9919eb041 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -124,7 +124,7 @@ def load_definitions(): default_title = re.sub(r'__\d+\.',u'.', default_title) # format is: track_number-title-123kbps.mp3 - m = re.match(".+-(?P<title>.+)-\d+kbps$", default_title) + m = re.match(".+-(?P<title>.+)-(\d+kbps|unknown)$", default_title) if m: new_title = m.group('title') else: new_title = re.sub(r'-\d+kbps$', u'', default_title) From 0a396378e52397c6c9d5ecf8a0c744760d808f27 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Fri, 26 Oct 2012 17:15:28 -0400 Subject: [PATCH 58/59] cc-4635: Fix --- python_apps/media-monitor2/media/metadata/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index 9919eb041..ec3bed48f 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -124,7 +124,7 @@ def load_definitions(): default_title = re.sub(r'__\d+\.',u'.', default_title) # format is: track_number-title-123kbps.mp3 - m = re.match(".+-(?P<title>.+)-(\d+kbps|unknown)$", default_title) + m = re.match(".+?-(?P<title>.+)-(\d+kbps|unknown)$", default_title) if m: new_title = m.group('title') else: new_title = re.sub(r'-\d+kbps$', u'', default_title) From f14320a769c9884bbba1718a769572c1e04573f0 Mon Sep 17 00:00:00 2001 From: James <james@sourcefabric-DX4840.(none)> Date: Fri, 26 Oct 2012 17:15:39 -0400 Subject: [PATCH 59/59] CC-4603: Dynamic Smart Blocks: adding to multiple shows at once generates the same content - fixed --- airtime_mvc/application/models/Scheduler.php | 43 +++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/airtime_mvc/application/models/Scheduler.php b/airtime_mvc/application/models/Scheduler.php index 664c40504..def7fb045 100644 --- a/airtime_mvc/application/models/Scheduler.php +++ b/airtime_mvc/application/models/Scheduler.php @@ -366,12 +366,11 @@ class Application_Model_Scheduler * @param array $fileIds * @param array $playlistIds */ - private function insertAfter($scheduleItems, $schedFiles, $adjustSched = true) + private function insertAfter($scheduleItems, $schedFiles, $adjustSched = true, $mediaItems = null) { try { - $affectedShowInstances = array(); - + //dont want to recalculate times for moved items. $excludeIds = array(); foreach ($schedFiles as $file) { @@ -384,7 +383,17 @@ class Application_Model_Scheduler foreach ($scheduleItems as $schedule) { $id = intval($schedule["id"]); - + + // if mediaItmes is passed in, we want to create contents + // at the time of insert. This is for dyanmic blocks or + // playlist that contains dynamic blocks + if ($mediaItems != null) { + $schedFiles = array(); + foreach ($mediaItems as $media) { + $schedFiles = array_merge($schedFiles, $this->retrieveMediaFiles($media["id"], $media["type"])); + } + } + if ($id !== 0) { $schedItem = CcScheduleQuery::create()->findPK($id, $this->con); $instance = $schedItem->getCcShowInstances($this->con); @@ -527,10 +536,32 @@ class Application_Model_Scheduler $this->validateRequest($scheduleItems); + $requireDynamicContentCreation = false; + foreach ($mediaItems as $media) { - $schedFiles = array_merge($schedFiles, $this->retrieveMediaFiles($media["id"], $media["type"])); + if ($media['type'] == "playlist") { + $pl = new Application_Model_Playlist($media['id']); + if ($pl->hasDynamicBlock()) { + $requireDynamicContentCreation = true; + break; + } + } else if ($media['type'] == "block") { + $bl = new Application_Model_Block($media['id']); + if (!$bl->isStatic()) { + $requireDynamicContentCreation = true; + break; + } + } + } + + if ($requireDynamicContentCreation) { + $this->insertAfter($scheduleItems, $schedFiles, $adjustSched, $mediaItems); + } else { + foreach ($mediaItems as $media) { + $schedFiles = array_merge($schedFiles, $this->retrieveMediaFiles($media["id"], $media["type"])); + } + $this->insertAfter($scheduleItems, $schedFiles, $adjustSched); } - $this->insertAfter($scheduleItems, $schedFiles, $adjustSched); $this->con->commit();