diff --git a/airtime_mvc/application/models/RabbitMq.php b/airtime_mvc/application/models/RabbitMq.php index c91342373..cba216f8e 100644 --- a/airtime_mvc/application/models/RabbitMq.php +++ b/airtime_mvc/application/models/RabbitMq.php @@ -25,7 +25,8 @@ class Application_Model_RabbitMq $CC_CONFIG["rabbitmq"]["password"], $CC_CONFIG["rabbitmq"]["vhost"]); $channel = $conn->channel(); - $channel->access_request($CC_CONFIG["rabbitmq"]["vhost"], false, false, true, true); + $channel->access_request($CC_CONFIG["rabbitmq"]["vhost"], false, false, + true, true); $EXCHANGE = 'airtime-pypo'; $channel->exchange_declare($EXCHANGE, 'direct', false, true); @@ -50,7 +51,8 @@ class Application_Model_RabbitMq $CC_CONFIG["rabbitmq"]["password"], $CC_CONFIG["rabbitmq"]["vhost"]); $channel = $conn->channel(); - $channel->access_request($CC_CONFIG["rabbitmq"]["vhost"], false, false, true, true); + $channel->access_request($CC_CONFIG["rabbitmq"]["vhost"], false, false, + true, true); $EXCHANGE = 'airtime-media-monitor'; $channel->exchange_declare($EXCHANGE, 'direct', false, true); @@ -73,7 +75,8 @@ class Application_Model_RabbitMq $CC_CONFIG["rabbitmq"]["password"], $CC_CONFIG["rabbitmq"]["vhost"]); $channel = $conn->channel(); - $channel->access_request($CC_CONFIG["rabbitmq"]["vhost"], false, false, true, true); + $channel->access_request($CC_CONFIG["rabbitmq"]["vhost"], false, false, + true, true); $EXCHANGE = 'airtime-pypo'; $channel->exchange_declare($EXCHANGE, 'direct', false, true); @@ -84,7 +87,8 @@ class Application_Model_RabbitMq $temp['event_type'] = $event_type; $temp['server_timezone'] = Application_Model_Preference::GetTimezone(); if ($event_type == "update_recorder_schedule") { - $temp['shows'] = Application_Model_Show::getShows($now, $end_timestamp, $excludeInstance=NULL, $onlyRecord=TRUE); + $temp['shows'] = Application_Model_Show::getShows($now, + $end_timestamp, $excludeInstance=NULL, $onlyRecord=TRUE); } $data = json_encode($temp); $msg = new AMQPMessage($data, array('content_type' => 'text/plain')); diff --git a/python_apps/media-monitor2/media/metadata/__init__.py b/python_apps/media-monitor2/media/metadata/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/python_apps/media-monitor2/media/metadata/__init__.py @@ -0,0 +1 @@ + diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py new file mode 100644 index 000000000..f77ea51b6 --- /dev/null +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +import media.monitor.process as md +from os.path import normpath +from media.monitor.pure import format_length, file_md5 + +with md.metadata('MDATA_KEY_DURATION') as t: + t.default(u'0.0') + t.depends('length') + t.translate(lambda k: format_length(k['length'])) + +with md.metadata('MDATA_KEY_MIME') as t: + t.default(u'') + t.depends('mime') + t.translate(lambda k: k['mime'].replace('-','/')) + +with md.metadata('MDATA_KEY_BITRATE') as t: + t.default(u'') + t.depends('bitrate') + t.translate(lambda k: k['bitrate']) + +with md.metadata('MDATA_KEY_SAMPLERATE') as t: + t.default(u'0') + t.depends('sample_rate') + t.translate(lambda k: k['sample_rate']) + +with md.metadata('MDATA_KEY_FTYPE'): + t.depends('ftype') # i don't think this field even exists + t.default(u'audioclip') + t.translate(lambda k: k['ftype']) # but just in case + +with md.metadata("MDATA_KEY_CREATOR") as t: + t.depends("artist") + # A little kludge to make sure that we have some value for when we parse + # MDATA_KEY_TITLE + t.default(u"") + t.max_length(512) + +with md.metadata("MDATA_KEY_SOURCE") as t: + t.depends("album") + t.max_length(512) + +with md.metadata("MDATA_KEY_GENRE") as t: + t.depends("genre") + t.max_length(64) + +with md.metadata("MDATA_KEY_MOOD") as t: + t.depends("mood") + t.max_length(64) + +with md.metadata("MDATA_KEY_TRACKNUMBER") as t: + t.depends("tracknumber") + +with md.metadata("MDATA_KEY_BPM") as t: + t.depends("bpm") + t.max_length(8) + +with md.metadata("MDATA_KEY_LABEL") as t: + t.depends("organization") + t.max_length(512) + +with md.metadata("MDATA_KEY_COMPOSER") as t: + t.depends("composer") + t.max_length(512) + +with md.metadata("MDATA_KEY_ENCODER") as t: + t.depends("encodedby") + t.max_length(512) + +with md.metadata("MDATA_KEY_CONDUCTOR") as t: + t.depends("conductor") + t.max_length(512) + +with md.metadata("MDATA_KEY_YEAR") as t: + t.depends("date") + t.max_length(16) + +with md.metadata("MDATA_KEY_URL") as t: + t.depends("website") + +with md.metadata("MDATA_KEY_ISRC") as t: + t.depends("isrc") + t.max_length(512) + +with md.metadata("MDATA_KEY_COPYRIGHT") as t: + t.depends("copyright") + t.max_length(512) + +with md.metadata("MDATA_KEY_FILEPATH") as t: + t.depends('path') + t.translate(lambda k: normpath(k['path'])) + +with md.metadata("MDATA_KEY_MD5") as t: + t.depends('path') + t.optional(False) + t.translate(lambda k: file_md5(k['path'], max_length=100)) + +# MDATA_KEY_TITLE is the annoying special case +with md.metadata('MDATA_KEY_TITLE') as t: + # Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is + # defaulted to "" from definitions above + t.depends('title','MDATA_KEY_CREATOR') + t.max_length(512) + diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py new file mode 100644 index 000000000..a4c361468 --- /dev/null +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +from contextlib import contextmanager +from media.monitor.pure import truncate_to_length, toposort +import mutagen + + +class MetadataAbsent(Exception): + def __init__(self, name): self.name = name + def __str__(self): return "Could not obtain element '%s'" % self.name + +class MetadataElement(object): + def __init__(self,name): + self.name = name + # "Sane" defaults + self.__deps = set() + self.__normalizer = lambda x: x + self.__optional = True + self.__default = None + self.__is_normalized = lambda _ : True + self.__max_length = -1 + + def max_length(self,l): + self.__max_length = l + + def optional(self, setting): + self.__optional = setting + + def is_optional(self): + return self.__optional + + def depends(self, *deps): + self.__deps = set(deps) + + def dependencies(self): + return self.__deps + + def translate(self, f): + self.__translator = f + + def is_normalized(self, f): + self.__is_normalized = f + + def normalize(self, f): + self.__normalizer = f + + def default(self,v): + self.__default = v + + def get_default(self): + if hasattr(self.__default, '__call__'): return self.__default() + else: return self.__default + + def has_default(self): + return self.__default is not None + + def path(self): + return self.__path + + def __slice_deps(self, d): + return dict( (k,v) for k,v in d.iteritems() if k in self.__deps) + + def __str__(self): + return "%s(%s)" % (self.name, ' '.join(list(self.__deps))) + + def read_value(self, path, original, running={}): + # If value is present and normalized then we don't touch it + if self.name in original: + v = original[self.name] + if self.__is_normalized(v): return v + else: return self.__normalizer(v) + + # A dictionary slice with all the dependencies and their values + dep_slice_orig = self.__slice_deps(original) + dep_slice_running = self.__slice_deps(running) + full_deps = dict( dep_slice_orig.items() + + dep_slice_running.items() ) + + # check if any dependencies are absent + if len(full_deps) != len(self.__deps) or len(self.__deps) == 0: + # If we have a default value then use that. Otherwise throw an + # exception + if self.has_default(): return self.get_default() + else: raise MetadataAbsent(self.name) + # We have all dependencies. Now for actual for parsing + r = self.__normalizer( self.__translator(full_deps) ) + if self.__max_length != -1: + r = truncate_to_length(r, self.__max_length) + return r + +def normalize_mutagen(path): + """ + Consumes a path and reads the metadata using mutagen. normalizes some of + the metadata that isn't read through the mutagen hash + """ + m = mutagen.File(path, easy=True) + md = {} + for k,v in m.iteritems(): + if type(v) is list: md[k] = v[0] + else: md[k] = v + # populate special metadata values + md['length'] = getattr(m.info, u'length', 0.0) + md['bitrate'] = getattr(m.info, 'bitrate', u'') + md['sample_rate'] = getattr(m.info, 'sample_rate', 0) + md['mime'] = m.mime[0] if len(m.mime) > 0 else u'' + md['path'] = path + return md + +class MetadataReader(object): + def __init__(self): + self.clear() + + def register_metadata(self,m): + self.__mdata_name_map[m.name] = m + d = dict( (name,m.dependencies()) for name,m in + self.__mdata_name_map.iteritems() ) + new_list = list( toposort(d) ) + self.__metadata = [ self.__mdata_name_map[name] for name in new_list + if name in self.__mdata_name_map] + + def clear(self): + self.__mdata_name_map = {} + self.__metadata = [] + + def read(self, path, muta_hash): + normalized_metadata = {} + for mdata in self.__metadata: + try: + normalized_metadata[mdata.name] = mdata.read_value( + path, muta_hash, normalized_metadata) + except MetadataAbsent: + if not mdata.is_optional(): raise + return normalized_metadata + +global_reader = MetadataReader() + +@contextmanager +def metadata(name): + t = MetadataElement(name) + yield t + global_reader.register_metadata(t) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index e7e2a57a5..fedd61e62 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -2,6 +2,7 @@ import copy import subprocess import os +import math import shutil import re import sys @@ -11,6 +12,9 @@ import operator as op from os.path import normpath from itertools import takewhile +# you need to import reduce in python 3 +try: from functools import reduce +except: pass from configobj import ConfigObj from media.monitor.exceptions import FailedToSetLocale, FailedToCreateDir @@ -84,6 +88,10 @@ def is_file_supported(path): # TODO : In the future we would like a better way to find out whether a show # has been recorded def is_airtime_recorded(md): + """ + Takes a metadata dictionary and returns True if it belongs to a file that + was recorded by Airtime. + """ if not 'MDATA_KEY_CREATOR' in md: return False return md['MDATA_KEY_CREATOR'] == u'Airtime Show Recorder' @@ -253,11 +261,13 @@ def normalized_metadata(md, original_path): if new_md['MDATA_KEY_BPM'] is None: del new_md['MDATA_KEY_BPM'] - if is_airtime_recorded(new_md): - hour,minute,second,name = new_md['MDATA_KEY_TITLE'].split("-",3) - new_md['MDATA_KEY_TITLE'] = u'%s-%s-%s:%s:%s' % \ - (name, new_md['MDATA_KEY_YEAR'], hour, minute, second) + #hour,minute,second,name = new_md['MDATA_KEY_TITLE'].split("-",3) + #new_md['MDATA_KEY_TITLE'] = u'%s-%s-%s:%s:%s' % \ + #(name, new_md['MDATA_KEY_YEAR'], hour, minute, second) + # We changed show recorder to output correct metadata for recorded + # shows + pass else: # Read title from filename if it does not exist default_title = no_extension_basename(original_path) @@ -265,9 +275,14 @@ def normalized_metadata(md, original_path): default_title = u'' new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_TITLE'], default=default_title) + new_md['MDATA_KEY_TITLE'] = re.sub(r'-\d+kbps$', u'', + new_md['MDATA_KEY_TITLE']) + # TODO : wtf is this for again? new_md['MDATA_KEY_TITLE'] = re.sub(r'-?%s-?' % unicode_unknown, u'', new_md['MDATA_KEY_TITLE']) + # ugly mother fucking band aid until enterprise metadata framework is + # working return new_md def organized_path(old_path, root_path, orig_md): @@ -280,8 +295,6 @@ def organized_path(old_path, root_path, orig_md): """ filepath = None ext = extension(old_path) - # The blocks for each if statement look awfully similar. Perhaps there is a - # way to simplify this code def default_f(dictionary, key): if key in dictionary: return len(dictionary[key]) == 0 else: return True @@ -291,6 +304,8 @@ def organized_path(old_path, root_path, orig_md): # MDATA_KEY_BITRATE is in bytes/second i.e. (256000) we want to turn this # into 254kbps + # Some metadata elements cannot be empty, hence we default them to some + # value just so that we can create a correct path normal_md = default_to_f(orig_md, path_md, unicode_unknown, default_f) try: formatted = str(int(normal_md['MDATA_KEY_BITRATE']) / 1000) @@ -299,13 +314,15 @@ def organized_path(old_path, root_path, orig_md): normal_md['MDATA_KEY_BITRATE'] = unicode_unknown if is_airtime_recorded(normal_md): - title_re = re.match("(?P.+)-(?P\d+-\d+-\d+-\d+:\d+:\d+)$", - normal_md['MDATA_KEY_TITLE']) + # normal_md['MDATA_KEY_TITLE'] = 'show_name-yyyy-mm-dd-hh:mm:ss' + r = "(?P.+)-(?P\d+-\d+-\d+)-(?P