From cae224593667e473fb0add0a15b79cfe7016fa59 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 4 Sep 2012 16:52:22 -0400 Subject: [PATCH 01/15] MM2: Cleaned up pure.py a little --- airtime_mvc/application/models/RabbitMq.php | 12 +++-- .../media-monitor2/media/monitor/pure.py | 46 ++++++++++++++++++- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/airtime_mvc/application/models/RabbitMq.php b/airtime_mvc/application/models/RabbitMq.php index c91342373..cba216f8e 100644 --- a/airtime_mvc/application/models/RabbitMq.php +++ b/airtime_mvc/application/models/RabbitMq.php @@ -25,7 +25,8 @@ class Application_Model_RabbitMq $CC_CONFIG["rabbitmq"]["password"], $CC_CONFIG["rabbitmq"]["vhost"]); $channel = $conn->channel(); - $channel->access_request($CC_CONFIG["rabbitmq"]["vhost"], false, false, true, true); + $channel->access_request($CC_CONFIG["rabbitmq"]["vhost"], false, false, + true, true); $EXCHANGE = 'airtime-pypo'; $channel->exchange_declare($EXCHANGE, 'direct', false, true); @@ -50,7 +51,8 @@ class Application_Model_RabbitMq $CC_CONFIG["rabbitmq"]["password"], $CC_CONFIG["rabbitmq"]["vhost"]); $channel = $conn->channel(); - $channel->access_request($CC_CONFIG["rabbitmq"]["vhost"], false, false, true, true); + $channel->access_request($CC_CONFIG["rabbitmq"]["vhost"], false, false, + true, true); $EXCHANGE = 'airtime-media-monitor'; $channel->exchange_declare($EXCHANGE, 'direct', false, true); @@ -73,7 +75,8 @@ class Application_Model_RabbitMq $CC_CONFIG["rabbitmq"]["password"], $CC_CONFIG["rabbitmq"]["vhost"]); $channel = $conn->channel(); - $channel->access_request($CC_CONFIG["rabbitmq"]["vhost"], false, false, true, true); + $channel->access_request($CC_CONFIG["rabbitmq"]["vhost"], false, false, + true, true); $EXCHANGE = 'airtime-pypo'; $channel->exchange_declare($EXCHANGE, 'direct', false, true); @@ -84,7 +87,8 @@ class Application_Model_RabbitMq $temp['event_type'] = $event_type; $temp['server_timezone'] = Application_Model_Preference::GetTimezone(); if ($event_type == "update_recorder_schedule") { - $temp['shows'] = Application_Model_Show::getShows($now, $end_timestamp, $excludeInstance=NULL, $onlyRecord=TRUE); + $temp['shows'] = Application_Model_Show::getShows($now, + $end_timestamp, $excludeInstance=NULL, $onlyRecord=TRUE); } $data = json_encode($temp); $msg = new AMQPMessage($data, array('content_type' => 'text/plain')); diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index e7e2a57a5..3c2da02f9 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -2,6 +2,7 @@ import copy import subprocess import os +import math import shutil import re import sys @@ -11,6 +12,9 @@ import operator as op from os.path import normpath from itertools import takewhile +# you need to import reduce in python 3 +try: from functools import reduce +except: pass from configobj import ConfigObj from media.monitor.exceptions import FailedToSetLocale, FailedToCreateDir @@ -280,8 +284,6 @@ def organized_path(old_path, root_path, orig_md): """ filepath = None ext = extension(old_path) - # The blocks for each if statement look awfully similar. Perhaps there is a - # way to simplify this code def default_f(dictionary, key): if key in dictionary: return len(dictionary[key]) == 0 else: return True @@ -465,6 +467,46 @@ def file_playable(pathname): return_code = subprocess.call(command, shell=True) return (return_code == 0) +def toposort(data): + for k, v in data.items(): + v.discard(k) # Ignore self dependencies + extra_items_in_deps = reduce(set.union, data.values()) - set(data.keys()) + data.update({item:set() for item in extra_items_in_deps}) + while True: + ordered = set(item for item,dep in data.items() if not dep) + if not ordered: break + for e in sorted(ordered): yield e + data = dict((item,(dep - ordered)) for item,dep in data.items() + if item not in ordered) + assert not data, "A cyclic dependency exists amongst %r" % data + +def truncate_to_length(item, length): + """ + Truncates 'item' to 'length' + """ + if isinstance(item, int): item = str(item) + if isinstance(item, basestring): + if len(item) > length: return item[0:length] + else: return item + +def format_length(mutagen_length): + """ + Convert mutagen length to airtime length + """ + t = float(mutagen_length) + h = int(math.floor(t / 3600)) + t = t % 3600 + m = int(math.floor(t / 60)) + s = t % 60 + # will be ss.uuu + s = str(s) + seconds = s.split(".") + s = seconds[0] + # have a maximum of 6 subseconds. + if len(seconds[1]) >= 6: ss = seconds[1][0:6] + else: ss = seconds[1][0:] + return "%s:%s:%s.%s" % (h, m, s, ss) + if __name__ == '__main__': import doctest doctest.testmod() From 982dd1e2a7e0a22541ff3e2c3f54466bc875cbb2 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Tue, 4 Sep 2012 17:21:59 -0400 Subject: [PATCH 02/15] CC-4348: Prepared statements - part 4 -StoredFile.php --- airtime_mvc/application/models/StoredFile.php | 79 +++++++++++-------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/airtime_mvc/application/models/StoredFile.php b/airtime_mvc/application/models/StoredFile.php index 319336941..2e41e4e65 100644 --- a/airtime_mvc/application/models/StoredFile.php +++ b/airtime_mvc/application/models/StoredFile.php @@ -322,10 +322,21 @@ class Application_Model_StoredFile { global $CC_CONFIG; $con = Propel::getConnection(); + $sql = "SELECT playlist_id " - ." FROM ".$CC_CONFIG['playistTable'] - ." WHERE file_id='{$this->id}'"; - $ids = $con->query($sql)->fetchAll(); + ." FROM cc_playlist" + ." WHERE file_id = :file_id"; + + $stmt = $con->prepare($sql); + $stmt->bindParam(':file_id', $this->id, PDO::PARAM_INT); + + if ($stmt->execute()) { + $ids = $stmt->fetchAll(); + } else { + $msg = implode(',', $stmt->errorInfo()); + throw new Exception("Error: $msg"); + } + $playlists = array(); if (is_array($ids) && count($ids) > 0) { foreach ($ids as $id) { @@ -951,7 +962,7 @@ class Application_Model_StoredFile $uid = $user->getId(); } $id_file = "$audio_stor.identifier"; - if (file_put_contents($id_file,$uid) === false) { + if (file_put_contents($id_file, $uid) === false) { Logging::info("Could not write file to identify user: '$uid'"); Logging::info("Id file path: '$id_file'"); Logging::info("Defaulting to admin (no identification file was @@ -1003,7 +1014,7 @@ class Application_Model_StoredFile global $CC_CONFIG; $con = Propel::getConnection(); - $sql = "SELECT count(*) as cnt FROM ".$CC_CONFIG["filesTable"]." WHERE file_exists"; + $sql = "SELECT count(*) as cnt FROM cc_files WHERE file_exists"; return $con->query($sql)->fetchColumn(0); } @@ -1012,53 +1023,59 @@ class Application_Model_StoredFile * * Enter description here ... * @param $dir_id - if this is not provided, it returns all files with full path constructed. - * @param $propelObj - if this is true, it returns array of proepl obj */ - public static function listAllFiles($dir_id=null, $all, $propelObj=false) + public static function listAllFiles($dir_id=null, $all) { $con = Propel::getConnection(); - $file_exists = $all ? "" : "and f.file_exists = 'TRUE'"; - - if ($propelObj) { - $sql = "SELECT m.directory || f.filepath as fp" - ." FROM CC_MUSIC_DIRS m" - ." LEFT JOIN CC_FILES f" - ." ON m.id = f.directory WHERE m.id = $dir_id $file_exists"; - } else { - $sql = "SELECT filepath as fp" - ." FROM CC_FILES as f" - ." WHERE f.directory = $dir_id $file_exists"; + $sql = "SELECT filepath as fp" + ." FROM CC_FILES as f" + ." WHERE f.directory = :dir_id"; + + if (!$all) { + $sql .= " AND f.file_exists = 'TRUE'"; } - $rows = $con->query($sql)->fetchAll(); + $stmt = $con->prepare($sql); + $stmt->bindParam(':dir_id', $dir_id); + + if ($stmt->execute()) { + $rows = $stmt->fetchAll(); + } else { + $msg = implode(',', $stmt->errorInfo()); + throw new Exception("Error: $msg"); + } + $results = array(); foreach ($rows as $row) { - if ($propelObj) { - $results[] = Application_Model_StoredFile::RecallByFilepath($row["fp"]); - } else { - $results[] = $row["fp"]; - } + $results[] = $row["fp"]; } return $results; } //TODO: MERGE THIS FUNCTION AND "listAllFiles" -MK - public static function listAllFiles2($dir_id=null, $limit=null) + public static function listAllFiles2($dir_id=null, $limit="ALL") { $con = Propel::getConnection(); $sql = "SELECT id, filepath as fp" ." FROM CC_FILES" - ." WHERE directory = $dir_id" + ." WHERE directory = :dir_id" ." AND file_exists = 'TRUE'" - ." AND replay_gain is NULL"; - if (!is_null($limit) && is_int($limit)) { - $sql .= " LIMIT $limit"; - } + ." AND replay_gain is NULL" + ." LIMIT :lim"; - $rows = $con->query($sql, PDO::FETCH_ASSOC)->fetchAll(); + $stmt = $con->prepare($sql); + $stmt->bindParam(':dir_id', $dir_id); + $stmt->bindParam(':lim', $limit); + + if ($stmt->execute()) { + $rows = $stmt->fetchAll(PDO::FETCH_ASSOC); + } else { + $msg = implode(',', $stmt->errorInfo()); + throw new Exception("Error: $msg"); + } return $rows; } From ac547342f123054861681b3b0eb7c678011c8a96 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Tue, 4 Sep 2012 17:22:21 -0400 Subject: [PATCH 03/15] Logging typo --- airtime_mvc/application/models/Block.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtime_mvc/application/models/Block.php b/airtime_mvc/application/models/Block.php index 678b81a69..8014352fe 100644 --- a/airtime_mvc/application/models/Block.php +++ b/airtime_mvc/application/models/Block.php @@ -446,7 +446,7 @@ EOT; $pos = $pos + 1; } } catch (Exception $e) { - Logging::log($e->getMessage()); + Logging::info($e->getMessage()); } } From b08874ad97c60327d26d3ea8561f1e0e8255f9e2 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 4 Sep 2012 17:24:29 -0400 Subject: [PATCH 04/15] Changed pypo recordeder to output title metadata with media monitor compatible output. To avoid extra parsing. --- python_apps/pypo/recorder.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/python_apps/pypo/recorder.py b/python_apps/pypo/recorder.py index 4acff83e7..b2309dc2e 100644 --- a/python_apps/pypo/recorder.py +++ b/python_apps/pypo/recorder.py @@ -149,9 +149,11 @@ class ShowRecorder(Thread): #set some metadata for our file daemon recorded_file = mutagen.File(filepath, easy = True) - recorded_file['title'] = record_time + "-" + self.show_name + #recorded_file['title'] = record_time + "-" + self.show_name recorded_file['artist'] = artist recorded_file['date'] = md[0] + recorded_file['title'] = "%s-%s-%s" % (self.show_name, + recorded_file['date'], md[1]) #recorded_file['date'] = md[0].split("-")[0] #You cannot pass ints into the metadata of a file. Even tracknumber needs to be a string recorded_file['tracknumber'] = unicode(self.show_instance) @@ -218,7 +220,8 @@ class Recorder(Thread): show_end = getDateTimeObj(show[u'ends']) time_delta = show_end - show_starts - temp_shows_to_record[show[u'starts']] = [time_delta, show[u'instance_id'], show[u'name'], m['server_timezone']] + temp_shows_to_record[show[u'starts']] = [time_delta, + show[u'instance_id'], show[u'name'], m['server_timezone']] self.shows_to_record = temp_shows_to_record def get_time_till_next_show(self): @@ -270,12 +273,12 @@ class Recorder(Thread): self.logger.error('Exception: %s', e) self.logger.error("traceback: %s", top) - """ - Main loop of the thread: - Wait for schedule updates from RabbitMQ, but in case there arent any, - poll the server to get the upcoming schedule. - """ def run(self): + """ + Main loop of the thread: + Wait for schedule updates from RabbitMQ, but in case there arent any, + poll the server to get the upcoming schedule. + """ try: self.logger.info("Started...") # Bootstrap: since we are just starting up, we need to grab the From ed368b82c0fccbe1ffffeb8391a33a0a433a15b7 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 4 Sep 2012 17:25:13 -0400 Subject: [PATCH 05/15] Removed unnecessary songs from media monitor. --- python_apps/media-monitor2/media/monitor/pure.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 3c2da02f9..f66f7b4a0 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -259,9 +259,12 @@ def normalized_metadata(md, original_path): if is_airtime_recorded(new_md): - hour,minute,second,name = new_md['MDATA_KEY_TITLE'].split("-",3) - new_md['MDATA_KEY_TITLE'] = u'%s-%s-%s:%s:%s' % \ - (name, new_md['MDATA_KEY_YEAR'], hour, minute, second) + #hour,minute,second,name = new_md['MDATA_KEY_TITLE'].split("-",3) + #new_md['MDATA_KEY_TITLE'] = u'%s-%s-%s:%s:%s' % \ + #(name, new_md['MDATA_KEY_YEAR'], hour, minute, second) + # We changed show recorder to output correct metadata for recorded + # shows + pass else: # Read title from filename if it does not exist default_title = no_extension_basename(original_path) From 5c9d0b7db550f5a3c7d584dd1142a8db153c100f Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 5 Sep 2012 10:20:10 -0400 Subject: [PATCH 06/15] mm2: changed metadata handling --- python_apps/media-monitor2/media/monitor/pure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index f66f7b4a0..94eb5fffb 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -268,8 +268,8 @@ def normalized_metadata(md, original_path): else: # Read title from filename if it does not exist default_title = no_extension_basename(original_path) - if re.match(".+-%s-.+$" % unicode_unknown, default_title): - default_title = u'' + #if re.match(".+-%s-.+$" % unicode_unknown, default_title): + #default_title = u'' new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_TITLE'], default=default_title) From ab9fbc48aea5a3dc938ce2f650c9dd6c08262911 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 5 Sep 2012 10:24:10 -0400 Subject: [PATCH 07/15] MM2: Added docstrings --- python_apps/media-monitor2/media/monitor/pure.py | 16 +++++++++++++++- python_apps/pypo/recorder.py | 2 ++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 94eb5fffb..00a8b70c6 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -88,6 +88,10 @@ def is_file_supported(path): # TODO : In the future we would like a better way to find out whether a show # has been recorded def is_airtime_recorded(md): + """ + Takes a metadata dictionary and returns True if it belongs to a file that + was recorded by Airtime. + """ if not 'MDATA_KEY_CREATOR' in md: return False return md['MDATA_KEY_CREATOR'] == u'Airtime Show Recorder' @@ -456,7 +460,9 @@ def owner_id(original_path): return owner_id def file_playable(pathname): - + """ + Returns True if 'pathname' is playable by liquidsoap. False otherwise. + """ #when there is an single apostrophe inside of a string quoted by #apostrophes, we can only escape it by replace that apostrophe with '\''. #This breaks the string into two, and inserts an escaped single quote in @@ -471,6 +477,14 @@ def file_playable(pathname): return (return_code == 0) def toposort(data): + """ + Topological sort on 'data' where 'data' is of the form: + data = [ + 'one' : set('two','three'), + 'two' : set('three'), + 'three' : set() + ] + """ for k, v in data.items(): v.discard(k) # Ignore self dependencies extra_items_in_deps = reduce(set.union, data.values()) - set(data.keys()) diff --git a/python_apps/pypo/recorder.py b/python_apps/pypo/recorder.py index b2309dc2e..5b735c290 100644 --- a/python_apps/pypo/recorder.py +++ b/python_apps/pypo/recorder.py @@ -37,6 +37,8 @@ except Exception, e: print ('Error loading config file: %s', e) sys.exit() +# TODO : add docstrings everywhere in this module + def getDateTimeObj(time): # TODO : clean up for this function later. # - use tuples to parse result from split (instead of indices) From 6c52a229b9e3a0d1445c27e7ed483ba305bd8f51 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 5 Sep 2012 10:32:27 -0400 Subject: [PATCH 08/15] mm2: added back metadata handling. Added new "Enterprise Metadata Framework" --- .../media-monitor2/media/metadata/__init__.py | 1 + .../media/metadata/definitions.py | 103 +++++++++++++ .../media-monitor2/media/metadata/process.py | 140 ++++++++++++++++++ .../media-monitor2/media/monitor/pure.py | 4 +- 4 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 python_apps/media-monitor2/media/metadata/__init__.py create mode 100644 python_apps/media-monitor2/media/metadata/definitions.py create mode 100644 python_apps/media-monitor2/media/metadata/process.py diff --git a/python_apps/media-monitor2/media/metadata/__init__.py b/python_apps/media-monitor2/media/metadata/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/python_apps/media-monitor2/media/metadata/__init__.py @@ -0,0 +1 @@ + diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py new file mode 100644 index 000000000..f77ea51b6 --- /dev/null +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +import media.monitor.process as md +from os.path import normpath +from media.monitor.pure import format_length, file_md5 + +with md.metadata('MDATA_KEY_DURATION') as t: + t.default(u'0.0') + t.depends('length') + t.translate(lambda k: format_length(k['length'])) + +with md.metadata('MDATA_KEY_MIME') as t: + t.default(u'') + t.depends('mime') + t.translate(lambda k: k['mime'].replace('-','/')) + +with md.metadata('MDATA_KEY_BITRATE') as t: + t.default(u'') + t.depends('bitrate') + t.translate(lambda k: k['bitrate']) + +with md.metadata('MDATA_KEY_SAMPLERATE') as t: + t.default(u'0') + t.depends('sample_rate') + t.translate(lambda k: k['sample_rate']) + +with md.metadata('MDATA_KEY_FTYPE'): + t.depends('ftype') # i don't think this field even exists + t.default(u'audioclip') + t.translate(lambda k: k['ftype']) # but just in case + +with md.metadata("MDATA_KEY_CREATOR") as t: + t.depends("artist") + # A little kludge to make sure that we have some value for when we parse + # MDATA_KEY_TITLE + t.default(u"") + t.max_length(512) + +with md.metadata("MDATA_KEY_SOURCE") as t: + t.depends("album") + t.max_length(512) + +with md.metadata("MDATA_KEY_GENRE") as t: + t.depends("genre") + t.max_length(64) + +with md.metadata("MDATA_KEY_MOOD") as t: + t.depends("mood") + t.max_length(64) + +with md.metadata("MDATA_KEY_TRACKNUMBER") as t: + t.depends("tracknumber") + +with md.metadata("MDATA_KEY_BPM") as t: + t.depends("bpm") + t.max_length(8) + +with md.metadata("MDATA_KEY_LABEL") as t: + t.depends("organization") + t.max_length(512) + +with md.metadata("MDATA_KEY_COMPOSER") as t: + t.depends("composer") + t.max_length(512) + +with md.metadata("MDATA_KEY_ENCODER") as t: + t.depends("encodedby") + t.max_length(512) + +with md.metadata("MDATA_KEY_CONDUCTOR") as t: + t.depends("conductor") + t.max_length(512) + +with md.metadata("MDATA_KEY_YEAR") as t: + t.depends("date") + t.max_length(16) + +with md.metadata("MDATA_KEY_URL") as t: + t.depends("website") + +with md.metadata("MDATA_KEY_ISRC") as t: + t.depends("isrc") + t.max_length(512) + +with md.metadata("MDATA_KEY_COPYRIGHT") as t: + t.depends("copyright") + t.max_length(512) + +with md.metadata("MDATA_KEY_FILEPATH") as t: + t.depends('path') + t.translate(lambda k: normpath(k['path'])) + +with md.metadata("MDATA_KEY_MD5") as t: + t.depends('path') + t.optional(False) + t.translate(lambda k: file_md5(k['path'], max_length=100)) + +# MDATA_KEY_TITLE is the annoying special case +with md.metadata('MDATA_KEY_TITLE') as t: + # Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is + # defaulted to "" from definitions above + t.depends('title','MDATA_KEY_CREATOR') + t.max_length(512) + diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py new file mode 100644 index 000000000..a4c361468 --- /dev/null +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +from contextlib import contextmanager +from media.monitor.pure import truncate_to_length, toposort +import mutagen + + +class MetadataAbsent(Exception): + def __init__(self, name): self.name = name + def __str__(self): return "Could not obtain element '%s'" % self.name + +class MetadataElement(object): + def __init__(self,name): + self.name = name + # "Sane" defaults + self.__deps = set() + self.__normalizer = lambda x: x + self.__optional = True + self.__default = None + self.__is_normalized = lambda _ : True + self.__max_length = -1 + + def max_length(self,l): + self.__max_length = l + + def optional(self, setting): + self.__optional = setting + + def is_optional(self): + return self.__optional + + def depends(self, *deps): + self.__deps = set(deps) + + def dependencies(self): + return self.__deps + + def translate(self, f): + self.__translator = f + + def is_normalized(self, f): + self.__is_normalized = f + + def normalize(self, f): + self.__normalizer = f + + def default(self,v): + self.__default = v + + def get_default(self): + if hasattr(self.__default, '__call__'): return self.__default() + else: return self.__default + + def has_default(self): + return self.__default is not None + + def path(self): + return self.__path + + def __slice_deps(self, d): + return dict( (k,v) for k,v in d.iteritems() if k in self.__deps) + + def __str__(self): + return "%s(%s)" % (self.name, ' '.join(list(self.__deps))) + + def read_value(self, path, original, running={}): + # If value is present and normalized then we don't touch it + if self.name in original: + v = original[self.name] + if self.__is_normalized(v): return v + else: return self.__normalizer(v) + + # A dictionary slice with all the dependencies and their values + dep_slice_orig = self.__slice_deps(original) + dep_slice_running = self.__slice_deps(running) + full_deps = dict( dep_slice_orig.items() + + dep_slice_running.items() ) + + # check if any dependencies are absent + if len(full_deps) != len(self.__deps) or len(self.__deps) == 0: + # If we have a default value then use that. Otherwise throw an + # exception + if self.has_default(): return self.get_default() + else: raise MetadataAbsent(self.name) + # We have all dependencies. Now for actual for parsing + r = self.__normalizer( self.__translator(full_deps) ) + if self.__max_length != -1: + r = truncate_to_length(r, self.__max_length) + return r + +def normalize_mutagen(path): + """ + Consumes a path and reads the metadata using mutagen. normalizes some of + the metadata that isn't read through the mutagen hash + """ + m = mutagen.File(path, easy=True) + md = {} + for k,v in m.iteritems(): + if type(v) is list: md[k] = v[0] + else: md[k] = v + # populate special metadata values + md['length'] = getattr(m.info, u'length', 0.0) + md['bitrate'] = getattr(m.info, 'bitrate', u'') + md['sample_rate'] = getattr(m.info, 'sample_rate', 0) + md['mime'] = m.mime[0] if len(m.mime) > 0 else u'' + md['path'] = path + return md + +class MetadataReader(object): + def __init__(self): + self.clear() + + def register_metadata(self,m): + self.__mdata_name_map[m.name] = m + d = dict( (name,m.dependencies()) for name,m in + self.__mdata_name_map.iteritems() ) + new_list = list( toposort(d) ) + self.__metadata = [ self.__mdata_name_map[name] for name in new_list + if name in self.__mdata_name_map] + + def clear(self): + self.__mdata_name_map = {} + self.__metadata = [] + + def read(self, path, muta_hash): + normalized_metadata = {} + for mdata in self.__metadata: + try: + normalized_metadata[mdata.name] = mdata.read_value( + path, muta_hash, normalized_metadata) + except MetadataAbsent: + if not mdata.is_optional(): raise + return normalized_metadata + +global_reader = MetadataReader() + +@contextmanager +def metadata(name): + t = MetadataElement(name) + yield t + global_reader.register_metadata(t) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 00a8b70c6..0571db552 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -272,8 +272,8 @@ def normalized_metadata(md, original_path): else: # Read title from filename if it does not exist default_title = no_extension_basename(original_path) - #if re.match(".+-%s-.+$" % unicode_unknown, default_title): - #default_title = u'' + if re.match(".+-%s-.+$" % unicode_unknown, default_title): + default_title = u'' new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_TITLE'], default=default_title) From d12833a71eb14b91290584d1c18c55a111177ea4 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 5 Sep 2012 10:36:32 -0400 Subject: [PATCH 09/15] Pypo debugging --- python_apps/pypo/recorder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python_apps/pypo/recorder.py b/python_apps/pypo/recorder.py index 5b735c290..c8e0f3750 100644 --- a/python_apps/pypo/recorder.py +++ b/python_apps/pypo/recorder.py @@ -159,6 +159,9 @@ class ShowRecorder(Thread): #recorded_file['date'] = md[0].split("-")[0] #You cannot pass ints into the metadata of a file. Even tracknumber needs to be a string recorded_file['tracknumber'] = unicode(self.show_instance) + self.logger.info("self.start_time: %s" % self.start_time) + self.logger.info("title:(%s).date:(%s)" % (recorded_file['title'], + recorded_file['date']) ) recorded_file.save() except Exception, e: From f4848a8306c5aa7260911564df6c3e8af1f0faa7 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 5 Sep 2012 10:59:56 -0400 Subject: [PATCH 10/15] cc-4241: Tweaked metadata handling again --- .../media-monitor2/media/monitor/pure.py | 9 ++++++--- python_apps/pypo/recorder.py | 20 ++++++------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 0571db552..91f8320fd 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -300,6 +300,8 @@ def organized_path(old_path, root_path, orig_md): # MDATA_KEY_BITRATE is in bytes/second i.e. (256000) we want to turn this # into 254kbps + # Some metadata elements cannot be empty, hence we default them to some + # value just so that we can create a correct path normal_md = default_to_f(orig_md, path_md, unicode_unknown, default_f) try: formatted = str(int(normal_md['MDATA_KEY_BITRATE']) / 1000) @@ -308,10 +310,11 @@ def organized_path(old_path, root_path, orig_md): normal_md['MDATA_KEY_BITRATE'] = unicode_unknown if is_airtime_recorded(normal_md): - title_re = re.match("(?P.+)-(?P\d+-\d+-\d+-\d+:\d+:\d+)$", - normal_md['MDATA_KEY_TITLE']) + # normal_md['MDATA_KEY_TITLE'] = 'show_name-yyyy-mm-dd-hh:mm:ss' + r = "(?P.+)-(?P\d+-\d+-\d+)-(?P