From 39e32743e13180c839d829bce65fab1745105580 Mon Sep 17 00:00:00 2001 From: Daniel James Date: Tue, 19 Jun 2012 16:23:09 +0100 Subject: [PATCH 001/108] Fixed typos in changelog --- changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog b/changelog index 087af1103..ebe61ddda 100644 --- a/changelog +++ b/changelog @@ -19,7 +19,7 @@ * Removing a watched directory and adding it again preserves playlists & shows with those files. * An icon in the playlist shows whether a file is missing on disk, warning the user that the playlist will not go according to plan. * Media monitor detects add and removal of watched temporary local storage (USB disks for example) and network drives. - * Broadcast Log - export play count of tracks within a given time range.  Useful for royalty reporting purposes. + * Broadcast Log - export play count of tracks within a given time range. Useful for royalty reporting purposes. * Minor Improvements: * Ability to turn off the broadcast. * Editing metadata in the library will update the metadata on disk. @@ -30,7 +30,7 @@ * Repeating shows default to "No End" * Ability to "View on Soundcloud" for recorded shows in the calendar * "Listen" preview player no longer falls behind the broadcast (you can only mute the stream now, not stop it) - * Tracks that cannot be played will be rejected on upload and put in to the directory "/srv/airtime/store/problem_files" (but currently it will not tell you that it rejected them - sorry\!) + * Tracks that cannot be played will be rejected on upload and put in to the directory "/srv/airtime/stor/problem_files" (but currently it will not tell you that it rejected them - sorry\!) * Library is automatically refreshed when media import is finished * Show "Disk Full" message when trying to upload a file that wont fit on the disk * Reduced CPU utilization for OGG streams From 1e8c8f81572a28974f04a09f7fe1fa9805890a27 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Mon, 9 Jul 2012 11:21:12 -0400 Subject: [PATCH 002/108] add a few try catch blocks --- .../airtimefilemonitor/airtimemetadata.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py b/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py index e7b6ad01d..cea950cba 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py @@ -221,9 +221,20 @@ class AirtimeMetadata: md['MDATA_KEY_COPYRIGHT'] = self.truncate_to_length(md['MDATA_KEY_COPYRIGHT'], 512) #end of db truncation checks. - md['MDATA_KEY_BITRATE'] = getattr(file_info.info, "bitrate", None) - md['MDATA_KEY_SAMPLERATE'] = getattr(file_info.info, "sample_rate", None) - self.logger.info( "Bitrate: %s , Samplerate: %s", md['MDATA_KEY_BITRATE'], md['MDATA_KEY_SAMPLERATE'] ) + try: + md['MDATA_KEY_BITRATE'] = getattr(file_info.info, "bitrate", "0") + except Exception as e: + self.logger.warn("Could not get Bitrate") + md['MDATA_KEY_BITRATE'] = "0" + + try: + md['MDATA_KEY_SAMPLERATE'] = getattr(file_info.info, "sample_rate", "0") + except Exception as e: + self.logger.warn("Could not get Samplerate") + md['MDATA_KEY_SAMPLERATE'] = "0" + + self.logger.info("Bitrate: %s , Samplerate: %s", md['MDATA_KEY_BITRATE'], md['MDATA_KEY_SAMPLERATE']) + try: md['MDATA_KEY_DURATION'] = self.format_length(file_info.info.length) except Exception as e: self.logger.warn("File: '%s' raises: %s", filepath, str(e)) From 6ce52f081a0075fba226ba63cef981aaf5a26e9d Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Mon, 9 Jul 2012 12:20:41 -0400 Subject: [PATCH 003/108] CC-4091: More comprehensive error message for installs on systems with incorrect locale -done --- install_minimal/airtime-install | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/install_minimal/airtime-install b/install_minimal/airtime-install index 86691c427..8c4541685 100755 --- a/install_minimal/airtime-install +++ b/install_minimal/airtime-install @@ -107,11 +107,28 @@ echo "* Making sure /etc/default/locale is set properly" set +e update-locale cat /etc/default/locale | grep -i "LANG=.*UTF-\?8" -set -e if [ "$?" != "0" ]; then - echo "non UTF-8 default locale found in /etc/default/locale." + echo -e " * Fail\n" + echo "A non UTF-8 default locale found in /etc/default/locale. Airtime requires +a UTF-8 locale to run. To fix this please do the following: + +Ubuntu: +Put line 'en_US.UTF-8 UTF-8' (or similar) without quotes to '/var/lib/locales/supported.d/local', +replacing any existing lines. +A list of supported locales is available in '/usr/share/i18n/SUPPORTED' +Then run 'sudo dpkg-reconfigure locales' + +Debian: +Run 'sudo dpkg-reconfigure locales' and use the interface to select 'en_US.UTF-8 UTF-8' (or similar). +On the second page select this new locale as the default. + +After these changes have been made simply run install again. + +Now exiting install... +" exit 1 fi +set -e # Check if airtime exists already From ee89ac660375481bfca7ba99b49406d6e822e591 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 19 Oct 2012 12:16:04 -0400 Subject: [PATCH 004/108] CC-4589: Small inconsistency in config files after upgrade --- .../upgrades/airtime-2.2.0/common/UpgradeCommon.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/install_minimal/upgrades/airtime-2.2.0/common/UpgradeCommon.php b/install_minimal/upgrades/airtime-2.2.0/common/UpgradeCommon.php index 77c297a65..496a576a2 100644 --- a/install_minimal/upgrades/airtime-2.2.0/common/UpgradeCommon.php +++ b/install_minimal/upgrades/airtime-2.2.0/common/UpgradeCommon.php @@ -111,6 +111,10 @@ class UpgradeCommon{ $old = "list_all_db_files = 'list-all-files/format/json/api_key/%%api_key%%/dir_id/%%dir_id%%'"; $new = "list_all_db_files = 'list-all-files/format/json/api_key/%%api_key%%/dir_id/%%dir_id%%/all/%%all%%'"; exec("sed -i \"s#$old#$new#g\" /etc/airtime/api_client.cfg"); + + $old = "update_start_playing_url = 'notify-media-item-start-play/api_key/%%api_key%%/media_id/%%media_id%%/schedule_id/%%schedule_id%%'"; + $new = "update_start_playing_url = 'notify-media-item-start-play/api_key/%%api_key%%/media_id/%%media_id%%/'"; + exec("sed -i \"s#$old#$new#g\" /etc/airtime/api_client.cfg"); } /** From a1e653b41f0657ca8d366897e0c824cd993fe7a5 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 19 Oct 2012 13:22:00 -0400 Subject: [PATCH 005/108] Deep optimization --- airtime_mvc/application/models/Show.php | 7 ++++--- .../application/models/ShowInstance.php | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/airtime_mvc/application/models/Show.php b/airtime_mvc/application/models/Show.php index 5da348b52..0006057f7 100644 --- a/airtime_mvc/application/models/Show.php +++ b/airtime_mvc/application/models/Show.php @@ -1743,7 +1743,8 @@ SQL; $days = $interval->format('%a'); $shows = Application_Model_Show::getShows($p_start, $p_end); $nowEpoch = time(); - + $content_count = Application_Model_ShowInstance::getContentCount( + $p_start, $p_end); $timezone = date_default_timezone_get(); foreach ($shows as $show) { @@ -1789,9 +1790,9 @@ SQL; $showInstance = new Application_Model_ShowInstance( $show["instance_id"]); - //$showContent = $showInstance->getShowListContent(); - $options["show_empty"] = ($showInstance->showEmpty()) ? 1 : 0; + $options["show_empty"] = (array_key_exists($show['instance_id'], + $content_count)) ? 1 : 0; $events[] = &self::makeFullCalendarEvent($show, $options, $startsDT, $endsDT, $startsEpochStr, $endsEpochStr); diff --git a/airtime_mvc/application/models/ShowInstance.php b/airtime_mvc/application/models/ShowInstance.php index e657494f9..8854f3896 100644 --- a/airtime_mvc/application/models/ShowInstance.php +++ b/airtime_mvc/application/models/ShowInstance.php @@ -662,6 +662,27 @@ SQL; } + + public static function getContentCount($p_start, $p_end) + { + $sql = << :p_start::TIMESTAMP + AND starts < :p_end::TIMESTAMP +GROUP BY instance_id; +SQL; + + $counts = Application_Common_Database::prepareAndExecute( $sql, array( + ':p_start' => $p_start->format("Y-m-d G:i:s"), + ':p_end' => $p_end->format("Y-m-d G:i:s")) + , 'all'); + + return $counts; + + } + public function showEmpty() { $sql = << Date: Fri, 19 Oct 2012 13:24:06 -0400 Subject: [PATCH 006/108] Removed trailing whitespace --- airtime_mvc/application/models/ShowInstance.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/airtime_mvc/application/models/ShowInstance.php b/airtime_mvc/application/models/ShowInstance.php index 8854f3896..5b60e1f70 100644 --- a/airtime_mvc/application/models/ShowInstance.php +++ b/airtime_mvc/application/models/ShowInstance.php @@ -671,16 +671,16 @@ SELECT instance_id, FROM cc_schedule WHERE ends > :p_start::TIMESTAMP AND starts < :p_end::TIMESTAMP -GROUP BY instance_id; -SQL; +GROUP BY instance_id +SQL; - $counts = Application_Common_Database::prepareAndExecute( $sql, array( - ':p_start' => $p_start->format("Y-m-d G:i:s"), + $counts = Application_Common_Database::prepareAndExecute( $sql, array( + ':p_start' => $p_start->format("Y-m-d G:i:s"), ':p_end' => $p_end->format("Y-m-d G:i:s")) - , 'all'); - - return $counts; - + , 'all'); + + return $counts; + } public function showEmpty() From 8f998fd2c89edc5334f3ae1d138f04aaee0bc0e2 Mon Sep 17 00:00:00 2001 From: denise Date: Fri, 19 Oct 2012 13:42:24 -0400 Subject: [PATCH 007/108] CC-4595: Fade in/out setting doesn't work -fixed --- airtime_mvc/application/models/airtime/CcSchedule.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airtime_mvc/application/models/airtime/CcSchedule.php b/airtime_mvc/application/models/airtime/CcSchedule.php index e051df6c3..db7fb19a7 100644 --- a/airtime_mvc/application/models/airtime/CcSchedule.php +++ b/airtime_mvc/application/models/airtime/CcSchedule.php @@ -127,9 +127,9 @@ class CcSchedule extends BaseCcSchedule { } if ($microsecond == 0) { - $this->fadein = $dt->format('H:i:s.u'); + $this->fade_in = $dt->format('H:i:s.u'); } else { - $this->fadein = $dt->format('H:i:s').".".$microsecond; + $this->fade_in = $dt->format('H:i:s').".".$microsecond; } $this->modifiedColumns[] = CcSchedulePeer::FADE_IN; @@ -164,9 +164,9 @@ class CcSchedule extends BaseCcSchedule { } if ($microsecond == 0) { - $this->fadeout = $dt->format('H:i:s.u'); + $this->fade_out = $dt->format('H:i:s.u'); } else { - $this->fadeout = $dt->format('H:i:s').".".$microsecond; + $this->fade_out = $dt->format('H:i:s').".".$microsecond; } $this->modifiedColumns[] = CcSchedulePeer::FADE_OUT; From 96ff4404350360af4c51b2b6b33298d3cd8fca84 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 19 Oct 2012 14:18:34 -0400 Subject: [PATCH 008/108] Removed space --- airtime_mvc/application/models/ShowInstance.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtime_mvc/application/models/ShowInstance.php b/airtime_mvc/application/models/ShowInstance.php index 5b60e1f70..a73d1f7f0 100644 --- a/airtime_mvc/application/models/ShowInstance.php +++ b/airtime_mvc/application/models/ShowInstance.php @@ -674,7 +674,7 @@ WHERE ends > :p_start::TIMESTAMP GROUP BY instance_id SQL; - $counts = Application_Common_Database::prepareAndExecute( $sql, array( + $counts = Application_Common_Database::prepareAndExecute($sql, array( ':p_start' => $p_start->format("Y-m-d G:i:s"), ':p_end' => $p_end->format("Y-m-d G:i:s")) , 'all'); From 1146e1f269fb60b1c4112460213fc73c3b8d89e0 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 19 Oct 2012 14:27:57 -0400 Subject: [PATCH 009/108] CC-4215: Smart Playlist: Exception happens when generate with criteria Length is 2012-08-10 14:00:00 - fixed --- airtime_mvc/application/forms/SmartBlockCriteria.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtime_mvc/application/forms/SmartBlockCriteria.php b/airtime_mvc/application/forms/SmartBlockCriteria.php index 59c375bf7..55a28c79a 100644 --- a/airtime_mvc/application/forms/SmartBlockCriteria.php +++ b/airtime_mvc/application/forms/SmartBlockCriteria.php @@ -454,7 +454,7 @@ class Application_Form_SmartBlockCriteria extends Zend_Form_SubForm $column = CcFilesPeer::getTableMap()->getColumnByPhpName($criteria2PeerMap[$d['sp_criteria_field']]); // validation on type of column if ($d['sp_criteria_field'] == 'length') { - if (!preg_match("/(\d{2}):(\d{2}):(\d{2})/", $d['sp_criteria_value'])) { + if (!preg_match("/^(\d{2}):(\d{2}):(\d{2})/", $d['sp_criteria_value'])) { $element->addError("'Length' should be in '00:00:00' format"); $isValid = false; } From 77b430ce01e35ffc3d72fd639651e7613123d650 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 19 Oct 2012 15:14:52 -0400 Subject: [PATCH 010/108] CC-4564: Webstream Book same webstream twice in Scheduler, the 2nd one doesn't get played -fixed --- .../pypo/liquidsoap_scripts/ls_lib.liq | 113 +++++++++--------- .../pypo/liquidsoap_scripts/ls_script.liq | 8 +- 2 files changed, 61 insertions(+), 60 deletions(-) diff --git a/python_apps/pypo/liquidsoap_scripts/ls_lib.liq b/python_apps/pypo/liquidsoap_scripts/ls_lib.liq index 4e9167638..16d688a4f 100644 --- a/python_apps/pypo/liquidsoap_scripts/ls_lib.liq +++ b/python_apps/pypo/liquidsoap_scripts/ls_lib.liq @@ -413,7 +413,7 @@ def create_dynamic_source(uri) = # We register both source and output # in the list of sources dyn_sources := - list.append([(uri,s),(uri,active_dyn_out)], !dyn_sources) + list.append([(current_dyn_id, s),(current_dyn_id, active_dyn_out)], !dyn_sources) notify([("schedule_table_id", !current_dyn_id)]) "Done!" @@ -421,7 +421,62 @@ end # A function to destroy a dynamic source -def destroy_dynamic_source_all(uri) = +def destroy_dynamic_source(id) = + # We need to find the source in the list, + # remove it and destroy it. Currently, the language + # lacks some nice operators for that so we do it + # the functional way + + # This function is executed on every item in the list + # of dynamic sources + def parse_list(ret, current_element) = + # ret is of the form: (matching_sources, remaining_sources) + # We extract those two: + matching_sources = fst(ret) + remaining_sources = snd(ret) + + # current_element is of the form: ("uri", source) so + # we check the first element + current_id = fst(current_element) + if current_id == id then + # In this case, we add the source to the list of + # matched sources + (list.append( [snd(current_element)], + matching_sources), + remaining_sources) + else + # In this case, we put the element in the list of remaining + # sources + (matching_sources, + list.append([current_element], + remaining_sources)) + end + end + + # Now we execute the function: + result = list.fold(parse_list, ([], []), !dyn_sources) + matching_sources = fst(result) + remaining_sources = snd(result) + + # We store the remaining sources in dyn_sources + dyn_sources := remaining_sources + + # If no source matched, we return an error + if list.length(matching_sources) == 0 then + "Error: no matching sources!" + else + # We stop all sources + list.iter(source.shutdown, matching_sources) + # And return + "Done!" + end +end + + + + +# A function to destroy a dynamic source +def destroy_dynamic_source_all() = # We need to find the source in the list, # remove it and destroy it. Currently, the language # lacks some nice operators for that so we do it @@ -466,57 +521,3 @@ end - -# A function to destroy a dynamic source -def destroy_dynamic_source(uri) = - # We need to find the source in the list, - # remove it and destroy it. Currently, the language - # lacks some nice operators for that so we do it - # the functional way - - # This function is executed on every item in the list - # of dynamic sources - def parse_list(ret, current_element) = - # ret is of the form: (matching_sources, remaining_sources) - # We extract those two: - matching_sources = fst(ret) - remaining_sources = snd(ret) - - # current_element is of the form: ("uri", source) so - # we check the first element - current_uri = fst(current_element) - if current_uri == uri then - # In this case, we add the source to the list of - # matched sources - (list.append( [snd(current_element)], - matching_sources), - remaining_sources) - else - # In this case, we put the element in the list of remaining - # sources - (matching_sources, - list.append([current_element], - remaining_sources)) - end - end - - # Now we execute the function: - result = list.fold(parse_list, ([], []), !dyn_sources) - matching_sources = fst(result) - remaining_sources = snd(result) - - # We store the remaining sources in dyn_sources - dyn_sources := remaining_sources - - # If no source matched, we return an error - if list.length(matching_sources) == 0 then - "Error: no matching sources!" - else - # We stop all sources - list.iter(source.shutdown, matching_sources) - # And return - "Done!" - end -end - - diff --git a/python_apps/pypo/liquidsoap_scripts/ls_script.liq b/python_apps/pypo/liquidsoap_scripts/ls_script.liq index fbb8e9870..0d60cd9c3 100644 --- a/python_apps/pypo/liquidsoap_scripts/ls_script.liq +++ b/python_apps/pypo/liquidsoap_scripts/ls_script.liq @@ -99,17 +99,17 @@ server.register(namespace="dynamic_source", description="Start a new dynamic source.", usage="start ", "read_start", - fun (s) -> begin log("dynamic_source.read_start") create_dynamic_source(s) end) + fun (uri) -> begin log("dynamic_source.read_start") create_dynamic_source(uri) end) server.register(namespace="dynamic_source", description="Stop a dynamic source.", - usage="stop ", + usage="stop ", "read_stop", fun (s) -> begin log("dynamic_source.read_stop") destroy_dynamic_source(s) end) server.register(namespace="dynamic_source", description="Stop a dynamic source.", - usage="stop ", + usage="stop ", "read_stop_all", - fun (s) -> begin log("dynamic_source.read_stop") destroy_dynamic_source_all(s) end) + fun (s) -> begin log("dynamic_source.read_stop") destroy_dynamic_source_all() end) default = amplify(id="silence_src", 0.00001, noise()) default = rewrite_metadata([("artist","Airtime"), ("title", "offline")], default) From 1d4b9055722e3a94efa417b354670567d967fc4d Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 19 Oct 2012 15:18:48 -0400 Subject: [PATCH 011/108] CC-4564: Webstream Book same webstream twice in Scheduler, the 2nd one doesn't get played -small syntax error in last commit --- python_apps/pypo/liquidsoap_scripts/ls_lib.liq | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/pypo/liquidsoap_scripts/ls_lib.liq b/python_apps/pypo/liquidsoap_scripts/ls_lib.liq index 16d688a4f..9d48eebbb 100644 --- a/python_apps/pypo/liquidsoap_scripts/ls_lib.liq +++ b/python_apps/pypo/liquidsoap_scripts/ls_lib.liq @@ -413,7 +413,7 @@ def create_dynamic_source(uri) = # We register both source and output # in the list of sources dyn_sources := - list.append([(current_dyn_id, s),(current_dyn_id, active_dyn_out)], !dyn_sources) + list.append([(!current_dyn_id, s),(!current_dyn_id, active_dyn_out)], !dyn_sources) notify([("schedule_table_id", !current_dyn_id)]) "Done!" From 9dfee6aa688098f5c102f11c5b1a6979bb45f3dc Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 19 Oct 2012 15:39:45 -0400 Subject: [PATCH 012/108] CC-4564: Webstream: Book same webstream twice in Scheduler, the 2nd one doesn't get played -fixed --- airtime_mvc/application/models/Schedule.php | 1 + python_apps/pypo/pypopush.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/airtime_mvc/application/models/Schedule.php b/airtime_mvc/application/models/Schedule.php index 1102efaff..5a18d7b79 100644 --- a/airtime_mvc/application/models/Schedule.php +++ b/airtime_mvc/application/models/Schedule.php @@ -713,6 +713,7 @@ SQL; 'end' => $stream_end, 'uri' => $uri, 'type' => 'stream_buffer_end', + 'row_id' => $item["id"], 'independent_event' => true ); self::appendScheduleItem($data, $stream_end, $schedule_item); diff --git a/python_apps/pypo/pypopush.py b/python_apps/pypo/pypopush.py index 87e1704bf..ad28abef9 100644 --- a/python_apps/pypo/pypopush.py +++ b/python_apps/pypo/pypopush.py @@ -455,6 +455,7 @@ class PypoPush(Thread): tn = telnetlib.Telnet(LS_HOST, LS_PORT) msg = 'dynamic_source.id %s\n' % media_item['row_id'] + self.logger.debug(msg) tn.write(msg) #example: dynamic_source.read_start http://87.230.101.24:80/top100station.mp3 @@ -523,7 +524,7 @@ class PypoPush(Thread): tn = telnetlib.Telnet(LS_HOST, LS_PORT) #dynamic_source.stop http://87.230.101.24:80/top100station.mp3 - msg = 'dynamic_source.read_stop %s\n' % media_item['uri'].encode('latin-1') + msg = 'dynamic_source.read_stop %s\n' % media_item['row_id'] self.logger.debug(msg) tn.write(msg) From f30d636b17d960601f3bdec2df73a2108e1dc410 Mon Sep 17 00:00:00 2001 From: denise Date: Fri, 19 Oct 2012 16:19:32 -0400 Subject: [PATCH 013/108] CC-4581: Confirmation dialog for removing an item from a show refers to deletion --- airtime_mvc/public/js/airtime/showbuilder/builder.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtime_mvc/public/js/airtime/showbuilder/builder.js b/airtime_mvc/public/js/airtime/showbuilder/builder.js index 9839df06c..fc8463481 100644 --- a/airtime_mvc/public/js/airtime/showbuilder/builder.js +++ b/airtime_mvc/public/js/airtime/showbuilder/builder.js @@ -283,7 +283,7 @@ var AIRTIME = (function(AIRTIME){ mod.fnRemove = function(aItems) { mod.disableUI(); - if (confirm("Delete selected item(s)?")) { + if (confirm("Remove selected scheduled item(s)?")) { $.post( "/showbuilder/schedule-remove", {"items": aItems, "format": "json"}, mod.fnItemCallback From bd5fef0bcd0e86e140660a474fd15ff755ea842e Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 19 Oct 2012 16:29:31 -0400 Subject: [PATCH 014/108] CC-4487: Webstream could not be heard unless you restart playout as well -fixed --- .../pypo/liquidsoap_scripts/ls_lib.liq | 5 +++ .../pypo/liquidsoap_scripts/ls_script.liq | 9 +++- python_apps/pypo/pypopush.py | 43 +++++++++++++++---- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/python_apps/pypo/liquidsoap_scripts/ls_lib.liq b/python_apps/pypo/liquidsoap_scripts/ls_lib.liq index 9d48eebbb..9e7903751 100644 --- a/python_apps/pypo/liquidsoap_scripts/ls_lib.liq +++ b/python_apps/pypo/liquidsoap_scripts/ls_lib.liq @@ -402,6 +402,11 @@ def set_dynamic_source_id(id) = string_of(!current_dyn_id) end +def get_dynamic_source_id() = + string_of(!current_dyn_id) +end + + # Function to create a playlist source and output it. def create_dynamic_source(uri) = # The playlist source diff --git a/python_apps/pypo/liquidsoap_scripts/ls_script.liq b/python_apps/pypo/liquidsoap_scripts/ls_script.liq index 0d60cd9c3..7a6d8c61d 100644 --- a/python_apps/pypo/liquidsoap_scripts/ls_script.liq +++ b/python_apps/pypo/liquidsoap_scripts/ls_script.liq @@ -19,7 +19,7 @@ queue = amplify(1., override="replay_gain", queue) #live stream setup set("harbor.bind_addr", "0.0.0.0") -current_dyn_id = ref '' +current_dyn_id = ref '-1' pypo_data = ref '0' stream_metadata_type = ref 0 @@ -95,6 +95,13 @@ server.register(namespace="dynamic_source", usage="id ", "id", fun (s) -> begin log("dynamic_source.id") set_dynamic_source_id(s) end) + +server.register(namespace="dynamic_source", + description="Get the cc_schedule row id", + usage="get_id", + "get_id", + fun (s) -> begin log("dynamic_source.get_id") get_dynamic_source_id() end) + server.register(namespace="dynamic_source", description="Start a new dynamic source.", usage="start ", diff --git a/python_apps/pypo/pypopush.py b/python_apps/pypo/pypopush.py index ad28abef9..82d7593f0 100644 --- a/python_apps/pypo/pypopush.py +++ b/python_apps/pypo/pypopush.py @@ -78,6 +78,7 @@ class PypoPush(Thread): #We get to the following lines only if a schedule was received. liquidsoap_queue_approx = self.get_queue_items_from_liquidsoap() + liquidsoap_stream_id = self.get_current_stream_id_from_liquidsoap() tnow = datetime.utcnow() current_event_chain, original_chain = self.get_current_chain(chains, tnow) @@ -92,7 +93,7 @@ class PypoPush(Thread): #is scheduled. We need to verify whether the schedule we just received matches #what Liquidsoap is playing, and if not, correct it. - self.handle_new_schedule(media_schedule, liquidsoap_queue_approx, current_event_chain) + self.handle_new_schedule(media_schedule, liquidsoap_queue_approx, liquidsoap_stream_id, current_event_chain) #At this point everything in the present has been taken care of and Liquidsoap @@ -134,6 +135,25 @@ class PypoPush(Thread): loops = 0 loops += 1 + def get_current_stream_id_from_liquidsoap(self): + response = "-1" + try: + self.telnet_lock.acquire() + tn = telnetlib.Telnet(LS_HOST, LS_PORT) + + msg = 'dynamic_source.get_id\n' + tn.write(msg) + response = tn.read_until("\r\n").strip(" \r\n") + tn.write('exit\n') + tn.read_all() + except Exception, e: + self.logger.error("Error connecting to Liquidsoap: %s", e) + response = [] + finally: + self.telnet_lock.release() + + return response + def get_queue_items_from_liquidsoap(self): """ This function connects to Liquidsoap to find what media items are in its queue. @@ -175,7 +195,7 @@ class PypoPush(Thread): return liquidsoap_queue_approx - def is_correct_current_item(self, media_item, liquidsoap_queue_approx): + def is_correct_current_item(self, media_item, liquidsoap_queue_approx, liquidsoap_stream_id): correct = False if media_item is None: correct = (len(liquidsoap_queue_approx) == 0 and self.current_stream_info is None) @@ -188,10 +208,7 @@ class PypoPush(Thread): liquidsoap_queue_approx[0]['row_id'] == media_item['row_id'] and \ liquidsoap_queue_approx[0]['end'] == media_item['end'] elif is_stream(media_item): - if self.current_stream_info is None: - correct = False - else: - correct = self.current_stream_info['row_id'] == media_item['row_id'] + correct = liquidsoap_stream_id == str(media_item['row_id']) self.logger.debug("Is current item correct?: %s", str(correct)) return correct @@ -202,7 +219,7 @@ class PypoPush(Thread): self.remove_from_liquidsoap_queue(0, None) self.stop_web_stream_all() - def handle_new_schedule(self, media_schedule, liquidsoap_queue_approx, current_event_chain): + def handle_new_schedule(self, media_schedule, liquidsoap_queue_approx, liquidsoap_stream_id, current_event_chain): """ This function's purpose is to gracefully handle situations where Liquidsoap already has a track in its queue, but the schedule @@ -220,7 +237,7 @@ class PypoPush(Thread): if len(current_event_chain) > 0: current_item = current_event_chain[0] - if not self.is_correct_current_item(current_item, liquidsoap_queue_approx): + if not self.is_correct_current_item(current_item, liquidsoap_queue_approx, liquidsoap_stream_id): self.clear_all_liquidsoap_items() if is_stream(current_item): if current_item['row_id'] != self.current_prebuffering_stream_id: @@ -234,7 +251,7 @@ class PypoPush(Thread): #we've changed the queue, so let's refetch it liquidsoap_queue_approx = self.get_queue_items_from_liquidsoap() - elif not self.is_correct_current_item(None, liquidsoap_queue_approx): + elif not self.is_correct_current_item(None, liquidsoap_queue_approx, liquidsoap_stream_id): #Liquidsoap is playing something even though it shouldn't be self.clear_all_liquidsoap_items() @@ -509,6 +526,10 @@ class PypoPush(Thread): self.logger.debug(msg) tn.write(msg) + msg = 'dynamic_source.id -1\n' + self.logger.debug(msg) + tn.write(msg) + tn.write("exit\n") self.logger.debug(tn.read_all()) @@ -528,6 +549,10 @@ class PypoPush(Thread): self.logger.debug(msg) tn.write(msg) + msg = 'dynamic_source.id -1\n' + self.logger.debug(msg) + tn.write(msg) + tn.write("exit\n") self.logger.debug(tn.read_all()) From 58535bef88aea186a6b07337614096791e9421eb Mon Sep 17 00:00:00 2001 From: James Date: Fri, 19 Oct 2012 16:40:32 -0400 Subject: [PATCH 015/108] CC-4580: No contents in rebroadcast show - fixed --- airtime_mvc/application/controllers/ApiController.php | 5 ++++- python_apps/media-monitor2/media/monitor/events.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/airtime_mvc/application/controllers/ApiController.php b/airtime_mvc/application/controllers/ApiController.php index 6572774cc..1a1876ce9 100644 --- a/airtime_mvc/application/controllers/ApiController.php +++ b/airtime_mvc/application/controllers/ApiController.php @@ -490,6 +490,10 @@ class ApiController extends Zend_Controller_Action $file->setFileExistsFlag(true); $file->setMetadata($md); } + if ($md['is_record'] != 0) { + $this->uploadRecordedActionParam($md['MDATA_KEY_TRACKNUMBER'], $file->getId()); + } + } elseif ($mode == "modify") { $filepath = $md['MDATA_KEY_FILEPATH']; $file = Application_Model_StoredFile::RecallByFilepath($filepath); @@ -562,7 +566,6 @@ class ApiController extends Zend_Controller_Action // least 1 digit if ( !preg_match('/^md\d+$/', $k) ) { continue; } $info_json = json_decode($raw_json, $assoc = true); - unset( $info_json["is_record"] ); // Log invalid requests if ( !array_key_exists('mode', $info_json) ) { Logging::info("Received bad request(key=$k), no 'mode' parameter. Bad request is:"); diff --git a/python_apps/media-monitor2/media/monitor/events.py b/python_apps/media-monitor2/media/monitor/events.py index a8ccc3b54..0b2a92e14 100644 --- a/python_apps/media-monitor2/media/monitor/events.py +++ b/python_apps/media-monitor2/media/monitor/events.py @@ -199,6 +199,7 @@ class NewFile(BaseEvent, HasMetaData): """ req_dict = self.metadata.extract() req_dict['mode'] = u'create' + req_dict['is_record'] = self.metadata.is_recorded() self.assign_owner(req_dict) req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path ) return [req_dict] From ef78cefcd96b1e9472cb5ea0af0b0a726dde2c1a Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 19 Oct 2012 16:49:01 -0400 Subject: [PATCH 016/108] CC-4487: Webstream could not be heard unless you restart playout as well -cleanup of unused variables --- python_apps/pypo/pypopush.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/python_apps/pypo/pypopush.py b/python_apps/pypo/pypopush.py index 82d7593f0..5bcce21b6 100644 --- a/python_apps/pypo/pypopush.py +++ b/python_apps/pypo/pypopush.py @@ -55,7 +55,6 @@ class PypoPush(Thread): self.pushed_objects = {} self.logger = logging.getLogger('push') - self.current_stream_info = None self.current_prebuffering_stream_id = None def main(self): @@ -198,7 +197,7 @@ class PypoPush(Thread): def is_correct_current_item(self, media_item, liquidsoap_queue_approx, liquidsoap_stream_id): correct = False if media_item is None: - correct = (len(liquidsoap_queue_approx) == 0 and self.current_stream_info is None) + correct = (len(liquidsoap_queue_approx) == 0 and liquidsoap_stream_id == "-1") else: if is_file(media_item): if len(liquidsoap_queue_approx) == 0: @@ -230,7 +229,6 @@ class PypoPush(Thread): file_chain = filter(lambda item: (item["type"] == "file"), current_event_chain) stream_chain = filter(lambda item: (item["type"] == "stream_output_start"), current_event_chain) - self.logger.debug(self.current_stream_info) self.logger.debug(current_event_chain) #Take care of the case where the current playing may be incorrect @@ -507,7 +505,6 @@ class PypoPush(Thread): self.logger.debug(tn.read_all()) self.current_prebuffering_stream_id = None - self.current_stream_info = media_item except Exception, e: self.logger.error(str(e)) finally: @@ -533,7 +530,6 @@ class PypoPush(Thread): tn.write("exit\n") self.logger.debug(tn.read_all()) - self.current_stream_info = None except Exception, e: self.logger.error(str(e)) finally: @@ -556,7 +552,6 @@ class PypoPush(Thread): tn.write("exit\n") self.logger.debug(tn.read_all()) - self.current_stream_info = None except Exception, e: self.logger.error(str(e)) finally: @@ -575,7 +570,6 @@ class PypoPush(Thread): tn.write("exit\n") self.logger.debug(tn.read_all()) - self.current_stream_info = None except Exception, e: self.logger.error(str(e)) finally: From a9df8cd7f21525747fb2a9297cbf66ff8a88dd08 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 19 Oct 2012 17:30:54 -0400 Subject: [PATCH 017/108] CC-4610: Creating recording show will generate lots of exception in zendphp.log - fixed --- airtime_mvc/application/models/Schedule.php | 1 - airtime_mvc/application/models/ShowBuilder.php | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/airtime_mvc/application/models/Schedule.php b/airtime_mvc/application/models/Schedule.php index 5a18d7b79..1f2e98810 100644 --- a/airtime_mvc/application/models/Schedule.php +++ b/airtime_mvc/application/models/Schedule.php @@ -1128,7 +1128,6 @@ SQL; } } else { if ($isAdminOrPM) { - Logging::info( $data ); Application_Model_Show::create($data); } diff --git a/airtime_mvc/application/models/ShowBuilder.php b/airtime_mvc/application/models/ShowBuilder.php index e1e2382aa..90a67a1d6 100644 --- a/airtime_mvc/application/models/ShowBuilder.php +++ b/airtime_mvc/application/models/ShowBuilder.php @@ -198,7 +198,9 @@ class Application_Model_ShowBuilder } elseif (intval($p_item["si_record"]) === 1) { $row["record"] = true; - if (Application_Model_Preference::GetUploadToSoundcloudOption()) { + // at the time of creating on show, the recorded file is not in the DB yet. + // therefore, 'si_file_id' is null. So we need to check it. + if (Application_Model_Preference::GetUploadToSoundcloudOption() && isset($p_item['si_file_id'])) { $file = Application_Model_StoredFile::Recall( $p_item['si_file_id']); if (isset($file)) { @@ -398,7 +400,7 @@ class Application_Model_ShowBuilder //see if the displayed show instances have changed. (deleted, //empty schedule etc) - if ($outdated === false && count($instances) + if ($outdated === false && count($instances) !== count($currentInstances)) { Logging::debug("show instances have changed."); $outdated = true; @@ -459,7 +461,7 @@ class Application_Model_ShowBuilder $display_items[] = $row; } - if ($current_id !== -1 && + if ($current_id !== -1 && !in_array($current_id, $this->showInstances)) { $this->showInstances[] = $current_id; } From e0402c88cf8bc9d24460af93d4a32299e693c7ab Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 22 Oct 2012 15:20:03 -0400 Subject: [PATCH 018/108] cc-4613: fixed --- airtime_mvc/application/models/Show.php | 2 +- airtime_mvc/application/models/ShowInstance.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airtime_mvc/application/models/Show.php b/airtime_mvc/application/models/Show.php index 0006057f7..41ffbeefd 100644 --- a/airtime_mvc/application/models/Show.php +++ b/airtime_mvc/application/models/Show.php @@ -1792,7 +1792,7 @@ SQL; $show["instance_id"]); $options["show_empty"] = (array_key_exists($show['instance_id'], - $content_count)) ? 1 : 0; + $content_count)) ? 0 : 1; $events[] = &self::makeFullCalendarEvent($show, $options, $startsDT, $endsDT, $startsEpochStr, $endsEpochStr); diff --git a/airtime_mvc/application/models/ShowInstance.php b/airtime_mvc/application/models/ShowInstance.php index a73d1f7f0..3d1d1d055 100644 --- a/airtime_mvc/application/models/ShowInstance.php +++ b/airtime_mvc/application/models/ShowInstance.php @@ -681,7 +681,7 @@ SQL; return $counts; - } + } public function showEmpty() { From feac48a0035326ce1b8c0d8a8ed415b980e89043 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 22 Oct 2012 15:34:33 -0400 Subject: [PATCH 019/108] cc-4613: fixed --- airtime_mvc/application/models/Show.php | 2 ++ airtime_mvc/application/models/ShowInstance.php | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/airtime_mvc/application/models/Show.php b/airtime_mvc/application/models/Show.php index 41ffbeefd..5fce268c6 100644 --- a/airtime_mvc/application/models/Show.php +++ b/airtime_mvc/application/models/Show.php @@ -1747,6 +1747,8 @@ SQL; $p_start, $p_end); $timezone = date_default_timezone_get(); + Logging::info( $content_count ); + foreach ($shows as $show) { $options = array(); diff --git a/airtime_mvc/application/models/ShowInstance.php b/airtime_mvc/application/models/ShowInstance.php index 3d1d1d055..b6930766d 100644 --- a/airtime_mvc/application/models/ShowInstance.php +++ b/airtime_mvc/application/models/ShowInstance.php @@ -679,7 +679,12 @@ SQL; ':p_end' => $p_end->format("Y-m-d G:i:s")) , 'all'); - return $counts; + + $real_counts = array(); + foreach ($counts as $c) { + $real_counts[$c['instance_id']] = $c['instance_count']; + } + return $real_counts; } From b7b849a48f71848c6c7ea447cfbb60d89d0b3162 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 22 Oct 2012 15:35:01 -0400 Subject: [PATCH 020/108] CC-4612: Create index for cc_schedule during upgrade to improve the performance when cc_schedule's getting bigger - fixed --- install_minimal/upgrades/airtime-2.2.0/data/upgrade.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/install_minimal/upgrades/airtime-2.2.0/data/upgrade.sql b/install_minimal/upgrades/airtime-2.2.0/data/upgrade.sql index 36b4f5734..771c3b13c 100644 --- a/install_minimal/upgrades/airtime-2.2.0/data/upgrade.sql +++ b/install_minimal/upgrades/airtime-2.2.0/data/upgrade.sql @@ -140,6 +140,12 @@ ALTER TABLE cc_playlistcontents ALTER TABLE cc_schedule ADD COLUMN stream_id integer; +CREATE INDEX cc_schedule_instance_id_idx + ON cc_schedule + USING btree + (instance_id); + + ALTER TABLE cc_subjs ADD COLUMN cell_phone character varying(255); From abf4319d67bc49eaf5fc9f20edc4fc19bbc0a9ec Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 22 Oct 2012 15:55:24 -0400 Subject: [PATCH 021/108] Removed logging --- airtime_mvc/application/models/Show.php | 2 -- airtime_mvc/application/models/ShowInstance.php | 1 - 2 files changed, 3 deletions(-) diff --git a/airtime_mvc/application/models/Show.php b/airtime_mvc/application/models/Show.php index 5fce268c6..41ffbeefd 100644 --- a/airtime_mvc/application/models/Show.php +++ b/airtime_mvc/application/models/Show.php @@ -1747,8 +1747,6 @@ SQL; $p_start, $p_end); $timezone = date_default_timezone_get(); - Logging::info( $content_count ); - foreach ($shows as $show) { $options = array(); diff --git a/airtime_mvc/application/models/ShowInstance.php b/airtime_mvc/application/models/ShowInstance.php index b6930766d..533118c29 100644 --- a/airtime_mvc/application/models/ShowInstance.php +++ b/airtime_mvc/application/models/ShowInstance.php @@ -679,7 +679,6 @@ SQL; ':p_end' => $p_end->format("Y-m-d G:i:s")) , 'all'); - $real_counts = array(); foreach ($counts as $c) { $real_counts[$c['instance_id']] = $c['instance_count']; From 780a05c2aa4121b2c5e2beb49ad50e1184c29949 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Mon, 22 Oct 2012 16:23:33 -0400 Subject: [PATCH 022/108] CC-2236: Overbooked shows do not respect default fade time -once a release we manage to break fades :) fixed. --- python_apps/pypo/liquidsoap_scripts/ls_lib.liq | 2 +- python_apps/pypo/liquidsoap_scripts/ls_script.liq | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python_apps/pypo/liquidsoap_scripts/ls_lib.liq b/python_apps/pypo/liquidsoap_scripts/ls_lib.liq index 9e7903751..5c5919e2e 100644 --- a/python_apps/pypo/liquidsoap_scripts/ls_lib.liq +++ b/python_apps/pypo/liquidsoap_scripts/ls_lib.liq @@ -27,7 +27,7 @@ def append_title(m) = end end -def crossfade(s) +def crossfade_airtime(s) #duration is automatically overwritten by metadata fields passed in #with audio s = fade.in(type="log", duration=0., s) diff --git a/python_apps/pypo/liquidsoap_scripts/ls_script.liq b/python_apps/pypo/liquidsoap_scripts/ls_script.liq index 7a6d8c61d..2c1ef9dee 100644 --- a/python_apps/pypo/liquidsoap_scripts/ls_script.liq +++ b/python_apps/pypo/liquidsoap_scripts/ls_script.liq @@ -44,9 +44,9 @@ web_stream = on_metadata(notify_stream, web_stream) output.dummy(fallible=true, web_stream) queue = on_metadata(notify, queue) -queue = map_metadata(update=false, append_title, queue) +queue = map_metadata(update=true, append_title, queue) # the crossfade function controls fade in/out -queue = crossfade(queue) +queue = crossfade_airtime(queue) output.dummy(fallible=true, queue) From 795ffbbf631443b315819a8ec4968f3f6ad772e2 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Mon, 22 Oct 2012 16:36:57 -0400 Subject: [PATCH 023/108] CC-2236: Overbooked shows do not respect default fade time -move map_metadata after crossfade function --- python_apps/pypo/liquidsoap_scripts/ls_script.liq | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python_apps/pypo/liquidsoap_scripts/ls_script.liq b/python_apps/pypo/liquidsoap_scripts/ls_script.liq index 2c1ef9dee..1f476b919 100644 --- a/python_apps/pypo/liquidsoap_scripts/ls_script.liq +++ b/python_apps/pypo/liquidsoap_scripts/ls_script.liq @@ -43,10 +43,11 @@ web_stream = input.harbor("test-harbor", port=8999, password=stream_harbor_pass) web_stream = on_metadata(notify_stream, web_stream) output.dummy(fallible=true, web_stream) -queue = on_metadata(notify, queue) -queue = map_metadata(update=true, append_title, queue) + # the crossfade function controls fade in/out queue = crossfade_airtime(queue) +queue = on_metadata(notify, queue) +queue = map_metadata(update=false, append_title, queue) output.dummy(fallible=true, queue) From c51fb8a04030035cdde8ed2fe70aad83aee871b9 Mon Sep 17 00:00:00 2001 From: denise Date: Mon, 22 Oct 2012 16:37:06 -0400 Subject: [PATCH 024/108] CC-4549: Tooltip for source username cannot be shared between input and output settings -fixed --- .../scripts/form/preferences_livestream.phtml | 2 +- airtime_mvc/public/css/styles.css | 3 ++- .../js/airtime/preferences/streamsetting.js | 21 +++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/airtime_mvc/application/views/scripts/form/preferences_livestream.phtml b/airtime_mvc/application/views/scripts/form/preferences_livestream.phtml index 00221bf4e..e2dd5d7dc 100644 --- a/airtime_mvc/application/views/scripts/form/preferences_livestream.phtml +++ b/airtime_mvc/application/views/scripts/form/preferences_livestream.phtml @@ -49,7 +49,7 @@
diff --git a/airtime_mvc/public/css/styles.css b/airtime_mvc/public/css/styles.css index 29d694d28..b14b77251 100644 --- a/airtime_mvc/public/css/styles.css +++ b/airtime_mvc/public/css/styles.css @@ -104,7 +104,8 @@ select { line-height:16px !important; } -.airtime_auth_help_icon, .custom_auth_help_icon, .stream_username_help_icon, .playlist_type_help_icon { +.airtime_auth_help_icon, .custom_auth_help_icon, .stream_username_help_icon, +.playlist_type_help_icon, .master_username_help_icon { cursor: help; position: relative; display:inline-block; zoom:1; diff --git a/airtime_mvc/public/js/airtime/preferences/streamsetting.js b/airtime_mvc/public/js/airtime/preferences/streamsetting.js index 35f7b373e..b25e2d157 100644 --- a/airtime_mvc/public/js/airtime/preferences/streamsetting.js +++ b/airtime_mvc/public/js/airtime/preferences/streamsetting.js @@ -333,6 +333,27 @@ $(document).ready(function() { }) $(".stream_username_help_icon").qtip({ + content: { + text: "If your Icecast server expects a username of 'source', this field can be left blank." + }, + hide: { + delay: 500, + fixed: true + }, + style: { + border: { + width: 0, + radius: 4 + }, + classes: "ui-tooltip-dark ui-tooltip-rounded" + }, + position: { + my: "left bottom", + at: "right center" + }, + }) + + $(".master_username_help_icon").qtip({ content: { text: "If your live streaming client does not ask for a username, this field should be 'source'." }, From 8b9b005708021cd1bd618c6ddbe0a7506b5c49fa Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Mon, 22 Oct 2012 17:29:26 -0400 Subject: [PATCH 025/108] CC-4612: Create index for cc_schedule during upgrade to improve the performance when cc_schedule's getting bigger - also create one during fresh install --- airtime_mvc/build/schema.xml | 5 +++++ airtime_mvc/build/sql/schema.sql | 2 ++ 2 files changed, 7 insertions(+) diff --git a/airtime_mvc/build/schema.xml b/airtime_mvc/build/schema.xml index d56927199..3d0957a50 100644 --- a/airtime_mvc/build/schema.xml +++ b/airtime_mvc/build/schema.xml @@ -91,6 +91,8 @@ + @@ -332,6 +334,9 @@ + + + diff --git a/airtime_mvc/build/sql/schema.sql b/airtime_mvc/build/sql/schema.sql index 332715b22..597ebf417 100644 --- a/airtime_mvc/build/sql/schema.sql +++ b/airtime_mvc/build/sql/schema.sql @@ -429,6 +429,8 @@ COMMENT ON TABLE "cc_schedule" IS ''; SET search_path TO public; +CREATE INDEX "cc_schedule_instance_id_idx" ON "cc_schedule" ("instance_id"); + ----------------------------------------------------------------------------- -- cc_sess ----------------------------------------------------------------------------- From 50d09d09d39dbe7f4d09bc6d17c459507c563882 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 23 Oct 2012 12:08:01 -0400 Subject: [PATCH 026/108] cc-4304: fixed ticket by deleting playlists when user is deleted. also removed useless index on file_exists --- airtime_mvc/build/schema.xml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/airtime_mvc/build/schema.xml b/airtime_mvc/build/schema.xml index 3d0957a50..fc3e6a51f 100644 --- a/airtime_mvc/build/schema.xml +++ b/airtime_mvc/build/schema.xml @@ -91,11 +91,6 @@ - - - -
@@ -208,7 +203,7 @@ - +
From 5024a3083c056d207fe1559e4e4687c40bf2e349 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Tue, 23 Oct 2012 12:18:52 -0400 Subject: [PATCH 027/108] cc-4487: keep playing webstream on liquidsoap restart --- install_minimal/upgrades/airtime-2.2.0/etc/api_client.cfg.220 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install_minimal/upgrades/airtime-2.2.0/etc/api_client.cfg.220 b/install_minimal/upgrades/airtime-2.2.0/etc/api_client.cfg.220 index 2e0e938e8..06e92edb2 100644 --- a/install_minimal/upgrades/airtime-2.2.0/etc/api_client.cfg.220 +++ b/install_minimal/upgrades/airtime-2.2.0/etc/api_client.cfg.220 @@ -117,3 +117,5 @@ get_files_without_replay_gain = 'get-files-without-replay-gain/api_key/%%api_key update_replay_gain_value = 'update-replay-gain-value/api_key/%%api_key%%' notify_webstream_data = 'notify-webstream-data/api_key/%%api_key%%/media_id/%%media_id%%/format/json' + +notify_liquidsoap_started = 'rabbitmq-do-push/api_key/%%api_key%%/format/json' From dff5bb4e2be704de6edb2d63bf8f5bdad3072c70 Mon Sep 17 00:00:00 2001 From: james Date: Tue, 23 Oct 2012 12:24:38 -0400 Subject: [PATCH 028/108] CC-4620: Now Playing page loading is very slow because 1M records in cc_schedule table - fixed --- airtime_mvc/application/models/Schedule.php | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/airtime_mvc/application/models/Schedule.php b/airtime_mvc/application/models/Schedule.php index 1f2e98810..233128433 100644 --- a/airtime_mvc/application/models/Schedule.php +++ b/airtime_mvc/application/models/Schedule.php @@ -287,7 +287,14 @@ SQL; SQL; $filesJoin = <<= '{$p_start}' + AND sched.starts < '{$p_end}') + OR (sched.ends > '{$p_start}' + AND sched.ends <= '{$p_end}') + OR (sched.starts <= '{$p_start}' + AND sched.ends >= '{$p_end}')) + ) SQL; @@ -307,7 +314,14 @@ SQL; SQL; $streamJoin = <<= '{$p_start}' + AND sched.starts < '{$p_end}') + OR (sched.ends > '{$p_start}' + AND sched.ends <= '{$p_end}') + OR (sched.starts <= '{$p_start}' + AND sched.ends >= '{$p_end}')) + ) LEFT JOIN cc_subjs AS sub ON (ws.creator_id = sub.id) SQL; From 2d6be404d491bb6b3cf4c32760093ce8fba82d97 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Tue, 23 Oct 2012 13:45:15 -0400 Subject: [PATCH 029/108] cc-4304: fixed ticket by deleting playlists when user is deleted. also removed useless index on file_exists -regen propel files --- .../application/models/airtime/map/CcPlaylistTableMap.php | 2 +- .../application/models/airtime/map/CcSubjsTableMap.php | 2 +- airtime_mvc/application/models/airtime/om/BaseCcSubjsPeer.php | 3 +++ airtime_mvc/build/sql/schema.sql | 4 +--- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/airtime_mvc/application/models/airtime/map/CcPlaylistTableMap.php b/airtime_mvc/application/models/airtime/map/CcPlaylistTableMap.php index 3162280d0..8dba50bbd 100644 --- a/airtime_mvc/application/models/airtime/map/CcPlaylistTableMap.php +++ b/airtime_mvc/application/models/airtime/map/CcPlaylistTableMap.php @@ -53,7 +53,7 @@ class CcPlaylistTableMap extends TableMap { */ public function buildRelations() { - $this->addRelation('CcSubjs', 'CcSubjs', RelationMap::MANY_TO_ONE, array('creator_id' => 'id', ), null, null); + $this->addRelation('CcSubjs', 'CcSubjs', RelationMap::MANY_TO_ONE, array('creator_id' => 'id', ), 'CASCADE', null); $this->addRelation('CcPlaylistcontents', 'CcPlaylistcontents', RelationMap::ONE_TO_MANY, array('id' => 'playlist_id', ), 'CASCADE', null); } // buildRelations() diff --git a/airtime_mvc/application/models/airtime/map/CcSubjsTableMap.php b/airtime_mvc/application/models/airtime/map/CcSubjsTableMap.php index 7ed8469d8..ffbe8cded 100644 --- a/airtime_mvc/application/models/airtime/map/CcSubjsTableMap.php +++ b/airtime_mvc/application/models/airtime/map/CcSubjsTableMap.php @@ -63,7 +63,7 @@ class CcSubjsTableMap extends TableMap { $this->addRelation('CcFilesRelatedByDbEditedby', 'CcFiles', RelationMap::ONE_TO_MANY, array('id' => 'editedby', ), null, null); $this->addRelation('CcPerms', 'CcPerms', RelationMap::ONE_TO_MANY, array('id' => 'subj', ), 'CASCADE', null); $this->addRelation('CcShowHosts', 'CcShowHosts', RelationMap::ONE_TO_MANY, array('id' => 'subjs_id', ), 'CASCADE', null); - $this->addRelation('CcPlaylist', 'CcPlaylist', RelationMap::ONE_TO_MANY, array('id' => 'creator_id', ), null, null); + $this->addRelation('CcPlaylist', 'CcPlaylist', RelationMap::ONE_TO_MANY, array('id' => 'creator_id', ), 'CASCADE', null); $this->addRelation('CcBlock', 'CcBlock', RelationMap::ONE_TO_MANY, array('id' => 'creator_id', ), null, null); $this->addRelation('CcPref', 'CcPref', RelationMap::ONE_TO_MANY, array('id' => 'subjid', ), 'CASCADE', null); $this->addRelation('CcSess', 'CcSess', RelationMap::ONE_TO_MANY, array('id' => 'userid', ), 'CASCADE', null); diff --git a/airtime_mvc/application/models/airtime/om/BaseCcSubjsPeer.php b/airtime_mvc/application/models/airtime/om/BaseCcSubjsPeer.php index 68403579f..dbd9978d7 100644 --- a/airtime_mvc/application/models/airtime/om/BaseCcSubjsPeer.php +++ b/airtime_mvc/application/models/airtime/om/BaseCcSubjsPeer.php @@ -404,6 +404,9 @@ abstract class BaseCcSubjsPeer { // Invalidate objects in CcShowHostsPeer instance pool, // since one or more of them may be deleted by ON DELETE CASCADE/SETNULL rule. CcShowHostsPeer::clearInstancePool(); + // Invalidate objects in CcPlaylistPeer instance pool, + // since one or more of them may be deleted by ON DELETE CASCADE/SETNULL rule. + CcPlaylistPeer::clearInstancePool(); // Invalidate objects in CcPrefPeer instance pool, // since one or more of them may be deleted by ON DELETE CASCADE/SETNULL rule. CcPrefPeer::clearInstancePool(); diff --git a/airtime_mvc/build/sql/schema.sql b/airtime_mvc/build/sql/schema.sql index 597ebf417..d7e1107a4 100644 --- a/airtime_mvc/build/sql/schema.sql +++ b/airtime_mvc/build/sql/schema.sql @@ -105,8 +105,6 @@ CREATE INDEX "cc_files_md5_idx" ON "cc_files" ("md5"); CREATE INDEX "cc_files_name_idx" ON "cc_files" ("name"); -CREATE INDEX "cc_files_file_exists_idx" ON "cc_files" ("file_exists"); - ----------------------------------------------------------------------------- -- cc_perms ----------------------------------------------------------------------------- @@ -691,7 +689,7 @@ ALTER TABLE "cc_show_hosts" ADD CONSTRAINT "cc_perm_show_fkey" FOREIGN KEY ("sho ALTER TABLE "cc_show_hosts" ADD CONSTRAINT "cc_perm_host_fkey" FOREIGN KEY ("subjs_id") REFERENCES "cc_subjs" ("id") ON DELETE CASCADE; -ALTER TABLE "cc_playlist" ADD CONSTRAINT "cc_playlist_createdby_fkey" FOREIGN KEY ("creator_id") REFERENCES "cc_subjs" ("id"); +ALTER TABLE "cc_playlist" ADD CONSTRAINT "cc_playlist_createdby_fkey" FOREIGN KEY ("creator_id") REFERENCES "cc_subjs" ("id") ON DELETE CASCADE; ALTER TABLE "cc_playlistcontents" ADD CONSTRAINT "cc_playlistcontents_file_id_fkey" FOREIGN KEY ("file_id") REFERENCES "cc_files" ("id") ON DELETE CASCADE; From 943d5f2a72e0256abc24d29a5fd1d9a9120cef0d Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Tue, 23 Oct 2012 13:52:11 -0400 Subject: [PATCH 030/108] cc-4304: fixed ticket by deleting playlists when user is deleted. also removed useless index on file_exists -add database modifications to upgrade.sql script --- .../upgrades/airtime-2.2.0/data/upgrade.sql | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/install_minimal/upgrades/airtime-2.2.0/data/upgrade.sql b/install_minimal/upgrades/airtime-2.2.0/data/upgrade.sql index 771c3b13c..9f524fa39 100644 --- a/install_minimal/upgrades/airtime-2.2.0/data/upgrade.sql +++ b/install_minimal/upgrades/airtime-2.2.0/data/upgrade.sql @@ -14,6 +14,13 @@ INSERT INTO cc_stream_setting (keyname, value, type) VALUES ('s2_channels', 'ste INSERT INTO cc_stream_setting (keyname, value, type) VALUES ('s3_channels', 'stereo', 'string'); +CREATE FUNCTION airtime_to_int(chartoconvert character varying) RETURNS integer + AS + 'SELECT CASE WHEN trim($1) SIMILAR TO ''[0-9]+'' THEN CAST(trim($1) AS integer) ELSE NULL END;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; + --clean up database of scheduled items that weren't properly deleted in 2.1.x --due to a bug DELETE @@ -27,14 +34,9 @@ WHERE id IN ALTER TABLE cc_files DROP CONSTRAINT cc_files_gunid_idx; -DROP TABLE cc_access; +DROP INDEX cc_files_file_exists_idx; -CREATE FUNCTION airtime_to_int(chartoconvert character varying) RETURNS integer - AS - 'SELECT CASE WHEN trim($1) SIMILAR TO ''[0-9]+'' THEN CAST(trim($1) AS integer) ELSE NULL END;' - LANGUAGE SQL - IMMUTABLE - RETURNS NULL ON NULL INPUT; +DROP TABLE cc_access; CREATE SEQUENCE cc_block_id_seq START WITH 1 @@ -176,6 +178,9 @@ ALTER TABLE cc_blockcontents ALTER TABLE cc_blockcriteria ADD CONSTRAINT cc_blockcontents_block_id_fkey FOREIGN KEY (block_id) REFERENCES cc_block(id) ON DELETE CASCADE; +ALTER TABLE cc_playlist + ADD CONSTRAINT cc_playlist_createdby_fkey FOREIGN KEY (creator_id) REFERENCES cc_subjs(id) ON DELETE CASCADE; + ALTER TABLE cc_playlistcontents ADD CONSTRAINT cc_playlistcontents_block_id_fkey FOREIGN KEY (block_id) REFERENCES cc_block(id) ON DELETE CASCADE; From a73aef6cd89a1ad06fcd08433a41b299daab4c98 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Tue, 23 Oct 2012 14:20:41 -0400 Subject: [PATCH 031/108] cc-4304: fixed ticket by deleting playlists when user is deleted. also removed useless index on file_exists -add database modifications to upgrade.sql script --- .../upgrades/airtime-2.2.0/data/upgrade.sql | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/install_minimal/upgrades/airtime-2.2.0/data/upgrade.sql b/install_minimal/upgrades/airtime-2.2.0/data/upgrade.sql index 9f524fa39..95e621874 100644 --- a/install_minimal/upgrades/airtime-2.2.0/data/upgrade.sql +++ b/install_minimal/upgrades/airtime-2.2.0/data/upgrade.sql @@ -178,9 +178,6 @@ ALTER TABLE cc_blockcontents ALTER TABLE cc_blockcriteria ADD CONSTRAINT cc_blockcontents_block_id_fkey FOREIGN KEY (block_id) REFERENCES cc_block(id) ON DELETE CASCADE; -ALTER TABLE cc_playlist - ADD CONSTRAINT cc_playlist_createdby_fkey FOREIGN KEY (creator_id) REFERENCES cc_subjs(id) ON DELETE CASCADE; - ALTER TABLE cc_playlistcontents ADD CONSTRAINT cc_playlistcontents_block_id_fkey FOREIGN KEY (block_id) REFERENCES cc_block(id) ON DELETE CASCADE; @@ -190,6 +187,33 @@ ALTER TABLE cc_schedule ALTER TABLE cc_webstream_metadata ADD CONSTRAINT cc_schedule_inst_fkey FOREIGN KEY (instance_id) REFERENCES cc_schedule(id) ON DELETE CASCADE; + + + +ALTER TABLE cc_playlist + DROP CONSTRAINT cc_playlist_createdby_fkey; + +ALTER SEQUENCE cc_block_id_seq + OWNED BY cc_block.id; + +ALTER SEQUENCE cc_blockcontents_id_seq + OWNED BY cc_blockcontents.id; + +ALTER SEQUENCE cc_blockcriteria_id_seq + OWNED BY cc_blockcriteria.id; + +ALTER SEQUENCE cc_webstream_id_seq + OWNED BY cc_webstream.id; + +ALTER SEQUENCE cc_webstream_metadata_id_seq + OWNED BY cc_webstream_metadata.id; + +ALTER TABLE cc_playlist + ADD CONSTRAINT cc_playlist_createdby_fkey FOREIGN KEY (creator_id) REFERENCES cc_subjs(id) ON DELETE CASCADE; + + + + DROP FUNCTION airtime_to_int(chartoconvert character varying); UPDATE cc_files From 4daaa776e27d820a8665e3d9669e1eaefd2ec8cf Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Tue, 23 Oct 2012 17:45:55 -0400 Subject: [PATCH 032/108] CC-4620: Now Playing page loading is very slow because 1M records in cc_schedule table -fixed --- airtime_mvc/application/models/Schedule.php | 45 +++++++++++-------- .../application/models/ShowBuilder.php | 3 +- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/airtime_mvc/application/models/Schedule.php b/airtime_mvc/application/models/Schedule.php index 233128433..b668f13e5 100644 --- a/airtime_mvc/application/models/Schedule.php +++ b/airtime_mvc/application/models/Schedule.php @@ -263,6 +263,15 @@ SQL; global $CC_CONFIG; $con = Propel::getConnection(); + $p_start_str = $p_start->format("Y-m-d H:i:s"); + $p_end_str = $p_end->format("Y-m-d H:i:s"); + + + //We need to search 24 hours before and after the show times so that that we + //capture all of the show's contents. + $p_track_start= $p_start->sub(new DateInterval("PT24H"))->format("Y-m-d H:i:s"); + $p_track_end = $p_end->add(new DateInterval("PT24H"))->format("Y-m-d H:i:s"); + $templateSql = <<= '{$p_start}' - AND sched.starts < '{$p_end}') - OR (sched.ends > '{$p_start}' - AND sched.ends <= '{$p_end}') - OR (sched.starts <= '{$p_start}' - AND sched.ends >= '{$p_end}')) + AND ((sched.starts >= '{$p_track_start}' + AND sched.starts < '{$p_track_end}') + OR (sched.ends > '{$p_track_start}' + AND sched.ends <= '{$p_track_end}') + OR (sched.starts <= '{$p_track_start}' + AND sched.ends >= '{$p_track_end}')) ) SQL; @@ -315,12 +324,12 @@ SQL; $streamJoin = <<= '{$p_start}' - AND sched.starts < '{$p_end}') - OR (sched.ends > '{$p_start}' - AND sched.ends <= '{$p_end}') - OR (sched.starts <= '{$p_start}' - AND sched.ends >= '{$p_end}')) + AND ((sched.starts >= '{$p_track_start}' + AND sched.starts < '{$p_track_end}') + OR (sched.ends > '{$p_track_start}' + AND sched.ends <= '{$p_track_end}') + OR (sched.starts <= '{$p_track_start}' + AND sched.ends >= '{$p_track_end}')) ) LEFT JOIN cc_subjs AS sub ON (ws.creator_id = sub.id) SQL; @@ -358,12 +367,12 @@ SELECT showt.name AS show_name, JOIN cc_show AS showt ON (showt.id = si.show_id) WHERE si.modified_instance = FALSE $showPredicate - AND ((si.starts >= '{$p_start}' - AND si.starts < '{$p_end}') - OR (si.ends > '{$p_start}' - AND si.ends <= '{$p_end}') - OR (si.starts <= '{$p_start}' - AND si.ends >= '{$p_end}')) + AND ((si.starts >= '{$p_start_str}' + AND si.starts < '{$p_end_str}') + OR (si.ends > '{$p_start_str}' + AND si.ends <= '{$p_end_str}') + OR (si.starts <= '{$p_start_str}' + AND si.ends >= '{$p_end_str}')) ORDER BY si_starts, sched_starts; SQL; diff --git a/airtime_mvc/application/models/ShowBuilder.php b/airtime_mvc/application/models/ShowBuilder.php index 90a67a1d6..8436c3011 100644 --- a/airtime_mvc/application/models/ShowBuilder.php +++ b/airtime_mvc/application/models/ShowBuilder.php @@ -423,8 +423,7 @@ class Application_Model_ShowBuilder } $scheduled_items = Application_Model_Schedule::GetScheduleDetailItems( - $this->startDT->format("Y-m-d H:i:s"), $this->endDT->format( - "Y-m-d H:i:s"), $shows); + $this->startDT, $this->endDT, $shows); for ($i = 0, $rows = count($scheduled_items); $i < $rows; $i++) { From f9f3f9527ead8c5c76e57b7af87e463422157c95 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Thu, 25 Oct 2012 12:25:02 -0400 Subject: [PATCH 033/108] CC-4624: phone_home_stat is broken in Ubuntu 12.10, with php 5.4 -fixed --- utils/airtime-check-system.php | 2 ++ utils/airtime-log.php | 2 ++ utils/phone_home_stat.php | 2 ++ utils/soundcloud-uploader.php | 2 ++ 4 files changed, 8 insertions(+) diff --git a/utils/airtime-check-system.php b/utils/airtime-check-system.php index 91fe19c5e..39d9b19b7 100644 --- a/utils/airtime-check-system.php +++ b/utils/airtime-check-system.php @@ -2,6 +2,8 @@ AirtimeCheck::ExitIfNotRoot(); +date_default_timezone_set("UTC"); + $sapi_type = php_sapi_name(); $showColor = !in_array("--no-color", $argv); diff --git a/utils/airtime-log.php b/utils/airtime-log.php index b98cc34c0..7a1050e72 100644 --- a/utils/airtime-log.php +++ b/utils/airtime-log.php @@ -2,6 +2,8 @@ exitIfNotRoot(); +date_default_timezone_set("UTC"); + $airtimeIni = getAirtimeConf(); $airtime_base_dir = $airtimeIni['general']['airtime_dir']; diff --git a/utils/phone_home_stat.php b/utils/phone_home_stat.php index 47929f5f7..d25c546f7 100644 --- a/utils/phone_home_stat.php +++ b/utils/phone_home_stat.php @@ -2,6 +2,8 @@ exitIfNotRoot(); +date_default_timezone_set("UTC"); + $values = parse_ini_file('/etc/airtime/airtime.conf', true); // Name of the web server user diff --git a/utils/soundcloud-uploader.php b/utils/soundcloud-uploader.php index 2660d32e2..2b4615a8c 100644 --- a/utils/soundcloud-uploader.php +++ b/utils/soundcloud-uploader.php @@ -1,4 +1,6 @@ Date: Thu, 25 Oct 2012 16:04:12 -0400 Subject: [PATCH 034/108] added throttling --- python_apps/media-monitor2/media/monitor/organizer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/monitor/organizer.py b/python_apps/media-monitor2/media/monitor/organizer.py index 713bd2156..0532b79b0 100644 --- a/python_apps/media-monitor2/media/monitor/organizer.py +++ b/python_apps/media-monitor2/media/monitor/organizer.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import time import media.monitor.pure as mmp import media.monitor.owners as owners from media.monitor.handler import ReportHandler @@ -72,6 +72,8 @@ class Organizer(ReportHandler,Loggable): directory=d) return cb + time.sleep(0.1) + mmp.magic_move(event.path, new_path, after_dir_make=new_dir_watch(dirname(new_path))) From 9811c670195b6321d5ba01d2df24680fd0cfb912 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 16:10:33 -0400 Subject: [PATCH 035/108] more throttling --- python_apps/media-monitor2/media/monitor/organizer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python_apps/media-monitor2/media/monitor/organizer.py b/python_apps/media-monitor2/media/monitor/organizer.py index 0532b79b0..176a4f2a4 100644 --- a/python_apps/media-monitor2/media/monitor/organizer.py +++ b/python_apps/media-monitor2/media/monitor/organizer.py @@ -77,6 +77,8 @@ class Organizer(ReportHandler,Loggable): mmp.magic_move(event.path, new_path, after_dir_make=new_dir_watch(dirname(new_path))) + time.sleep(0.1) + # The reason we need to go around saving the owner in this ass # backwards way is bewcause we are unable to encode the owner id # into the file itself so that the StoreWatchListener listener can From b89862f304e15c9a1086aea3ee988a3104a6e1ea Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 16:34:00 -0400 Subject: [PATCH 036/108] tweaked throttling --- python_apps/media-monitor2/media/monitor/organizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/organizer.py b/python_apps/media-monitor2/media/monitor/organizer.py index 176a4f2a4..bb3c0cb02 100644 --- a/python_apps/media-monitor2/media/monitor/organizer.py +++ b/python_apps/media-monitor2/media/monitor/organizer.py @@ -72,12 +72,12 @@ class Organizer(ReportHandler,Loggable): directory=d) return cb - time.sleep(0.1) + time.sleep(0.05) mmp.magic_move(event.path, new_path, after_dir_make=new_dir_watch(dirname(new_path))) - time.sleep(0.1) + time.sleep(0.05) # The reason we need to go around saving the owner in this ass # backwards way is bewcause we are unable to encode the owner id From 5579a99d2bb8051604d649811622edb475b34f47 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Thu, 25 Oct 2012 16:57:02 -0400 Subject: [PATCH 037/108] added changelog for 2.2.0 --- changelog | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/changelog b/changelog index 96eb74af9..115265768 100644 --- a/changelog +++ b/changelog @@ -1,3 +1,14 @@ +2.2.0 - October 25th, 2012 + * New features + * Smart Playlists + * Webstream rebroadcasts + * Replaygain support + * FLAC + WAV support (AAC if you compile your own Liquidsoap) + * Huge performance increase on library import + * User ownership of files + * Stereo/mono streams + * Rescan watched folders button (useful for network drives where keeping in sync is more difficult) + 2.1.3 - July 4th, 2012 * Changes * Clarify inputs and output labels under stream settings From 5b051aae32c5c8179ac9ab1aeb73543cb1256060 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 17:20:02 -0400 Subject: [PATCH 038/108] changed throttling --- python_apps/media-monitor2/media/monitor/listeners.py | 6 ++++-- python_apps/media-monitor2/media/monitor/manager.py | 2 +- python_apps/media-monitor2/media/monitor/organizer.py | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/listeners.py b/python_apps/media-monitor2/media/monitor/listeners.py index 72b8ca8aa..d30a65147 100644 --- a/python_apps/media-monitor2/media/monitor/listeners.py +++ b/python_apps/media-monitor2/media/monitor/listeners.py @@ -48,11 +48,13 @@ class BaseListener(object): class OrganizeListener(BaseListener, pyinotify.ProcessEvent, Loggable): def process_IN_CLOSE_WRITE(self, event): #self.logger.info("===> handling: '%s'" % str(event)) - self.process_to_organize(event) + #self.process_to_organize(event) + pass # got cookie def process_IN_MOVED_TO(self, event): #self.logger.info("===> handling: '%s'" % str(event)) - self.process_to_organize(event) + #self.process_to_organize(event) + pass def process_default(self, event): pass diff --git a/python_apps/media-monitor2/media/monitor/manager.py b/python_apps/media-monitor2/media/monitor/manager.py index 77bcf10ec..ec705d5af 100644 --- a/python_apps/media-monitor2/media/monitor/manager.py +++ b/python_apps/media-monitor2/media/monitor/manager.py @@ -18,7 +18,7 @@ class ManagerTimeout(threading.Thread,Loggable): secnods. This used to be just a work around for cc-4235 but recently became a permanent solution because it's "cheap" and reliable """ - def __init__(self, manager, interval=3): + def __init__(self, manager, interval=1.5): # TODO : interval should be read from config and passed here instead # of just using the hard coded value threading.Thread.__init__(self) diff --git a/python_apps/media-monitor2/media/monitor/organizer.py b/python_apps/media-monitor2/media/monitor/organizer.py index bb3c0cb02..c6bda1cfa 100644 --- a/python_apps/media-monitor2/media/monitor/organizer.py +++ b/python_apps/media-monitor2/media/monitor/organizer.py @@ -72,12 +72,12 @@ class Organizer(ReportHandler,Loggable): directory=d) return cb - time.sleep(0.05) + time.sleep(0.02) mmp.magic_move(event.path, new_path, after_dir_make=new_dir_watch(dirname(new_path))) - time.sleep(0.05) + time.sleep(0.02) # The reason we need to go around saving the owner in this ass # backwards way is bewcause we are unable to encode the owner id From 3dd5b2869db158a3e77f238f883550ac67f343fa Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 18:38:15 -0400 Subject: [PATCH 039/108] Added lsof to check if file is locked --- .../media-monitor2/media/monitor/listeners.py | 17 ++++++----------- .../media-monitor2/media/monitor/manager.py | 4 ++-- .../media-monitor2/media/monitor/organizer.py | 5 ----- .../media-monitor2/media/monitor/pure.py | 7 +++++++ 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/listeners.py b/python_apps/media-monitor2/media/monitor/listeners.py index d30a65147..4c860a97e 100644 --- a/python_apps/media-monitor2/media/monitor/listeners.py +++ b/python_apps/media-monitor2/media/monitor/listeners.py @@ -48,17 +48,11 @@ class BaseListener(object): class OrganizeListener(BaseListener, pyinotify.ProcessEvent, Loggable): def process_IN_CLOSE_WRITE(self, event): #self.logger.info("===> handling: '%s'" % str(event)) - #self.process_to_organize(event) - pass - # got cookie + self.process_to_organize(event) + def process_IN_MOVED_TO(self, event): #self.logger.info("===> handling: '%s'" % str(event)) - #self.process_to_organize(event) - pass - - def process_default(self, event): - pass - #self.logger.info("===> Not handling: '%s'" % str(event)) + self.process_to_organize(event) def flush_events(self, path): """ @@ -69,8 +63,9 @@ class OrganizeListener(BaseListener, pyinotify.ProcessEvent, Loggable): for f in mmp.walk_supported(path, clean_empties=True): self.logger.info("Bootstrapping: File in 'organize' directory: \ '%s'" % f) - dispatcher.send(signal=self.signal, sender=self, - event=OrganizeFile(f)) + if not mmp.file_locked(f): + dispatcher.send(signal=self.signal, sender=self, + event=OrganizeFile(f)) flushed += 1 #self.logger.info("Flushed organized directory with %d files" % flushed) diff --git a/python_apps/media-monitor2/media/monitor/manager.py b/python_apps/media-monitor2/media/monitor/manager.py index ec705d5af..33dc90468 100644 --- a/python_apps/media-monitor2/media/monitor/manager.py +++ b/python_apps/media-monitor2/media/monitor/manager.py @@ -26,7 +26,7 @@ class ManagerTimeout(threading.Thread,Loggable): self.interval = interval def run(self): while True: - time.sleep(self.interval) # every 3 seconds + time.sleep(self.interval) self.manager.flush_organize() class Manager(Loggable): @@ -178,7 +178,7 @@ class Manager(Loggable): # the OrganizeListener instance will walk path and dispatch an organize # event for every file in that directory self.organize['organize_listener'].flush_events(new_path) - self.__add_watch(new_path, self.organize['organize_listener']) + #self.__add_watch(new_path, self.organize['organize_listener']) def flush_organize(self): path = self.organize['organize_path'] diff --git a/python_apps/media-monitor2/media/monitor/organizer.py b/python_apps/media-monitor2/media/monitor/organizer.py index c6bda1cfa..ea6851356 100644 --- a/python_apps/media-monitor2/media/monitor/organizer.py +++ b/python_apps/media-monitor2/media/monitor/organizer.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import time import media.monitor.pure as mmp import media.monitor.owners as owners from media.monitor.handler import ReportHandler @@ -72,13 +71,9 @@ class Organizer(ReportHandler,Loggable): directory=d) return cb - time.sleep(0.02) - mmp.magic_move(event.path, new_path, after_dir_make=new_dir_watch(dirname(new_path))) - time.sleep(0.02) - # The reason we need to go around saving the owner in this ass # backwards way is bewcause we are unable to encode the owner id # into the file itself so that the StoreWatchListener listener can diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index ec8f1695a..877cbdc72 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import copy +from subprocess import Popen, PIPE import subprocess import os import math @@ -165,6 +166,12 @@ def walk_supported(directory, clean_empties=False): for fp in full_paths: yield fp if clean_empties: clean_empty_dirs(directory) + +def file_locked(path): + cmd = "lsof %s" % path + f = Popen(cmd, shell=True, stdout=PIPE).stdout + return bool(f.readlines()) + def magic_move(old, new, after_dir_make=lambda : None): """ Moves path old to new and constructs the necessary to directories for new From 4cb4b2d3d5b0af5df461dbe0da84b27fb95d7904 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 26 Oct 2012 00:04:38 -0400 Subject: [PATCH 040/108] updated credits file for 2.2.0 --- CREDITS | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CREDITS b/CREDITS index d303519ef..2594f4dc5 100644 --- a/CREDITS +++ b/CREDITS @@ -1,3 +1,32 @@ +======= +CREDITS +======= +Version 2.2.0 +------------- +Martin Konecny (martin.konecny@sourcefabric.org) + Role: Developer Team Lead + +Naomi Aro (naomi.aro@sourcefabric.org) + Role: Software Developer + +James Moon (james.moon@sourcefabric.org) + Role: Software Developer + +Denise Rigato (denise.rigato@sourcefabric.org) + Role: Software Developer + +Rudi Grinberg (rudi.grinberg@sourcefabric.org) + Role: Software Developer + +Cliff Wang (cliff.wang@sourcefabric.org) + Role: QA + +Mikayel Karapetian (michael.karapetian@sourcefabric.org) + Role: QA + +Daniel James (daniel.james@sourcefabric.org) + Role: Documentor & QA + ======= CREDITS ======= From e5bbed378213581cb1a0cdba5228e9638f7e8ba7 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 26 Oct 2012 00:35:44 -0400 Subject: [PATCH 041/108] remove CR line endings --- widgets/sample_page.html | 114 +++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/widgets/sample_page.html b/widgets/sample_page.html index 94cc51396..cba9c37ed 100644 --- a/widgets/sample_page.html +++ b/widgets/sample_page.html @@ -1,57 +1,57 @@ - - - - -Airtime widgets - - - - - - - -

"Now Playing" Widget

-This widget displays what is currently playing: -
-
-
-
-
-

"Today's Program" Widget

-This widget displays what is being played today: -
-
-
-
-
-

"Weekly Program" Widget

-This widget displays all the shows for the entire week: -
-
-
- - - + + + + +Airtime widgets + + + + + + + +

"Now Playing" Widget

+This widget displays what is currently playing: +
+
+
+
+
+

"Today's Program" Widget

+This widget displays what is being played today: +
+
+
+
+
+

"Weekly Program" Widget

+This widget displays all the shows for the entire week: +
+
+
+ + + From e285bcfba1e64266c3f53577cbcab2a85e29142f Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 26 Oct 2012 12:54:10 -0400 Subject: [PATCH 042/108] update liquidsoap library scripts -git commit hash f776e7f4c5d3729cbcf863381f3cadd96c578a5b --- .../library/external-todo.liq | 314 ------------------ .../liquidsoap_scripts/library/externals.liq | 1 + .../pypo/liquidsoap_scripts/library/http.liq | 16 +- .../liquidsoap_scripts/library/pervasives.liq | 3 +- .../pypo/liquidsoap_scripts/library/utils.liq | 59 ++-- .../liquidsoap_scripts/library/video_text.liq | 2 + 6 files changed, 46 insertions(+), 349 deletions(-) delete mode 100644 python_apps/pypo/liquidsoap_scripts/library/external-todo.liq diff --git a/python_apps/pypo/liquidsoap_scripts/library/external-todo.liq b/python_apps/pypo/liquidsoap_scripts/library/external-todo.liq deleted file mode 100644 index 24ae18cdc..000000000 --- a/python_apps/pypo/liquidsoap_scripts/library/external-todo.liq +++ /dev/null @@ -1,314 +0,0 @@ -# These operators need to be updated.. - - -# Stream data from mplayer -# @category Source / Input -# @param s data URI. -# @param ~restart restart on exit. -# @param ~restart_on_error restart on exit with error. -# @param ~buffer Duration of the pre-buffered data. -# @param ~max Maximum duration of the buffered data. -def input.mplayer(~id="input.mplayer", - ~restart=true,~restart_on_error=false, - ~buffer=0.2,~max=10.,s) = - input.external(id=id,restart=restart, - restart_on_error=restart_on_error, - buffer=buffer,max=max, - "mplayer -really-quiet -ao pcm:file=/dev/stdout \ - -vc null -vo null #{quote(s)} 2>/dev/null") -end - - -# Output the stream using aplay. -# Using this turns "root.sync" to false -# since aplay will do the synchronisation -# @category Source / Output -# @param ~id Output's ID -# @param ~device Alsa pcm device name -# @param ~restart_on_crash Restart external process on crash. If false, liquidsoap will stop. -# @param ~fallible Allow the child source to fail, in which case the output will be (temporarily) stopped. -# @param ~on_start Callback executed when outputting starts. -# @param ~on_stop Callback executed when outputting stops. -# @param s Source to play -def output.aplay(~id="output.aplay",~device="default", - ~fallible=false,~on_start={()},~on_stop={()}, - ~restart_on_crash=false,s) - def aplay_p(m) = - "aplay -D #{device}" - end - log(label=id,level=3,"Setting root.sync to false") - set("root.sync",false) - output.pipe.external(id=id, - fallible=fallible,on_start=on_start,on_stop=on_stop, - restart_on_crash=restart_on_crash, - restart_on_new_track=false, - process=aplay_p,s) -end - -%ifdef output.icecast.external -# Output to icecast using the lame command line encoder. -# @category Source / Output -# @param ~id Output's ID -# @param ~start Start output threads on operator initialization. -# @param ~restart Restart output after a failure. By default, liquidsoap will stop if the output failed. -# @param ~restart_delay Delay, in seconds, before attempting new connection, if restart is enabled. -# @param ~restart_on_crash Restart external process on crash. If false, liquidsoap will stop. -# @param ~restart_on_new_track Restart encoder upon new track. -# @param ~restart_encoder_delay Restart the encoder after this delay, in seconds. -# @param ~user User for shout source connection. Useful only in special cases, like with per-mountpoint users. -# @param ~lame The lame binary -# @param ~bitrate Encoder bitrate -# @param ~swap Swap audio samples. Depends on local machine's endianess and lame's version. Test this parameter if you experience garbaged mp3 audio data. On intel 32 and 64 architectures, the parameter should be "true" for lame version >= 3.98. -# @param ~dumpfile Dump stream to file, for debugging purpose. Disabled if empty. -# @param ~protocol Protocol of the streaming server: 'http' for Icecast, 'icy' for Shoutcast. -# @param ~fallible Allow the child source to fail, in which case the output will be (temporarily) stopped. -# @param ~on_start Callback executed when outputting starts. -# @param ~on_stop Callback executed when outputting stops. -# @param s The source to output -def output.icecast.lame( - ~id="output.icecast.lame",~start=true, - ~restart=false,~restart_delay=3, - ~host="localhost",~port=8000, - ~user="source",~password="hackme", - ~genre="Misc",~url="http://savonet.sf.net/", - ~description="Liquidsoap Radio!",~public=true, - ~dumpfile="",~mount="Use [name]", - ~name="Use [mount]",~protocol="http", - ~lame="lame",~bitrate=128,~swap=false, - ~fallible=false,~on_start={()},~on_stop={()}, - ~restart_on_crash=false,~restart_on_new_track=false, - ~restart_encoder_delay=3600,~headers=[],s) - samplerate = get(default=44100,"frame.samplerate") - samplerate = float_of_int(samplerate) / 1000. - channels = get(default=2,"frame.channels") - swap = if swap then "-x" else "" end - mode = - if channels == 2 then - "j" # Encoding in joint stereo.. - else - "m" - end - # Metadata update is set by ICY with icecast - def lame_p(m) - "#{lame} -b #{bitrate} -r --bitwidth 16 -s #{samplerate} \ - --signed -m #{mode} --nores #{swap} -t - -" - end - output.icecast.external(id=id, - process=lame_p,bitrate=bitrate,start=start, - restart=restart,restart_delay=restart_delay, - host=host,port=port,user=user,password=password, - genre=genre,url=url,description=description, - public=public,dumpfile=dumpfile,restart_encoder_delay=restart_encoder_delay, - name=name,mount=mount,protocol=protocol, - header=false,restart_on_crash=restart_on_crash, - restart_on_new_track=restart_on_new_track,headers=headers, - fallible=fallible,on_start=on_start,on_stop=on_stop, - s) -end - -# Output to shoutcast using the lame encoder. -# @category Source / Output -# @param ~id Output's ID -# @param ~start Start output threads on operator initialization. -# @param ~restart Restart output after a failure. By default, liquidsoap will stop if the output failed. -# @param ~restart_delay Delay, in seconds, before attempting new connection, if restart is enabled. -# @param ~restart_on_crash Restart external process on crash. If false, liquidsoap will stop. -# @param ~restart_on_new_track Restart encoder upon new track. -# @param ~restart_encoder_delay Restart the encoder after this delay, in seconds. -# @param ~user User for shout source connection. Useful only in special cases, like with per-mountpoint users. -# @param ~lame The lame binary -# @param ~bitrate Encoder bitrate -# @param ~icy_reset Reset shoutcast source buffer upon connecting (necessary for NSV). -# @param ~dumpfile Dump stream to file, for debugging purpose. Disabled if empty. -# @param ~fallible Allow the child source to fail, in which case the output will be (temporarily) stopped. -# @param ~on_start Callback executed when outputting starts. -# @param ~on_stop Callback executed when outputting stops. -# @param s The source to output -def output.shoutcast.lame( - ~id="output.shoutcast.mp3",~start=true, - ~restart=false,~restart_delay=3, - ~host="localhost",~port=8000, - ~user="source",~password="hackme", - ~genre="Misc",~url="http://savonet.sf.net/", - ~description="Liquidsoap Radio!",~public=true, - ~dumpfile="",~name="Use [mount]",~icy_reset=true, - ~lame="lame",~aim="",~icq="",~irc="", - ~fallible=false,~on_start={()},~on_stop={()}, - ~restart_on_crash=false,~restart_on_new_track=false, - ~restart_encoder_delay=3600,~bitrate=128,s) = - icy_reset = if icy_reset then "1" else "0" end - headers = [("icy-aim",aim),("icy-irc",irc), - ("icy-icq",icq),("icy-reset",icy_reset)] - output.icecast.lame( - id=id, headers=headers, lame=lame, - bitrate=bitrate, start=start, - restart=restart, restart_encoder_delay=restart_encoder_delay, - host=host, port=port, user=user, password=password, - genre=genre, url=url, description=description, - public=public, dumpfile=dumpfile, - restart_on_crash=restart_on_crash, - restart_on_new_track=restart_on_new_track, - name=name, mount="/", protocol="icy", - fallible=fallible,on_start=on_start,on_stop=on_stop, - s) -end - -# Output to icecast using the flac command line encoder. -# @category Source / Output -# @param ~id Output's ID -# @param ~start Start output threads on operator initialization. -# @param ~restart Restart output after a failure. By default, liquidsoap will stop if the output failed. -# @param ~restart_delay Delay, in seconds, before attempting new connection, if restart is enabled. -# @param ~restart_on_crash Restart external process on crash. If false, liquidsoap will stop. -# @param ~restart_on_new_track Restart encoder upon new track. If false, the resulting stream will have a single track. -# @param ~restart_encoder_delay Restart the encoder after this delay, in seconds. -# @param ~user User for shout source connection. Useful only in special cases, like with per-mountpoint users. -# @param ~flac The flac binary -# @param ~quality Encoder quality (0..8) -# @param ~dumpfile Dump stream to file, for debugging purpose. Disabled if empty. -# @param ~protocol Protocol of the streaming server: 'http' for Icecast, 'icy' for Shoutcast. -# @param ~fallible Allow the child source to fail, in which case the output will be (temporarily) stopped. -# @param ~on_start Callback executed when outputting starts. -# @param ~on_stop Callback executed when outputting stops. -# @param s The source to output -def output.icecast.flac( - ~id="output.icecast.flac",~start=true, - ~restart=false,~restart_delay=3, - ~host="localhost",~port=8000, - ~user="source",~password="hackme", - ~genre="Misc",~url="http://savonet.sf.net/", - ~description="Liquidsoap Radio!",~public=true, - ~dumpfile="",~mount="Use [name]", - ~name="Use [mount]",~protocol="http", - ~flac="flac",~quality=6, - ~restart_on_crash=false, - ~restart_on_new_track=true, - ~restart_encoder_delay=(-1), - ~fallible=false,~on_start={()},~on_stop={()}, - s) - # We will use raw format, to - # bypass input length value in WAV - # header (input length is not known) - channels = get(default=2,"frame.channels") - samplerate = get(default=44100,"frame.samplerate") - def flac_p(m)= - def option(x) = - "-T #{quote(fst(x))}=#{quote(snd(x))}" - end - m = list.map(option,m) - m = string.concat(separator=" ",m) - "#{flac} --force-raw-format --endian=little --channels=#{channels} \ - --bps=16 --sample-rate=#{samplerate} --sign=signed #{m} \ - -#{quality} --ogg -c -" - end - output.icecast.external(id=id, - process=flac_p,bitrate=(-1),start=start, - restart=restart,restart_delay=restart_delay, - host=host,port=port,user=user,password=password, - genre=genre,url=url,description=description, - public=public,dumpfile=dumpfile, - name=name,mount=mount,protocol=protocol, - fallible=fallible,on_start=on_start,on_stop=on_stop, - restart_on_new_track=restart_on_new_track, - format="ogg",header=false,icy_metadata=false, - restart_on_crash=restart_on_crash, - restart_encoder_delay=restart_encoder_delay, - s) -end - -# Output to icecast using the aacplusenc command line encoder. -# @category Source / Output -# @param ~id Output's ID -# @param ~start Start output threads on operator initialization. -# @param ~restart Restart output after a failure. By default, liquidsoap will stop if the output failed. -# @param ~restart_delay Delay, in seconds, before attempting new connection, if restart is enabled. -# @param ~restart_on_crash Restart external process on crash. If false, liquidsoap will stop. -# @param ~restart_on_new_track Restart encoder upon new track. -# @param ~restart_encoder_delay Restart the encoder after this delay, in seconds. -# @param ~user User for shout source connection. Useful only in special cases, like with per-mountpoint users. -# @param ~aacplusenc The aacplusenc binary -# @param ~bitrate Encoder bitrate -# @param ~dumpfile Dump stream to file, for debugging purpose. Disabled if empty. -# @param ~protocol Protocol of the streaming server: 'http' for Icecast, 'icy' for Shoutcast. -# @param ~fallible Allow the child source to fail, in which case the output will be (temporarily) stopped. -# @param ~on_start Callback executed when outputting starts. -# @param ~on_stop Callback executed when outputting stops. -# @param s The source to output -def output.icecast.aacplusenc( - ~id="output.icecast.aacplusenc",~start=true, - ~restart=false,~restart_delay=3, - ~host="localhost",~port=8000, - ~user="source",~password="hackme", - ~genre="Misc",~url="http://savonet.sf.net/", - ~description="Liquidsoap Radio!",~public=true, - ~dumpfile="",~mount="Use [name]", - ~name="Use [mount]",~protocol="http", - ~aacplusenc="aacplusenc",~bitrate=64, - ~fallible=false,~on_start={()},~on_stop={()}, - ~restart_on_crash=false,~restart_on_new_track=false, - ~restart_encoder_delay=3600,~headers=[],s) - # Metadata update is set by ICY with icecast - def aacplusenc_p(m) - "#{aacplusenc} - - #{bitrate}" - end - output.icecast.external(id=id, - process=aacplusenc_p,bitrate=bitrate,start=start, - restart=restart,restart_delay=restart_delay, - host=host,port=port,user=user,password=password, - genre=genre,url=url,description=description, - public=public,dumpfile=dumpfile, - name=name,mount=mount,protocol=protocol, - fallible=fallible,on_start=on_start,on_stop=on_stop, - header=true,restart_on_crash=restart_on_crash, - restart_on_new_track=restart_on_new_track,headers=headers, - restart_encoder_delay=restart_encoder_delay,format="audio/aacp",s) -end - -# Output to shoutcast using the aacplusenc encoder. -# @category Source / Output -# @param ~id Output's ID -# @param ~start Start output threads on operator initialization. -# @param ~restart Restart output after a failure. By default, liquidsoap will stop if the output failed. -# @param ~restart_delay Delay, in seconds, before attempting new connection, if restart is enabled. -# @param ~restart_on_crash Restart external process on crash. If false, liquidsoap will stop. -# @param ~restart_on_new_track Restart encoder upon new track. -# @param ~restart_encoder_delay Restart the encoder after this delay, in seconds. -# @param ~user User for shout source connection. Useful only in special cases, like with per-mountpoint users. -# @param ~aacplusenc The aacplusenc binary -# @param ~bitrate Encoder bitrate -# @param ~icy_reset Reset shoutcast source buffer upon connecting (necessary for NSV). -# @param ~dumpfile Dump stream to file, for debugging purpose. Disabled if empty. -# @param ~fallible Allow the child source to fail, in which case the output will be (temporarily) stopped. -# @param ~on_start Callback executed when outputting starts. -# @param ~on_stop Callback executed when outputting stops. -# @param s The source to output -def output.shoutcast.aacplusenc( - ~id="output.shoutcast.aacplusenc",~start=true, - ~restart=false,~restart_delay=3, - ~host="localhost",~port=8000, - ~user="source",~password="hackme", - ~genre="Misc",~url="http://savonet.sf.net/", - ~description="Liquidsoap Radio!",~public=true, - ~fallible=false,~on_start={()},~on_stop={()}, - ~dumpfile="",~name="Use [mount]",~icy_reset=true, - ~aim="",~icq="",~irc="",~aacplusenc="aacplusenc", - ~restart_on_crash=false,~restart_on_new_track=false, - ~restart_encoder_delay=3600,~bitrate=64,s) = - icy_reset = if icy_reset then "1" else "0" end - headers = [("icy-aim",aim),("icy-irc",irc), - ("icy-icq",icq),("icy-reset",icy_reset)] - output.icecast.aacplusenc( - id=id, headers=headers, aacplusenc=aacplusenc, - bitrate=bitrate, start=start, - restart=restart, restart_delay=restart_delay, - host=host, port=port, user=user, password=password, - genre=genre, url=url, description=description, - public=public, dumpfile=dumpfile, - fallible=fallible,on_start=on_start,on_stop=on_stop, - restart_on_crash=restart_on_crash, restart_encoder_delay=restart_encoder_delay, - restart_on_new_track=restart_on_new_track, - name=name, mount="/", protocol="icy", - s) -end -%endif - diff --git a/python_apps/pypo/liquidsoap_scripts/library/externals.liq b/python_apps/pypo/liquidsoap_scripts/library/externals.liq index 6e1b98a60..b56f13d0c 100644 --- a/python_apps/pypo/liquidsoap_scripts/library/externals.liq +++ b/python_apps/pypo/liquidsoap_scripts/library/externals.liq @@ -4,6 +4,7 @@ # Enable external Musepack decoder. Requires the # mpcdec binary in the path. Does not work on # Win32. +# @category Liquidsoap def enable_external_mpc_decoder() = # A list of know extensions and content-type for Musepack. # Values from http://en.wikipedia.org/wiki/Musepack diff --git a/python_apps/pypo/liquidsoap_scripts/library/http.liq b/python_apps/pypo/liquidsoap_scripts/library/http.liq index 109baf41e..53644ce38 100644 --- a/python_apps/pypo/liquidsoap_scripts/library/http.liq +++ b/python_apps/pypo/liquidsoap_scripts/library/http.liq @@ -13,18 +13,12 @@ def http_response(~protocol="HTTP/1.1", ~headers=[], ~data="") = status = http_codes[string_of(code)] - # Set content-length if needed and not set by the - # user. + # Set content-length and connection: close headers = - if data != "" and - not list.mem_assoc("Content-Length",headers) - then - list.append([("Content-Length", - "#{string.length(data)}")], - headers) - else - headers - end + list.append(headers, + [("Content-Length", "#{string.length(data)}"), + ("Connection", "close")]) + headers = list.map(fun (x) -> "#{fst(x)}: #{snd(x)}",headers) headers = string.concat(separator="\r\n",headers) # If no headers are provided, we should avoid diff --git a/python_apps/pypo/liquidsoap_scripts/library/pervasives.liq b/python_apps/pypo/liquidsoap_scripts/library/pervasives.liq index bd894766b..722138018 100644 --- a/python_apps/pypo/liquidsoap_scripts/library/pervasives.liq +++ b/python_apps/pypo/liquidsoap_scripts/library/pervasives.liq @@ -4,4 +4,5 @@ %include "lastfm.liq" %include "flows.liq" %include "http.liq" -%include "video_text.liq" \ No newline at end of file +%include "video_text.liq" +%include "gstreamer.liq" diff --git a/python_apps/pypo/liquidsoap_scripts/library/utils.liq b/python_apps/pypo/liquidsoap_scripts/library/utils.liq index 3d82d8997..a35af94f3 100644 --- a/python_apps/pypo/liquidsoap_scripts/library/utils.liq +++ b/python_apps/pypo/liquidsoap_scripts/library/utils.liq @@ -21,10 +21,14 @@ end # @param a Key to look for # @param l List of pairs (key,value) def list.mem_assoc(a,l) - v = list.assoc(a,l) - # We check for existence, since "" may indicate - # either a binding (a,"") or no binding.. - list.mem((a,v),l) + def f(cur, el) = + if not cur then + fst(el) == a + else + cur + end + end + list.fold(f, false, l) end # Remove a pair from an associative list @@ -164,8 +168,7 @@ def out(s) output.prefered(mksafe(s)) end -# Special track insensitive fallback that -# always skip current song before switching. +# Special track insensitive fallback that always skips current song before switching. # @category Source / Track Processing # @param ~input The input source # @param f The fallback source @@ -212,14 +215,17 @@ end # Simple crossfade. # @category Source / Track Processing # @param ~start_next Duration in seconds of the crossed end of track. -# @param ~fade_in Duration of the fade in for next track -# @param ~fade_out Duration of the fade out for previous track -# @param s The source to use -def crossfade(~id="",~start_next,~fade_in,~fade_out,s) +# @param ~fade_in Duration of the fade in for next track. +# @param ~fade_out Duration of the fade out for previous track. +# @param ~conservative Always prepare for a premature end-of-track. +# @param s The source to use. +def crossfade(~id="",~conservative=true, + ~start_next=5.,~fade_in=3.,~fade_out=3., + s) s = fade.in(duration=fade_in,s) s = fade.out(duration=fade_out,s) fader = fun (a,b) -> add(normalize=false,[b,a]) - cross(id=id,conservative=true,duration=start_next,fader,s) + cross(id=id,conservative=conservative,duration=start_next,fader,s) end # Append speech-synthesized tracks reading the metadata. @@ -242,8 +248,7 @@ def helium(s) end %endif -# Return true if process exited with 0 code. -# Command should return quickly. +# Return true if process exited with 0 code. Command should return quickly. # @category System # @param command Command to test def test_process(command) @@ -277,12 +282,9 @@ def url.split(uri) = end end -# Register a server/telnet command to -# update a source's metadata. Returns -# a new source, which will receive the -# updated metadata. It behaves just like -# the pre-1.0 insert_metadata() operator, -# i.e. insert key1="val1",key2="val2",... +# Register a server/telnet command to update a source's metadata. Returns +# a new source, which will receive the updated metadata. The command has +# the following format: insert key1="val1",key2="val2",... # @category Source / Track Processing # @param ~id Force the value of the source ID. def server.insert_metadata(~id="",s) = @@ -424,15 +426,15 @@ end # @param ~conservative Always prepare for a premature end-of-track. # @param ~default Transition used when no rule applies \ # (default: sequence). -# @param ~high Value, in dB, for loud sound level -# @param ~medium Value, in dB, for medium sound level +# @param ~high Value, in dB, for loud sound level. +# @param ~medium Value, in dB, for medium sound level. # @param ~margin Margin to detect sources that have too different \ # sound level for crossing. # @param s The input source. def smart_crossfade (~start_next=5.,~fade_in=3.,~fade_out=3., ~default=(fun (a,b) -> sequence([a, b])), ~high=-15., ~medium=-32., ~margin=4., - ~width=2.,~conservative=false,s) + ~width=2.,~conservative=true,s) fade.out = fade.out(type="sin",duration=fade_out) fade.in = fade.in(type="sin",duration=fade_in) add = fun (a,b) -> add(normalize=false,[b, a]) @@ -549,7 +551,18 @@ def playlist.reloadable(~id="",~random=false,~on_done={()},uri) if request.resolve(playlist) then playlist = request.filename(playlist) files = playlist.parse(playlist) - list.map(snd,files) + def file_request(el) = + meta = fst(el) + file = snd(el) + s = list.fold(fun (cur, el) -> + "#{cur},#{fst(el)}=#{string.escape(snd(el))}", "", meta) + if s == "" then + file + else + "annotate:#{s}:#{file}" + end + end + list.map(file_request,files) else log(label=id,"Couldn't read playlist: request resolution failed.") [] diff --git a/python_apps/pypo/liquidsoap_scripts/library/video_text.liq b/python_apps/pypo/liquidsoap_scripts/library/video_text.liq index b372a1f7b..8c017a93d 100644 --- a/python_apps/pypo/liquidsoap_scripts/library/video_text.liq +++ b/python_apps/pypo/liquidsoap_scripts/library/video_text.liq @@ -1,5 +1,6 @@ %ifdef video.add_text.gd # Add a scrolling line of text on video frames. +# @category Source / Video Processing # @param ~id Force the value of the source ID. # @param ~color Text color (in 0xRRGGBB format). # @param ~cycle Cycle text. @@ -22,6 +23,7 @@ end %ifdef video.add_text.sdl # Add a scrolling line of text on video frames. +# @category Source / Video Processing # @param ~id Force the value of the source ID. # @param ~color Text color (in 0xRRGGBB format). # @param ~cycle Cycle text. From cad6a7e7b0783e24cddbb12db40cff9fa0c07efb Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 9 Oct 2012 12:25:40 -0400 Subject: [PATCH 043/108] added perliminary stuff for emf --- python_apps/media-monitor2/media/monitor/metadata.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index bd42e434c..7b7d6d184 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -185,6 +185,15 @@ class Metadata(Loggable): # TODO : Simplify the way all of these rules are handled right now it's # extremely unclear and needs to be refactored. #if full_mutagen is None: raise BadSongFile(fpath) + try: # emf stuff for testing: + import media.metadata.process as md + import pprint.pformat as pf + if full_mutagen: + normalized = md.global_reader.read('fpath', full_mutagen) + self.logger.info(pf(normalized)) + except Exception as e: + self.logger.unexpected_exception(e) + if full_mutagen is None: full_mutagen = FakeMutagen(fpath) self.__metadata = Metadata.airtime_dict(full_mutagen) # Now we extra the special values that are calculated from the mutagen @@ -209,6 +218,7 @@ class Metadata(Loggable): # from the file? self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100) + def is_recorded(self): """ returns true if the file has been created by airtime through recording From a88b7255ff9ec2d223bacc3cfaa380471e8db26b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 9 Oct 2012 12:36:52 -0400 Subject: [PATCH 044/108] removed automatic loading of emf definitions --- .../media/metadata/definitions.py | 171 +++++++++--------- .../media-monitor2/media/monitor/metadata.py | 12 +- 2 files changed, 97 insertions(+), 86 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index 51ef6fd14..cfeb0d2fb 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -3,109 +3,116 @@ import media.monitor.process as md from os.path import normpath from media.monitor.pure import format_length, file_md5 -with md.metadata('MDATA_KEY_DURATION') as t: - t.default(u'0.0') - t.depends('length') - t.translate(lambda k: format_length(k['length'])) +defs_loaded = False -with md.metadata('MDATA_KEY_MIME') as t: - t.default(u'') - t.depends('mime') - t.translate(lambda k: k['mime'].replace('-','/')) +def is_defs_loaded(): + global defs_loaded + return defs_loaded -with md.metadata('MDATA_KEY_BITRATE') as t: - t.default(u'') - t.depends('bitrate') - t.translate(lambda k: k['bitrate']) +def load_definitions(): + with md.metadata('MDATA_KEY_DURATION') as t: + t.default(u'0.0') + t.depends('length') + t.translate(lambda k: format_length(k['length'])) -with md.metadata('MDATA_KEY_SAMPLERATE') as t: - t.default(u'0') - t.depends('sample_rate') - t.translate(lambda k: k['sample_rate']) + with md.metadata('MDATA_KEY_MIME') as t: + t.default(u'') + t.depends('mime') + t.translate(lambda k: k['mime'].replace('-','/')) -with md.metadata('MDATA_KEY_FTYPE'): - t.depends('ftype') # i don't think this field even exists - t.default(u'audioclip') - t.translate(lambda k: k['ftype']) # but just in case + with md.metadata('MDATA_KEY_BITRATE') as t: + t.default(u'') + t.depends('bitrate') + t.translate(lambda k: k['bitrate']) -with md.metadata("MDATA_KEY_CREATOR") as t: - t.depends("artist") - # A little kludge to make sure that we have some value for when we parse - # MDATA_KEY_TITLE - t.default(u"") - t.max_length(512) + with md.metadata('MDATA_KEY_SAMPLERATE') as t: + t.default(u'0') + t.depends('sample_rate') + t.translate(lambda k: k['sample_rate']) -with md.metadata("MDATA_KEY_SOURCE") as t: - t.depends("album") - t.max_length(512) + with md.metadata('MDATA_KEY_FTYPE'): + t.depends('ftype') # i don't think this field even exists + t.default(u'audioclip') + t.translate(lambda k: k['ftype']) # but just in case -with md.metadata("MDATA_KEY_GENRE") as t: - t.depends("genre") - t.max_length(64) + with md.metadata("MDATA_KEY_CREATOR") as t: + t.depends("artist") + # A little kludge to make sure that we have some value for when we parse + # MDATA_KEY_TITLE + t.default(u"") + t.max_length(512) -with md.metadata("MDATA_KEY_MOOD") as t: - t.depends("mood") - t.max_length(64) + with md.metadata("MDATA_KEY_SOURCE") as t: + t.depends("album") + t.max_length(512) -with md.metadata("MDATA_KEY_TRACKNUMBER") as t: - t.depends("tracknumber") + with md.metadata("MDATA_KEY_GENRE") as t: + t.depends("genre") + t.max_length(64) -with md.metadata("MDATA_KEY_BPM") as t: - t.depends("bpm") - t.max_length(8) + with md.metadata("MDATA_KEY_MOOD") as t: + t.depends("mood") + t.max_length(64) -with md.metadata("MDATA_KEY_LABEL") as t: - t.depends("organization") - t.max_length(512) + with md.metadata("MDATA_KEY_TRACKNUMBER") as t: + t.depends("tracknumber") -with md.metadata("MDATA_KEY_COMPOSER") as t: - t.depends("composer") - t.max_length(512) + with md.metadata("MDATA_KEY_BPM") as t: + t.depends("bpm") + t.max_length(8) -with md.metadata("MDATA_KEY_ENCODER") as t: - t.depends("encodedby") - t.max_length(512) + with md.metadata("MDATA_KEY_LABEL") as t: + t.depends("organization") + t.max_length(512) -with md.metadata("MDATA_KEY_CONDUCTOR") as t: - t.depends("conductor") - t.max_length(512) + with md.metadata("MDATA_KEY_COMPOSER") as t: + t.depends("composer") + t.max_length(512) -with md.metadata("MDATA_KEY_YEAR") as t: - t.depends("date") - t.max_length(16) + with md.metadata("MDATA_KEY_ENCODER") as t: + t.depends("encodedby") + t.max_length(512) -with md.metadata("MDATA_KEY_URL") as t: - t.depends("website") + with md.metadata("MDATA_KEY_CONDUCTOR") as t: + t.depends("conductor") + t.max_length(512) -with md.metadata("MDATA_KEY_ISRC") as t: - t.depends("isrc") - t.max_length(512) + with md.metadata("MDATA_KEY_YEAR") as t: + t.depends("date") + t.max_length(16) -with md.metadata("MDATA_KEY_COPYRIGHT") as t: - t.depends("copyright") - t.max_length(512) + with md.metadata("MDATA_KEY_URL") as t: + t.depends("website") -with md.metadata("MDATA_KEY_FILEPATH") as t: - t.depends('path') - t.translate(lambda k: normpath(k['path'])) + with md.metadata("MDATA_KEY_ISRC") as t: + t.depends("isrc") + t.max_length(512) -with md.metadata("MDATA_KEY_MD5") as t: - t.depends('path') - t.optional(False) - t.translate(lambda k: file_md5(k['path'], max_length=100)) + with md.metadata("MDATA_KEY_COPYRIGHT") as t: + t.depends("copyright") + t.max_length(512) -# owner is handled differently by (by events.py) + with md.metadata("MDATA_KEY_FILEPATH") as t: + t.depends('path') + t.translate(lambda k: normpath(k['path'])) -with md.metadata('MDATA_KEY_ORIGINAL_PATH') as t: - t.depends('original_path') + with md.metadata("MDATA_KEY_MD5") as t: + t.depends('path') + t.optional(False) + t.translate(lambda k: file_md5(k['path'], max_length=100)) -# MDATA_KEY_TITLE is the annoying special case -with md.metadata('MDATA_KEY_TITLE') as t: - # Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is - # defaulted to "" from definitions above - t.depends('title','MDATA_KEY_CREATOR') - t.max_length(512) + # owner is handled differently by (by events.py) -with md.metadata('MDATA_KEY_LABEL') as t: - t.depends('label') - t.max_length(512) + with md.metadata('MDATA_KEY_ORIGINAL_PATH') as t: + t.depends('original_path') + + # MDATA_KEY_TITLE is the annoying special case + with md.metadata('MDATA_KEY_TITLE') as t: + # Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is + # defaulted to "" from definitions above + t.depends('title','MDATA_KEY_CREATOR') + t.max_length(512) + + with md.metadata('MDATA_KEY_LABEL') as t: + t.depends('label') + t.max_length(512) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 7b7d6d184..3e7fa893d 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -11,6 +11,12 @@ from media.monitor.log import Loggable from media.monitor.pure import format_length, truncate_to_length import media.monitor.pure as mmp +# emf related stuff +from media.metadata.process import global_reader +import media.metadata.definitions as defs +from pprint import pformat +defs.load_definitions() + """ list of supported easy tags in mutagen version 1.20 ['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry', @@ -186,11 +192,9 @@ class Metadata(Loggable): # extremely unclear and needs to be refactored. #if full_mutagen is None: raise BadSongFile(fpath) try: # emf stuff for testing: - import media.metadata.process as md - import pprint.pformat as pf if full_mutagen: - normalized = md.global_reader.read('fpath', full_mutagen) - self.logger.info(pf(normalized)) + normalized = global_reader.read('fpath', full_mutagen) + self.logger.info(pformat(normalized)) except Exception as e: self.logger.unexpected_exception(e) From 10c4a71866bbb0e9c0992f93b79515077a3dcb0e Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 9 Oct 2012 14:46:01 -0400 Subject: [PATCH 045/108] Added default translator for mdata elements that don't specify it --- .../media/metadata/definitions.py | 2 +- .../media-monitor2/media/metadata/process.py | 19 ++++++++++++++++++- .../media-monitor2/media/monitor/metadata.py | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index cfeb0d2fb..68ada6034 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -import media.monitor.process as md +import media.metadata.process as md from os.path import normpath from media.monitor.pure import format_length, file_md5 diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index a4c361468..6cebc0269 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from contextlib import contextmanager from media.monitor.pure import truncate_to_length, toposort +from media.monitor.log import Loggable import mutagen @@ -8,7 +9,12 @@ class MetadataAbsent(Exception): def __init__(self, name): self.name = name def __str__(self): return "Could not obtain element '%s'" % self.name -class MetadataElement(object): +class MetadataElement(Loggable): + + def __default_translator(k): + e = [ x for x in self.dependencies() ][0] + return k[e] + def __init__(self,name): self.name = name # "Sane" defaults @@ -19,6 +25,8 @@ class MetadataElement(object): self.__is_normalized = lambda _ : True self.__max_length = -1 + + def max_length(self,l): self.__max_length = l @@ -82,6 +90,15 @@ class MetadataElement(object): if self.has_default(): return self.get_default() else: raise MetadataAbsent(self.name) # We have all dependencies. Now for actual for parsing + + # Only case where we can select a default translator + if not self.__translator: + if len(self.dependencies()) == 1: + self.translate(MetadataElement.__default_translator) + else: + self.logger.info("Could not set more than 1 translator with \ + more than 1 dependancies") + r = self.__normalizer( self.__translator(full_deps) ) if self.__max_length != -1: r = truncate_to_length(r, self.__max_length) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 3e7fa893d..6cce20e34 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -196,7 +196,7 @@ class Metadata(Loggable): normalized = global_reader.read('fpath', full_mutagen) self.logger.info(pformat(normalized)) except Exception as e: - self.logger.unexpected_exception(e) + self.unexpected_exception(e) if full_mutagen is None: full_mutagen = FakeMutagen(fpath) self.__metadata = Metadata.airtime_dict(full_mutagen) From b7b11feae00dd1ef6b12a8fcf6bfe3d79e7f882e Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 9 Oct 2012 18:02:03 -0400 Subject: [PATCH 046/108] emf improvements --- .../media/metadata/definitions.py | 11 +++---- .../media-monitor2/media/metadata/process.py | 30 +++++++++++-------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index 68ada6034..f3399e24c 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -96,17 +96,18 @@ def load_definitions(): t.depends('path') t.translate(lambda k: normpath(k['path'])) - with md.metadata("MDATA_KEY_MD5") as t: - t.depends('path') - t.optional(False) - t.translate(lambda k: file_md5(k['path'], max_length=100)) + #with md.metadata("MDATA_KEY_MD5") as t: + #t.depends('path') + #t.optional(False) + #t.translate(lambda k: file_md5(k['path'], max_length=100)) # owner is handled differently by (by events.py) with md.metadata('MDATA_KEY_ORIGINAL_PATH') as t: t.depends('original_path') - # MDATA_KEY_TITLE is the annoying special case + # MDATA_KEY_TITLE is the annoying special case b/c we sometimes read it + # from file name with md.metadata('MDATA_KEY_TITLE') as t: # Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is # defaulted to "" from definitions above diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index 6cebc0269..88a8b5cc6 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -11,10 +11,6 @@ class MetadataAbsent(Exception): class MetadataElement(Loggable): - def __default_translator(k): - e = [ x for x in self.dependencies() ][0] - return k[e] - def __init__(self,name): self.name = name # "Sane" defaults @@ -24,8 +20,7 @@ class MetadataElement(Loggable): self.__default = None self.__is_normalized = lambda _ : True self.__max_length = -1 - - + self.__translator = None def max_length(self,l): self.__max_length = l @@ -71,6 +66,7 @@ class MetadataElement(Loggable): return "%s(%s)" % (self.name, ' '.join(list(self.__deps))) def read_value(self, path, original, running={}): + self.logger.info("Trying to read: %s" % str(original)) # If value is present and normalized then we don't touch it if self.name in original: v = original[self.name] @@ -83,21 +79,29 @@ class MetadataElement(Loggable): full_deps = dict( dep_slice_orig.items() + dep_slice_running.items() ) + full_deps['path'] = path + # check if any dependencies are absent - if len(full_deps) != len(self.__deps) or len(self.__deps) == 0: + if len(full_deps) < len(self.__deps) or len(self.__deps) == 0: # If we have a default value then use that. Otherwise throw an # exception if self.has_default(): return self.get_default() else: raise MetadataAbsent(self.name) # We have all dependencies. Now for actual for parsing + def def_translate(dep): + def wrap(k): + e = [ x for x in dep ][0] + return k[e] + return wrap + # Only case where we can select a default translator - if not self.__translator: - if len(self.dependencies()) == 1: - self.translate(MetadataElement.__default_translator) - else: - self.logger.info("Could not set more than 1 translator with \ - more than 1 dependancies") + if self.__translator is None: + self.translate(def_translate(self.dependencies())) + if len(self.dependencies()) > 2: # dependencies include themselves + self.logger.info("Ignoring some dependencies in translate %s" + % self.name) + self.logger.info(self.dependencies()) r = self.__normalizer( self.__translator(full_deps) ) if self.__max_length != -1: From 55567d1de03d7f993b33b93794a4fb4d93356a4a Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 10 Oct 2012 14:40:46 -0400 Subject: [PATCH 047/108] Aded diff_dict function to help locate differences between emf and old metadata parsing. --- .../media-monitor2/media/monitor/pure.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 877cbdc72..81b9cba49 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -68,6 +68,23 @@ class IncludeOnly(object): return _wrap + +def diff_dict(d1, d2, width=30): + """ + returns a formatted diff of 2 dictionaries + """ + out = "" + all_keys = d1.keys() + d2.keys() + for k in all_keys: + v1, v2 = d1.get(k), d2.get(k) + + # default values + if v1 is None: v1 = "N/A" + if v2 is None: v2 = "N/A" + + if d1[k] != d2[k]: + out += "%s%s%s" % (k, d1[k], d2[k]) + def partition(f, alist): """ Partition is very similar to filter except that it also returns the From 31b2a29392120de85fab4d05f3e1aaa7e76c838c Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 10 Oct 2012 14:41:12 -0400 Subject: [PATCH 048/108] Added docstirng for __slice_deps --- .../media-monitor2/media/metadata/process.py | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index 88a8b5cc6..d8be46bdf 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -60,35 +60,49 @@ class MetadataElement(Loggable): return self.__path def __slice_deps(self, d): + """ + returns a dictionary of all the key value pairs in d that are also + present in self.__deps + """ return dict( (k,v) for k,v in d.iteritems() if k in self.__deps) def __str__(self): return "%s(%s)" % (self.name, ' '.join(list(self.__deps))) def read_value(self, path, original, running={}): - self.logger.info("Trying to read: %s" % str(original)) - # If value is present and normalized then we don't touch it + + # If value is present and normalized then we only check if it's + # normalized or not. We normalize if it's not normalized already + if self.name in original: v = original[self.name] if self.__is_normalized(v): return v else: return self.__normalizer(v) - # A dictionary slice with all the dependencies and their values + # We slice out only the dependencies that are required for the metadata + # element. dep_slice_orig = self.__slice_deps(original) dep_slice_running = self.__slice_deps(running) + # TODO : remove this later + dep_slice_special = self.__slice_deps({'path' : path}) + # We combine all required dependencies into a single dictionary + # that we will pass to the translator full_deps = dict( dep_slice_orig.items() - + dep_slice_running.items() ) - - full_deps['path'] = path + + dep_slice_running.items() + + dep_slice_special.items()) # check if any dependencies are absent - if len(full_deps) < len(self.__deps) or len(self.__deps) == 0: + # note: there is no point checking the case that len(full_deps) > + # len(self.__deps) because we make sure to "slice out" any supefluous + # dependencies above. + if len(full_deps) != len(self.dependencies()) or \ + len(self.dependencies()) == 0: # If we have a default value then use that. Otherwise throw an # exception if self.has_default(): return self.get_default() else: raise MetadataAbsent(self.name) - # We have all dependencies. Now for actual for parsing + # We have all dependencies. Now for actual for parsing def def_translate(dep): def wrap(k): e = [ x for x in dep ][0] From 4bbc1fa95bf9afe8a47d2eaec2b68b88567d4c67 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 10 Oct 2012 14:41:39 -0400 Subject: [PATCH 049/108] Added comparison for emf and non emf metadata values. --- .../media-monitor2/media/monitor/metadata.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 6cce20e34..2b6c07a83 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -191,12 +191,6 @@ class Metadata(Loggable): # TODO : Simplify the way all of these rules are handled right now it's # extremely unclear and needs to be refactored. #if full_mutagen is None: raise BadSongFile(fpath) - try: # emf stuff for testing: - if full_mutagen: - normalized = global_reader.read('fpath', full_mutagen) - self.logger.info(pformat(normalized)) - except Exception as e: - self.unexpected_exception(e) if full_mutagen is None: full_mutagen = FakeMutagen(fpath) self.__metadata = Metadata.airtime_dict(full_mutagen) @@ -222,6 +216,17 @@ class Metadata(Loggable): # from the file? self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100) + try: # emf stuff for testing: + if full_mutagen: + normalized = global_reader.read_mutagen(fpath) + self.logger.info("EMF--------------------") + self.logger.info(pformat(normalized)) + self.logger.info("OLD--------------------") + self.logger.info(pformat(self.__metadata)) + self.logger.info("-----------------------") + + except Exception as e: + self.unexpected_exception(e) def is_recorded(self): """ From a2792b01acd7e13720b7c4674543585f67f94db5 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 10 Oct 2012 14:42:07 -0400 Subject: [PATCH 050/108] Got rid of emf bug where a list of 0 elements might be accessed. --- python_apps/media-monitor2/media/metadata/process.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index d8be46bdf..51a3c732d 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -130,10 +130,11 @@ def normalize_mutagen(path): m = mutagen.File(path, easy=True) md = {} for k,v in m.iteritems(): - if type(v) is list: md[k] = v[0] + if type(v) is list: + if len(v) > 0: md[k] = v[0] else: md[k] = v # populate special metadata values - md['length'] = getattr(m.info, u'length', 0.0) + md['length'] = getattr(m.info, 'length', 0.0) md['bitrate'] = getattr(m.info, 'bitrate', u'') md['sample_rate'] = getattr(m.info, 'sample_rate', 0) md['mime'] = m.mime[0] if len(m.mime) > 0 else u'' From 445ec23b8e0efe4fcfdce0834ceaaaf8ddfad006 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 10 Oct 2012 14:42:22 -0400 Subject: [PATCH 051/108] Added utility method for reading mutagen info through emf. --- python_apps/media-monitor2/media/metadata/process.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index 51a3c732d..1b62646a4 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -167,6 +167,9 @@ class MetadataReader(object): if not mdata.is_optional(): raise return normalized_metadata + def read_mutagen(self, path): + return self.read(path, normalize_mutagen(path)) + global_reader = MetadataReader() @contextmanager From 4242ed38be7c5a002f5f2f5af93bb269da9a3b5d Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 11:14:53 -0400 Subject: [PATCH 052/108] Added more testing for emf --- python_apps/media-monitor2/tests/test_emf.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 python_apps/media-monitor2/tests/test_emf.py diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py new file mode 100644 index 000000000..69fc566db --- /dev/null +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +import unittest + +from media.metadata.process import global_reader +import media.metadata.definitions as defs +defs.load_definitions() + +class TestMMP(unittest.TestCase): + def test_sanity(self): + m = global_reader.read_mutagen("/home/rudi/music/Nightingale.mp3") + self.assertTrue( len(m) > 0 ) From 09a8dfefc1d5f88ec4a4edfd50a458e1040ef6a4 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 13:55:03 -0400 Subject: [PATCH 053/108] Fixed metadata definitions in emf --- .../media-monitor2/media/metadata/definitions.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index f3399e24c..f0288bc61 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -30,7 +30,7 @@ def load_definitions(): t.depends('sample_rate') t.translate(lambda k: k['sample_rate']) - with md.metadata('MDATA_KEY_FTYPE'): + with md.metadata('MDATA_KEY_FTYPE') as t: t.depends('ftype') # i don't think this field even exists t.default(u'audioclip') t.translate(lambda k: k['ftype']) # but just in case @@ -92,9 +92,9 @@ def load_definitions(): t.depends("copyright") t.max_length(512) - with md.metadata("MDATA_KEY_FILEPATH") as t: + with md.metadata("MDATA_KEY_ORIGINAL_PATH") as t: t.depends('path') - t.translate(lambda k: normpath(k['path'])) + t.translate(lambda k: unicode(normpath(k['path']))) #with md.metadata("MDATA_KEY_MD5") as t: #t.depends('path') @@ -103,15 +103,13 @@ def load_definitions(): # owner is handled differently by (by events.py) - with md.metadata('MDATA_KEY_ORIGINAL_PATH') as t: - t.depends('original_path') - # MDATA_KEY_TITLE is the annoying special case b/c we sometimes read it # from file name with md.metadata('MDATA_KEY_TITLE') as t: # Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is # defaulted to "" from definitions above t.depends('title','MDATA_KEY_CREATOR') + t.translate(lambda k: k['title']) t.max_length(512) with md.metadata('MDATA_KEY_LABEL') as t: From 027153b88224376fddfc2a665f067dc249309baa Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 13:56:54 -0400 Subject: [PATCH 054/108] Fixed emf handling of bad definitions --- python_apps/media-monitor2/media/metadata/process.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index 1b62646a4..ca24feeb0 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -74,6 +74,7 @@ class MetadataElement(Loggable): # If value is present and normalized then we only check if it's # normalized or not. We normalize if it's not normalized already + if self.name in original: v = original[self.name] if self.__is_normalized(v): return v @@ -141,11 +142,18 @@ def normalize_mutagen(path): md['path'] = path return md + +class OverwriteMetadataElement(Exception): + def __init__(self, m): self.m = m + def __str__(self): return "Trying to overwrite: %s" % self.m + class MetadataReader(object): def __init__(self): self.clear() def register_metadata(self,m): + if m in self.__mdata_name_map: + raise OverwriteMetadataElement(m) self.__mdata_name_map[m.name] = m d = dict( (name,m.dependencies()) for name,m in self.__mdata_name_map.iteritems() ) From 591d2d741f814ed63cc4d7ce89bebf73fb23ce17 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 13:57:37 -0400 Subject: [PATCH 055/108] Added original path element in metadata --- python_apps/media-monitor2/media/monitor/pure.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 81b9cba49..1bb4231ae 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -312,6 +312,7 @@ def normalized_metadata(md, original_path): # TODO : wtf is this for again? new_md['MDATA_KEY_TITLE'] = re.sub(r'-?%s-?' % unicode_unknown, u'', new_md['MDATA_KEY_TITLE']) + new_md['MDATA_KEY_ORIGINAL_PATH'] = original_path return new_md def organized_path(old_path, root_path, orig_md): From 7db61cc4b4b1089f8de5746e8d63b360540cec78 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 13:58:05 -0400 Subject: [PATCH 056/108] fixed tests --- .../media-monitor2/media/monitor/metadata.py | 20 +++++++++---------- python_apps/media-monitor2/tests/test_emf.py | 14 ++++++++++++- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 2b6c07a83..30511c50d 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -216,17 +216,17 @@ class Metadata(Loggable): # from the file? self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100) - try: # emf stuff for testing: - if full_mutagen: - normalized = global_reader.read_mutagen(fpath) - self.logger.info("EMF--------------------") - self.logger.info(pformat(normalized)) - self.logger.info("OLD--------------------") - self.logger.info(pformat(self.__metadata)) - self.logger.info("-----------------------") + #try: # emf stuff for testing: + #if full_mutagen: + #normalized = global_reader.read_mutagen(fpath) + #self.logger.info("EMF--------------------") + #self.logger.info(pformat(normalized)) + #self.logger.info("OLD--------------------") + #self.logger.info(pformat(self.__metadata)) + #self.logger.info("-----------------------") - except Exception as e: - self.unexpected_exception(e) + #except Exception as e: + #self.unexpected_exception(e) def is_recorded(self): """ diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py index 69fc566db..355f93dd9 100644 --- a/python_apps/media-monitor2/tests/test_emf.py +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -1,11 +1,23 @@ # -*- coding: utf-8 -*- import unittest +from pprint import pprint as pp from media.metadata.process import global_reader +from media.monitor.metadata import Metadata + import media.metadata.definitions as defs defs.load_definitions() class TestMMP(unittest.TestCase): def test_sanity(self): - m = global_reader.read_mutagen("/home/rudi/music/Nightingale.mp3") + path = "/home/rudi/music/Nightingale.mp3" + m = global_reader.read_mutagen(path) self.assertTrue( len(m) > 0 ) + n = Metadata(path) + self.assertEqual(n.extract(), m) + pp(n.extract()) + print("--------------") + pp(m) + + +if __name__ == '__main__': unittest.main() From 482e2475ca6d49ca042921b06698e7259b164f7b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 14:24:33 -0400 Subject: [PATCH 057/108] cleaned up test --- python_apps/media-monitor2/media/metadata/definitions.py | 8 ++++---- python_apps/media-monitor2/tests/test_emf.py | 3 --- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index f0288bc61..97b3f6841 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -96,10 +96,10 @@ def load_definitions(): t.depends('path') t.translate(lambda k: unicode(normpath(k['path']))) - #with md.metadata("MDATA_KEY_MD5") as t: - #t.depends('path') - #t.optional(False) - #t.translate(lambda k: file_md5(k['path'], max_length=100)) + with md.metadata("MDATA_KEY_MD5") as t: + t.depends('path') + t.optional(False) + t.translate(lambda k: file_md5(k['path'], max_length=100)) # owner is handled differently by (by events.py) diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py index 355f93dd9..b4f9ef03b 100644 --- a/python_apps/media-monitor2/tests/test_emf.py +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -15,9 +15,6 @@ class TestMMP(unittest.TestCase): self.assertTrue( len(m) > 0 ) n = Metadata(path) self.assertEqual(n.extract(), m) - pp(n.extract()) - print("--------------") - pp(m) if __name__ == '__main__': unittest.main() From d00c74fdbe8f6779553f66ecbdb4c44279a52f36 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 14:32:56 -0400 Subject: [PATCH 058/108] Renamed test --- python_apps/media-monitor2/tests/test_emf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py index b4f9ef03b..545c059fd 100644 --- a/python_apps/media-monitor2/tests/test_emf.py +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -9,12 +9,11 @@ import media.metadata.definitions as defs defs.load_definitions() class TestMMP(unittest.TestCase): - def test_sanity(self): + def test_old_metadata(self): path = "/home/rudi/music/Nightingale.mp3" m = global_reader.read_mutagen(path) self.assertTrue( len(m) > 0 ) n = Metadata(path) self.assertEqual(n.extract(), m) - if __name__ == '__main__': unittest.main() From cefc5c99d93dd23fbdd3f0301e3573ea866db20e Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 15:39:35 -0400 Subject: [PATCH 059/108] Added special handling for title in emf --- .../media/metadata/definitions.py | 29 +++++++++++++++++-- .../media-monitor2/media/metadata/process.py | 1 + python_apps/media-monitor2/tests/test_emf.py | 6 ++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index 97b3f6841..a4f93848e 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- import media.metadata.process as md +import re from os.path import normpath -from media.monitor.pure import format_length, file_md5 +from media.monitor.pure import format_length, file_md5, is_airtime_recorded, \ + no_extension_basename defs_loaded = False @@ -105,11 +107,32 @@ def load_definitions(): # MDATA_KEY_TITLE is the annoying special case b/c we sometimes read it # from file name + + + # must handle 3 cases: + # 1. regular case (not recorded + title is present) + # 2. title is absent (read from file) + # 3. recorded file + def tr_title(k): + unicode_unknown = u"unknown" + new_title = u"" + if is_airtime_recorded(k) or k['title'] != u"": + new_title = k['title'] + else: + default_title = no_extension_basename(k['path']) + default_title = re.sub(r'__\d+\.',u'.', default_title) + if re.match(".+-%s-.+$" % unicode_unknown, default_title): + default_title = u'' + new_title = default_title + new_title = re.sub(r'-\d+kbps$', u'', new_title) + return new_title + with md.metadata('MDATA_KEY_TITLE') as t: # Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is # defaulted to "" from definitions above - t.depends('title','MDATA_KEY_CREATOR') - t.translate(lambda k: k['title']) + t.depends('title','MDATA_KEY_CREATOR','path') + t.optional(False) + t.translate(tr_title) t.max_length(512) with md.metadata('MDATA_KEY_LABEL') as t: diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index ca24feeb0..cde28cfaa 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -140,6 +140,7 @@ def normalize_mutagen(path): md['sample_rate'] = getattr(m.info, 'sample_rate', 0) md['mime'] = m.mime[0] if len(m.mime) > 0 else u'' md['path'] = path + if 'title' not in md: md['title'] = u'' return md diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py index 545c059fd..3114e2f41 100644 --- a/python_apps/media-monitor2/tests/test_emf.py +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -16,4 +16,10 @@ class TestMMP(unittest.TestCase): n = Metadata(path) self.assertEqual(n.extract(), m) + def test_recorded(self): + recorded_file = "./15:15:00-Untitled Show-256kbps.ogg" + m = global_reader.read_mutagen(recorded_file) + pp(m) + + if __name__ == '__main__': unittest.main() From ff8969efb997c7aefbbcd06593587ce7e3b03073 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 12 Oct 2012 15:54:38 -0400 Subject: [PATCH 060/108] Improved handling for paths and added tests for it --- .../media-monitor2/media/metadata/definitions.py | 3 ++- .../media-monitor2/media/metadata/process.py | 3 ++- python_apps/media-monitor2/media/monitor/pure.py | 2 +- python_apps/media-monitor2/tests/test_emf.py | 13 ++++++++++--- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index a4f93848e..ee4175033 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -20,7 +20,8 @@ def load_definitions(): with md.metadata('MDATA_KEY_MIME') as t: t.default(u'') t.depends('mime') - t.translate(lambda k: k['mime'].replace('-','/')) + # Is this necessary? + t.translate(lambda k: k['mime'].replace('audio/vorbis','audio/ogg')) with md.metadata('MDATA_KEY_BITRATE') as t: t.default(u'') diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index cde28cfaa..85a462919 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from contextlib import contextmanager from media.monitor.pure import truncate_to_length, toposort +from os.path import normpath from media.monitor.log import Loggable import mutagen @@ -139,7 +140,7 @@ def normalize_mutagen(path): md['bitrate'] = getattr(m.info, 'bitrate', u'') md['sample_rate'] = getattr(m.info, 'sample_rate', 0) md['mime'] = m.mime[0] if len(m.mime) > 0 else u'' - md['path'] = path + md['path'] = normpath(path) if 'title' not in md: md['title'] = u'' return md diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 1bb4231ae..fd1c55a13 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -312,7 +312,7 @@ def normalized_metadata(md, original_path): # TODO : wtf is this for again? new_md['MDATA_KEY_TITLE'] = re.sub(r'-?%s-?' % unicode_unknown, u'', new_md['MDATA_KEY_TITLE']) - new_md['MDATA_KEY_ORIGINAL_PATH'] = original_path + new_md['MDATA_KEY_ORIGINAL_PATH'] = normpath(original_path) return new_md def organized_path(old_path, root_path, orig_md): diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py index 3114e2f41..0213fa918 100644 --- a/python_apps/media-monitor2/tests/test_emf.py +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- import unittest -from pprint import pprint as pp +#from pprint import pprint as pp from media.metadata.process import global_reader from media.monitor.metadata import Metadata @@ -9,6 +9,13 @@ import media.metadata.definitions as defs defs.load_definitions() class TestMMP(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + + def metadatas(self,f): + return global_reader.read_mutagen(f), Metadata(f).extract() + def test_old_metadata(self): path = "/home/rudi/music/Nightingale.mp3" m = global_reader.read_mutagen(path) @@ -18,8 +25,8 @@ class TestMMP(unittest.TestCase): def test_recorded(self): recorded_file = "./15:15:00-Untitled Show-256kbps.ogg" - m = global_reader.read_mutagen(recorded_file) - pp(m) + emf, old = self.metadatas(recorded_file) + self.assertEqual(emf, old) if __name__ == '__main__': unittest.main() From 5a593112b38209dc44c3cd43629eb5c0bdcf132b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 12:20:08 -0400 Subject: [PATCH 061/108] Use emf instead of omf --- python_apps/media-monitor2/media/monitor/metadata.py | 4 +++- python_apps/media-monitor2/tests/test_emf.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 30511c50d..2bd059323 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -14,7 +14,6 @@ import media.monitor.pure as mmp # emf related stuff from media.metadata.process import global_reader import media.metadata.definitions as defs -from pprint import pformat defs.load_definitions() """ @@ -173,6 +172,9 @@ class Metadata(Loggable): for e in exceptions: raise e def __init__(self, fpath): + self.__metadata = global_reader.read_mutagen(fpath) + + def __init__2(self, fpath): # Forcing the unicode through try : fpath = fpath.decode("utf-8") except : pass diff --git a/python_apps/media-monitor2/tests/test_emf.py b/python_apps/media-monitor2/tests/test_emf.py index 0213fa918..140c0aa8b 100644 --- a/python_apps/media-monitor2/tests/test_emf.py +++ b/python_apps/media-monitor2/tests/test_emf.py @@ -28,5 +28,4 @@ class TestMMP(unittest.TestCase): emf, old = self.metadatas(recorded_file) self.assertEqual(emf, old) - if __name__ == '__main__': unittest.main() From 7255241e6cbe293445d5ca098c557efd1aff3b1b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 12:27:58 -0400 Subject: [PATCH 062/108] Removed old commented code --- python_apps/media-monitor2/media/monitor/metadata.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 2bd059323..199bd7420 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -218,18 +218,6 @@ class Metadata(Loggable): # from the file? self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100) - #try: # emf stuff for testing: - #if full_mutagen: - #normalized = global_reader.read_mutagen(fpath) - #self.logger.info("EMF--------------------") - #self.logger.info(pformat(normalized)) - #self.logger.info("OLD--------------------") - #self.logger.info(pformat(self.__metadata)) - #self.logger.info("-----------------------") - - #except Exception as e: - #self.unexpected_exception(e) - def is_recorded(self): """ returns true if the file has been created by airtime through recording From 0638d2147365e394a342f7daed475f134ee0cd03 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 14:48:16 -0400 Subject: [PATCH 063/108] Refactored code --- .../media-monitor2/media/metadata/process.py | 28 ++++++++++++++++++- .../media-monitor2/media/monitor/metadata.py | 5 ++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/metadata/process.py b/python_apps/media-monitor2/media/metadata/process.py index 85a462919..b500d029d 100644 --- a/python_apps/media-monitor2/media/metadata/process.py +++ b/python_apps/media-monitor2/media/metadata/process.py @@ -2,9 +2,28 @@ from contextlib import contextmanager from media.monitor.pure import truncate_to_length, toposort from os.path import normpath +from media.monitor.exceptions import BadSongFile from media.monitor.log import Loggable +import media.monitor.pure as mmp +from collections import namedtuple import mutagen +class FakeMutagen(dict): + """ + Need this fake mutagen object so that airtime_special functions + return a proper default value instead of throwing an exceptions for + files that mutagen doesn't recognize + """ + FakeInfo = namedtuple('FakeInfo','length bitrate') + def __init__(self,path): + self.path = path + self.mime = ['audio/wav'] + self.info = FakeMutagen.FakeInfo(0.0, '') + dict.__init__(self) + def set_length(self,l): + old_bitrate = self.info.bitrate + self.info = FakeMutagen.FakeInfo(l, old_bitrate) + class MetadataAbsent(Exception): def __init__(self, name): self.name = name @@ -129,7 +148,14 @@ def normalize_mutagen(path): Consumes a path and reads the metadata using mutagen. normalizes some of the metadata that isn't read through the mutagen hash """ - m = mutagen.File(path, easy=True) + if not mmp.file_playable(path): raise BadSongFile(path) + try : m = mutagen.File(path, easy=True) + except Exception : raise BadSongFile(path) + if m is None: m = FakeMutagen(path) + try: + if mmp.extension(path) == 'wav': + m.set_length(mmp.read_wave_duration(path)) + except Exception: raise BadSongFile(path) md = {} for k,v in m.iteritems(): if type(v) is list: diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 199bd7420..a496af61a 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -48,6 +48,8 @@ airtime2mutagen = { "MDATA_KEY_COPYRIGHT" : "copyright", } + +# TODO :Remove FakeMutagen class. Moved to media.metadata.process class FakeMutagen(dict): """ Need this fake mutagen object so that airtime_special functions @@ -172,6 +174,9 @@ class Metadata(Loggable): for e in exceptions: raise e def __init__(self, fpath): + # Forcing the unicode through + try : fpath = fpath.decode("utf-8") + except : pass self.__metadata = global_reader.read_mutagen(fpath) def __init__2(self, fpath): From 9a1948707c8ea2d0c0aa769ae0f5d048bff32032 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 14:51:51 -0400 Subject: [PATCH 064/108] Removed duplicate code --- .../media-monitor2/media/monitor/metadata.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index a496af61a..30bab7419 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -49,23 +49,6 @@ airtime2mutagen = { } -# TODO :Remove FakeMutagen class. Moved to media.metadata.process -class FakeMutagen(dict): - """ - Need this fake mutagen object so that airtime_special functions - return a proper default value instead of throwing an exceptions for - files that mutagen doesn't recognize - """ - FakeInfo = namedtuple('FakeInfo','length bitrate') - def __init__(self,path): - self.path = path - self.mime = ['audio/wav'] - self.info = FakeMutagen.FakeInfo(0.0, '') - dict.__init__(self) - def set_length(self,l): - old_bitrate = self.info.bitrate - self.info = FakeMutagen.FakeInfo(l, old_bitrate) - # Some airtime attributes are special because they must use the mutagen object # itself to calculate the value that they need. The lambda associated with each # key should attempt to extract the corresponding value from the mutagen object From 6a8d902a61e9dacea8ccecef1bbe74eb46c5448d Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 14:52:38 -0400 Subject: [PATCH 065/108] Removed omf --- .../media-monitor2/media/monitor/metadata.py | 45 ------------------- 1 file changed, 45 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 30bab7419..9754c1b0e 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -2,7 +2,6 @@ import mutagen import os import copy -from collections import namedtuple from mutagen.easymp4 import EasyMP4KeyError from mutagen.easyid3 import EasyID3KeyError @@ -162,50 +161,6 @@ class Metadata(Loggable): except : pass self.__metadata = global_reader.read_mutagen(fpath) - def __init__2(self, fpath): - # Forcing the unicode through - try : fpath = fpath.decode("utf-8") - except : pass - - if not mmp.file_playable(fpath): raise BadSongFile(fpath) - - try : full_mutagen = mutagen.File(fpath, easy=True) - except Exception : raise BadSongFile(fpath) - - self.path = fpath - if not os.path.exists(self.path): - self.logger.info("Attempting to read metadata of file \ - that does not exist. Setting metadata to {}") - self.__metadata = {} - return - # TODO : Simplify the way all of these rules are handled right now it's - # extremely unclear and needs to be refactored. - #if full_mutagen is None: raise BadSongFile(fpath) - - if full_mutagen is None: full_mutagen = FakeMutagen(fpath) - self.__metadata = Metadata.airtime_dict(full_mutagen) - # Now we extra the special values that are calculated from the mutagen - # object itself: - - if mmp.extension(fpath) == 'wav': - full_mutagen.set_length(mmp.read_wave_duration(fpath)) - - for special_key,f in airtime_special.iteritems(): - try: - new_val = f(full_mutagen) - if new_val is not None: - self.__metadata[special_key] = new_val - except Exception as e: - self.logger.info("Could not get special key %s for %s" % - (special_key, fpath)) - self.logger.info(str(e)) - # Finally, we "normalize" all the metadata here: - self.__metadata = mmp.normalized_metadata(self.__metadata, fpath) - # Now we must load the md5: - # TODO : perhaps we shouldn't hard code how many bytes we're reading - # from the file? - self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100) - def is_recorded(self): """ returns true if the file has been created by airtime through recording From 66b5bf304f0c692d3d59955162c6ccc3152db6ed Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 14:58:03 -0400 Subject: [PATCH 066/108] changed test strings --- python_apps/media-monitor2/tests/test_api_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/tests/test_api_client.py b/python_apps/media-monitor2/tests/test_api_client.py index 26f1a25ef..18595f860 100644 --- a/python_apps/media-monitor2/tests/test_api_client.py +++ b/python_apps/media-monitor2/tests/test_api_client.py @@ -19,8 +19,8 @@ class TestApiClient(unittest.TestCase): self.apc.register_component("api-client-tester") # All of the following requests should error out in some way self.bad_requests = [ - { 'mode' : 'dang it', 'is_record' : 0 }, - { 'mode' : 'damn frank', 'is_record' : 1 }, + { 'mode' : 'foo', 'is_record' : 0 }, + { 'mode' : 'bar', 'is_record' : 1 }, { 'no_mode' : 'at_all' }, ] def test_bad_requests(self): From dc361c11367f5bc5ab4f4d42107f9ce41a29124b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 16 Oct 2012 14:59:06 -0400 Subject: [PATCH 067/108] Aligned constants --- airtime_mvc/application/configs/constants.php | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/airtime_mvc/application/configs/constants.php b/airtime_mvc/application/configs/constants.php index 1d1ed55e4..eb5cb3989 100644 --- a/airtime_mvc/application/configs/constants.php +++ b/airtime_mvc/application/configs/constants.php @@ -1,40 +1,40 @@ Date: Tue, 23 Oct 2012 12:06:30 -0400 Subject: [PATCH 068/108] Removed unused method --- .../media-monitor2/media/monitor/metadata.py | 36 ++----------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 9754c1b0e..5bbbafaeb 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -7,7 +7,7 @@ from mutagen.easyid3 import EasyID3KeyError from media.monitor.exceptions import BadSongFile, InvalidMetadataElement from media.monitor.log import Loggable -from media.monitor.pure import format_length, truncate_to_length +from media.monitor.pure import format_length import media.monitor.pure as mmp # emf related stuff @@ -89,6 +89,7 @@ class Metadata(Loggable): # little bit messy. Some of the handling is in m.m.pure while the rest is # here. Also interface is not very consistent + # TODO : what is this shit? maybe get rid of it? @staticmethod def fix_title(path): # If we have no title in path we will format it @@ -99,39 +100,6 @@ class Metadata(Loggable): m[u'title'] = new_title m.save() - @staticmethod - def airtime_dict(d): - """ - Converts mutagen dictionary 'd' into airtime dictionary - """ - temp_dict = {} - for m_key, m_val in d.iteritems(): - # TODO : some files have multiple fields for the same metadata. - # genre is one example. In that case mutagen will return a list - # of values - - if isinstance(m_val, list): - # TODO : does it make more sense to just skip the element in - # this case? - if len(m_val) == 0: assign_val = '' - else: assign_val = m_val[0] - else: assign_val = m_val - - temp_dict[ m_key ] = assign_val - airtime_dictionary = {} - for muta_k, muta_v in temp_dict.iteritems(): - # We must check if we can actually translate the mutagen key into - # an airtime key before doing the conversion - if muta_k in mutagen2airtime: - airtime_key = mutagen2airtime[muta_k] - # Apply truncation in the case where airtime_key is in our - # truncation table - muta_v = \ - truncate_to_length(muta_v, truncate_table[airtime_key])\ - if airtime_key in truncate_table else muta_v - airtime_dictionary[ airtime_key ] = muta_v - return airtime_dictionary - @staticmethod def write_unsafe(path,md): """ From 54e6bec16af71eb9ba26832e2acd1144a20d1f59 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 23 Oct 2012 15:00:09 -0400 Subject: [PATCH 069/108] Reshuffled/removed some unit tests to get rid of omf --- .../media-monitor2/tests/test_metadata.py | 6 -- python_apps/media-monitor2/tests/test_pure.py | 70 ++----------------- 2 files changed, 7 insertions(+), 69 deletions(-) diff --git a/python_apps/media-monitor2/tests/test_metadata.py b/python_apps/media-monitor2/tests/test_metadata.py index 6f24240b0..4b05a9fce 100644 --- a/python_apps/media-monitor2/tests/test_metadata.py +++ b/python_apps/media-monitor2/tests/test_metadata.py @@ -42,10 +42,4 @@ class TestMetadata(unittest.TestCase): x1 = 123456 print("Formatting '%s' to '%s'" % (x1, mmm.format_length(x1))) - def test_truncate_to_length(self): - s1 = "testing with non string literal" - s2 = u"testing with unicode literal" - self.assertEqual( len(mmm.truncate_to_length(s1, 5)), 5) - self.assertEqual( len(mmm.truncate_to_length(s2, 8)), 8) - if __name__ == '__main__': unittest.main() diff --git a/python_apps/media-monitor2/tests/test_pure.py b/python_apps/media-monitor2/tests/test_pure.py index 64d09dc62..6bc5d4906 100644 --- a/python_apps/media-monitor2/tests/test_pure.py +++ b/python_apps/media-monitor2/tests/test_pure.py @@ -2,7 +2,6 @@ import unittest import os import media.monitor.pure as mmp -from media.monitor.metadata import Metadata class TestMMP(unittest.TestCase): def setUp(self): @@ -34,68 +33,6 @@ class TestMMP(unittest.TestCase): sd = mmp.default_to(dictionary=sd, keys=def_keys, default='DEF') for k in def_keys: self.assertEqual( sd[k], 'DEF' ) - def test_normalized_metadata(self): - #Recorded show test first - orig = Metadata.airtime_dict({ - 'date' : [u'2012-08-21'], - 'tracknumber' : [u'2'], - 'title' : [u'record-2012-08-21-11:29:00'], - 'artist' : [u'Airtime Show Recorder'] - }) - orga = Metadata.airtime_dict({ - 'date' : [u'2012-08-21'], - 'tracknumber' : [u'2'], - 'artist' : [u'Airtime Show Recorder'], - 'title' : [u'record-2012-08-21-11:29:00'] - }) - orga['MDATA_KEY_FTYPE'] = u'audioclip' - orig['MDATA_KEY_BITRATE'] = u'256000' - orga['MDATA_KEY_BITRATE'] = u'256000' - old_path = "/home/rudi/recorded/2012-08-21-11:29:00.ogg" - normalized = mmp.normalized_metadata(orig, old_path) - normalized['MDATA_KEY_BITRATE'] = u'256000' - - self.assertEqual( orga, normalized ) - - organized_base_name = "11:29:00-record-256kbps.ogg" - base = "/srv/airtime/stor/" - organized_path = mmp.organized_path(old_path,base, normalized) - self.assertEqual(os.path.basename(organized_path), organized_base_name) - - def test_normalized_metadata2(self): - """ - cc-4305 - """ - orig = Metadata.airtime_dict({ - 'date' : [u'2012-08-27'], - 'tracknumber' : [u'3'], - 'title' : [u'18-11-00-Untitled Show'], - 'artist' : [u'Airtime Show Recorder'] - }) - old_path = "/home/rudi/recorded/doesnt_really_matter.ogg" - normalized = mmp.normalized_metadata(orig, old_path) - normalized['MDATA_KEY_BITRATE'] = u'256000' - opath = mmp.organized_path(old_path, "/srv/airtime/stor/", - normalized) - # TODO : add a better test than this... - self.assertTrue( len(opath) > 0 ) - - def test_normalized_metadata3(self): - """ - Test the case where the metadata is empty - """ - orig = Metadata.airtime_dict({}) - paths_unknown_title = [ - ("/testin/unknown-unknown-unknown.mp3",""), - ("/testin/01-unknown-123kbps.mp3",""), - ("/testin/02-unknown-140kbps.mp3",""), - ("/testin/unknown-unknown-123kbps.mp3",""), - ("/testin/unknown-bibimbop-unknown.mp3","bibimbop"), - ] - for p,res in paths_unknown_title: - normalized = mmp.normalized_metadata(orig, p) - self.assertEqual( normalized['MDATA_KEY_TITLE'], res) - def test_file_md5(self): p = os.path.realpath(__file__) m1 = mmp.file_md5(p) @@ -116,6 +53,13 @@ class TestMMP(unittest.TestCase): self.assertEqual( mmp.parse_int("123asf"), "123" ) self.assertEqual( mmp.parse_int("asdf"), None ) + def test_truncate_to_length(self): + s1 = "testing with non string literal" + s2 = u"testing with unicode literal" + self.assertEqual( len(mmp.truncate_to_length(s1, 5)), 5) + self.assertEqual( len(mmp.truncate_to_length(s2, 8)), 8) + + def test_owner_id(self): start_path = "testing.mp3" id_path = "testing.mp3.identifier" From 1359bb402e2af565bc94a9c83a443c291ac02fc5 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 23 Oct 2012 15:03:37 -0400 Subject: [PATCH 070/108] Removed useless comment --- python_apps/media-monitor2/media/monitor/pure.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index fd1c55a13..fa14089d7 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -22,7 +22,6 @@ from configobj import ConfigObj from media.monitor.exceptions import FailedToSetLocale, FailedToCreateDir -#supported_extensions = [u"mp3", u"ogg", u"oga"] supported_extensions = [u"mp3", u"ogg", u"oga", u"flac", u"wav", u'm4a', u'mp4'] From 29c96d2efe9f29c799d5f38eb7224beeb7e500b3 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 23 Oct 2012 15:24:27 -0400 Subject: [PATCH 071/108] Removed md5 related test --- python_apps/media-monitor2/tests/test_metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python_apps/media-monitor2/tests/test_metadata.py b/python_apps/media-monitor2/tests/test_metadata.py index 4b05a9fce..7a32b61a8 100644 --- a/python_apps/media-monitor2/tests/test_metadata.py +++ b/python_apps/media-monitor2/tests/test_metadata.py @@ -26,7 +26,6 @@ class TestMetadata(unittest.TestCase): i += 1 print("Sample metadata: '%s'" % md) self.assertTrue( len( md.keys() ) > 0 ) - self.assertTrue( 'MDATA_KEY_MD5' in md ) utf8 = md_full.utf8() for k,v in md.iteritems(): if hasattr(utf8[k], 'decode'): From 9169893e4d0deaebb19d5790d068367dcb4a683a Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 23 Oct 2012 16:44:11 -0400 Subject: [PATCH 072/108] Alignment of method definitions --- python_apps/media-monitor2/media/monitor/syncdb.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/syncdb.py b/python_apps/media-monitor2/media/monitor/syncdb.py index 0c14fb038..cc8abc294 100644 --- a/python_apps/media-monitor2/media/monitor/syncdb.py +++ b/python_apps/media-monitor2/media/monitor/syncdb.py @@ -53,11 +53,11 @@ class AirtimeDB(Loggable): """ return self.id_to_dir[ dir_id ] - def storage_path(self): return self.base_storage - def organize_path(self): return self.storage_paths['organize'] - def problem_path(self): return self.storage_paths['problem_files'] - def import_path(self): return self.storage_paths['imported'] - def recorded_path(self): return self.storage_paths['recorded'] + def storage_path(self) : return self.base_storage + def organize_path(self) : return self.storage_paths['organize'] + def problem_path(self) : return self.storage_paths['problem_files'] + def import_path(self) : return self.storage_paths['imported'] + def recorded_path(self) : return self.storage_paths['recorded'] def list_watched(self): """ From f718070d3cda1165f338888216ec76617e06364a Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 23 Oct 2012 17:00:22 -0400 Subject: [PATCH 073/108] Uncoupled the api_client instance from the RequestSync object --- .../media-monitor2/media/monitor/watchersyncer.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index e7a2cf9c0..191a0f8a1 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -18,10 +18,18 @@ class RequestSync(threading.Thread,Loggable): to airtime. In the process it packs the requests and retries for some number of times """ - def __init__(self, watcher, requests): + + @classmethod + def create_with_api_client(cls, watcher, requests): + apiclient = ac.AirtimeApiClient.create_right_config() + self = cls(watcher, requests, apiclient) + return self + + def __init__(self, watcher, requests, apiclient): threading.Thread.__init__(self) self.watcher = watcher self.requests = requests + self.apiclient = apiclient self.retries = 1 self.request_wait = 0.3 @@ -209,7 +217,8 @@ class WatchSyncer(ReportHandler,Loggable): requests = copy.copy(self.__queue) def launch_request(): # Need shallow copy here - t = RequestSync(watcher=self, requests=requests) + t = RequestSync.create_with_api_client(watcher=self, + requests=requests) t.start() self.__current_thread = t self.__requests.append(launch_request) From 86b262ed8f89b0fd0b07b4140a1803457bdecc5c Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 11:08:21 -0400 Subject: [PATCH 074/108] uncoupled threading from making a request --- .../media/monitor/watchersyncer.py | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index 191a0f8a1..6ed9ba594 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -6,13 +6,23 @@ import copy from media.monitor.handler import ReportHandler from media.monitor.log import Loggable from media.monitor.exceptions import BadSongFile -from media.monitor.pure import LazyProperty from media.monitor.eventcontractor import EventContractor from media.monitor.events import EventProxy import api_clients.api_client as ac -class RequestSync(threading.Thread,Loggable): + +class ThreadedRequestSync(threading.Thread, Loggable): + def __init__(self, rs): + threading.Thread.__init__(self) + self.rs = rs + self.daemon = True + self.start() + + def run(self): + self.rs.run_request() + +class RequestSync(Loggable): """ This class is responsible for making the api call to send a request to airtime. In the process it packs the requests and retries for @@ -26,18 +36,13 @@ class RequestSync(threading.Thread,Loggable): return self def __init__(self, watcher, requests, apiclient): - threading.Thread.__init__(self) self.watcher = watcher self.requests = requests self.apiclient = apiclient self.retries = 1 self.request_wait = 0.3 - @LazyProperty - def apiclient(self): - return ac.AirtimeApiClient.create_right_config() - - def run(self): + def run_request(self): self.logger.info("Attempting request with %d items." % len(self.requests)) # Note that we must attach the appropriate mode to every @@ -59,6 +64,8 @@ class RequestSync(threading.Thread,Loggable): request_event.path) def make_req(): self.apiclient.send_media_monitor_requests( packed_requests ) + # TODO : none of the shit below is necessary. get rid of it + # eventually for try_index in range(0,self.retries): try: make_req() # most likely we did not get json response as we expected @@ -76,7 +83,7 @@ class RequestSync(threading.Thread,Loggable): break else: self.logger.info("Failed to send request after '%d' tries..." % self.retries) - self.watcher.flag_done() + self.watcher.flag_done() # poor man's condition variable class TimeoutWatcher(threading.Thread,Loggable): """ @@ -217,9 +224,8 @@ class WatchSyncer(ReportHandler,Loggable): requests = copy.copy(self.__queue) def launch_request(): # Need shallow copy here - t = RequestSync.create_with_api_client(watcher=self, - requests=requests) - t.start() + t = ThreadedRequestSync( RequestSync.create_with_api_client( + watcher=self, requests=requests) ) self.__current_thread = t self.__requests.append(launch_request) self.__reset_queue() From 289d5575ece18521d4a21747a1634860a0d5e03e Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 11:18:41 -0400 Subject: [PATCH 075/108] Cleaned the shit out of RequestSync. Removed old comments and useless code --- .../media/monitor/watchersyncer.py | 40 +++++-------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index 6ed9ba594..6ab6509bf 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -28,7 +28,6 @@ class RequestSync(Loggable): to airtime. In the process it packs the requests and retries for some number of times """ - @classmethod def create_with_api_client(cls, watcher, requests): apiclient = ac.AirtimeApiClient.create_right_config() @@ -39,17 +38,10 @@ class RequestSync(Loggable): self.watcher = watcher self.requests = requests self.apiclient = apiclient - self.retries = 1 - self.request_wait = 0.3 def run_request(self): self.logger.info("Attempting request with %d items." % len(self.requests)) - # Note that we must attach the appropriate mode to every - # response. Also Not forget to attach the 'is_record' to any - # requests that are related to recorded shows - # TODO : recorded shows aren't flagged right - # Is this retry shit even necessary? Consider getting rid of this. packed_requests = [] for request_event in self.requests: try: @@ -62,27 +54,17 @@ class RequestSync(Loggable): if hasattr(request_event, 'path'): self.logger.info("Possibly related to path: '%s'" % request_event.path) - def make_req(): - self.apiclient.send_media_monitor_requests( packed_requests ) - # TODO : none of the shit below is necessary. get rid of it - # eventually - for try_index in range(0,self.retries): - try: make_req() - # most likely we did not get json response as we expected - except ValueError: - self.logger.info("ApiController.php probably crashed, we \ - diagnose this from the fact that it did not return \ - valid json") - self.logger.info("Trying again after %f seconds" % - self.request_wait) - time.sleep( self.request_wait ) - except Exception as e: self.unexpected_exception(e) - else: - self.logger.info("Request worked on the '%d' try" % - (try_index + 1)) - break - else: self.logger.info("Failed to send request after '%d' tries..." % - self.retries) + try: self.apiclient.send_media_monitor_requests( packed_requests ) + # most likely we did not get json response as we expected + except ValueError: + self.logger.info("ApiController.php probably crashed, we \ + diagnose this from the fact that it did not return \ + valid json") + self.logger.info("Trying again after %f seconds" % + self.request_wait) + except Exception as e: self.unexpected_exception(e) + else: + self.logger.info("Request was successful") self.watcher.flag_done() # poor man's condition variable class TimeoutWatcher(threading.Thread,Loggable): From a6ab91333f278375286e3be0bb635565b781970e Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 11:24:23 -0400 Subject: [PATCH 076/108] formatting --- python_apps/media-monitor2/media/monitor/watchersyncer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index 6ab6509bf..d7567a535 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -63,8 +63,7 @@ class RequestSync(Loggable): self.logger.info("Trying again after %f seconds" % self.request_wait) except Exception as e: self.unexpected_exception(e) - else: - self.logger.info("Request was successful") + else: self.logger.info("Request was successful") self.watcher.flag_done() # poor man's condition variable class TimeoutWatcher(threading.Thread,Loggable): From 38cbd7d7e42ac7020954e9b36223a4c4cc4b11bd Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 11:52:47 -0400 Subject: [PATCH 077/108] added emf unit tests --- .../media-monitor2/tests/test_metadata_def.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 python_apps/media-monitor2/tests/test_metadata_def.py diff --git a/python_apps/media-monitor2/tests/test_metadata_def.py b/python_apps/media-monitor2/tests/test_metadata_def.py new file mode 100644 index 000000000..e666ef68a --- /dev/null +++ b/python_apps/media-monitor2/tests/test_metadata_def.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +import unittest + +import media.metadata.process as md + +class TestMetadataDef(unittest.TestCase): + def test_simple(self): + + with md.metadata('MDATA_TESTING') as t: + t.optional(True) + t.depends('ONE','TWO') + t.default('unknown') + t.translate(lambda kw: kw['ONE'] + kw['TWO']) + + h = { 'ONE' : "testing", 'TWO' : "123" } + result = md.global_reader.read('test_path',h) + self.assertTrue( 'MDATA_TESTING' in result ) + self.assertEqual( result['MDATA_TESTING'], 'testing123' ) + h1 = { 'ONE' : 'big testing', 'two' : 'nothing' } + result1 = md.global_reader.read('bs path', h1) + self.assertEqual( result1['MDATA_TESTING'], 'unknown' ) + + def test_topo(self): + with md.metadata('MDATA_TESTING') as t: + t.depends('shen','sheni') + t.default('megitzda') + t.translate(lambda kw: kw['shen'] + kw['sheni']) + + with md.metadata('shen') as t: + t.default('vaxo') + + with md.metadata('sheni') as t: + t.default('gio') + + with md.metadata('vaxo') as t: + t.depends('shevetsi') + + v = md.global_reader.read('bs mang', {}) + self.assertEqual(v['MDATA_TESTING'], 'vaxogio') + self.assertTrue( 'vaxo' not in v ) + + md.global_reader.clear() + +if __name__ == '__main__': unittest.main() From 0422127689ef4a073090d766aae66fefea04a434 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 12:54:28 -0400 Subject: [PATCH 078/108] Added unit tests for requestsync --- .../media-monitor2/tests/test_requestsync.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 python_apps/media-monitor2/tests/test_requestsync.py diff --git a/python_apps/media-monitor2/tests/test_requestsync.py b/python_apps/media-monitor2/tests/test_requestsync.py new file mode 100644 index 000000000..a26b86b46 --- /dev/null +++ b/python_apps/media-monitor2/tests/test_requestsync.py @@ -0,0 +1,48 @@ +import unittest +from mock import MagicMock + +from media.monitor.watchersyncer import RequestSync + +class TestRequestSync(unittest.TestCase): + + def apc_mock(self): + fake_apc = MagicMock() + fake_apc.send_media_monitor_requests = MagicMock() + return fake_apc + + def watcher_mock(self): + fake_watcher = MagicMock() + fake_watcher.flag_done = MagicMock() + return fake_watcher + + def request_mock(self): + fake_request = MagicMock() + fake_request.safe_pack = MagicMock(return_value=[]) + return fake_request + + def test_send_media_monitor(self): + fake_apc = self.apc_mock() + fake_requests = [ self.request_mock() for x in range(1,5) ] + fake_watcher = self.watcher_mock() + rs = RequestSync(fake_watcher, fake_requests, fake_apc) + rs.run_request() + self.assertEquals(fake_apc.send_media_monitor_requests.call_count, 1) + + def test_flag_done(self): + fake_apc = self.apc_mock() + fake_requests = [ self.request_mock() for x in range(1,5) ] + fake_watcher = self.watcher_mock() + rs = RequestSync(fake_watcher, fake_requests, fake_apc) + rs.run_request() + self.assertEquals(fake_watcher.flag_done.call_count, 1) + + def test_safe_pack(self): + fake_apc = self.apc_mock() + fake_requests = [ self.request_mock() for x in range(1,5) ] + fake_watcher = self.watcher_mock() + rs = RequestSync(fake_watcher, fake_requests, fake_apc) + rs.run_request() + for req in fake_requests: + self.assertEquals(req.safe_pack.call_count, 1) + +if __name__ == '__main__': unittest.main() From 212e3bd30e614de05adbaa7d6f1b0ce69a8490a0 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:07:51 -0400 Subject: [PATCH 079/108] created a module for requests (refactorings) --- .../media-monitor2/media/monitor/request.py | 62 +++++++++++++++++++ .../media/monitor/watchersyncer.py | 58 +---------------- .../media-monitor2/tests/test_requestsync.py | 2 +- 3 files changed, 64 insertions(+), 58 deletions(-) create mode 100644 python_apps/media-monitor2/media/monitor/request.py diff --git a/python_apps/media-monitor2/media/monitor/request.py b/python_apps/media-monitor2/media/monitor/request.py new file mode 100644 index 000000000..934290a05 --- /dev/null +++ b/python_apps/media-monitor2/media/monitor/request.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +import threading + +from media.monitor.exceptions import BadSongFile +from media.monitor.log import Loggable +import api_clients.api_client as ac + +class ThreadedRequestSync(threading.Thread, Loggable): + def __init__(self, rs): + threading.Thread.__init__(self) + self.rs = rs + self.daemon = True + self.start() + + def run(self): + self.rs.run_request() + +class RequestSync(Loggable): + """ + This class is responsible for making the api call to send a request + to airtime. In the process it packs the requests and retries for + some number of times + """ + @classmethod + def create_with_api_client(cls, watcher, requests): + apiclient = ac.AirtimeApiClient.create_right_config() + self = cls(watcher, requests, apiclient) + return self + + def __init__(self, watcher, requests, apiclient): + self.watcher = watcher + self.requests = requests + self.apiclient = apiclient + + def run_request(self): + self.logger.info("Attempting request with %d items." % + len(self.requests)) + packed_requests = [] + for request_event in self.requests: + try: + for request in request_event.safe_pack(): + if isinstance(request, BadSongFile): + self.logger.info("Bad song file: '%s'" % request.path) + else: packed_requests.append(request) + except Exception as e: + self.unexpected_exception( e ) + if hasattr(request_event, 'path'): + self.logger.info("Possibly related to path: '%s'" % + request_event.path) + try: self.apiclient.send_media_monitor_requests( packed_requests ) + # most likely we did not get json response as we expected + except ValueError: + self.logger.info("ApiController.php probably crashed, we \ + diagnose this from the fact that it did not return \ + valid json") + self.logger.info("Trying again after %f seconds" % + self.request_wait) + except Exception as e: self.unexpected_exception(e) + else: self.logger.info("Request was successful") + self.watcher.flag_done() # poor man's condition variable + diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index d7567a535..9f83c6526 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -8,63 +8,7 @@ from media.monitor.log import Loggable from media.monitor.exceptions import BadSongFile from media.monitor.eventcontractor import EventContractor from media.monitor.events import EventProxy - -import api_clients.api_client as ac - - -class ThreadedRequestSync(threading.Thread, Loggable): - def __init__(self, rs): - threading.Thread.__init__(self) - self.rs = rs - self.daemon = True - self.start() - - def run(self): - self.rs.run_request() - -class RequestSync(Loggable): - """ - This class is responsible for making the api call to send a request - to airtime. In the process it packs the requests and retries for - some number of times - """ - @classmethod - def create_with_api_client(cls, watcher, requests): - apiclient = ac.AirtimeApiClient.create_right_config() - self = cls(watcher, requests, apiclient) - return self - - def __init__(self, watcher, requests, apiclient): - self.watcher = watcher - self.requests = requests - self.apiclient = apiclient - - def run_request(self): - self.logger.info("Attempting request with %d items." % - len(self.requests)) - packed_requests = [] - for request_event in self.requests: - try: - for request in request_event.safe_pack(): - if isinstance(request, BadSongFile): - self.logger.info("Bad song file: '%s'" % request.path) - else: packed_requests.append(request) - except Exception as e: - self.unexpected_exception( e ) - if hasattr(request_event, 'path'): - self.logger.info("Possibly related to path: '%s'" % - request_event.path) - try: self.apiclient.send_media_monitor_requests( packed_requests ) - # most likely we did not get json response as we expected - except ValueError: - self.logger.info("ApiController.php probably crashed, we \ - diagnose this from the fact that it did not return \ - valid json") - self.logger.info("Trying again after %f seconds" % - self.request_wait) - except Exception as e: self.unexpected_exception(e) - else: self.logger.info("Request was successful") - self.watcher.flag_done() # poor man's condition variable +from media.monitor.request import ThreadedRequestSync, RequestSync class TimeoutWatcher(threading.Thread,Loggable): """ diff --git a/python_apps/media-monitor2/tests/test_requestsync.py b/python_apps/media-monitor2/tests/test_requestsync.py index a26b86b46..2570bc34e 100644 --- a/python_apps/media-monitor2/tests/test_requestsync.py +++ b/python_apps/media-monitor2/tests/test_requestsync.py @@ -1,7 +1,7 @@ import unittest from mock import MagicMock -from media.monitor.watchersyncer import RequestSync +from media.monitor.request import RequestSync class TestRequestSync(unittest.TestCase): From fb28c3a6c5c7dc9ad3780d12f219f449c3756cc2 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:32:27 -0400 Subject: [PATCH 080/108] Formatting --- python_apps/media-monitor2/media/monitor/watchersyncer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index 9f83c6526..b008b10d0 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -71,8 +71,7 @@ class WatchSyncer(ReportHandler,Loggable): #self.push_queue( event ) except BadSongFile as e: self.fatal_exception("Received bas song file '%s'" % e.path, e) - except Exception as e: - self.unexpected_exception(e) + except Exception as e: self.unexpected_exception(e) else: self.logger.info("Received event that does not implement packing.\ Printing its representation:") From a42210e321f6a3e563ee655d8e6197756c620f38 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:38:11 -0400 Subject: [PATCH 081/108] added comment for destructor --- python_apps/media-monitor2/media/monitor/watchersyncer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index b008b10d0..d2df9ed3a 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -157,7 +157,8 @@ class WatchSyncer(ReportHandler,Loggable): def __reset_queue(self): self.__queue = [] def __del__(self): - # Ideally we would like to do a little more to ensure safe shutdown + #this destructor is completely untested and it's unclear whether + #it's even doing anything useful. consider removing it if self.events_in_queue(): self.logger.warn("Terminating with events still in the queue...") if self.requests_in_queue(): From a0f83c4db0600d99890fcf0b81f6d437aec02817 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:40:13 -0400 Subject: [PATCH 082/108] removed big unused method --- .../media-monitor2/media/monitor/pure.py | 48 ------------------- 1 file changed, 48 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index fa14089d7..462daf07a 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -265,54 +265,6 @@ def parse_int(s): try : return str(reduce(op.add, takewhile(lambda x: x.isdigit(), s))) except: return None -def normalized_metadata(md, original_path): - """ - consumes a dictionary of metadata and returns a new dictionary with the - formatted meta data. We also consume original_path because we must set - MDATA_KEY_CREATOR based on in it sometimes - """ - new_md = copy.deepcopy(md) - # replace all slashes with dashes - #for k,v in new_md.iteritems(): new_md[k] = unicode(v).replace('/','-') - # Specific rules that are applied in a per attribute basis - format_rules = { - 'MDATA_KEY_TRACKNUMBER' : parse_int, - 'MDATA_KEY_FILEPATH' : lambda x: os.path.normpath(x), - 'MDATA_KEY_BPM' : lambda x: x[0:8], - 'MDATA_KEY_MIME' : lambda x: x.replace('audio/vorbis','audio/ogg'), - # Whenever 0 is reported we change it to empty - #'MDATA_KEY_BITRATE' : lambda x: '' if str(x) == '0' else x - } - - new_md = remove_whitespace(new_md) # remove whitespace fields - # Format all the fields in format_rules - new_md = apply_rules_dict(new_md, format_rules) - # set filetype to audioclip by default - new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_FTYPE'], - default=u'audioclip') - - # Try to parse bpm but delete the whole key if that fails - if 'MDATA_KEY_BPM' in new_md: - new_md['MDATA_KEY_BPM'] = parse_int(new_md['MDATA_KEY_BPM']) - if new_md['MDATA_KEY_BPM'] is None: - del new_md['MDATA_KEY_BPM'] - - if not is_airtime_recorded(new_md): - # Read title from filename if it does not exist - default_title = no_extension_basename(original_path) - default_title = re.sub(r'__\d+\.',u'.', default_title) - if re.match(".+-%s-.+$" % unicode_unknown, default_title): - default_title = u'' - new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_TITLE'], - default=default_title) - new_md['MDATA_KEY_TITLE'] = re.sub(r'-\d+kbps$', u'', - new_md['MDATA_KEY_TITLE']) - - # TODO : wtf is this for again? - new_md['MDATA_KEY_TITLE'] = re.sub(r'-?%s-?' % unicode_unknown, u'', - new_md['MDATA_KEY_TITLE']) - new_md['MDATA_KEY_ORIGINAL_PATH'] = normpath(original_path) - return new_md def organized_path(old_path, root_path, orig_md): """ From 958b489fca148833214a4e698b434bfba191ebc0 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:43:15 -0400 Subject: [PATCH 083/108] formatted comments and removed unused function --- .../media-monitor2/media/monitor/pure.py | 31 +++---------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 462daf07a..1c747fdf0 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -66,24 +66,6 @@ class IncludeOnly(object): return func(moi, event, *args, **kwargs) return _wrap - - -def diff_dict(d1, d2, width=30): - """ - returns a formatted diff of 2 dictionaries - """ - out = "" - all_keys = d1.keys() + d2.keys() - for k in all_keys: - v1, v2 = d1.get(k), d2.get(k) - - # default values - if v1 is None: v1 = "N/A" - if v2 is None: v2 = "N/A" - - if d1[k] != d2[k]: - out += "%s%s%s" % (k, d1[k], d2[k]) - def partition(f, alist): """ Partition is very similar to filter except that it also returns the @@ -452,9 +434,8 @@ def owner_id(original_path): return owner_id def file_playable(pathname): - """ - Returns True if 'pathname' is playable by liquidsoap. False otherwise. - """ + """ Returns True if 'pathname' is playable by liquidsoap. False + otherwise. """ # when there is an single apostrophe inside of a string quoted by # apostrophes, we can only escape it by replace that apostrophe with # '\''. This breaks the string into two, and inserts an escaped @@ -490,18 +471,14 @@ def toposort(data): assert not data, "A cyclic dependency exists amongst %r" % data def truncate_to_length(item, length): - """ - Truncates 'item' to 'length' - """ + """ Truncates 'item' to 'length' """ if isinstance(item, int): item = str(item) if isinstance(item, basestring): if len(item) > length: return item[0:length] else: return item def format_length(mutagen_length): - """ - Convert mutagen length to airtime length - """ + """ Convert mutagen length to airtime length """ t = float(mutagen_length) h = int(math.floor(t / 3600)) t = t % 3600 From 5ed86e26665b8f090b22c89f7559ce1704f01147 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:44:51 -0400 Subject: [PATCH 084/108] added docstring for wave duration --- python_apps/media-monitor2/media/monitor/pure.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 1c747fdf0..1d8abf8f3 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -91,14 +91,13 @@ def is_file_supported(path): # TODO : In the future we would like a better way to find out whether a show # has been recorded def is_airtime_recorded(md): - """ - Takes a metadata dictionary and returns True if it belongs to a file that - was recorded by Airtime. - """ + """ Takes a metadata dictionary and returns True if it belongs to a + file that was recorded by Airtime. """ if not 'MDATA_KEY_CREATOR' in md: return False return md['MDATA_KEY_CREATOR'] == u'Airtime Show Recorder' def read_wave_duration(path): + """ Read the length of .wav file (mutagen does not handle this) """ with contextlib.closing(wave.open(path,'r')) as f: frames = f.getnframes() rate = f.getframerate() @@ -106,9 +105,7 @@ def read_wave_duration(path): return duration def clean_empty_dirs(path): - """ - walks path and deletes every empty directory it finds - """ + """ walks path and deletes every empty directory it finds """ # TODO : test this function if path.endswith('/'): clean_empty_dirs(path[0:-1]) else: From cccbfa2c3465df9b45d260a0863af6be1723e04b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:47:23 -0400 Subject: [PATCH 085/108] Formatted docstrings --- .../media-monitor2/media/monitor/pure.py | 66 +++++++------------ 1 file changed, 24 insertions(+), 42 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 1d8abf8f3..ad8ce336b 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -150,11 +150,10 @@ def no_extension_basename(path): else: return '.'.join(base.split(".")[0:-1]) def walk_supported(directory, clean_empties=False): - """ - A small generator wrapper around os.walk to only give us files that support - the extensions we are considering. When clean_empties is True we - recursively delete empty directories left over in directory after the walk. - """ + """ A small generator wrapper around os.walk to only give us files + that support the extensions we are considering. When clean_empties + is True we recursively delete empty directories left over in + directory after the walk. """ for root, dirs, files in os.walk(directory): full_paths = ( os.path.join(root, name) for name in files if is_file_supported(name) ) @@ -168,10 +167,8 @@ def file_locked(path): return bool(f.readlines()) def magic_move(old, new, after_dir_make=lambda : None): - """ - Moves path old to new and constructs the necessary to directories for new - along the way - """ + """ Moves path old to new and constructs the necessary to + directories for new along the way """ new_dir = os.path.dirname(new) if not os.path.exists(new_dir): os.makedirs(new_dir) # We need this crusty hack because anytime a directory is created we must @@ -181,18 +178,15 @@ def magic_move(old, new, after_dir_make=lambda : None): shutil.move(old,new) def move_to_dir(dir_path,file_path): - """ - moves a file at file_path into dir_path/basename(filename) - """ + """ moves a file at file_path into dir_path/basename(filename) """ bs = os.path.basename(file_path) magic_move(file_path, os.path.join(dir_path, bs)) def apply_rules_dict(d, rules): - """ - Consumes a dictionary of rules that maps some keys to lambdas which it - applies to every matching element in d and returns a new dictionary with - the rules applied. If a rule returns none then it's not applied - """ + """ Consumes a dictionary of rules that maps some keys to lambdas + which it applies to every matching element in d and returns a new + dictionary with the rules applied. If a rule returns none then it's + not applied """ new_d = copy.deepcopy(d) for k, rule in rules.iteritems(): if k in d: @@ -207,17 +201,14 @@ def default_to_f(dictionary, keys, default, condition): return new_d def default_to(dictionary, keys, default): - """ - Checks if the list of keys 'keys' exists in 'dictionary'. If not then it - returns a new dictionary with all those missing keys defaults to 'default' - """ + """ Checks if the list of keys 'keys' exists in 'dictionary'. If + not then it returns a new dictionary with all those missing keys + defaults to 'default' """ cnd = lambda dictionary, key: key not in dictionary return default_to_f(dictionary, keys, default, cnd) def remove_whitespace(dictionary): - """ - Remove values that empty whitespace in the dictionary - """ + """ Remove values that empty whitespace in the dictionary """ nd = copy.deepcopy(dictionary) bad_keys = [] for k,v in nd.iteritems(): @@ -303,10 +294,9 @@ def organized_path(old_path, root_path, orig_md): # TODO : Get rid of this function and every one of its uses. We no longer use # the md5 signature of a song for anything def file_md5(path,max_length=100): - """ - Get md5 of file path (if it exists). Use only max_length characters to save - time and memory. Pass max_length=-1 to read the whole file (like in mm1) - """ + """ Get md5 of file path (if it exists). Use only max_length + characters to save time and memory. Pass max_length=-1 to read the + whole file (like in mm1) """ if os.path.exists(path): with open(path, 'rb') as f: m = hashlib.md5() @@ -322,16 +312,12 @@ def encode_to(obj, encoding='utf-8'): return obj def convert_dict_value_to_utf8(md): - """ - formats a dictionary to send as a request to api client - """ + """ formats a dictionary to send as a request to api client """ return dict([(item[0], encode_to(item[1], "utf-8")) for item in md.items()]) def get_system_locale(locale_path='/etc/default/locale'): - """ - Returns the configuration object for the system's default locale. Normally - requires root access. - """ + """ Returns the configuration object for the system's default + locale. Normally requires root access. """ if os.path.exists(locale_path): try: config = ConfigObj(locale_path) @@ -341,9 +327,7 @@ def get_system_locale(locale_path='/etc/default/locale'): permissions issue?" % locale_path) def configure_locale(config): - """ - sets the locale according to the system's locale. - """ + """ sets the locale according to the system's locale. """ current_locale = locale.getlocale() if current_locale[1] is None: default_locale = locale.getdefaultlocale() @@ -360,10 +344,8 @@ def configure_locale(config): def fondle(path,times=None): # TODO : write unit tests for this - """ - touch a file to change the last modified date. Beware of calling this - function on the same file from multiple threads. - """ + """ touch a file to change the last modified date. Beware of calling + this function on the same file from multiple threads. """ with file(path, 'a'): os.utime(path, times) def last_modified(path): From 41b1b357eb8e9fc130a282df2d381d8c9a9fe3b0 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:48:01 -0400 Subject: [PATCH 086/108] formatted more docstrings --- .../media-monitor2/media/monitor/pure.py | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index ad8ce336b..45f4cc66e 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -349,20 +349,16 @@ def fondle(path,times=None): with file(path, 'a'): os.utime(path, times) def last_modified(path): - """ - return the time of the last time mm2 was ran. path refers to the index file - whose date modified attribute contains this information. In the case when - the file does not exist we set this time 0 so that any files on the - filesystem were modified after it - """ + """ return the time of the last time mm2 was ran. path refers to the + index file whose date modified attribute contains this information. + In the case when the file does not exist we set this time 0 so that + any files on the filesystem were modified after it """ if os.path.exists(path): return os.path.getmtime(path) else: return 0 def expand_storage(store): - """ - A storage directory usually consists of 4 different subdirectories. This - function returns their paths - """ + """ A storage directory usually consists of 4 different + subdirectories. This function returns their paths """ store = os.path.normpath(store) return { 'organize' : os.path.join(store, 'organize'), @@ -372,10 +368,8 @@ def expand_storage(store): } def create_dir(path): - """ - will try and make sure that path exists at all costs. raises an exception - if it fails at this task. - """ + """ will try and make sure that path exists at all costs. raises an + exception if it fails at this task. """ if not os.path.exists(path): try : os.makedirs(path) except Exception as e : raise FailedToCreateDir(path, e) @@ -393,11 +387,10 @@ def sub_path(directory,f): return common == normalized def owner_id(original_path): - """ - Given 'original_path' return the file name of the of 'identifier' file. - return the id that is contained in it. If no file is found or nothing is - read then -1 is returned. File is deleted after the number has been read - """ + """ Given 'original_path' return the file name of the of + 'identifier' file. return the id that is contained in it. If no file + is found or nothing is read then -1 is returned. File is deleted + after the number has been read """ fname = "%s.identifier" % original_path owner_id = -1 try: From b2650b9a21dc80639915903e3a549e29d746bc0d Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:49:01 -0400 Subject: [PATCH 087/108] Added comment --- python_apps/media-monitor2/media/monitor/pure.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_apps/media-monitor2/media/monitor/pure.py b/python_apps/media-monitor2/media/monitor/pure.py index 45f4cc66e..a879f449f 100644 --- a/python_apps/media-monitor2/media/monitor/pure.py +++ b/python_apps/media-monitor2/media/monitor/pure.py @@ -220,6 +220,7 @@ def remove_whitespace(dictionary): return nd def parse_int(s): + # TODO : this function isn't used anywhere yet but it may useful for emf """ Tries very hard to get some sort of integer result from s. Defaults to 0 when it fails From 0f1e843017662cc909e1d12d76f27f9d81303b69 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:49:49 -0400 Subject: [PATCH 088/108] Formatted docstring --- python_apps/media-monitor2/media/monitor/log.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/log.py b/python_apps/media-monitor2/media/monitor/log.py index 5d632bbf4..cf4efb79a 100644 --- a/python_apps/media-monitor2/media/monitor/log.py +++ b/python_apps/media-monitor2/media/monitor/log.py @@ -6,23 +6,19 @@ from media.monitor.pure import LazyProperty appname = 'root' def setup_logging(log_path): - """ - Setup logging by writing log to 'log_path' - """ + """ Setup logging by writing log to 'log_path' """ #logger = logging.getLogger(appname) logging.basicConfig(filename=log_path, level=logging.DEBUG) def get_logger(): - """ - in case we want to use the common logger from a procedural interface - """ + """ in case we want to use the common logger from a procedural + interface """ return logging.getLogger() class Loggable(object): - """ - Any class that wants to log can inherit from this class and automatically - get a logger attribute that can be used like: self.logger.info(...) etc. - """ + """ Any class that wants to log can inherit from this class and + automatically get a logger attribute that can be used like: + self.logger.info(...) etc. """ __metaclass__ = abc.ABCMeta @LazyProperty def logger(self): return get_logger() From 2e54fd64d3ab0312a98840ce7cc8fdb134437ac9 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 14:50:08 -0400 Subject: [PATCH 089/108] Formatted more docstrings --- python_apps/media-monitor2/media/monitor/log.py | 9 +++------ python_apps/media-monitor2/media/monitor/request.py | 8 +++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/log.py b/python_apps/media-monitor2/media/monitor/log.py index cf4efb79a..7e75c719d 100644 --- a/python_apps/media-monitor2/media/monitor/log.py +++ b/python_apps/media-monitor2/media/monitor/log.py @@ -24,15 +24,12 @@ class Loggable(object): def logger(self): return get_logger() def unexpected_exception(self,e): - """ - Default message for 'unexpected' exceptions - """ + """ Default message for 'unexpected' exceptions """ self.fatal_exception("'Unexpected' exception has occured:", e) def fatal_exception(self, message, e): - """ - Prints an exception 'e' with 'message'. Also outputs the traceback. - """ + """ Prints an exception 'e' with 'message'. Also outputs the + traceback. """ self.logger.error( message ) self.logger.error( str(e) ) self.logger.error( traceback.format_exc() ) diff --git a/python_apps/media-monitor2/media/monitor/request.py b/python_apps/media-monitor2/media/monitor/request.py index 934290a05..a2f3cdc8f 100644 --- a/python_apps/media-monitor2/media/monitor/request.py +++ b/python_apps/media-monitor2/media/monitor/request.py @@ -17,11 +17,9 @@ class ThreadedRequestSync(threading.Thread, Loggable): self.rs.run_request() class RequestSync(Loggable): - """ - This class is responsible for making the api call to send a request - to airtime. In the process it packs the requests and retries for - some number of times - """ + """ This class is responsible for making the api call to send a + request to airtime. In the process it packs the requests and retries + for some number of times """ @classmethod def create_with_api_client(cls, watcher, requests): apiclient = ac.AirtimeApiClient.create_right_config() From 10a6a4d541e47cee48f552e4c04a7586f20cac40 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 15:03:59 -0400 Subject: [PATCH 090/108] formatted docstrings. fixed decorator with functools --- .../media-monitor2/media/monitor/listeners.py | 2 ++ .../media-monitor2/media/monitor/organizer.py | 22 ++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/listeners.py b/python_apps/media-monitor2/media/monitor/listeners.py index 4c860a97e..b33a5c1a9 100644 --- a/python_apps/media-monitor2/media/monitor/listeners.py +++ b/python_apps/media-monitor2/media/monitor/listeners.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import pyinotify from pydispatch import dispatcher +from functools import wraps import media.monitor.pure as mmp from media.monitor.pure import IncludeOnly @@ -31,6 +32,7 @@ class FileMediator(object): def unignore(path): FileMediator.ignored_set.remove(path) def mediate_ignored(fn): + @wraps(fn) def wrapped(self, event, *args,**kwargs): event.pathname = unicode(event.pathname, "utf-8") if FileMediator.is_ignored(event.pathname): diff --git a/python_apps/media-monitor2/media/monitor/organizer.py b/python_apps/media-monitor2/media/monitor/organizer.py index ea6851356..ce1849b90 100644 --- a/python_apps/media-monitor2/media/monitor/organizer.py +++ b/python_apps/media-monitor2/media/monitor/organizer.py @@ -10,14 +10,12 @@ from os.path import dirname import os.path class Organizer(ReportHandler,Loggable): - """ - Organizer is responsible to to listening to OrganizeListener events - and committing the appropriate changes to the filesystem. It does - not in any interact with WatchSyncer's even when the the WatchSyncer - is a "storage directory". The "storage" directory picks up all of - its events through pyinotify. (These events are fed to it through - StoreWatchListener) - """ + """ Organizer is responsible to to listening to OrganizeListener + events and committing the appropriate changes to the filesystem. + It does not in any interact with WatchSyncer's even when the the + WatchSyncer is a "storage directory". The "storage" directory picks + up all of its events through pyinotify. (These events are fed to it + through StoreWatchListener) """ # Commented out making this class a singleton because it's just a band aid # for the real issue. The real issue being making multiple Organizer @@ -41,11 +39,9 @@ class Organizer(ReportHandler,Loggable): super(Organizer, self).__init__(signal=self.channel, weak=False) def handle(self, sender, event): - """ - Intercept events where a new file has been added to the organize - directory and place it in the correct path (starting with - self.target_path) - """ + """ Intercept events where a new file has been added to the + organize directory and place it in the correct path (starting + with self.target_path) """ # Only handle this event type assert isinstance(event, OrganizeFile), \ "Organizer can only handle OrganizeFile events.Given '%s'" % event From 0135fbe9dd457c0c6762bec91999e70d6dde5859 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 25 Oct 2012 16:20:15 -0400 Subject: [PATCH 091/108] Fixed code formatting --- python_apps/media-monitor2/mm2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 17731cad8..ea1178a2f 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -59,7 +59,8 @@ def main(global_config, api_client_config, log_config, try: with open(config['index_path'], 'w') as f: f.write(" ") except Exception as e: - log.info("Failed to create index file with exception: %s" % str(e)) + log.info("Failed to create index file with exception: %s" \ + % str(e)) else: log.info("Created index file, reloading configuration:") main( global_config, api_client_config, log_config, From b8a5c6ce955ffd7339249dad7c8e7f64a92a1c21 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 26 Oct 2012 15:44:58 -0400 Subject: [PATCH 092/108] cc-4634: Fixed --- python_apps/media-monitor2/media/monitor/metadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_apps/media-monitor2/media/monitor/metadata.py b/python_apps/media-monitor2/media/monitor/metadata.py index 5bbbafaeb..d5dba3b51 100644 --- a/python_apps/media-monitor2/media/monitor/metadata.py +++ b/python_apps/media-monitor2/media/monitor/metadata.py @@ -114,6 +114,7 @@ class Metadata(Loggable): if airtime_k in airtime2mutagen: # The unicode cast here is mostly for integers that need to be # strings + if airtime_v is None: continue try: song_file[ airtime2mutagen[airtime_k] ] = unicode(airtime_v) except (EasyMP4KeyError, EasyID3KeyError) as e: From 7c48683492772523cac454aad89f45d148a1bf16 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 26 Oct 2012 16:02:32 -0400 Subject: [PATCH 093/108] cc-4635: Fix --- python_apps/media-monitor2/media/metadata/definitions.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index ee4175033..c53d091f3 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -122,8 +122,16 @@ def load_definitions(): else: default_title = no_extension_basename(k['path']) default_title = re.sub(r'__\d+\.',u'.', default_title) + + # format is: track_number-title-123kbps.mp3 + m = re.match(".+-(?P.+)-\d+kbps", default_title) + if m: + if m.group('title') == unicode_unknown: new_title = '' + else: new_title = m.group('title') + if re.match(".+-%s-.+$" % unicode_unknown, default_title): default_title = u'' + new_title = default_title new_title = re.sub(r'-\d+kbps$', u'', new_title) return new_title From 1ee228558ac3dffcf95fb889af65800f776ebe25 Mon Sep 17 00:00:00 2001 From: James <james@sourcefabric-DX4840.(none)> Date: Fri, 26 Oct 2012 16:24:17 -0400 Subject: [PATCH 094/108] CC-4574: Audio preview fails with certain characters in metadata - fixed --- airtime_mvc/public/js/airtime/common/common.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtime_mvc/public/js/airtime/common/common.js b/airtime_mvc/public/js/airtime/common/common.js index 3125d8d55..19de0d0a9 100644 --- a/airtime_mvc/public/js/airtime/common/common.js +++ b/airtime_mvc/public/js/airtime/common/common.js @@ -54,7 +54,7 @@ function open_audio_preview(type, id, audioFileTitle, audioFileArtist) { audioFileTitle = audioFileTitle.substring(0,index); } - openPreviewWindow('audiopreview/audio-preview/audioFileID/'+id+'/audioFileArtist/'+encodeURIComponent(audioFileArtist)+'/audioFileTitle/'+encodeURIComponent(audioFileTitle)+'/type/'+type); + openPreviewWindow('audiopreview/audio-preview/audioFileID/'+id+'/audioFileArtist/'+encodeURIComponent(encodeURIComponent(audioFileArtist))+'/audioFileTitle/'+encodeURIComponent(encodeURIComponent(audioFileTitle))+'/type/'+type); _preview_window.focus(); } From e667a721adfb8350e092d9dd05b66df9bb72b160 Mon Sep 17 00:00:00 2001 From: James <james@sourcefabric-DX4840.(none)> Date: Fri, 26 Oct 2012 16:26:48 -0400 Subject: [PATCH 095/108] - comment for some code --- airtime_mvc/public/js/airtime/common/common.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airtime_mvc/public/js/airtime/common/common.js b/airtime_mvc/public/js/airtime/common/common.js index 19de0d0a9..0d9883b66 100644 --- a/airtime_mvc/public/js/airtime/common/common.js +++ b/airtime_mvc/public/js/airtime/common/common.js @@ -54,6 +54,9 @@ function open_audio_preview(type, id, audioFileTitle, audioFileArtist) { audioFileTitle = audioFileTitle.substring(0,index); } + // The reason that we need to encode artist and title string is that + // sometime they contain '/' or '\' and apache reject %2f or %5f + // so the work around is to encode it twice. openPreviewWindow('audiopreview/audio-preview/audioFileID/'+id+'/audioFileArtist/'+encodeURIComponent(encodeURIComponent(audioFileArtist))+'/audioFileTitle/'+encodeURIComponent(encodeURIComponent(audioFileTitle))+'/type/'+type); _preview_window.focus(); From 7cd8541696372bb15e83d64f1c46686d72b7fa80 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Fri, 26 Oct 2012 16:41:29 -0400 Subject: [PATCH 096/108] cc-4635: Fix --- python_apps/media-monitor2/media/metadata/definitions.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index c53d091f3..0e7282b20 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -124,16 +124,12 @@ def load_definitions(): default_title = re.sub(r'__\d+\.',u'.', default_title) # format is: track_number-title-123kbps.mp3 - m = re.match(".+-(?P<title>.+)-\d+kbps", default_title) + m = re.match(".+-(?P<title>.+)-\d+kbps$", default_title) if m: if m.group('title') == unicode_unknown: new_title = '' else: new_title = m.group('title') + else: new_title = re.sub(r'-\d+kbps$', u'', new_title) - if re.match(".+-%s-.+$" % unicode_unknown, default_title): - default_title = u'' - - new_title = default_title - new_title = re.sub(r'-\d+kbps$', u'', new_title) return new_title with md.metadata('MDATA_KEY_TITLE') as t: From e10f81aec85b77a327aa7938fea1c90340103fb8 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Fri, 26 Oct 2012 16:48:03 -0400 Subject: [PATCH 097/108] cc-4635: Fix --- python_apps/media-monitor2/media/metadata/definitions.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index 0e7282b20..a6d8221df 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -115,7 +115,7 @@ def load_definitions(): # 2. title is absent (read from file) # 3. recorded file def tr_title(k): - unicode_unknown = u"unknown" + #unicode_unknown = u"unknown" new_title = u"" if is_airtime_recorded(k) or k['title'] != u"": new_title = k['title'] @@ -125,9 +125,7 @@ def load_definitions(): # format is: track_number-title-123kbps.mp3 m = re.match(".+-(?P<title>.+)-\d+kbps$", default_title) - if m: - if m.group('title') == unicode_unknown: new_title = '' - else: new_title = m.group('title') + if m: new_title = m.group('title') else: new_title = re.sub(r'-\d+kbps$', u'', new_title) return new_title From 72928bb5e65f1a7ab68212a2f71b2cd15b05ca92 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Fri, 26 Oct 2012 16:59:32 -0400 Subject: [PATCH 098/108] cc-4635: Fix --- python_apps/media-monitor2/media/metadata/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index a6d8221df..31d28b563 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -126,7 +126,7 @@ def load_definitions(): # format is: track_number-title-123kbps.mp3 m = re.match(".+-(?P<title>.+)-\d+kbps$", default_title) if m: new_title = m.group('title') - else: new_title = re.sub(r'-\d+kbps$', u'', new_title) + else: new_title = re.sub(r'-\d+kbps$', u'', default_title) return new_title From c4f0491ff643ca8b17038118c41be32123e15255 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Fri, 26 Oct 2012 17:04:42 -0400 Subject: [PATCH 099/108] cc-4635: Fix --- python_apps/media-monitor2/media/metadata/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index 31d28b563..9919eb041 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -124,7 +124,7 @@ def load_definitions(): default_title = re.sub(r'__\d+\.',u'.', default_title) # format is: track_number-title-123kbps.mp3 - m = re.match(".+-(?P<title>.+)-\d+kbps$", default_title) + m = re.match(".+-(?P<title>.+)-(\d+kbps|unknown)$", default_title) if m: new_title = m.group('title') else: new_title = re.sub(r'-\d+kbps$', u'', default_title) From 0a396378e52397c6c9d5ecf8a0c744760d808f27 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Fri, 26 Oct 2012 17:15:28 -0400 Subject: [PATCH 100/108] cc-4635: Fix --- python_apps/media-monitor2/media/metadata/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/metadata/definitions.py b/python_apps/media-monitor2/media/metadata/definitions.py index 9919eb041..ec3bed48f 100644 --- a/python_apps/media-monitor2/media/metadata/definitions.py +++ b/python_apps/media-monitor2/media/metadata/definitions.py @@ -124,7 +124,7 @@ def load_definitions(): default_title = re.sub(r'__\d+\.',u'.', default_title) # format is: track_number-title-123kbps.mp3 - m = re.match(".+-(?P<title>.+)-(\d+kbps|unknown)$", default_title) + m = re.match(".+?-(?P<title>.+)-(\d+kbps|unknown)$", default_title) if m: new_title = m.group('title') else: new_title = re.sub(r'-\d+kbps$', u'', default_title) From f14320a769c9884bbba1718a769572c1e04573f0 Mon Sep 17 00:00:00 2001 From: James <james@sourcefabric-DX4840.(none)> Date: Fri, 26 Oct 2012 17:15:39 -0400 Subject: [PATCH 101/108] CC-4603: Dynamic Smart Blocks: adding to multiple shows at once generates the same content - fixed --- airtime_mvc/application/models/Scheduler.php | 43 +++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/airtime_mvc/application/models/Scheduler.php b/airtime_mvc/application/models/Scheduler.php index 664c40504..def7fb045 100644 --- a/airtime_mvc/application/models/Scheduler.php +++ b/airtime_mvc/application/models/Scheduler.php @@ -366,12 +366,11 @@ class Application_Model_Scheduler * @param array $fileIds * @param array $playlistIds */ - private function insertAfter($scheduleItems, $schedFiles, $adjustSched = true) + private function insertAfter($scheduleItems, $schedFiles, $adjustSched = true, $mediaItems = null) { try { - $affectedShowInstances = array(); - + //dont want to recalculate times for moved items. $excludeIds = array(); foreach ($schedFiles as $file) { @@ -384,7 +383,17 @@ class Application_Model_Scheduler foreach ($scheduleItems as $schedule) { $id = intval($schedule["id"]); - + + // if mediaItmes is passed in, we want to create contents + // at the time of insert. This is for dyanmic blocks or + // playlist that contains dynamic blocks + if ($mediaItems != null) { + $schedFiles = array(); + foreach ($mediaItems as $media) { + $schedFiles = array_merge($schedFiles, $this->retrieveMediaFiles($media["id"], $media["type"])); + } + } + if ($id !== 0) { $schedItem = CcScheduleQuery::create()->findPK($id, $this->con); $instance = $schedItem->getCcShowInstances($this->con); @@ -527,10 +536,32 @@ class Application_Model_Scheduler $this->validateRequest($scheduleItems); + $requireDynamicContentCreation = false; + foreach ($mediaItems as $media) { - $schedFiles = array_merge($schedFiles, $this->retrieveMediaFiles($media["id"], $media["type"])); + if ($media['type'] == "playlist") { + $pl = new Application_Model_Playlist($media['id']); + if ($pl->hasDynamicBlock()) { + $requireDynamicContentCreation = true; + break; + } + } else if ($media['type'] == "block") { + $bl = new Application_Model_Block($media['id']); + if (!$bl->isStatic()) { + $requireDynamicContentCreation = true; + break; + } + } + } + + if ($requireDynamicContentCreation) { + $this->insertAfter($scheduleItems, $schedFiles, $adjustSched, $mediaItems); + } else { + foreach ($mediaItems as $media) { + $schedFiles = array_merge($schedFiles, $this->retrieveMediaFiles($media["id"], $media["type"])); + } + $this->insertAfter($scheduleItems, $schedFiles, $adjustSched); } - $this->insertAfter($scheduleItems, $schedFiles, $adjustSched); $this->con->commit(); From 3cb2346af95a1b11436e774bf9bd8b595e6a97d8 Mon Sep 17 00:00:00 2001 From: Martin Konecny <martin.konecny@gmail.com> Date: Fri, 26 Oct 2012 18:23:28 -0400 Subject: [PATCH 102/108] add pcre debian package for compiling liquidsoap --- dev_tools/fabric/fab_liquidsoap_compile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev_tools/fabric/fab_liquidsoap_compile.py b/dev_tools/fabric/fab_liquidsoap_compile.py index 221aa41c6..98f816e47 100644 --- a/dev_tools/fabric/fab_liquidsoap_compile.py +++ b/dev_tools/fabric/fab_liquidsoap_compile.py @@ -181,7 +181,8 @@ libmad-ocaml-dev libtaglib-ocaml-dev libalsa-ocaml-dev libtaglib-ocaml-dev libvo libspeex-dev libspeexdsp-dev speex libladspa-ocaml-dev festival festival-dev \ libsamplerate-dev libxmlplaylist-ocaml-dev libxmlrpc-light-ocaml-dev libflac-dev \ libxml-dom-perl libxml-dom-xpath-perl patch autoconf libmp3lame-dev \ -libcamomile-ocaml-dev libcamlimages-ocaml-dev libtool libpulse-dev libjack-dev camlidl libfaad-dev''') +libcamomile-ocaml-dev libcamlimages-ocaml-dev libtool libpulse-dev libjack-dev +camlidl libfaad-dev libpcre-ocaml-dev''') root = '/home/martin/src' do_run('mkdir -p %s' % root) From d6a4f2f7136fa6c1f165fe7ef7eedfb67f79b7ba Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Fri, 26 Oct 2012 18:45:27 -0400 Subject: [PATCH 103/108] Cleaned up soundcloud --- airtime_mvc/application/models/Soundcloud.php | 115 +++++++++--------- 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/airtime_mvc/application/models/Soundcloud.php b/airtime_mvc/application/models/Soundcloud.php index 2b1068e57..70148c05c 100644 --- a/airtime_mvc/application/models/Soundcloud.php +++ b/airtime_mvc/application/models/Soundcloud.php @@ -25,66 +25,67 @@ class Application_Model_Soundcloud public function uploadTrack($filepath, $filename, $description, $tags=array(), $release=null, $genre=null) - { - if ($this->getToken()) { - if (count($tags)) { - $tags = join(" ", $tags); - $tags = $tags." ".Application_Model_Preference::GetSoundCloudTags(); - } else { - $tags = Application_Model_Preference::GetSoundCloudTags(); - } + { - $downloadable = Application_Model_Preference::GetSoundCloudDownloadbleOption() == '1'; - - $track_data = array( - 'track[sharing]' => 'private', - 'track[title]' => $filename, - 'track[asset_data]' => '@' . $filepath, - 'track[tag_list]' => $tags, - 'track[description]' => $description, - 'track[downloadable]' => $downloadable, - - ); - - if (isset($release)) { - $release = str_replace(" ", "-", $release); - $release = str_replace(":", "-", $release); - - //YYYY-MM-DD-HH-mm-SS - $release = explode("-", $release); - $track_data['track[release_year]'] = $release[0]; - $track_data['track[release_month]'] = $release[1]; - $track_data['track[release_day]'] = $release[2]; - } - - if (isset($genre) && $genre != "") { - $track_data['track[genre]'] = $genre; - } else { - $default_genre = Application_Model_Preference::GetSoundCloudGenre(); - if ($default_genre != "") { - $track_data['track[genre]'] = $default_genre; - } - } - - $track_type = Application_Model_Preference::GetSoundCloudTrackType(); - if ($track_type != "") { - $track_data['track[track_type]'] = $track_type; - } - - $license = Application_Model_Preference::GetSoundCloudLicense(); - if ($license != "") { - $track_data['track[license]'] = $license; - } - - $response = json_decode( - $this->_soundcloud->post('tracks', $track_data), - true - ); - - return $response; - } else { + if (!$this->getToken()) { throw new NoSoundCloundToken(); + } + if (count($tags)) { + $tags = join(" ", $tags); + $tags = $tags." ".Application_Model_Preference::GetSoundCloudTags(); + } else { + $tags = Application_Model_Preference::GetSoundCloudTags(); } + + $downloadable = Application_Model_Preference::GetSoundCloudDownloadbleOption() == '1'; + + $track_data = array( + 'track[sharing]' => 'private', + 'track[title]' => $filename, + 'track[asset_data]' => '@' . $filepath, + 'track[tag_list]' => $tags, + 'track[description]' => $description, + 'track[downloadable]' => $downloadable, + + ); + + if (isset($release)) { + $release = str_replace(" ", "-", $release); + $release = str_replace(":", "-", $release); + + //YYYY-MM-DD-HH-mm-SS + $release = explode("-", $release); + $track_data['track[release_year]'] = $release[0]; + $track_data['track[release_month]'] = $release[1]; + $track_data['track[release_day]'] = $release[2]; + } + + if (isset($genre) && $genre != "") { + $track_data['track[genre]'] = $genre; + } else { + $default_genre = Application_Model_Preference::GetSoundCloudGenre(); + if ($default_genre != "") { + $track_data['track[genre]'] = $default_genre; + } + } + + $track_type = Application_Model_Preference::GetSoundCloudTrackType(); + if ($track_type != "") { + $track_data['track[track_type]'] = $track_type; + } + + $license = Application_Model_Preference::GetSoundCloudLicense(); + if ($license != "") { + $track_data['track[license]'] = $license; + } + + $response = json_decode( + $this->_soundcloud->post('tracks', $track_data), + true + ); + + return $response; + } public static function uploadSoundcloud($id) From 3b51c93766383f73b7193368e21ba1c99708014a Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Mon, 29 Oct 2012 11:25:05 -0400 Subject: [PATCH 104/108] Refactored pyponotify. Removed code duplication and made runner reusable by other scripts. --- python_apps/pypo/pyponotify.py | 61 +++++++++++++--------------------- 1 file changed, 24 insertions(+), 37 deletions(-) diff --git a/python_apps/pypo/pyponotify.py b/python_apps/pypo/pyponotify.py index 446da8d49..9c2f1688c 100644 --- a/python_apps/pypo/pyponotify.py +++ b/python_apps/pypo/pyponotify.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import traceback """ Python part of radio playout (pypo) @@ -102,6 +103,24 @@ class Notify: logger.debug('# Calling server to update webstream data #') logger.debug('#################################################') response = self.api_client.notify_webstream_data(data, media_id) + logger.debug("Response: " + json.dumps(response)) + + def run_with_options(self, options): + if options.error and options.stream_id: + self.notify_liquidsoap_status(options.error, options.stream_id, options.time) + elif options.connect and options.stream_id: + self.notify_liquidsoap_status("OK", options.stream_id, options.time) + elif options.source_name and options.source_status: + self.notify_source_status(options.source_name, options.source_status) + elif options.webstream: + self.notify_webstream_data(options.webstream, options.media_id) + elif options.media_id: + self.notify_media_start_playing(options.media_id) + elif options.liquidsoap_started: + self.notify_liquidsoap_started() + else: + logger.debug("Unrecognized option in options(%s). Doing nothing" \ + % str(options)) if __name__ == '__main__': @@ -112,41 +131,9 @@ if __name__ == '__main__': print '#########################################' # initialize - if options.error and options.stream_id: - try: - n = Notify() - n.notify_liquidsoap_status(options.error, options.stream_id, options.time) - except Exception, e: - print e - elif options.connect and options.stream_id: - try: - n = Notify() - n.notify_liquidsoap_status("OK", options.stream_id, options.time) - except Exception, e: - print e - elif options.source_name and options.source_status: - try: - n = Notify() - n.notify_source_status(options.source_name, options.source_status) - except Exception, e: - print e - elif options.webstream: - try: - n = Notify() - n.notify_webstream_data(options.webstream, options.media_id) - except Exception, e: - print e - elif options.media_id: - - try: - n = Notify() - n.notify_media_start_playing(options.media_id) - except Exception, e: - print e - elif options.liquidsoap_started: - try: - n = Notify() - n.notify_liquidsoap_started() - except Exception, e: - print e + try: + n = Notify() + n.run_with_options(options) + except Exception as e: + print( traceback.format_exc() ) From 35a9e77062c5400039ce224d7d01c928162d5163 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Mon, 29 Oct 2012 11:40:23 -0400 Subject: [PATCH 105/108] Formatted spaces. --- utils/airtime-import/airtime-import.py | 34 +++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/utils/airtime-import/airtime-import.py b/utils/airtime-import/airtime-import.py index 5f1d305ca..408bd91ac 100644 --- a/utils/airtime-import/airtime-import.py +++ b/utils/airtime-import/airtime-import.py @@ -21,7 +21,7 @@ logger.addHandler(ch) if (os.geteuid() != 0): print 'Must be a root user.' sys.exit() - + # loading config file try: config = ConfigObj('/etc/airtime/media-monitor.cfg') @@ -66,12 +66,12 @@ def copy_or_move_files_to(paths, dest, flag): print "Cannot find file or path: %s" % path except Exception as e: print "Error: ", e - + def format_dir_string(path): if(path[-1] != '/'): path = path+'/' return path - + def helper_get_stor_dir(): res = api_client.list_all_watched_dirs() if(res is None): @@ -87,18 +87,18 @@ def checkOtherOption(args): for i in args: if(i[0] == '-'): return True - + def errorIfMultipleOption(args, msg=''): if(checkOtherOption(args)): if(msg != ''): raise OptionValueError(msg) else: raise OptionValueError("This option cannot be combined with other options") - + def printHelp(): storage_dir = helper_get_stor_dir() if(storage_dir is None): - storage_dir = "Unknown" + storage_dir = "Unknown" else: storage_dir += "imported/" print """ @@ -111,15 +111,15 @@ There are two ways to import audio files into Airtime: Copied or moved files will be placed into the folder: %s - + Files will be automatically organized into the structure "Artist/Album/TrackNumber-TrackName-Bitrate.file_extension". 2) Use airtime-import to add a folder to the Airtime library ("watch" a folder). - + All the files in the watched folder will be imported to Airtime and the folder will be monitored to automatically detect any changes. Hence any - changes done in the folder(add, delete, edit a file) will trigger + changes done in the folder(add, delete, edit a file) will trigger updates in Airtime library. """ % storage_dir parser.print_help() @@ -209,7 +209,7 @@ def WatchRemoveAction(option, opt, value, parser): print "Removing the watch folder failed: %s" % res['msg']['error'] else: print "The given path is not a directory: %s" % path - + def StorageSetAction(option, opt, value, parser): bypass = False isF = '-f' in parser.rargs @@ -231,12 +231,12 @@ def StorageSetAction(option, opt, value, parser): confirm = confirm or 'N' if(confirm == 'n' or confirm =='N'): sys.exit(1) - + if(len(parser.rargs) > 1): raise OptionValueError("Too many arguments. This option requires exactly one argument.") elif(len(parser.rargs) == 0 ): raise OptionValueError("No argument found. This option requires exactly one argument.") - + path = parser.rargs[0] if (path[0] == "/" or path[0] == "~"): path = os.path.realpath(path) @@ -254,17 +254,17 @@ def StorageSetAction(option, opt, value, parser): print "Setting storage folder failed: %s" % res['msg']['error'] else: print "The given path is not a directory: %s" % path - + def StorageGetAction(option, opt, value, parser): errorIfMultipleOption(parser.rargs) if(len(parser.rargs) > 0): raise OptionValueError("This option does not take any arguments.") print helper_get_stor_dir() - + class OptionValueError(RuntimeError): def __init__(self, msg): self.msg = msg - + usage = """[-c|--copy FILE/DIR [FILE/DIR...]] [-m|--move FILE/DIR [FILE/DIR...]] [--watch-add DIR] [--watch-list] [--watch-remove DIR] [--storage-dir-set DIR] [--storage-dir-get]""" @@ -293,7 +293,7 @@ if('-h' in sys.argv): if(len(sys.argv) == 1 or '-' not in sys.argv[1]): printHelp() sys.exit() - + try: (option, args) = parser.parse_args() except Exception, e: @@ -306,7 +306,7 @@ except Exception, e: except SystemExit: printHelp() sys.exit() - + if option.help: printHelp() sys.exit() From 31a7b8f94353f4d683276094b3c005eb0003f006 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Mon, 29 Oct 2012 11:40:41 -0400 Subject: [PATCH 106/108] Removed usesless try catch from send_media_monitor_requests. --- python_apps/api_clients/api_client.py | 85 ++++++++++++--------------- 1 file changed, 39 insertions(+), 46 deletions(-) diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index 130724f66..5b855686a 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -422,53 +422,46 @@ class AirtimeApiClient(): def send_media_monitor_requests(self, action_list, dry=False): """ - Send a gang of media monitor events at a time. actions_list is a list - of dictionaries where every dictionary is representing an action. Every - action dict must contain a 'mode' key that says what kind of action it - is and an optional 'is_record' key that says whether the show was - recorded or not. The value of this key does not matter, only if it's - present or not. + Send a gang of media monitor events at a time. actions_list is a + list of dictionaries where every dictionary is representing an + action. Every action dict must contain a 'mode' key that says + what kind of action it is and an optional 'is_record' key that + says whether the show was recorded or not. The value of this key + does not matter, only if it's present or not. """ - logger = self.logger - try: - url = self.construct_url('reload_metadata_group') - # We are assuming that action_list is a list of dictionaries such - # that every dictionary represents the metadata of a file along - # with a special mode key that is the action to be executed by the - # controller. - valid_actions = [] - # We could get a list of valid_actions in a much shorter way using - # filter but here we prefer a little more verbosity to help - # debugging - for action in action_list: - if not 'mode' in action: - self.logger.debug("Warning: Trying to send a request element without a 'mode'") - self.logger.debug("Here is the the request: '%s'" % str(action) ) - else: - # We alias the value of is_record to true or false no - # matter what it is based on if it's absent in the action - if 'is_record' not in action: - action['is_record'] = 0 - valid_actions.append(action) - # Note that we must prefix every key with: mdX where x is a number - # Is there a way to format the next line a little better? The - # parenthesis make the code almost unreadable - md_list = dict((("md%d" % i), json.dumps(convert_dict_value_to_utf8(md))) \ - for i,md in enumerate(valid_actions)) - # For testing we add the following "dry" parameter to tell the - # controller not to actually do any changes - if dry: md_list['dry'] = 1 - self.logger.info("Pumping out %d requests..." % len(valid_actions)) - data = urllib.urlencode(md_list) - req = urllib2.Request(url, data) - response = self.get_response_from_server(req) - response = json.loads(response) - return response - except ValueError: raise - except Exception, e: - logger.error('Exception: %s', e) - logger.error("traceback: %s", traceback.format_exc()) - raise + url = self.construct_url('reload_metadata_group') + # We are assuming that action_list is a list of dictionaries such + # that every dictionary represents the metadata of a file along + # with a special mode key that is the action to be executed by the + # controller. + valid_actions = [] + # We could get a list of valid_actions in a much shorter way using + # filter but here we prefer a little more verbosity to help + # debugging + for action in action_list: + if not 'mode' in action: + self.logger.debug("Warning: Trying to send a request element without a 'mode'") + self.logger.debug("Here is the the request: '%s'" % str(action) ) + else: + # We alias the value of is_record to true or false no + # matter what it is based on if it's absent in the action + if 'is_record' not in action: + action['is_record'] = 0 + valid_actions.append(action) + # Note that we must prefix every key with: mdX where x is a number + # Is there a way to format the next line a little better? The + # parenthesis make the code almost unreadable + md_list = dict((("md%d" % i), json.dumps(convert_dict_value_to_utf8(md))) \ + for i,md in enumerate(valid_actions)) + # For testing we add the following "dry" parameter to tell the + # controller not to actually do any changes + if dry: md_list['dry'] = 1 + self.logger.info("Pumping out %d requests..." % len(valid_actions)) + data = urllib.urlencode(md_list) + req = urllib2.Request(url, data) + response = self.get_response_from_server(req) + response = json.loads(response) + return response #returns a list of all db files for a given directory in JSON format: #{"files":["path/to/file1", "path/to/file2"]} From 3cec987d100b14403ff8eb31dfee9946eaa7011d Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Mon, 29 Oct 2012 11:42:24 -0400 Subject: [PATCH 107/108] Added todo --- python_apps/api_clients/api_client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index 5b855686a..9ef49618d 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -20,6 +20,11 @@ import traceback AIRTIME_VERSION = "2.2.0" + +# TODO : Place these functions in some common module. Right now, media +# monitor uses the same functions and it would be better to reuse them +# instead of copy pasting them around + def to_unicode(obj, encoding='utf-8'): if isinstance(obj, basestring): if not isinstance(obj, unicode): From bf5bc3a1166ca19dcc60d35cfe5026851f8209e8 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg <rudi.grinberg@sourcefabric.org> Date: Mon, 29 Oct 2012 11:43:39 -0400 Subject: [PATCH 108/108] Changed APC to new-style classes --- python_apps/api_clients/api_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index 9ef49618d..5ca176ec2 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -44,7 +44,7 @@ def convert_dict_value_to_utf8(md): # Airtime API Client ################################################################################ -class AirtimeApiClient(): +class AirtimeApiClient(object): # This is a little hacky fix so that I don't have to pass the config object # everywhere where AirtimeApiClient needs to be initialized