From a744e1137050aa4939a243124f6d3fff47624cac Mon Sep 17 00:00:00 2001 From: Naomi Aro Date: Thu, 28 Jul 2011 11:13:00 +0200 Subject: [PATCH 1/7] CC-2600 : Upgrading from 1.8->1.9 wipe all shows content --- .../airtime-1.8.1/airtime-upgrade.php | 21 ------------------- .../airtime-1.8.2/airtime-upgrade.php | 20 ------------------ .../airtime-1.9.0/media-monitor-upgrade.py | 3 ++- 3 files changed, 2 insertions(+), 42 deletions(-) diff --git a/install_minimal/upgrades/airtime-1.8.1/airtime-upgrade.php b/install_minimal/upgrades/airtime-1.8.1/airtime-upgrade.php index d05985a20..c077e1c25 100644 --- a/install_minimal/upgrades/airtime-1.8.1/airtime-upgrade.php +++ b/install_minimal/upgrades/airtime-1.8.1/airtime-upgrade.php @@ -213,24 +213,6 @@ function MergeConfigFiles($configFiles, $suffix) } } -function InstallPhpCode() -{ - global $CC_CONFIG; - global $AIRTIME_SRC; - echo "* Installing PHP code to ".$CC_CONFIG['phpDir'].PHP_EOL; - exec("mkdir -p ".$CC_CONFIG['phpDir']); - exec("cp -R ".$AIRTIME_SRC."/* ".$CC_CONFIG['phpDir']); - -} - -function InstallBinaries() -{ - global $AIRTIME_UTILS; - echo "* Installing binaries to ".CONF_DIR_BINARIES.PHP_EOL; - exec("mkdir -p ".CONF_DIR_BINARIES); - exec("cp -R ".$AIRTIME_UTILS." ".CONF_DIR_BINARIES); -} - // Backup the config files $suffix = date("Ymdhis")."-1.8.1"; foreach ($configFiles as $conf) { @@ -245,6 +227,3 @@ echo "* Initializing INI files".PHP_EOL; MergeConfigFiles($configFiles, $suffix); $CC_CONFIG = LoadConfig($CC_CONFIG); - -InstallPhpCode(); -InstallBinaries(); diff --git a/install_minimal/upgrades/airtime-1.8.2/airtime-upgrade.php b/install_minimal/upgrades/airtime-1.8.2/airtime-upgrade.php index 6ed7d1ec7..788db3e26 100644 --- a/install_minimal/upgrades/airtime-1.8.2/airtime-upgrade.php +++ b/install_minimal/upgrades/airtime-1.8.2/airtime-upgrade.php @@ -213,23 +213,6 @@ function MergeConfigFiles($configFiles, $suffix) } } -function InstallPhpCode() -{ - global $CC_CONFIG; - global $AIRTIME_SRC; - echo "* Installing PHP code to ".$CC_CONFIG['phpDir'].PHP_EOL; - exec("mkdir -p ".$CC_CONFIG['phpDir']); - exec("cp -R ".$AIRTIME_SRC."/* ".$CC_CONFIG['phpDir']); -} - -function InstallBinaries() -{ - global $AIRTIME_UTILS; - echo "* Installing binaries to ".CONF_DIR_BINARIES.PHP_EOL; - exec("mkdir -p ".CONF_DIR_BINARIES); - exec("cp -R ".$AIRTIME_UTILS." ".CONF_DIR_BINARIES); -} - $suffix = date("Ymdhis")."-1.8.2"; foreach ($configFiles as $conf) { if (file_exists($conf)) { @@ -243,6 +226,3 @@ echo "* Initializing INI files".PHP_EOL; MergeConfigFiles($configFiles, $suffix); $CC_CONFIG = LoadConfig($CC_CONFIG); - -InstallPhpCode(); -InstallBinaries(); diff --git a/install_minimal/upgrades/airtime-1.9.0/media-monitor-upgrade.py b/install_minimal/upgrades/airtime-1.9.0/media-monitor-upgrade.py index 7ccc68bf1..7eebc9d93 100644 --- a/install_minimal/upgrades/airtime-1.9.0/media-monitor-upgrade.py +++ b/install_minimal/upgrades/airtime-1.9.0/media-monitor-upgrade.py @@ -38,7 +38,8 @@ mmc = MediaMonitorCommon(mmconfig) try: os.makedirs(organize_dir) except Exception, e: - print e + #organize dir already exists. ( really shouldn't though ) + pass #older versions of Airtime installed from repository at least had owner of stor dir as "root" mmc.set_needed_file_permissions(stor_dir, True) From 2f107a36f2eee49bc4cc0d0a6e5650c237a30c14 Mon Sep 17 00:00:00 2001 From: Naomi Aro Date: Thu, 28 Jul 2011 11:41:59 +0200 Subject: [PATCH 2/7] CC-2600 : Upgrading from 1.8->1.9 wipe all shows content --- install_minimal/upgrades/airtime-1.9.0/media-monitor-upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_minimal/upgrades/airtime-1.9.0/media-monitor-upgrade.py b/install_minimal/upgrades/airtime-1.9.0/media-monitor-upgrade.py index 7eebc9d93..52cd1e61c 100644 --- a/install_minimal/upgrades/airtime-1.9.0/media-monitor-upgrade.py +++ b/install_minimal/upgrades/airtime-1.9.0/media-monitor-upgrade.py @@ -51,7 +51,7 @@ pairs = [] for root, dirs, files in os.walk(mmconfig.storage_directory): for f in files: old_filepath = os.path.join(root, f) - new_filepath = mmc.organize_new_file(os.path.join(root, f)) + new_filepath = mmc.organize_new_file(old_filepath) pair = old_filepath, new_filepath pairs.append(pair) mmc.set_needed_file_permissions(new_filepath, False) From 7c87d2024abb7c0f72da1e060694583105736ecf Mon Sep 17 00:00:00 2001 From: Naomi Aro Date: Thu, 28 Jul 2011 11:48:23 +0200 Subject: [PATCH 3/7] CC-2600 : Upgrading from 1.8->1.9 wipe all shows content --- .../upgrades/airtime-1.9.0/media-monitor-upgrade.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/install_minimal/upgrades/airtime-1.9.0/media-monitor-upgrade.py b/install_minimal/upgrades/airtime-1.9.0/media-monitor-upgrade.py index 52cd1e61c..63ee5d837 100644 --- a/install_minimal/upgrades/airtime-1.9.0/media-monitor-upgrade.py +++ b/install_minimal/upgrades/airtime-1.9.0/media-monitor-upgrade.py @@ -52,9 +52,16 @@ for root, dirs, files in os.walk(mmconfig.storage_directory): for f in files: old_filepath = os.path.join(root, f) new_filepath = mmc.organize_new_file(old_filepath) - pair = old_filepath, new_filepath - pairs.append(pair) - mmc.set_needed_file_permissions(new_filepath, False) + + if new_filepath is not None: + pair = old_filepath, new_filepath + pairs.append(pair) + mmc.set_needed_file_permissions(new_filepath, False) + #incase file has a metadata problem. + else: + pair = old_filepath, old_filepath + pairs.append(pair) + mmc.set_needed_file_permissions(old_filepath, False) #need to set all the dirs in imported to be owned by www-data. command = "chown -R www-data " + stor_dir From 137ca79db85facca2414c691b57f1ca3a40878b1 Mon Sep 17 00:00:00 2001 From: Naomi Aro Date: Thu, 28 Jul 2011 12:21:20 +0200 Subject: [PATCH 4/7] CC-2600 : Upgrading from 1.8->1.9 wipe all shows content --- python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py b/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py index d420ec074..b9e7164f2 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py @@ -103,6 +103,7 @@ class AirtimeMetadata: def get_md_from_file(self, filepath): + self.logger.debug("testing upgrade") self.logger.info("getting info from filepath %s", filepath) try: From 16608990961199dd85006cfb3019a4f07e974846 Mon Sep 17 00:00:00 2001 From: Naomi Aro Date: Thu, 28 Jul 2011 15:51:51 +0200 Subject: [PATCH 5/7] CC-2600 : Upgrading from 1.8->1.9 wipe all shows content check if there were any files in stor. --- .../airtime-1.9.0/airtime-upgrade.php | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/install_minimal/upgrades/airtime-1.9.0/airtime-upgrade.php b/install_minimal/upgrades/airtime-1.9.0/airtime-upgrade.php index 582f12a09..47f851305 100644 --- a/install_minimal/upgrades/airtime-1.9.0/airtime-upgrade.php +++ b/install_minimal/upgrades/airtime-1.9.0/airtime-upgrade.php @@ -621,15 +621,18 @@ class Airtime190Upgrade{ exec($command, $output); print_r($output); - $oldAndNewFileNames = json_decode($output[0]); + if (isset($output[0])) { - $stor_dir_id = $propel_stor_dir->getId(); - foreach ($oldAndNewFileNames as $pair){ - $relPathNew = pg_escape_string(substr($pair[1], strlen($stor_dir))); - $absPathOld = pg_escape_string($pair[0]); - $sql = "UPDATE cc_files SET filepath = '$relPathNew', directory=$stor_dir_id WHERE filepath = '$absPathOld'"; - echo $sql.PHP_EOL; - Airtime190Upgrade::execSqlQuery($sql); + $oldAndNewFileNames = json_decode($output[0]); + + $stor_dir_id = $propel_stor_dir->getId(); + foreach ($oldAndNewFileNames as $pair){ + $relPathNew = pg_escape_string(substr($pair[1], strlen($stor_dir))); + $absPathOld = pg_escape_string($pair[0]); + $sql = "UPDATE cc_files SET filepath = '$relPathNew', directory=$stor_dir_id WHERE filepath = '$absPathOld'"; + echo $sql.PHP_EOL; + Airtime190Upgrade::execSqlQuery($sql); + } } echo "Upgrading Linked Files".PHP_EOL; From fb5c0803ce1eba37e952e238a6be8f19a53af6c2 Mon Sep 17 00:00:00 2001 From: Naomi Aro Date: Fri, 29 Jul 2011 11:41:22 +0200 Subject: [PATCH 6/7] replacing media monitor code in upgrade with latest. --- .../airtimefilemonitor/airtimemetadata.py | 26 ++++++++++++++++++- .../airtimefilemonitor/mediamonitorcommon.py | 12 ++++----- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/install_minimal/upgrades/airtime-1.9.0/airtimefilemonitor/airtimemetadata.py b/install_minimal/upgrades/airtime-1.9.0/airtimefilemonitor/airtimemetadata.py index d1a984cb9..b9e7164f2 100644 --- a/install_minimal/upgrades/airtime-1.9.0/airtimefilemonitor/airtimemetadata.py +++ b/install_minimal/upgrades/airtime-1.9.0/airtimefilemonitor/airtimemetadata.py @@ -3,6 +3,7 @@ import hashlib import mutagen import logging import math +import re """ list of supported easy tags in mutagen version 1.20 @@ -102,6 +103,7 @@ class AirtimeMetadata: def get_md_from_file(self, filepath): + self.logger.debug("testing upgrade") self.logger.info("getting info from filepath %s", filepath) try: @@ -133,9 +135,28 @@ class AirtimeMetadata: md['MDATA_KEY_TITLE'] = original_name #incase track number is in format u'4/11' + #need to also check that the tracknumber is even a tracknumber (cc-2582) if 'MDATA_KEY_TRACKNUMBER' in md: + try: + md['MDATA_KEY_TRACKNUMBER'] = int(md['MDATA_KEY_TRACKNUMBER']) + except Exception, e: + pass + if isinstance(md['MDATA_KEY_TRACKNUMBER'], basestring): - md['MDATA_KEY_TRACKNUMBER'] = md['MDATA_KEY_TRACKNUMBER'].split("/")[0] + match = re.search('^(\d*/\d*)?', md['MDATA_KEY_TRACKNUMBER']) + + if match.group(0) is not u'': + md['MDATA_KEY_TRACKNUMBER'] = int(md['MDATA_KEY_TRACKNUMBER'].split("/")[0]) + else: + del md['MDATA_KEY_TRACKNUMBER'] + + #make sure bpm is valid, need to check more types of formats for this tag to assure correct parsing. + if 'MDATA_KEY_BPM' in md: + if isinstance(md['MDATA_KEY_BPM'], basestring): + try: + md['MDATA_KEY_BPM'] = int(md['MDATA_KEY_BPM']) + except Exception, e: + del md['MDATA_KEY_BPM'] md['MDATA_KEY_BITRATE'] = file_info.info.bitrate md['MDATA_KEY_SAMPLERATE'] = file_info.info.sample_rate @@ -152,4 +173,7 @@ class AirtimeMetadata: if(isinstance(md[key], basestring)): md[key] = md[key].encode('utf-8') + self.logger.info("MD after parsing.") + self.logger.debug(md) + return md diff --git a/install_minimal/upgrades/airtime-1.9.0/airtimefilemonitor/mediamonitorcommon.py b/install_minimal/upgrades/airtime-1.9.0/airtimefilemonitor/mediamonitorcommon.py index d3125c5b5..fa1fcbd8f 100644 --- a/install_minimal/upgrades/airtime-1.9.0/airtimefilemonitor/mediamonitorcommon.py +++ b/install_minimal/upgrades/airtime-1.9.0/airtimefilemonitor/mediamonitorcommon.py @@ -207,18 +207,18 @@ class MediaMonitorCommon: return filepath - def execCommandAndReturnStdOut(self, command): - p = Popen(command, shell=True, stdout=PIPE) - stdout = p.communicate()[0] + def exec_command(self, command): + p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() if p.returncode != 0: self.logger.warn("command \n%s\n return with a non-zero return value", command) + self.logger.error(stderr) return stdout def scan_dir_for_new_files(self, dir): command = 'find "%s" -type f -iname "*.ogg" -o -iname "*.mp3" -readable' % dir.replace('"', '\\"') self.logger.debug(command) - stdout = self.execCommandAndReturnStdOut(command) - stdout = unicode(stdout, "utf_8") + stdout = self.exec_command(command) return stdout.splitlines() @@ -230,8 +230,6 @@ class MediaMonitorCommon: file_md = self.md_manager.get_md_from_file(pathname) if file_md is not None: - #is_recorded_show = 'MDATA_KEY_CREATOR' in file_md and \ - # file_md['MDATA_KEY_CREATOR'] == "AIRTIMERECORDERSOURCEFABRIC".encode('utf-8') filepath = self.create_file_path(pathname, file_md) self.logger.debug("Moving from %s to %s", pathname, filepath) From 1e6d9efad2fe35a498bb6181320e31f0cef67f9d Mon Sep 17 00:00:00 2001 From: Naomi Aro Date: Fri, 29 Jul 2011 11:57:06 +0200 Subject: [PATCH 7/7] CC-2612 : Upgrade: Push DB metadata into the file before upgrade might have too much data to log it. --- .../airtimefilemonitor/airtimemediamonitorbootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py b/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py index 42b111001..b34ca2d48 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py @@ -86,7 +86,7 @@ class AirtimeMediaMonitorBootstrap(): command = "find %s -type f -iname '*.ogg' -o -iname '*.mp3' -readable" % dir stdout = self.mmc.exec_command(command) - self.logger.info(stdout) + #self.logger.info(stdout) new_files = stdout.splitlines()