From 22c5da1629cb18c4eb20c1ff49d4e727dc0c4a8d Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 12:40:44 -0400 Subject: [PATCH 01/11] CC-3755: Found Exception in apache's error log -Return error code so that python services automatically retry later. --- airtime_mvc/application/models/Preference.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/airtime_mvc/application/models/Preference.php b/airtime_mvc/application/models/Preference.php index c9c1f17a3..90e87b9b2 100644 --- a/airtime_mvc/application/models/Preference.php +++ b/airtime_mvc/application/models/Preference.php @@ -61,7 +61,13 @@ class Application_Model_Preference ." VALUES ($id, '$key', $value)"; } } - return $con->exec($sql); + try { + $con->exec($sql); + } catch (Exception $e){ + Logging::log("Could not connect to database."); + header('HTTP/1.0 503 Service Unavailable'); + exit; + } } public static function GetValue($key, $isUserValue = false){ From 9092b7f7258de241b91f781257998217ae1f5d75 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 12:47:29 -0400 Subject: [PATCH 02/11] CC-3764: Media Library->Edit Metadata: Throw exception when trying to edit metadata -fixed --- airtime_mvc/application/models/StoredFile.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airtime_mvc/application/models/StoredFile.php b/airtime_mvc/application/models/StoredFile.php index 886f9a8ba..a6044c868 100644 --- a/airtime_mvc/application/models/StoredFile.php +++ b/airtime_mvc/application/models/StoredFile.php @@ -235,7 +235,9 @@ class Application_Model_StoredFile { foreach ($c['user'] as $constant => $value) { if (preg_match('/^MDATA_KEY/', $constant)) { if (isset($dbmd_copy[$value])) { - $md[$constant] = $this->getDbColMetadataValue($value); + $propelColumn = $dbmd_copy[$value]; + $method = "get$propelColumn"; + $md[$constant] = $this->_file->$method(); } } } From ebc8af0863e727ed73a1aa23fa4e1b603044ed28 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 13:03:05 -0400 Subject: [PATCH 03/11] CC-3749: Media Monitor should not load files pypo cannot read --- .../media-monitor/airtimefilemonitor/mediamonitorcommon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py index b28b1dfcd..0c38a7d45 100644 --- a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py +++ b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py @@ -84,6 +84,7 @@ class MediaMonitorCommon: # stat.S_IROTH - readable by all permission # stat.S_IXOTH - excutable by all permission # try to set permission + self.logger.info("%s has incorrect permissions. Will modify to be readable.", item) if self.is_parent_directory(item, self.config.storage_directory) or self.is_parent_directory(item, self.config.imported_directory) or self.is_parent_directory(item, self.config.organize_directory): if is_dir is True: os.chmod(item, 02777) @@ -98,8 +99,7 @@ class MediaMonitorCommon: bitor = stats.st_mode | stat.S_IROTH os.chmod(item, bitor) except Exception, e: - self.logger.error("Failed to change file's owner/group/permissions. %s", e) - self.logger.error("traceback: %s", traceback.format_exc()) + self.logger.warn("Failed to change owner/group/permissions for %s", item) return False finally: os.umask(omask) From 08217d59fc0986ba76637523a210aa099afde1cb Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 13:23:46 -0400 Subject: [PATCH 04/11] CC-3749: Media Monitor should not load files pypo cannot read -fixed --- .../airtimefilemonitor/mediamonitorcommon.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py index 0c38a7d45..89800c5c5 100644 --- a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py +++ b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py @@ -79,18 +79,24 @@ class MediaMonitorCommon: def set_needed_file_permissions(self, item, is_dir): try: omask = os.umask(0) - if not self.has_correct_permissions(item, 'www-data', 'www-data') or not self.has_correct_permissions(item, 'pypo', 'pypo'): + if not self.has_correct_permissions(item, 'www-data', 'www-data') \ + or not self.has_correct_permissions(item, 'pypo', 'pypo'): + # stats.st_mode is the original permission # stat.S_IROTH - readable by all permission # stat.S_IXOTH - excutable by all permission # try to set permission - self.logger.info("%s has incorrect permissions. Will modify to be readable.", item) - if self.is_parent_directory(item, self.config.storage_directory) or self.is_parent_directory(item, self.config.imported_directory) or self.is_parent_directory(item, self.config.organize_directory): + self.logger.warn("%s has incorrect permissions for reading. Skipping import.", item) + """ + if self.is_parent_directory(item, self.config.storage_directory) \ + or self.is_parent_directory(item, self.config.imported_directory) \ + or self.is_parent_directory(item, self.config.organize_directory): + if is_dir is True: os.chmod(item, 02777) else: os.chmod(item, 0666) - else : + else: # add world readable permission stats = os.stat(item) if is_dir is True: @@ -98,6 +104,7 @@ class MediaMonitorCommon: else: bitor = stats.st_mode | stat.S_IROTH os.chmod(item, bitor) + """ except Exception, e: self.logger.warn("Failed to change owner/group/permissions for %s", item) return False From fa373a683ea95da76b05a6244cffd4fc270518ff Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 13:37:43 -0400 Subject: [PATCH 05/11] CC-3749: Media Monitor should not load files pypo cannot read -fixed --- .../airtimefilemonitor/airtimenotifier.py | 12 +++---- .../airtimefilemonitor/airtimeprocessevent.py | 6 ++-- .../airtimefilemonitor/mediamonitorcommon.py | 33 +++---------------- 3 files changed, 14 insertions(+), 37 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py index 42f22b0ba..cbb404712 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py @@ -103,9 +103,9 @@ class AirtimeNotifier(Notifier): self.mmc.ensure_is_dir(self.config.imported_directory) self.mmc.ensure_is_dir(self.config.organize_directory) - self.mmc.set_needed_file_permissions(self.config.storage_directory, True) - self.mmc.set_needed_file_permissions(self.config.imported_directory, True) - self.mmc.set_needed_file_permissions(self.config.organize_directory, True) + self.mmc.is_readable(self.config.storage_directory, True) + self.mmc.is_readable(self.config.imported_directory, True) + self.mmc.is_readable(self.config.organize_directory, True) self.watch_directory(new_storage_directory) elif m['event_type'] == "file_delete": @@ -192,17 +192,17 @@ class AirtimeNotifier(Notifier): mm = self.proc_fun() - self.mmc.set_needed_file_permissions(directory, True) + self.mmc.is_readable(directory, True) for (path, dirs, files) in os.walk(directory): for d in dirs: - self.mmc.set_needed_file_permissions(os.path.join(path, d), True) + self.mmc.is_readable(os.path.join(path, d), True) for filename in files: full_filepath = os.path.join(path, filename) if self.mmc.is_audio_file(full_filepath): - if self.mmc.set_needed_file_permissions(full_filepath, False): + if self.mmc.is_readable(full_filepath, False): self.logger.info("importing %s", full_filepath) event = {'filepath': full_filepath, 'mode': self.config.MODE_CREATE, 'is_recorded_show': False} mm.multi_queue.put(event) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py index 0d6a6e602..947b16282 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py @@ -151,14 +151,14 @@ class AirtimeProcessEvent(ProcessEvent): self.logger.error('Exception: %s', e) self.logger.error("traceback: %s", traceback.format_exc()) - self.mmc.set_needed_file_permissions(pathname, dir) + self.mmc.is_readable(pathname, dir) is_recorded = self.mmc.is_parent_directory(pathname, self.config.recorded_directory) self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': pathname, 'is_recorded_show': is_recorded}) else: #event is because of a created directory if self.mmc.is_parent_directory(pathname, self.config.storage_directory): - self.mmc.set_needed_file_permissions(pathname, dir) + self.mmc.is_readable(pathname, dir) def process_IN_MODIFY(self, event): # if IN_MODIFY is followed by IN_CREATE, it's not true modify event @@ -237,7 +237,7 @@ class AirtimeProcessEvent(ProcessEvent): if event.pathname in filename: self.handle_mount_change() #if stuff dropped in stor via a UI move must change file permissions. - self.mmc.set_needed_file_permissions(event.pathname, event.dir) + self.mmc.is_readable(event.pathname, event.dir) if not event.dir: if self.mmc.is_audio_file(event.name): if event.cookie in self.temp_files: diff --git a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py index 89800c5c5..b6ed45a80 100644 --- a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py +++ b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py @@ -49,7 +49,7 @@ class MediaMonitorCommon: return False #check if file is readable by "nobody" - def has_correct_permissions(self, filepath, euid='nobody', egid='nogroup'): + def is_user_readable(self, filepath, euid='nobody', egid='nogroup'): try: uid = pwd.getpwnam(euid)[2] @@ -76,40 +76,17 @@ class MediaMonitorCommon: return readable # the function only changes the permission if its not readable by www-data - def set_needed_file_permissions(self, item, is_dir): + def is_readable(self, item, is_dir): try: - omask = os.umask(0) - if not self.has_correct_permissions(item, 'www-data', 'www-data') \ - or not self.has_correct_permissions(item, 'pypo', 'pypo'): + if not self.is_user_readable(item, 'www-data', 'www-data') \ + or not self.is_user_readable(item, 'pypo', 'pypo'): - # stats.st_mode is the original permission - # stat.S_IROTH - readable by all permission - # stat.S_IXOTH - excutable by all permission - # try to set permission self.logger.warn("%s has incorrect permissions for reading. Skipping import.", item) - """ - if self.is_parent_directory(item, self.config.storage_directory) \ - or self.is_parent_directory(item, self.config.imported_directory) \ - or self.is_parent_directory(item, self.config.organize_directory): - - if is_dir is True: - os.chmod(item, 02777) - else: - os.chmod(item, 0666) - else: - # add world readable permission - stats = os.stat(item) - if is_dir is True: - bitor = stats.st_mode | stat.S_IROTH | stat.S_IXOTH - else: - bitor = stats.st_mode | stat.S_IROTH - os.chmod(item, bitor) - """ + return False except Exception, e: self.logger.warn("Failed to change owner/group/permissions for %s", item) return False finally: - os.umask(omask) return True From d6a12053eb02d13dcbbdc8e2a0cd3c9dc82b12ec Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 13:39:48 -0400 Subject: [PATCH 06/11] CC-3752: Media Monitor DB sync -forgot to exit --- python_apps/media-monitor/media_monitor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_apps/media-monitor/media_monitor.py b/python_apps/media-monitor/media_monitor.py index d9dcf518c..7c39bba0a 100644 --- a/python_apps/media-monitor/media_monitor.py +++ b/python_apps/media-monitor/media_monitor.py @@ -51,6 +51,7 @@ def configure_locale(): if current_locale_encoding not in ['utf-8', 'utf8']: logger.error("Need a UTF-8 locale. Currently '%s'. Exiting..." % current_locale_encoding) + sys.exit(1) # configure logging try: From e13d06df4553d0d698d8abf25ad918dd86b3c02b Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 13:45:43 -0400 Subject: [PATCH 07/11] CC-3749: Media Monitor should not load files pypo cannot read -fixed --- .../airtimefilemonitor/airtimenotifier.py | 4 ---- .../airtimefilemonitor/mediamonitorcommon.py | 12 +++--------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py index cbb404712..c6dac73c5 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py @@ -194,10 +194,6 @@ class AirtimeNotifier(Notifier): self.mmc.is_readable(directory, True) for (path, dirs, files) in os.walk(directory): - - for d in dirs: - self.mmc.is_readable(os.path.join(path, d), True) - for filename in files: full_filepath = os.path.join(path, filename) diff --git a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py index b6ed45a80..8e68b88bf 100644 --- a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py +++ b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py @@ -78,17 +78,11 @@ class MediaMonitorCommon: # the function only changes the permission if its not readable by www-data def is_readable(self, item, is_dir): try: - if not self.is_user_readable(item, 'www-data', 'www-data') \ - or not self.is_user_readable(item, 'pypo', 'pypo'): - - self.logger.warn("%s has incorrect permissions for reading. Skipping import.", item) - return False + return self.is_user_readable(item, 'www-data', 'www-data') \ + and not self.is_user_readable(item, 'pypo', 'pypo'): except Exception, e: - self.logger.warn("Failed to change owner/group/permissions for %s", item) + self.logger.warn("Failed to check owner/group/permissions for %s", item) return False - finally: - return True - #checks if path is a directory, and if it doesnt exist, then creates it. #Otherwise prints error to log file. From 7e9d1a3308ecfa9bf6c933aec51fc61488601476 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 13:50:38 -0400 Subject: [PATCH 08/11] CC-3749: Media Monitor should not load files pypo cannot read -fixed --- .../media-monitor/airtimefilemonitor/mediamonitorcommon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py index 8e68b88bf..7d9069c05 100644 --- a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py +++ b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py @@ -79,7 +79,7 @@ class MediaMonitorCommon: def is_readable(self, item, is_dir): try: return self.is_user_readable(item, 'www-data', 'www-data') \ - and not self.is_user_readable(item, 'pypo', 'pypo'): + and not self.is_user_readable(item, 'pypo', 'pypo') except Exception, e: self.logger.warn("Failed to check owner/group/permissions for %s", item) return False From d98d23d483aafeed85c1292a0fd31ea9a0f050ea Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 15:04:54 -0400 Subject: [PATCH 09/11] CC-3749: Media Monitor should not load files pypo cannot read -fixed --- .../media-monitor/airtimefilemonitor/airtimenotifier.py | 1 - .../media-monitor/airtimefilemonitor/airtimeprocessevent.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py index c6dac73c5..8986ffc6f 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py @@ -150,7 +150,6 @@ class AirtimeNotifier(Notifier): file_md = None data = None - if (os.path.exists(filepath) and (mode == self.config.MODE_CREATE)): if file_md is None: mutagen = self.md_manager.get_md_from_file(filepath) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py index 947b16282..019f460e0 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py @@ -134,7 +134,7 @@ class AirtimeProcessEvent(ProcessEvent): #file is being overwritten/replaced in GUI. elif "goutputstream" in pathname: self.temp_files[pathname] = None - elif self.mmc.is_audio_file(pathname): + elif self.mmc.is_audio_file(pathname) and self.mmc.is_readable(pathname, False): if self.mmc.is_parent_directory(pathname, self.config.organize_directory): #file was created in /srv/airtime/stor/organize. Need to process and move #to /srv/airtime/stor/imported @@ -173,7 +173,7 @@ class AirtimeProcessEvent(ProcessEvent): # update timestamp on create_dict for the entry with pathname as the key if pathname in self.create_dict: self.create_dict[pathname] = time.time() - if not dir and not self.mmc.is_parent_directory(pathname, self.config.organize_directory): + if not dir and not self.mmc.is_parent_directory(pathname, self.config.organize_directory) and self.mmc.is_readable(pathname, False): self.logger.info("Modified: %s", pathname) if self.mmc.is_audio_file(name): self.file_events.append({'filepath': pathname, 'mode': self.config.MODE_MODIFY}) @@ -239,7 +239,7 @@ class AirtimeProcessEvent(ProcessEvent): #if stuff dropped in stor via a UI move must change file permissions. self.mmc.is_readable(event.pathname, event.dir) if not event.dir: - if self.mmc.is_audio_file(event.name): + if self.mmc.is_audio_file(event.name) and self.mmc.is_readable(full_filepath, False): if event.cookie in self.temp_files: self.file_events.append({'filepath': event.pathname, 'mode': self.config.MODE_MODIFY}) del self.temp_files[event.cookie] From 906b6e8bd0c7d118d962766741e01603e1d0ca6e Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 15:22:54 -0400 Subject: [PATCH 10/11] CC-3749: Media Monitor should not load files pypo cannot read -logic error --- .../media-monitor/airtimefilemonitor/mediamonitorcommon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py index 7d9069c05..fdd83de6b 100644 --- a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py +++ b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py @@ -79,7 +79,7 @@ class MediaMonitorCommon: def is_readable(self, item, is_dir): try: return self.is_user_readable(item, 'www-data', 'www-data') \ - and not self.is_user_readable(item, 'pypo', 'pypo') + and self.is_user_readable(item, 'pypo', 'pypo') except Exception, e: self.logger.warn("Failed to check owner/group/permissions for %s", item) return False From 676205bb00ee08694cee9720e4cf827e68d22d3b Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 15:49:33 -0400 Subject: [PATCH 11/11] CC-3750Media Library->Edit Metadata: media-monitor throw exception when trying to edit metadata with non-ascii characters -Improved save metadata function --- .../airtimefilemonitor/airtimemetadata.py | 17 ++++++++--------- .../airtimefilemonitor/airtimenotifier.py | 2 +- .../airtimefilemonitor/airtimeprocessevent.py | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py b/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py index cab57381c..1cef988b1 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py @@ -88,17 +88,16 @@ class AirtimeMetadata: try: airtime_file = mutagen.File(m['MDATA_KEY_FILEPATH'], easy=True) - for key in m.keys() : + for key in m: if key in self.airtime2mutagen: value = m[key] - if (value is not None): - self.logger.debug("Saving %s to file", key) - self.logger.debug(value) - if isinstance(value, basestring) and (len(value) > 0): - airtime_file[self.airtime2mutagen[key]] = api_client.encode_to(value, 'utf-8') - elif isinstance(value, int): - airtime_file[self.airtime2mutagen[key]] = str(value) - + + if value is not None: + value = unicode(value) + + if len(value) > 0: + self.logger.debug("Saving key '%s' with value '%s' to file", key, value) + airtime_file[self.airtime2mutagen[key]] = value airtime_file.save() except Exception, e: diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py index 8986ffc6f..e527d56cb 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py @@ -62,7 +62,7 @@ class AirtimeNotifier(Notifier): message.ack() self.logger.info("Received md from RabbitMQ: " + body) - m = json.loads(message.body) + m = json.loads(message.body) if m['event_type'] == "md_update": self.logger.info("AIRTIME NOTIFIER md update event") diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py index 019f460e0..419bbd685 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py @@ -173,7 +173,7 @@ class AirtimeProcessEvent(ProcessEvent): # update timestamp on create_dict for the entry with pathname as the key if pathname in self.create_dict: self.create_dict[pathname] = time.time() - if not dir and not self.mmc.is_parent_directory(pathname, self.config.organize_directory) and self.mmc.is_readable(pathname, False): + if not dir and not self.mmc.is_parent_directory(pathname, self.config.organize_directory): self.logger.info("Modified: %s", pathname) if self.mmc.is_audio_file(name): self.file_events.append({'filepath': pathname, 'mode': self.config.MODE_MODIFY})