From d260c66abc218cad68e2884ebb9169f6641cb0a4 Mon Sep 17 00:00:00 2001 From: martin Date: Mon, 4 Jul 2011 15:08:02 -0400 Subject: [PATCH 1/3] cc-2419: media monitor import on startup -added multiple dir support --- .../application/controllers/ApiController.php | 27 +++++++++++++++++++ airtime_mvc/application/models/MusicDir.php | 23 ++++++---------- python_apps/api_clients/api_client.py | 22 ++++++++++++++- .../airtimemediamonitorbootstrap.py | 14 ++++++---- python_apps/media-monitor/media-monitor.cfg | 3 +++ 5 files changed, 68 insertions(+), 21 deletions(-) diff --git a/airtime_mvc/application/controllers/ApiController.php b/airtime_mvc/application/controllers/ApiController.php index 0039a2475..fcd6c2b52 100644 --- a/airtime_mvc/application/controllers/ApiController.php +++ b/airtime_mvc/application/controllers/ApiController.php @@ -15,6 +15,7 @@ class ApiController extends Zend_Controller_Action ->addActionContext('media-item-status', 'json') ->addActionContext('reload-metadata', 'json') ->addActionContext('list-all-files', 'json') + ->addActionContext('list-all-watched-dirs', 'json') ->initContext(); } @@ -523,5 +524,31 @@ class ApiController extends Zend_Controller_Action $this->view->files = StoredFile::listAllFiles(); } + + public function listAllWatchedDirsAction() { + global $CC_CONFIG; + + $request = $this->getRequest(); + $api_key = $request->getParam('api_key'); + if (!in_array($api_key, $CC_CONFIG["apiKey"])) + { + header('HTTP/1.0 401 Unauthorized'); + print 'You are not allowed to access this resource.'; + exit; + } + + $result = array(); + + $arrWatchedDirs = MusicDir::getWatchedDirs(); + $storDir = MusicDir::getStorDir(); + + $result[] = $storDir->getDirectory(); + + foreach ($arrWatchedDirs as $watchedDir){ + $result[] = $watchedDir->getDirectory(); + } + + $this->view->dirs = $result; + } } diff --git a/airtime_mvc/application/models/MusicDir.php b/airtime_mvc/application/models/MusicDir.php index 6580ab99c..0534983a4 100644 --- a/airtime_mvc/application/models/MusicDir.php +++ b/airtime_mvc/application/models/MusicDir.php @@ -7,9 +7,9 @@ class MusicDir { */ private $_dir; - public function __construct() + public function __construct($dir) { - + $this->_dir = $dir; } public function getId() @@ -50,8 +50,7 @@ class MusicDir { $dir->setDirectory($p_path); $dir->save(); - $mus_dir = new MusicDir(); - $mus_dir->_dir = $dir; + $mus_dir = new MusicDir($dir); return $mus_dir; } @@ -65,8 +64,7 @@ class MusicDir { { $dir = CcMusicDirsQuery::create()->findPK($pk); - $mus_dir = new MusicDir(); - $mus_dir->_dir = $dir; + $mus_dir = new MusicDir($dir); return $mus_dir; } @@ -77,9 +75,7 @@ class MusicDir { ->filterByDirectory($p_path) ->findOne(); - $mus_dir = new MusicDir(); - $mus_dir->_dir = $dir; - + $mus_dir = new MusicDir($dir); return $mus_dir; } @@ -92,9 +88,7 @@ class MusicDir { ->find(); foreach($dirs as $dir) { - $tmp = new MusicDir(); - $tmp->_dir = $dir; - + $tmp = new MusicDir($dir); $result[] = $tmp; } @@ -107,7 +101,7 @@ class MusicDir { ->filterByType("stor") ->findOne(); - $mus_dir = new MusicDir(); + $mus_dir = new MusicDir($dir); $mus_dir->_dir = $dir; return $mus_dir; @@ -127,8 +121,7 @@ class MusicDir { foreach($dirs as $dir) { $directory = $dir->getDirectory(); if (substr($p_filepath, 0, strlen($directory)) === $directory) { - $mus_dir = new MusicDir(); - $mus_dir->_dir = $dir; + $mus_dir = new MusicDir($dir); return $mus_dir; } } diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index ad38dae68..7ed9acdd3 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -106,6 +106,9 @@ class ApiClientInterface: pass def list_all_db_files(self): + pass + + def list_all_watched_dirs(self): pass # Put here whatever tests you want to run to make sure your API is working @@ -421,7 +424,24 @@ class AirTimeApiClient(ApiClientInterface): logger.error("Exception: %s", e) return response - + + def list_all_watched_dirs(self): + logger = logging.getLogger() + try: + url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["list_all_watched_dirs"]) + + + url = url.replace("%%api_key%%", self.config["api_key"]) + logger.debug(url) + + req = urllib2.Request(url) + response = urllib2.urlopen(req).read() + response = json.loads(response) + except Exception, e: + response = None + logger.error("Exception: %s", e) + + return response ################################################################################ diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py b/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py index a36626b95..39e4d076b 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py @@ -22,13 +22,19 @@ class AirtimeMediaMonitorBootstrap(): went offline. We can do this by doing a hash of the directory metadata. """ def scan(self): - directories = ['/srv/airtime/stor'] + directories = self.get_list_of_watched_dirs(); + + self.logger.info("watched directories found: %s", directories) for dir in directories: self.check_for_diff(dir) def list_db_files(self): - return self.api_client.list_all_db_files() + return self.api_client.list_all_db_files() + + def get_list_of_watched_dirs(self): + json = self.api_client.list_all_watched_dirs() + return json["dirs"] def check_for_diff(self, dir): #set to hold new and/or modified files. We use a set to make it ok if files are added @@ -85,9 +91,7 @@ class AirtimeMediaMonitorBootstrap(): self.logger.info("Modified files: \n%s\n\n"%modified_files_set) #"touch" file timestamp - open("/var/tmp/airtime/media_monitor_boot","w") - #return - + open("/var/tmp/airtime/media_monitor_boot","w") for file_path in deleted_files_set: self.pe.handle_removed_file(file_path) diff --git a/python_apps/media-monitor/media-monitor.cfg b/python_apps/media-monitor/media-monitor.cfg index c31cd83e5..79354e67f 100644 --- a/python_apps/media-monitor/media-monitor.cfg +++ b/python_apps/media-monitor/media-monitor.cfg @@ -31,6 +31,9 @@ update_media_url = 'reload-metadata/format/json/api_key/%%api_key%%/mode/%%mode% # URL to tell Airtime we want a listing of all files it knows about list_all_db_files = 'list-all-files/format/json/api_key/%%api_key%%' +# URL to tell Airtime we want a listing of all dirs its watching (including the stor dir) +list_all_watched_dirs = 'list-all-watched-dirs/format/json/api_key/%%api_key%%' + ############################################ # RabbitMQ settings # ############################################ From c67e711a051258252671fb82ecb648dcc7ed8171 Mon Sep 17 00:00:00 2001 From: martin Date: Mon, 4 Jul 2011 15:40:09 -0400 Subject: [PATCH 2/3] cc-2419: media monitor import on startup -fixed syntax errors, added more logging. --- airtime_mvc/application/controllers/ApiController.php | 2 +- .../airtimefilemonitor/airtimemediamonitorbootstrap.py | 6 +++--- .../media-monitor/airtimefilemonitor/airtimemetadata.py | 1 + .../media-monitor/airtimefilemonitor/airtimeprocessevent.py | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/airtime_mvc/application/controllers/ApiController.php b/airtime_mvc/application/controllers/ApiController.php index fcd6c2b52..57bff257d 100644 --- a/airtime_mvc/application/controllers/ApiController.php +++ b/airtime_mvc/application/controllers/ApiController.php @@ -25,7 +25,7 @@ class ApiController extends Zend_Controller_Action } /** - * Returns Airtime version. i.e "1.7.0 alpha" + * Returns Airtime version. i.e "1.7.0-beta" * * First checks to ensure the correct API key was * supplied, then returns AIRTIME_VERSION as defined diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py b/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py index 39e4d076b..9e3a07fd6 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py @@ -59,10 +59,10 @@ class AirtimeMediaMonitorBootstrap(): all_files_set.add(file_path) - if os.path.exists("/var/tmp/airtime/media_monitor_boot"): + if os.path.exists("/var/tmp/airtime/.media_monitor_boot"): #find files that have been modified since the last time #media-monitor process started. - time_diff_sec = time.time() - os.path.getmtime("/var/tmp/airtime/media_monitor_boot") + time_diff_sec = time.time() - os.path.getmtime("/var/tmp/airtime/.media_monitor_boot") command = "find %s -type f -iname '*.ogg' -o -iname '*.mp3' -readable -mmin -%d" % (dir, time_diff_sec/60+1) else: command = "find %s -type f -iname '*.ogg' -o -iname '*.mp3' -readable" % dir @@ -91,7 +91,7 @@ class AirtimeMediaMonitorBootstrap(): self.logger.info("Modified files: \n%s\n\n"%modified_files_set) #"touch" file timestamp - open("/var/tmp/airtime/media_monitor_boot","w") + open("/var/tmp/airtime/.media_monitor_boot","w") for file_path in deleted_files_set: self.pe.handle_removed_file(file_path) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py b/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py index 7a3079c4a..d1a984cb9 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py @@ -113,6 +113,7 @@ class AirtimeMetadata: except Exception, e: self.logger.error("failed getting metadata from %s", filepath) + self.logger.error("Exception %s", e) return None diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py index 19e0a9925..c3a3a9ab1 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py @@ -289,7 +289,7 @@ class AirtimeProcessEvent(ProcessEvent): if self.is_parent_directory(event.pathname, storage_directory): file_md = self.md_manager.get_md_from_file(event.pathname) if file_md is not None: - filepath, is_recorded_show = self.create_file_path(event.pathname) + filepath, is_recorded_show = self.create_file_path(event.pathname, file_md) self.move_file(event.pathname, filepath) self.renamed_files[event.pathname] = filepath self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': filepath, 'data': file_md, 'is_recorded_show': False}) From df1dec2078bccd47ec116d4289a268a92796d6cd Mon Sep 17 00:00:00 2001 From: martin Date: Mon, 4 Jul 2011 17:37:05 -0400 Subject: [PATCH 3/3] cc-2419: media monitor import on startup -fixed support for watched dirs --- .../application/controllers/ApiController.php | 7 ++++--- airtime_mvc/application/models/StoredFile.php | 5 +++-- python_apps/api_clients/api_client.py | 5 +++-- .../airtimemediamonitorbootstrap.py | 12 +++++------ .../airtimefilemonitor/airtimenotifier.py | 20 ++++++++++++------- .../airtimefilemonitor/airtimeprocessevent.py | 9 ++++++++- python_apps/media-monitor/media-monitor.cfg | 2 +- 7 files changed, 38 insertions(+), 22 deletions(-) diff --git a/airtime_mvc/application/controllers/ApiController.php b/airtime_mvc/application/controllers/ApiController.php index 57bff257d..a1a96df8b 100644 --- a/airtime_mvc/application/controllers/ApiController.php +++ b/airtime_mvc/application/controllers/ApiController.php @@ -521,8 +521,9 @@ class ApiController extends Zend_Controller_Action print 'You are not allowed to access this resource.'; exit; } + $dir_id = $request->getParam('dir_id'); - $this->view->files = StoredFile::listAllFiles(); + $this->view->files = StoredFile::listAllFiles($dir_id); } public function listAllWatchedDirsAction() { @@ -542,10 +543,10 @@ class ApiController extends Zend_Controller_Action $arrWatchedDirs = MusicDir::getWatchedDirs(); $storDir = MusicDir::getStorDir(); - $result[] = $storDir->getDirectory(); + $result[$storDir->getId()] = $storDir->getDirectory(); foreach ($arrWatchedDirs as $watchedDir){ - $result[] = $watchedDir->getDirectory(); + $result[$watchedDir->getId()] = $watchedDir->getDirectory(); } $this->view->dirs = $result; diff --git a/airtime_mvc/application/models/StoredFile.php b/airtime_mvc/application/models/StoredFile.php index bd7c77628..1401e366a 100644 --- a/airtime_mvc/application/models/StoredFile.php +++ b/airtime_mvc/application/models/StoredFile.php @@ -819,14 +819,15 @@ class StoredFile { } - public static function listAllFiles(){ + public static function listAllFiles($dir_id){ global $CC_CONFIG, $CC_DBC; $sql = "SELECT m.directory || '/' || f.filepath as fp" ." FROM CC_MUSIC_DIRS m" ." LEFT JOIN CC_FILES f" ." ON m.id = f.directory" - ." WHERE m.id = f.directory"; + ." WHERE m.id = f.directory" + ." AND directory = $dir_id"; $rows = $CC_DBC->getAll($sql); $results = array(); diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index 7ed9acdd3..eb828d2df 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -105,7 +105,7 @@ class ApiClientInterface: def update_media_metadata(self, md): pass - def list_all_db_files(self): + def list_all_db_files(self, dir_id): pass def list_all_watched_dirs(self): @@ -409,12 +409,13 @@ class AirTimeApiClient(ApiClientInterface): return response - def list_all_db_files(self): + def list_all_db_files(self, dir_id): logger = logging.getLogger() try: url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["list_all_db_files"]) url = url.replace("%%api_key%%", self.config["api_key"]) + url = url.replace("%%dir_id%%", dir_id) req = urllib2.Request(url) response = urllib2.urlopen(req).read() diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py b/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py index 9e3a07fd6..e8c4b7367 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py @@ -26,17 +26,17 @@ class AirtimeMediaMonitorBootstrap(): self.logger.info("watched directories found: %s", directories) - for dir in directories: - self.check_for_diff(dir) + for id, dir in directories: + self.check_for_diff(id, dir) - def list_db_files(self): - return self.api_client.list_all_db_files() + def list_db_files(self, dir_id): + return self.api_client.list_all_db_files(dir_id) def get_list_of_watched_dirs(self): json = self.api_client.list_all_watched_dirs() return json["dirs"] - def check_for_diff(self, dir): + def check_for_diff(self, dir_id, dir): #set to hold new and/or modified files. We use a set to make it ok if files are added #twice. This is become some of the tests for new files return result sets that are not #mutually exclusive from each other. @@ -45,7 +45,7 @@ class AirtimeMediaMonitorBootstrap(): db_known_files_set = set() - files = self.list_db_files() + files = self.list_db_files(dir_id) for file in files['files']: db_known_files_set.add(file) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py index fd29a5e5a..361e8248d 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py @@ -64,11 +64,14 @@ class AirtimeNotifier(Notifier): self.md_manager.save_md_to_file(m) elif m['event_type'] == "new_watch": - self.logger.info("AIRTIME NOTIFIER add watched folder event " + m['directory']) - self.walk_newly_watched_directory(m['directory']) - mm = self.proc_fun() - mm.watch_directory(m['directory']) + if mm.has_correct_permissions(m['directory']): + self.logger.info("AIRTIME NOTIFIER add watched folder event " + m['directory']) + self.walk_newly_watched_directory(m['directory']) + + mm.watch_directory(m['directory']) + else: + self.logger.warn("filepath '%s' has does not have sufficient read permissions. Ignoring.", full_filepath) elif m['event_type'] == "remove_watch": watched_directory = m['directory'].encode('utf-8') @@ -170,7 +173,10 @@ class AirtimeNotifier(Notifier): full_filepath = path+"/"+filename if mm.is_audio_file(full_filepath): - self.logger.info("importing %s", full_filepath) - event = {'filepath': full_filepath, 'mode': self.config.MODE_CREATE, 'is_recorded_show': False} - mm.multi_queue.put(event) + if mm.has_correct_permissions(full_filepath): + self.logger.info("importing %s", full_filepath) + event = {'filepath': full_filepath, 'mode': self.config.MODE_CREATE, 'is_recorded_show': False} + mm.multi_queue.put(event) + else: + self.logger.warn("file '%s' has does not have sufficient read permissions. Ignoring.", full_filepath) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py index c3a3a9ab1..6ed9f1272 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py @@ -61,6 +61,13 @@ class AirtimeProcessEvent(ProcessEvent): return True else: return False + + #file needs to be readable by all users, and directories + #up to this file needs to be readable AND executable by all + #users. + def has_correct_permissions(self, filepath): + st = os.stat(filepath) + return bool(st.st_mode & stat.S_IROTH) def set_needed_file_permissions(self, item, is_dir): @@ -100,7 +107,7 @@ class AirtimeProcessEvent(ProcessEvent): omask = os.umask(0) os.rename(source, dest) except Exception, e: - self.logger.error("failed to move file.") + self.logger.error("failed to move file. %s", e) finally: os.umask(omask) diff --git a/python_apps/media-monitor/media-monitor.cfg b/python_apps/media-monitor/media-monitor.cfg index 79354e67f..f9c86d616 100644 --- a/python_apps/media-monitor/media-monitor.cfg +++ b/python_apps/media-monitor/media-monitor.cfg @@ -32,7 +32,7 @@ update_media_url = 'reload-metadata/format/json/api_key/%%api_key%%/mode/%%mode% list_all_db_files = 'list-all-files/format/json/api_key/%%api_key%%' # URL to tell Airtime we want a listing of all dirs its watching (including the stor dir) -list_all_watched_dirs = 'list-all-watched-dirs/format/json/api_key/%%api_key%%' +list_all_watched_dirs = 'list-all-watched-dirs/format/json/api_key/%%api_key%%/dir_id/%%dir_id%%' ############################################ # RabbitMQ settings #