CC-2571 : Media Monitor Bootstrap

fixing up character encoding issues.

airtime_mvc/application/models/StoredFile.php

@@ -735,7 +735,8 @@ class StoredFile {
 		$fileName = isset($_REQUEST["name"]) ? $_REQUEST["name"] : '';
 
 		// Clean the fileName for security reasons
-		$fileName = preg_replace('/[^\w\._]+/', '', $fileName);
+        //this needs fixing for songs not in ascii.
+		//$fileName = preg_replace('/[^\w\._]+/', '', $fileName);
 
 		// Create target dir
 		if (!file_exists($p_targetDir))

python_apps/api_clients/api_client.py

@@ -105,20 +105,20 @@ class ApiClientInterface:
         pass
 
     def update_media_metadata(self, md):
-        pass    
+        pass
-        
+
     def list_all_db_files(self, dir_id):
-        pass    
+        pass
-        
+
     def list_all_watched_dirs(self):
         pass
-    
+
     def add_watched_dir(self):
         pass
-    
+
     def remove_watched_dir(self):
         pass
-    
+
     def set_storage_dir(self):
         pass
 
@@ -423,96 +423,96 @@ class AirTimeApiClient(ApiClientInterface):
 
         except Exception, e:
             response = None
-            logger.error("Exception: %s", e)
+            logger.error("Exception with filepath %s: %s", md['MDATA_KEY_FILEPATH'], e)
 
         return response
-        
+
     #returns a list of all db files for a given directory in JSON format:
     #{"files":["path/to/file1", "path/to/file2"]}
-    #Note that these are relative paths to the given directory. The full 
+    #Note that these are relative paths to the given directory. The full
     #path is not returned.
     def list_all_db_files(self, dir_id):
         logger = logging.getLogger()
         try:
             url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["list_all_db_files"])
-            
+
             url = url.replace("%%api_key%%", self.config["api_key"])
             url = url.replace("%%dir_id%%", dir_id)
-            
+
             req = urllib2.Request(url)
             response = urllib2.urlopen(req).read()
             response = json.loads(response)
         except Exception, e:
             response = None
             logger.error("Exception: %s", e)
-            
+
         return response
-        
+
     def list_all_watched_dirs(self):
         logger = logging.getLogger()
         try:
             url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["list_all_watched_dirs"])
-            
+
             url = url.replace("%%api_key%%", self.config["api_key"])
-            
+
             req = urllib2.Request(url)
             response = urllib2.urlopen(req).read()
             response = json.loads(response)
         except Exception, e:
             response = None
             logger.error("Exception: %s", e)
-            
+
         return response
-    
+
     def add_watched_dir(self, path):
         logger = logging.getLogger()
         try:
             url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["add_watched_dir"])
-            
+
             url = url.replace("%%api_key%%", self.config["api_key"])
             url = url.replace("%%path%%", base64.b64encode(path))
-            
+
             req = urllib2.Request(url)
             response = urllib2.urlopen(req).read()
             response = json.loads(response)
         except Exception, e:
             response = None
             logger.error("Exception: %s", e)
-            
+
         return response
-    
+
     def remove_watched_dir(self, path):
         logger = logging.getLogger()
         try:
             url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["remove_watched_dir"])
-            
+
             url = url.replace("%%api_key%%", self.config["api_key"])
             url = url.replace("%%path%%", base64.b64encode(path))
-            
+
             req = urllib2.Request(url)
             response = urllib2.urlopen(req).read()
             response = json.loads(response)
         except Exception, e:
             response = None
             logger.error("Exception: %s", e)
-            
+
         return response
-    
+
     def set_storage_dir(self, path):
         logger = logging.getLogger()
         try:
             url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["set_storage_dir"])
-            
+
             url = url.replace("%%api_key%%", self.config["api_key"])
             url = url.replace("%%path%%", base64.b64encode(path))
-            
+
             req = urllib2.Request(url)
             response = urllib2.urlopen(req).read()
             response = json.loads(response)
         except Exception, e:
             response = None
             logger.error("Exception: %s", e)
-            
+
         return response
 
 

python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py

@@ -29,7 +29,7 @@ class AirtimeMediaMonitorBootstrap():
 
         for id, dir in directories.iteritems():
             self.logger.debug("%s, %s", id, dir)
-            self.sync_database_to_filesystem(id, dir)
+            self.sync_database_to_filesystem(id, dir.encode("utf-8"))
 
     """Gets a list of files that the Airtime database knows for a specific directory.
     You need to provide the directory's row ID, which is obtained when calling
@@ -69,7 +69,7 @@ class AirtimeMediaMonitorBootstrap():
         db_known_files_set = set()
         files = self.list_db_files(dir_id)
         for file in files['files']:
-            db_known_files_set.add(file)
+            db_known_files_set.add(file.encode('utf-8'))
 
         new_files = self.mmc.scan_dir_for_new_files(dir)
 
@@ -85,8 +85,8 @@ class AirtimeMediaMonitorBootstrap():
         else:
             command = "find %s -type f -iname '*.ogg' -o -iname '*.mp3' -readable" % dir
 
-        stdout = self.mmc.execCommandAndReturnStdOut(command)
+        stdout = self.mmc.exec_command(command)
-        stdout = unicode(stdout, "utf_8")
+        self.logger.info(stdout)
 
         new_files = stdout.splitlines()
 
@@ -120,16 +120,23 @@ class AirtimeMediaMonitorBootstrap():
             self.logger.warn(e)
 
         for file_path in deleted_files_set:
-            self.pe.handle_removed_file(False, "%s%s" % (dir, file_path))
+            self.logger.debug("deleted file")
+            full_file_path = "%s%s" % (dir, file_path)
+            self.logger.debug(full_file_path)
+            self.pe.handle_removed_file(False, full_file_path)
 
         for file_path in new_files_set:
-            file_path = "%s%s" % (dir, file_path)
+            self.logger.debug("new file")
-            if os.path.exists(file_path):
+            full_file_path = "%s%s" % (dir, file_path)
-                organized_filepath = self.pe.handle_created_file(False, file_path, os.path.basename(file_path))
+            self.logger.debug(full_file_path)
+            if os.path.exists(full_file_path):
+                organized_filepath = self.pe.handle_created_file(False, full_file_path, os.path.basename(full_file_path))
                 if organized_filepath is not None:
                     self.pe.handle_created_file(False, organized_filepath, os.path.basename(organized_filepath))
 
         for file_path in modified_files_set:
-            file_path = "%s%s" % (dir, file_path)
+            self.logger.debug("modified file")
-            if os.path.exists(file_path):
+            full_file_path = "%s%s" % (dir, file_path)
-                self.pe.handle_modified_file(False, file_path, os.path.basename(file_path))
+            self.logger.debug(full_file_path)
+            if os.path.exists(full_file_path):
+                self.pe.handle_modified_file(False, full_file_path, os.path.basename(full_file_path))

python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py

@@ -67,11 +67,14 @@ class AirtimeProcessEvent(ProcessEvent):
                     new_filepath = self.mmc.organize_new_file(pathname)
                     return new_filepath
                 else:
+                    self.logger.debug("setting file permissions")
                     self.mmc.set_needed_file_permissions(pathname, dir)
+                    self.logger.debug("checking if recorded")
                     if self.mmc.is_parent_directory(pathname, self.config.recorded_directory):
                         is_recorded = True
                     else :
                         is_recorded = False
+                    self.logger.debug("appending event")
                     self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': pathname, 'is_recorded_show': is_recorded})
 
         else:

python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py

@@ -207,18 +207,18 @@ class MediaMonitorCommon:
 
         return filepath
 
-    def execCommandAndReturnStdOut(self, command):
+    def exec_command(self, command):
-        p = Popen(command, shell=True, stdout=PIPE)
+        p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)
-        stdout = p.communicate()[0]
+        stdout, stderr = p.communicate()
         if p.returncode != 0:
             self.logger.warn("command \n%s\n return with a non-zero return value", command)
+            self.logger.error(stderr)
         return stdout
 
     def scan_dir_for_new_files(self, dir):
         command = 'find "%s" -type f -iname "*.ogg" -o -iname "*.mp3" -readable' % dir.replace('"', '\\"')
         self.logger.debug(command)
-        stdout = self.execCommandAndReturnStdOut(command)
+        stdout = self.exec_command(command)
-        stdout = unicode(stdout, "utf_8")
 
         return stdout.splitlines()
 
@@ -230,8 +230,6 @@ class MediaMonitorCommon:
         file_md = self.md_manager.get_md_from_file(pathname)
 
         if file_md is not None:
-            #is_recorded_show = 'MDATA_KEY_CREATOR' in file_md and \
-            #    file_md['MDATA_KEY_CREATOR'] == "AIRTIMERECORDERSOURCEFABRIC".encode('utf-8')
             filepath = self.create_file_path(pathname, file_md)
 
             self.logger.debug("Moving from %s to %s", pathname, filepath)