Merge branch 'devel' of dev.sourcefabric.org:airtime into devel

airtime_mvc/application/controllers/ApiController.php

@@ -408,7 +408,8 @@ class ApiController extends Zend_Controller_Action
     public function reloadMetadataAction() {
         global $CC_CONFIG;
 
-        $api_key = $this->_getParam('api_key');
+        $request = $this->getRequest();
+        $api_key = $request->getParam('api_key');
         if (!in_array($api_key, $CC_CONFIG["apiKey"]))
         {
             header('HTTP/1.0 401 Unauthorized');
@@ -416,8 +417,16 @@ class ApiController extends Zend_Controller_Action
             exit;
         }
 
-        $md = $this->_getParam('md');
+        $mode = $request->getParam('mode');
-        $mode = $this->_getParam('mode');
+        $params = $request->getParams();
+
+        $md = array();
+        //extract all file metadata params from the request.
+        foreach ($params as $key => $value) {
+            if (preg_match('/^MDATA_KEY/', $key)) {
+                $md[$key] = $value;
+            }
+        }
 
         if ($mode == "create") {
             $md5 = $md['MDATA_KEY_MD5'];

airtime_mvc/application/controllers/LibraryController.php

@@ -167,7 +167,7 @@ class LibraryController extends Zend_Controller_Action
 
                 $data = $file->getMetadata();
 
-                RabbitMq::SendFileMetaData($data);
+                RabbitMq::SendMessageToMediaMonitor("md_update", $data);
 
                 $this->_helper->redirector('index');
             }

airtime_mvc/application/controllers/PreferenceController.php

@@ -28,6 +28,7 @@ class PreferenceController extends Zend_Controller_Action
                 Application_Model_Preference::SetDefaultFade($values["preferences_general"]["stationDefaultFade"]);
                 Application_Model_Preference::SetStreamLabelFormat($values["preferences_general"]["streamFormat"]);
                 Application_Model_Preference::SetAllow3rdPartyApi($values["preferences_general"]["thirdPartyApi"]);
+                Application_Model_Preference::SetWatchedDirectory($values["preferences_general"]["watchedFolder"]);
 
                 Application_Model_Preference::SetDoSoundCloudUpload($values["preferences_soundcloud"]["UseSoundCloud"]);
                 Application_Model_Preference::SetSoundCloudUser($values["preferences_soundcloud"]["SoundCloudUser"]);
@@ -37,6 +38,10 @@ class PreferenceController extends Zend_Controller_Action
                 Application_Model_Preference::SetSoundCloudTrackType($values["preferences_soundcloud"]["SoundCloudTrackType"]);
                 Application_Model_Preference::SetSoundCloudLicense($values["preferences_soundcloud"]["SoundCloudLicense"]);
 
+                $data = array();
+                $data["directory"] = $values["preferences_general"]["watchedFolder"];
+                RabbitMq::SendMessageToMediaMonitor("new_watch", $data);
+
                 $this->view->statusMsg = "<div class='success'>Preferences updated.</div>";
             }
         }

airtime_mvc/application/forms/GeneralPreferences.php

@@ -58,6 +58,18 @@ class Application_Form_GeneralPreferences extends Zend_Form_SubForm
         $third_party_api->setValue(Application_Model_Preference::GetAllow3rdPartyApi());
         $third_party_api->setDecorators(array('ViewHelper'));
         $this->addElement($third_party_api);
+
+        //Default station fade
+        $this->addElement('text', 'watchedFolder', array(
+            'class'      => 'input_text',
+            'label'      => 'WatchedFolder:',
+            'required'   => false,
+            'filters'    => array('StringTrim'),
+            'value' => Application_Model_Preference::GetWatchedDirectory(),
+            'decorators' => array(
+                'ViewHelper'
+            )
+        ));
     }
 
 

airtime_mvc/application/models/Preference.php

@@ -189,5 +189,13 @@ class Application_Model_Preference
         }
     }
 
+    public static function SetWatchedDirectory($directory) {
+        Application_Model_Preference::SetValue("watched_directory", $directory);
+    }
+
+    public static function GetWatchedDirectory() {
+        return Application_Model_Preference::GetValue("watched_directory");
+    }
+
 }
 

airtime_mvc/application/models/RabbitMq.php

@@ -40,10 +40,12 @@ class RabbitMq
         }
     }
 
-    public static function SendFileMetaData($md)
+    public static function SendMessageToMediaMonitor($event_type, $md)
     {
         global $CC_CONFIG;
 
+        $md["event_type"] = $event_type;
+
         $conn = new AMQPConnection($CC_CONFIG["rabbitmq"]["host"],
                                          $CC_CONFIG["rabbitmq"]["port"],
                                          $CC_CONFIG["rabbitmq"]["user"],

airtime_mvc/application/views/scripts/form/preferences_general.phtml

@@ -73,6 +73,19 @@
                 </ul>
             <?php endif; ?>
        </dd>
+       <dt id="watchedFolder-label" class="block-display">
+            <label class="required" for="watchedFolder"><?php echo $this->element->getElement('watchedFolder')->getLabel() ?></label>
+        </dt>
+        <dd id="watchedFolder-element" class="block-display">
+            <?php echo $this->element->getElement('watchedFolder') ?>
+            <?php if($this->element->getElement('watchedFolder')->hasErrors()) : ?>
+                <ul class='errors'>
+                    <?php foreach($this->element->getElement('watchedFolder')->getMessages() as $error): ?>
+                        <li><?php echo $error; ?></li>
+                    <?php endforeach; ?>
+                </ul>
+            <?php endif; ?>
+        </dd>
 
     </dl>
 </fieldset>

python_apps/api_clients/api_client.py

@@ -31,24 +31,6 @@ def api_client_factory(config):
         logger.info('API Client "'+config["api_client"]+'" not supported.  Please check your config file.\n')
         sys.exit()
 
-def recursive_urlencode(d):
-    def recursion(d, base=None):
-        pairs = []
-
-        for key, value in d.items():
-            if hasattr(value, 'values'):
-                pairs += recursion(value, key)
-            else:
-                new_pair = None
-                if base:
-                    new_pair = "%s[%s]=%s" % (base, urllib.quote(unicode(key)), urllib.quote(unicode(value)))
-                else:
-                    new_pair = "%s=%s" % (urllib.quote(unicode(key)), urllib.quote(unicode(value)))
-                pairs.append(new_pair)
-        return pairs
-
-    return '&'.join(recursion(d))
-
 class ApiClientInterface:
 
     # Implementation: optional
@@ -402,11 +384,12 @@ class AirTimeApiClient(ApiClientInterface):
         response = None
         try:
             url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["update_media_url"])
-            logger.debug(url)
+
             url = url.replace("%%api_key%%", self.config["api_key"])
             url = url.replace("%%mode%%", mode)
+            logger.debug(url)
 
-            data = recursive_urlencode(md)
+            data = urllib.urlencode(md)
             req = urllib2.Request(url, data)
 
             response = urllib2.urlopen(req).read()

python_apps/media-monitor/MediaMonitor.py

@@ -10,9 +10,12 @@ import hashlib
 import json
 import shutil
 import math
+import socket
+import grp
+import pwd
 
 from collections import deque
-from pwd import getpwnam
+
 from subprocess import Popen, PIPE, STDOUT
 
 from configobj import ConfigObj
@@ -26,6 +29,8 @@ from kombu.connection import BrokerConnection
 from kombu.messaging import Exchange, Queue, Consumer, Producer
 from api_clients import api_client
 
+from multiprocessing import Process, Lock
+
 MODE_CREATE = "create"
 MODE_MODIFY = "modify"
 MODE_MOVED = "moved"
@@ -130,7 +135,7 @@ class MetadataExtractor:
                     value = m[key]
                     if ((value is not None) and (len(str(value)) > 0)):
                         airtime_file[self.airtime2mutagen[key]] = str(value)
-                        self.logger.info('setting %s = %s ', key, str(value))
+                        #self.logger.info('setting %s = %s ', key, str(value))
 
 
             airtime_file.save()
@@ -150,7 +155,17 @@ class MetadataExtractor:
             if key in attrs :
                 md[attrs[key]] = file_info[key][0]
 
-        #md['MDATA_KEY_TRACKNUMBER'] = "%02d" % (int(md['MDATA_KEY_TRACKNUMBER']))
+        if 'MDATA_KEY_TITLE' not in md:
+            #get rid of file extention from original name, name might have more than 1 '.' in it.
+            original_name = os.path.basename(filepath)
+            original_name = original_name.split(".")[0:-1]
+            original_name = ''.join(original_name)
+            md['MDATA_KEY_TITLE'] = original_name
+
+        #incase track number is in format u'4/11'
+        if 'MDATA_KEY_TRACKNUMBER' in md:
+            if isinstance(md['MDATA_KEY_TRACKNUMBER'], basestring):
+                md['MDATA_KEY_TRACKNUMBER'] = md['MDATA_KEY_TRACKNUMBER'].split("/")[0]
 
         md['MDATA_KEY_BITRATE'] = file_info.info.bitrate
         md['MDATA_KEY_SAMPLERATE'] = file_info.info.sample_rate
@@ -162,6 +177,11 @@ class MetadataExtractor:
         elif "vorbis" in md['MDATA_KEY_MIME']:
             md['MDATA_KEY_FTYPE'] = "audioclip"
 
+        #do this so object can be urlencoded properly.
+        for key in md.keys():
+            if(isinstance(md[key], basestring)):
+                md[key] = md[key].encode('utf-8')
+
         return md
 
 
@@ -179,7 +199,10 @@ class AirtimeNotifier(Notifier):
         consumer.consume()
 
         self.logger = logging.getLogger('root')
+        self.api_client = api_client.api_client_factory(config)
         self.md_manager = MetadataExtractor()
+        self.import_processes = {}
+        self.watched_folders = []
 
     def handle_message(self, body, message):
         # ACK the message to take it off the queue
@@ -187,7 +210,56 @@ class AirtimeNotifier(Notifier):
 
         self.logger.info("Received md from RabbitMQ: " + body)
         m =  json.loads(message.body)
-        self.md_manager.save_md_to_file(m)
+
+        if m['event_type'] == "md_update":
+            self.logger.info("AIRTIME NOTIFIER md update event")
+            self.md_manager.save_md_to_file(m)
+        elif m['event_type'] == "new_watch":
+            self.logger.info("AIRTIME NOTIFIER add watched folder event " + m['directory'])
+            #start a new process to walk through this folder and add the files to Airtime.
+            p = Process(target=self.walk_newly_watched_directory, args=(m['directory'],))
+            p.start()
+            self.import_processes[m['directory']] = p
+            #add this new folder to our list of watched folders
+            self.watched_folders.append(m['directory'])
+
+    def update_airtime(self, d):
+
+        filepath = d['filepath']
+        mode = d['mode']
+
+        data = None
+        md = {}
+        md['MDATA_KEY_FILEPATH'] = filepath
+
+        if (os.path.exists(filepath) and (mode == MODE_CREATE)):
+            mutagen = self.md_manager.get_md_from_file(filepath)
+            md.update(mutagen)
+            data = md
+        elif (os.path.exists(filepath) and (mode == MODE_MODIFY)):
+            mutagen = self.md_manager.get_md_from_file(filepath)
+            md.update(mutagen)
+            data = md
+        elif (mode == MODE_MOVED):
+            mutagen = self.md_manager.get_md_from_file(filepath)
+            md.update(mutagen)
+            data = md
+        elif (mode == MODE_DELETE):
+            data = md
+
+        if data is not None:
+            self.logger.info("Updating Change to Airtime " + filepath)
+            response = None
+            while response is None:
+                response = self.api_client.update_media_metadata(data, mode)
+                time.sleep(5)
+
+    def walk_newly_watched_directory(self, directory):
+
+        for (path, dirs, files) in os.walk(directory):
+            for filename in files:
+                full_filepath = path+"/"+filename
+                self.update_airtime({'filepath': full_filepath, 'mode': MODE_CREATE})
 
 
 class MediaMonitor(ProcessEvent):
@@ -219,6 +291,27 @@ class MediaMonitor(ProcessEvent):
     def is_parent_directory(self, filepath, directory):
         return (directory == filepath[0:len(directory)])
 
+    def set_needed_file_permissions(self, item, is_dir):
+
+        try:
+            omask = os.umask(0)
+
+            uid = pwd.getpwnam('pypo')[2]
+            gid = grp.getgrnam('www-data')[2]
+
+            os.chown(item, uid, gid)
+
+            if is_dir is True:
+                os.chmod(item, 02777)
+            else:
+                os.chmod(item, 0666)
+
+        except Exception, e:
+            self.logger.error("Failed to change file's owner/group/permissions.")
+            self.logger.error(item)
+        finally:
+            os.umask(omask)
+
     def ensure_dir(self, filepath):
         directory = os.path.dirname(filepath)
 
@@ -230,21 +323,38 @@ class MediaMonitor(ProcessEvent):
         finally:
             os.umask(omask)
 
+    def move_file(self, source, dest):
+
+        try:
+            omask = os.umask(0)
+            os.rename(source, dest)
+        except Exception, e:
+            self.logger.error("failed to move file.")
+        finally:
+            os.umask(omask)
+
     def create_unique_filename(self, filepath):
 
-        if(os.path.exists(filepath)):
+        try:
-            file_dir = os.path.dirname(filepath)
+            if(os.path.exists(filepath)):
-            filename = os.path.basename(filepath).split(".")[0]
+                self.logger.info("Path %s exists", filepath)
-            #will be in the format .ext
+                file_dir = os.path.dirname(filepath)
-            file_ext = os.path.splitext(filepath)[1]
+                filename = os.path.basename(filepath).split(".")[0]
-            i = 1;
+                #will be in the format .ext
-            while(True):
+                file_ext = os.path.splitext(filepath)[1]
-                new_filepath = "%s/%s(%s)%s" % (file_dir, filename, i, file_ext)
+                i = 1;
+                while(True):
+                    new_filepath = '%s/%s(%s)%s' % (file_dir, filename, i, file_ext)
+                    self.logger.error("Trying %s", new_filepath)
 
-                if(os.path.exists(new_filepath)):
+                    if(os.path.exists(new_filepath)):
-                    i = i+1;
+                        i = i+1;
-                else:
+                    else:
-                    filepath = new_filepath
+                        filepath = new_filepath
+                        break
+
+        except Exception, e:
+             self.logger.error("Exception %s", e)
 
         return filepath
 
@@ -260,23 +370,39 @@ class MediaMonitor(ProcessEvent):
 
             #will be in the format .ext
             file_ext = os.path.splitext(imported_filepath)[1]
+            file_ext = file_ext.encode('utf-8')
             md = self.md_manager.get_md_from_file(imported_filepath)
 
             path_md = ['MDATA_KEY_TITLE', 'MDATA_KEY_CREATOR', 'MDATA_KEY_SOURCE', 'MDATA_KEY_TRACKNUMBER', 'MDATA_KEY_BITRATE']
 
+            self.logger.info('Getting md')
+
             for m in path_md:
                 if m not in md:
-                    md[m] = 'unknown'
+                    md[m] = u'unknown'.encode('utf-8')
+                else:
+                    #get rid of any "/" which will interfere with the filepath.
+                    if isinstance(md[m], basestring):
+                        md[m] = md[m].replace("/", "-")
+
+            self.logger.info(md)
+
+            self.logger.info('Starting filepath creation')
 
             filepath = None
-            if (md['MDATA_KEY_TITLE'] == 'unknown'):
+            if (md['MDATA_KEY_TITLE'] == u'unknown'.encode('utf-8')):
-                filepath = "%s/%s/%s/%s-%s%s" % (storage_directory, md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], original_name, md['MDATA_KEY_BITRATE'], file_ext)
+                self.logger.info('unknown title')
-            elif(md['MDATA_KEY_TRACKNUMBER'] == 'unknown'):
+                filepath = '%s/%s/%s/%s-%s%s' % (storage_directory.encode('utf-8'), md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], original_name, md['MDATA_KEY_BITRATE'], file_ext)
-                filepath = "%s/%s/%s/%s-%s%s" % (storage_directory, md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext)
+            elif(md['MDATA_KEY_TRACKNUMBER'] == u'unknown'.encode('utf-8')):
+                self.logger.info('unknown track number')
+                filepath = '%s/%s/%s/%s-%s%s' % (storage_directory.encode('utf-8'), md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext)
             else:
-                filepath = "%s/%s/%s/%s-%s-%s%s" % (storage_directory, md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TRACKNUMBER'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext)
+                self.logger.info('full metadata')
+                filepath = '%s/%s/%s/%s-%s-%s%s' % (storage_directory.encode('utf-8'), md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TRACKNUMBER'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext)
 
+            self.logger.info(u'Created filepath: %s', filepath)
             filepath = self.create_unique_filename(filepath)
+            self.logger.info(u'Unique filepath: %s', filepath)
             self.ensure_dir(filepath)
 
         except Exception, e:
@@ -284,37 +410,6 @@ class MediaMonitor(ProcessEvent):
 
         return filepath
 
-    def update_airtime(self, d):
-
-        filepath = d['filepath']
-        mode = d['mode']
-
-        data = None
-        md = {}
-        md['MDATA_KEY_FILEPATH'] = filepath
-
-        if (os.path.exists(filepath) and (mode == MODE_CREATE)):
-            mutagen = self.md_manager.get_md_from_file(filepath)
-            md.update(mutagen)
-            data = {'md': md}
-        elif (os.path.exists(filepath) and (mode == MODE_MODIFY)):
-            mutagen = self.md_manager.get_md_from_file(filepath)
-            md.update(mutagen)
-            data = {'md': md}
-        elif (mode == MODE_MOVED):
-            mutagen = self.md_manager.get_md_from_file(filepath)
-            md.update(mutagen)
-            data = {'md': md}
-        elif (mode == MODE_DELETE):
-            data = {'md': md}
-
-        if data is not None:
-            self.logger.info("Updating Change to Airtime")
-            response = None
-            while response is None:
-                response = self.api_client.update_media_metadata(data, mode)
-                time.sleep(5)
-
     def is_temp_file(self, filename):
         info = filename.split(".")
 
@@ -342,18 +437,20 @@ class MediaMonitor(ProcessEvent):
                 global plupload_directory
                 #files that have been added through plupload have a placeholder already put in Airtime's database.
                 if not self.is_parent_directory(event.pathname, plupload_directory):
-                    md5 = self.md_manager.get_md5(event.pathname)
+                    if self.is_audio_file(event.pathname):
-                    response = self.api_client.check_media_status(md5)
+                        self.set_needed_file_permissions(event.pathname, event.dir)
+                        md5 = self.md_manager.get_md5(event.pathname)
+                        response = self.api_client.check_media_status(md5)
 
-                    #this file is new, md5 does not exist in Airtime.
+                        #this file is new, md5 does not exist in Airtime.
-                    if(response['airtime_status'] == 0):
+                        if(response['airtime_status'] == 0):
-                        filepath = self.create_file_path(event.pathname)
+                            filepath = self.create_file_path(event.pathname)
-                        os.rename(event.pathname, filepath)
+                            self.move_file(event.pathname, filepath)
-                        self.file_events.append({'mode': MODE_CREATE, 'filepath': filepath})
+                            self.file_events.append({'mode': MODE_CREATE, 'filepath': filepath})
 
-        #immediately add a watch on the new directory.
         else:
-            self.watch_directory(event.pathname)
+            self.set_needed_file_permissions(event.pathname, event.dir)
+
 
     def process_IN_MODIFY(self, event):
         if not event.dir:
@@ -375,6 +472,8 @@ class MediaMonitor(ProcessEvent):
 
     def process_IN_MOVED_TO(self, event):
         self.logger.info("%s: %s", event.maskname, event.pathname)
+        #if stuff dropped in stor via a UI move must change file permissions.
+        self.set_needed_file_permissions(event.pathname, event.dir)
         if not event.dir:
             if event.cookie in self.temp_files:
                 del self.temp_files[event.cookie]
@@ -388,7 +487,7 @@ class MediaMonitor(ProcessEvent):
                     #file renamed from /tmp/plupload does not have a path in our naming scheme yet.
                     md_filepath = self.create_file_path(event.pathname)
                     #move the file a second time to its correct Airtime naming schema.
-                    os.rename(event.pathname, md_filepath)
+                    self.move_file(event.pathname, md_filepath)
                     self.file_events.append({'filepath': md_filepath, 'mode': MODE_MOVED})
                 else:
                     self.file_events.append({'filepath': event.pathname, 'mode': MODE_MOVED})
@@ -397,7 +496,7 @@ class MediaMonitor(ProcessEvent):
                 #TODO need to pass in if md5 exists to this file creation function, identical files will just replace current files not have a (1) etc.
                 #file has been most likely dropped into stor folder from an unwatched location. (from gui, mv command not cp)
                 md_filepath = self.create_file_path(event.pathname)
-                os.rename(event.pathname, md_filepath)
+                self.move_file(event.pathname, md_filepath)
                 self.file_events.append({'mode': MODE_CREATE, 'filepath': md_filepath})
 
     def process_IN_DELETE(self, event):
@@ -410,12 +509,22 @@ class MediaMonitor(ProcessEvent):
 
     def notifier_loop_callback(self, notifier):
 
+        for watched_directory in notifier.import_processes.keys():
+            process = notifier.import_processes[watched_directory]
+            if not process.is_alive():
+                self.watch_directory(watched_directory)
+                del notifier.import_processes[watched_directory]
+
         while len(self.file_events) > 0:
+            self.logger.info("Processing a file event update to Airtime.")
             file_info = self.file_events.popleft()
-            self.update_airtime(file_info)
+            notifier.update_airtime(file_info)
 
         try:
             notifier.connection.drain_events(timeout=1)
+        #avoid logging a bunch of timeout messages.
+        except socket.timeout:
+            pass
         except Exception, e:
             self.logger.info("%s", e)
 

python_apps/media-monitor/airtime-media-monitor-init-d

@@ -9,8 +9,8 @@
 # Short-Description: Manage airtime-media-monitor daemon
 ### END INIT INFO
 
-USERID=pypo
+USERID=root
-GROUPID=pypo
+GROUPID=www-data
 NAME=Airtime
 
 DAEMON=/usr/bin/airtime-media-monitor

python_apps/media-monitor/install/media-monitor-install.py

@@ -63,6 +63,7 @@ try:
 
   print "Setting permissions"
   os.system("chmod -R 755 "+config["bin_dir"])
+  #os.system("chmod -R 755 "+config["bin_dir"]+"/airtime-media-monitor)
   os.system("chown -R pypo:pypo "+config["bin_dir"])
 
   print "Creating symbolic links"