Merge branch 'cc-1799-put-airtime-storage-into-a-human-readable-file-naming-convention' into devel

Conflicts:
	python_apps/media-monitor/MediaMonitor.py

airtime_mvc/application/configs/constants.php

@@ -4,19 +4,31 @@ define('AIRTIME_VERSION', '1.9.0-devel');
 define('AIRTIME_COPYRIGHT_DATE', '2010-2011');
 define('AIRTIME_REST_VERSION', '1.1');
 
-// Metadata Keys
+// Metadata Keys for files
-define('UI_MDATA_KEY_TITLE', 'dc:title');
+define('MDATA_KEY_FILEPATH', 'filepath');
-define('UI_MDATA_KEY_CREATOR', 'dc:creator');
+define('MDATA_KEY_MD5', 'md5');
-define('UI_MDATA_KEY_SOURCE', 'dc:source');
+define('MDATA_KEY_TITLE', 'track_title');
-define('UI_MDATA_KEY_DURATION', 'dcterms:extent');
+define('MDATA_KEY_CREATOR', 'artist_name');
-define('UI_MDATA_KEY_URL', 'ls:url');
+define('MDATA_KEY_SOURCE', 'album_title');
-define('UI_MDATA_KEY_FORMAT', 'dc:format');
+define('MDATA_KEY_DURATION', 'length');
-define('UI_MDATA_KEY_DESCRIPTION', 'dc:description');
+define('MDATA_KEY_MIME', 'mime');
-define('UI_MDATA_KEY_CHANNELS', 'ls:channels');
+define('MDATA_KEY_FTYPE', 'ftype');
-define('UI_MDATA_KEY_SAMPLERATE', 'ls:samplerate');
+define('MDATA_KEY_URL', 'url');
-define('UI_MDATA_KEY_BITRATE', 'ls:bitrate');
+define('MDATA_KEY_GENRE', 'genre');
-define('UI_MDATA_KEY_ENCODER', 'ls:encoder');
+define('MDATA_KEY_MOOD', 'mood');
-define('UI_MDATA_KEY_FILENAME', 'ls:filename');
+define('MDATA_KEY_LABEL', 'label');
+define('MDATA_KEY_COMPOSER', 'composer');
+define('MDATA_KEY_FORMAT', 'format');
+define('MDATA_KEY_DESCRIPTION', 'description');
+define('MDATA_KEY_SAMPLERATE', 'sample_rate');
+define('MDATA_KEY_BITRATE', 'bit_rate');
+define('MDATA_KEY_ENCODER', 'encoded_by');
+define('MDATA_KEY_ISRC', 'isrc_number');
+define('MDATA_KEY_COPYRIGHT', 'copyright');
+define('MDATA_KEY_YEAR', 'year');
+define('MDATA_KEY_BPM', 'bpm');
+define('MDATA_KEY_TRACKNUMBER', 'track_number');
+define('MDATA_KEY_CONDUCTOR', 'conductor');
 
 define('UI_MDATA_VALUE_FORMAT_FILE', 'File');
 define('UI_MDATA_VALUE_FORMAT_STREAM', 'live stream');

airtime_mvc/application/controllers/ApiController.php

@@ -10,6 +10,8 @@ class ApiController extends Zend_Controller_Action
         $context->addActionContext('version', 'json')
                 ->addActionContext('recorded-shows', 'json')
                 ->addActionContext('upload-recorded', 'json')
+                ->addActionContext('media-monitor-setup', 'json')
+                ->addActionContext('media-item-status', 'json')
                 ->addActionContext('reload-metadata', 'json')
                 ->initContext();
     }
@@ -77,7 +79,7 @@ class ApiController extends Zend_Controller_Action
         if (ctype_alnum($file_id) && strlen($file_id) == 32) {
           $media = StoredFile::RecallByGunid($file_id);
           if ($media != null && !PEAR::isError($media)) {
-            $filepath = $media->getRealFilePath();
+            $filepath = $media->getFilePath();
             if(!is_file($filepath))
             {
                 header($_SERVER["SERVER_PROTOCOL"]." 404 Not Found");
@@ -293,17 +295,17 @@ class ApiController extends Zend_Controller_Action
             print 'You are not allowed to access this resource.';
             exit;
         }
-        
+
        	$showCanceled = false;
         $show_instance  = $this->_getParam('show_instance');
-        
+
         $upload_dir = ini_get("upload_tmp_dir");
         $file = StoredFile::uploadFile($upload_dir);
-        
+
         $show_name = "";
         try {
             $show_inst = new ShowInstance($show_instance);
-            
+
             $show_inst->setRecordedFile($file->getId());
             $show_name = $show_inst->getName();
             $show_genre = $show_inst->getGenre();
@@ -317,12 +319,12 @@ class ApiController extends Zend_Controller_Action
             //the library), now lets just return.
             $showCanceled = true;
         }
-        
+
 		$tmpTitle = !(empty($show_name))?$show_name."-":"";
 		$tmpTitle .= $file->getName();
-		
+
 		$file->setMetadataValue(UI_MDATA_KEY_TITLE, $tmpTitle);
-        
+
         if (!$showCanceled && Application_Model_Preference::GetDoSoundCloudUpload())
         {
         	for ($i=0; $i<$CC_CONFIG['soundcloud-connection-retries']; $i++) {
@@ -353,8 +355,57 @@ class ApiController extends Zend_Controller_Action
         $this->view->id = $file->getId();
     }
 
-    public function reloadMetadataAction() {
+    public function mediaMonitorSetupAction() {
+        global $CC_CONFIG;
 
+        // disable the view and the layout
+        $this->view->layout()->disableLayout();
+        $this->_helper->viewRenderer->setNoRender(true);
+
+        $api_key = $this->_getParam('api_key');
+        if (!in_array($api_key, $CC_CONFIG["apiKey"]))
+        {
+            header('HTTP/1.0 401 Unauthorized');
+            print 'You are not allowed to access this resource.';
+            exit;
+        }
+
+        $plupload_dir = ini_get("upload_tmp_dir") . DIRECTORY_SEPARATOR . "plupload";
+
+        //need to make sure plupload dir exists so we can watch it.
+        if(!file_exists($plupload_dir)) {
+            @mkdir($plupload_dir, 0755);
+        }
+
+        $this->view->stor = $CC_CONFIG['storageDir'];
+        $this->view->plupload = $plupload_dir;
+    }
+
+    public function mediaItemStatusAction() {
+        global $CC_CONFIG;
+
+        $api_key = $this->_getParam('api_key');
+        if (!in_array($api_key, $CC_CONFIG["apiKey"]))
+        {
+            header('HTTP/1.0 401 Unauthorized');
+            print 'You are not allowed to access this resource.';
+            exit;
+        }
+
+        $md5 = $this->_getParam('md5');
+        $file = StoredFile::RecallByMd5($md5);
+
+        //New file added to Airtime
+        if (is_null($file)) {
+            $this->view->airtime_status = 0;
+        }
+        else {
+            $this->view->airtime_status = 1;
+        }
+
+    }
+
+    public function reloadMetadataAction() {
         global $CC_CONFIG;
 
         $api_key = $this->_getParam('api_key');
@@ -366,22 +417,62 @@ class ApiController extends Zend_Controller_Action
         }
 
         $md = $this->_getParam('md');
-        $filepath = $md['filepath'];
+        $mode = $this->_getParam('mode');
-        $filepath = str_replace("\\", "", $filepath);
+
-        $file = StoredFile::Recall(null, null, null, $filepath);
+        if ($mode == "create") {
-        if (PEAR::isError($file) || is_null($file)) {
+            $md5 = $md['MDATA_KEY_MD5'];
-            $this->view->response = "File not in Airtime's Database";
+            $file = StoredFile::RecallByMd5($md5);
-            return;
+
+            if (is_null($file)) {
+                $file = StoredFile::Insert($md);
+            }
+            else {
+                $this->view->error = "File already exists in Airtime.";
+                return;
+            }
+        }
+        else if ($mode == "modify") {
+            $filepath = $md['MDATA_KEY_FILEPATH'];
+            $filepath = str_replace("\\", "", $filepath);
+            $file = StoredFile::RecallByFilepath($filepath);
+
+            //File is not in database anymore.
+            if (is_null($file)) {
+                $this->view->error = "File does not exist in Airtime.";
+                return;
+            }
+            //Updating a metadata change.
+            else {
+                $file->setMetadata($md);
+            }
+        }
+        else if ($mode == "moved") {
+            $md5 = $md['MDATA_KEY_MD5'];
+            $file = StoredFile::RecallByMd5($md5);
+
+            if (is_null($file)) {
+                $this->view->error = "File doesn't exist in Airtime.";
+                return;
+            }
+            else {
+                $file->setMetadata($md);
+            }
+        }
+        else if ($mode == "delete") {
+            $filepath = $md['MDATA_KEY_FILEPATH'];
+            $filepath = str_replace("\\", "", $filepath);
+            $file = StoredFile::RecallByFilepath($filepath);
+
+            if (is_null($file)) {
+                $this->view->error = "File doesn't exist in Airtime.";
+                return;
+            }
+            else {
+                $file->delete();
+            }
         }
 
-        $res = $file->replaceDbMetadata($md);
+        $this->view->id = $file->getId();
-
-        if (PEAR::isError($res)) {
-            $this->view->response = "Metadata Change Failed";
-        }
-        else {
-            $this->view->response = "Success!";
-        }
     }
 }
 

airtime_mvc/application/controllers/LibraryController.php

@@ -78,7 +78,7 @@ class LibraryController extends Zend_Controller_Action
 	    	$file_id = $this->_getParam('id', null);
 	        $file = StoredFile::Recall($file_id);
 
-	        $url = $file->getFileURL().'/api_key/'.$CC_CONFIG["apiKey"][0].'/download/true';
+	        $url = $file->getFileUrl().'/api_key/'.$CC_CONFIG["apiKey"][0].'/download/true';
             $menu[] = array('action' => array('type' => 'gourl', 'url' => $url),
             				'title' => 'Download');
 
@@ -162,18 +162,18 @@ class LibraryController extends Zend_Controller_Action
             if ($form->isValid($request->getPost())) {
 
                 $formdata = $form->getValues();
-                $file->replaceDbMetadata($formdata);
+                $file->setDbColMetadata($formdata);
 
                 $data = $formdata;
-                $data['filepath'] = $file->getRealFilePath();
+                $data['filepath'] = $file->getFilePath();
-                //wait for 1.9.0 release
+
-                //RabbitMq::SendFileMetaData($data);
+                RabbitMq::SendFileMetaData($data);
 
                 $this->_helper->redirector('index');
             }
         }
 
-        $form->populate($file->md);
+        $form->populate($file->getDbColMetadata());
         $this->view->form = $form;
     }
 

airtime_mvc/application/controllers/PluploadController.php

@@ -25,9 +25,13 @@ class PluploadController extends Zend_Controller_Action
     public function uploadAction()
     {
         $upload_dir = ini_get("upload_tmp_dir") . DIRECTORY_SEPARATOR . "plupload";
-        $file = StoredFile::uploadFile($upload_dir);
+        $res = StoredFile::uploadFile($upload_dir);
 
-		die('{"jsonrpc" : "2.0", "id" : '.$file->getId().' }');
+        if (isset($res)) {
+            die('{"jsonrpc" : "2.0", "id" : '.$file->getMessage().' }');
+        }
+
+		die('{"jsonrpc" : "2.0"}');
     }
 }
 

airtime_mvc/application/models/RabbitMq.php

@@ -40,9 +40,6 @@ class RabbitMq
         }
     }
 
-/*
-*   wait for 1.9.0 release
-
     public static function SendFileMetaData($md)
     {
         global $CC_CONFIG;
@@ -64,7 +61,5 @@ class RabbitMq
         $channel->close();
         $conn->close();
     }
-*/
-
 }
 

install/airtime-install

@@ -11,11 +11,11 @@ SCRIPTPATH=`dirname $SCRIPT`
 
 echo -e "\n******************************** Install Begin *********************************"
 
-php ${SCRIPTPATH}/airtime-install.php $@
-
 echo -e "\n*** Creating Pypo User ***"
 python ${SCRIPTPATH}/../python_apps/create-pypo-user.py
 
+php ${SCRIPTPATH}/airtime-install.php $@
+
 echo -e "\n*** Pypo Installation ***"
 python ${SCRIPTPATH}/../python_apps/pypo/install/pypo-install.py
 

install/airtime-install.php

@@ -121,8 +121,6 @@ if ($db_install) {
 
 AirtimeInstall::InstallStorageDirectory();
 
-AirtimeInstall::ChangeDirOwnerToWebserver($CC_CONFIG["storageDir"]);
-
 AirtimeInstall::CreateSymlinksToUtils();
 
 AirtimeInstall::CreateZendPhpLogFile();

install/include/AirtimeInstall.php

@@ -116,42 +116,39 @@ class AirtimeInstall
         }
     }
 
-    public static function ChangeDirOwnerToWebserver($filePath)
-    {
-        global $CC_CONFIG;
-        echo "* Giving Apache permission to access $filePath".PHP_EOL;
-
-        $success = chgrp($filePath, $CC_CONFIG["webServerUser"]);
-        $fileperms=@fileperms($filePath);
-        $fileperms = $fileperms | 0x0010; // group write bit
-        $fileperms = $fileperms | 0x0400; // group sticky bit
-        chmod($filePath, $fileperms);
-    }
-
     public static function InstallStorageDirectory()
     {
         global $CC_CONFIG, $CC_DBC;
         echo "* Storage directory setup".PHP_EOL;
 
-        foreach (array('baseFilesDir', 'storageDir') as $d) {
+        $stor_dir = $CC_CONFIG['storageDir'];
-            if ( !file_exists($CC_CONFIG[$d]) ) {
+
-                @mkdir($CC_CONFIG[$d], 02775, true);
+        if (!file_exists($stor_dir)) {
-                if (file_exists($CC_CONFIG[$d])) {
+            @mkdir($stor_dir, 02777, true);
-                    $rp = realpath($CC_CONFIG[$d]);
+            if (file_exists($stor_dir)) {
-                    echo "* Directory $rp created".PHP_EOL;
+                $rp = realpath($stor_dir);
-                } else {
+                echo "* Directory $rp created".PHP_EOL;
-                    echo "* Failed creating {$CC_CONFIG[$d]}".PHP_EOL;
-                    exit(1);
-                }
-            } elseif (is_writable($CC_CONFIG[$d])) {
-                $rp = realpath($CC_CONFIG[$d]);
-                echo "* Skipping directory already exists: $rp".PHP_EOL;
             } else {
-                $rp = realpath($CC_CONFIG[$d]);
+                echo "* Failed creating {$stor_dir}".PHP_EOL;
-                echo "* WARNING: Directory already exists, but is not writable: $rp".PHP_EOL;
+                exit(1);
             }
-            $CC_CONFIG[$d] = $rp;
         }
+        else if (is_writable($stor_dir)) {
+            $rp = realpath($stor_dir);
+            echo "* Skipping directory already exists: $rp".PHP_EOL;
+        }
+        else {
+            $rp = realpath($stor_dir);
+            echo "* WARNING: Directory already exists, but is not writable: $rp".PHP_EOL;
+            return;
+        }
+
+        echo "* Giving Apache permission to access $rp".PHP_EOL;
+        $success = chgrp($rp, $CC_CONFIG["webServerUser"]);
+        $success = chown($rp, "pypo");
+        $success = chmod($rp, 02777);
+        $CC_CONFIG['storageDir'] = $rp;
+
     }
 
     public static function CreateDatabaseUser()
@@ -300,7 +297,7 @@ class AirtimeInstall
         echo "* Installing airtime-update-db-settings".PHP_EOL;
         $dir = AirtimeInstall::CONF_DIR_BINARIES."/utils/airtime-update-db-settings";
         exec("ln -s $dir /usr/bin/airtime-update-db-settings");
-        
+
         echo "* Installing airtime-check-system".PHP_EOL;
         $dir = AirtimeInstall::CONF_DIR_BINARIES."/utils/airtime-check-system";
         exec("ln -s $dir /usr/bin/airtime-check-system");

python_apps/api_clients/api_client.py

@@ -117,6 +117,9 @@ class ApiClientInterface:
     def upload_recorded_show(self):
         pass
 
+    def check_media_status(self, md5):
+        pass
+
     def update_media_metadata(self, md):
         pass
 
@@ -356,13 +359,52 @@ class AirTimeApiClient(ApiClientInterface):
 
         return response
 
-    def update_media_metadata(self, md):
+    def setup_media_monitor(self):
+        logger = logging.getLogger()
+
+        response = None
+        try:
+            url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["media_setup_url"])
+            url = url.replace("%%api_key%%", self.config["api_key"])
+            logger.debug(url)
+
+            response = urllib.urlopen(url)
+            response = json.loads(response.read())
+            logger.debug("Json Media Setup %s", response)
+
+        except Exception, e:
+            response = None
+            logger.error("Exception: %s", e)
+
+        return response
+
+    def check_media_status(self, md5):
+        logger = logging.getLogger()
+
+        response = None
+        try:
+            url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["media_status_url"])
+            url = url.replace("%%api_key%%", self.config["api_key"])
+            url = url.replace("%%md5%%", md5)
+            logger.debug(url)
+
+            response = urllib.urlopen(url)
+            response = json.loads(response.read())
+            logger.info("Json Media Status %s", response)
+
+        except Exception, e:
+            logger.error("Exception: %s", e)
+
+        return response
+
+    def update_media_metadata(self, md, mode):
         logger = logging.getLogger()
         response = None
         try:
             url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["update_media_url"])
             logger.debug(url)
             url = url.replace("%%api_key%%", self.config["api_key"])
+            url = url.replace("%%mode%%", mode)
 
             data = recursive_urlencode(md)
             req = urllib2.Request(url, data)
@@ -372,6 +414,7 @@ class AirTimeApiClient(ApiClientInterface):
             response = json.loads(response)
 
         except Exception, e:
+            response = None
             logger.error("Exception: %s", e)
 
         return response

python_apps/media-monitor/MediaMonitor.py

@@ -9,7 +9,10 @@ import sys
 import hashlib
 import json
 import shutil
+import math
 
+from collections import deque
+from pwd import getpwnam
 from subprocess import Popen, PIPE, STDOUT
 
 from configobj import ConfigObj
@@ -23,8 +26,13 @@ from kombu.connection import BrokerConnection
 from kombu.messaging import Exchange, Queue, Consumer, Producer
 from api_clients import api_client
 
+MODE_CREATE = "create"
+MODE_MODIFY = "modify"
+MODE_MOVED = "moved"
+MODE_DELETE = "delete"
+
 global storage_directory
-storage_directory = "/srv/airtime/stor"
+global plupload_directory
 
 # configure logging
 try:
@@ -46,35 +54,27 @@ list of supported easy tags in mutagen version 1.20
 ['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry', 'date', 'performer', 'musicbrainz_albumartistid', 'composer', 'encodedby', 'tracknumber', 'musicbrainz_albumid', 'album', 'asin', 'musicbrainz_artistid', 'mood', 'copyright', 'author', 'media', 'length', 'version', 'artistsort', 'titlesort', 'discsubtitle', 'website', 'musicip_fingerprint', 'conductor', 'compilation', 'barcode', 'performer:*', 'composersort', 'musicbrainz_discid', 'musicbrainz_albumtype', 'genre', 'isrc', 'discnumber', 'musicbrainz_trmid', 'replaygain_*_gain', 'musicip_puid', 'artist', 'title', 'bpm', 'musicbrainz_trackid', 'arranger', 'albumsort', 'replaygain_*_peak', 'organization']
 """
 
-
-def checkRabbitMQ(notifier):
-    try:
-        notifier.connection.drain_events(timeout=int(config["check_airtime_events"]))
-    except Exception, e:
-            logger = logging.getLogger('root')
-            logger.info("%s", e)
-
 class AirtimeNotifier(Notifier):
 
     def __init__(self, watch_manager, default_proc_fun=None, read_freq=0, threshold=0, timeout=None):
         Notifier.__init__(self, watch_manager, default_proc_fun, read_freq, threshold, timeout)
 
         self.airtime2mutagen = {\
-        "track_title": "title",\
+        "MDATA_KEY_TITLE": "title",\
-        "artist_name": "artist",\
+        "MDATA_KEY_CREATOR": "artist",\
-        "album_title": "album",\
+        "MDATA_KEY_SOURCE": "album",\
-        "genre": "genre",\
+        "MDATA_KEY_GENRE": "genre",\
-        "mood": "mood",\
+        "MDATA_KEY_MOOD": "mood",\
-        "track_number": "tracknumber",\
+        "MDATA_KEY_TRACKNUMBER": "tracknumber",\
-        "bpm": "bpm",\
+        "MDATA_KEY_BPM": "bpm",\
-        "label": "organization",\
+        "MDATA_KEY_LABEL": "organization",\
-        "composer": "composer",\
+        "MDATA_KEY_COMPOSER": "composer",\
-        "encoded_by": "encodedby",\
+        "MDATA_KEY_ENCODER": "encodedby",\
-        "conductor": "conductor",\
+        "MDATA_KEY_CONDUCTOR": "conductor",\
-        "year": "date",\
+        "MDATA_KEY_YEAR": "date",\
-        "info_url": "website",\
+        "MDATA_KEY_URL": "website",\
-        "isrc_number": "isrc",\
+        "MDATA_KEY_ISRC": "isrc",\
-        "copyright": "copyright",\
+        "MDATA_KEY_COPYRIGHT": "copyright",\
         }
 
         schedule_exchange = Exchange("airtime-media-monitor", "direct", durable=True, auto_delete=True)
@@ -112,27 +112,48 @@ class MediaMonitor(ProcessEvent):
         self.api_client = api_client.api_client_factory(config)
 
         self.mutagen2airtime = {\
-        "title": "track_title",\
+        "title": "MDATA_KEY_TITLE",\
-        "artist": "artist_name",\
+        "artist": "MDATA_KEY_CREATOR",\
-        "album": "album_title",\
+        "album": "MDATA_KEY_SOURCE",\
-        "genre": "genre",\
+        "genre": "MDATA_KEY_GENRE",\
-        "mood": "mood",\
+        "mood": "MDATA_KEY_MOOD",\
-        "tracknumber": "track_number",\
+        "tracknumber": "MDATA_KEY_TRACKNUMBER",\
-        "bpm": "bpm",\
+        "bpm": "MDATA_KEY_BPM",\
-        "organization": "label",\
+        "organization": "MDATA_KEY_LABEL",\
-        "composer": "composer",\
+        "composer": "MDATA_KEY_COMPOSER",\
-        "encodedby": "encoded_by",\
+        "encodedby": "MDATA_KEY_ENCODER",\
-        "conductor": "conductor",\
+        "conductor": "MDATA_KEY_CONDUCTOR",\
-        "date": "year",\
+        "date": "MDATA_KEY_YEAR",\
-        "website": "info_url",\
+        "website": "MDATA_KEY_URL",\
-        "isrc": "isrc_number",\
+        "isrc": "MDATA_KEY_ISRC",\
-        "copyright": "copyright",\
+        "copyright": "MDATA_KEY_COPYRIGHT",\
         }
 
         self.supported_file_formats = ['mp3', 'ogg']
         self.logger = logging.getLogger('root')
         self.temp_files = {}
-        self.imported_renamed_files = {}
+        self.moved_files = {}
+        self.file_events = deque()
+
+        self.mask =  pyinotify.IN_CREATE | \
+                pyinotify.IN_MODIFY | \
+                pyinotify.IN_MOVED_FROM | \
+                pyinotify.IN_MOVED_TO | \
+                pyinotify.IN_DELETE | \
+                pyinotify.IN_DELETE_SELF
+
+        self.wm = WatchManager()
+
+        schedule_exchange = Exchange("airtime-media-monitor", "direct", durable=True, auto_delete=True)
+        schedule_queue = Queue("media-monitor", exchange=schedule_exchange, key="filesystem")
+        connection = BrokerConnection(config["rabbitmq_host"], config["rabbitmq_user"], config["rabbitmq_password"], "/")
+        channel = connection.channel()
+
+    def watch_directory(self, directory):
+        return self.wm.add_watch(directory, self.mask, rec=True, auto_add=True)
+
+    def is_parent_directory(self, filepath, directory):
+        return (directory == filepath[0:len(directory)])
 
     def get_md5(self, filepath):
         f = open(filepath, 'rb')
@@ -142,20 +163,40 @@ class MediaMonitor(ProcessEvent):
 
         return md5
 
-    def ensure_dir(self, filepath):
+    ## mutagen_length is in seconds with the format (d+).dd
+    ## return format hh:mm:ss.uuu
+    def format_length(self, mutagen_length):
+        t = float(mutagen_length)
+        h = int(math.floor(t/3600))
+        t = t % 3600
+        m = int(math.floor(t/60))
 
+        s = t % 60
+        # will be ss.uuu
+        s = str(s)
+        s = s[:6]
+
+        length = "%s:%s:%s" % (h, m, s)
+
+        return length
+
+    def ensure_dir(self, filepath):
         directory = os.path.dirname(filepath)
 
-        if ((not os.path.exists(directory)) or ((os.path.exists(directory) and not os.path.isdir(directory)))):
+        try:
-            os.makedirs(directory, 02775)
+            omask = os.umask(0)
+            if ((not os.path.exists(directory)) or ((os.path.exists(directory) and not os.path.isdir(directory)))):
+                os.makedirs(directory, 02777)
+                self.watch_directory(directory)
+        finally:
+            os.umask(omask)
 
     def create_unique_filename(self, filepath):
 
-        file_dir = os.path.dirname(filepath)
-        filename = os.path.basename(filepath).split(".")[0]
-        file_ext = os.path.splitext(filepath)[1]
-
         if(os.path.exists(filepath)):
+            file_dir = os.path.dirname(filepath)
+            filename = os.path.basename(filepath).split(".")[0]
+            file_ext = os.path.splitext(filepath)[1]
             i = 1;
             while(True):
                 new_filepath = "%s/%s(%s).%s" % (file_dir, filename, i, file_ext)
@@ -165,79 +206,119 @@ class MediaMonitor(ProcessEvent):
                 else:
                     filepath = new_filepath
 
-        self.imported_renamed_files[filepath] = 0
-
         return filepath
 
     def create_file_path(self, imported_filepath):
 
         global storage_directory
 
-        original_name = os.path.basename(imported_filepath)
-        file_ext = os.path.splitext(imported_filepath)[1]
-        file_info = mutagen.File(imported_filepath, easy=True)
-
-        metadata = {'artist':None,
-                    'album':None,
-                    'title':None,
-                    'tracknumber':None}
-
-        for key in metadata.keys():
-            if key in file_info:
-                metadata[key] = file_info[key][0]
-
-        if metadata['artist'] is not None:
-            base = "%s/%s" % (storage_directory, metadata['artist'])
-            if metadata['album'] is not None:
-                base = "%s/%s" % (base, metadata['album'])
-            if metadata['title'] is not None:
-                if metadata['tracknumber'] is not None:
-                    metadata['tracknumber'] = "%02d" % (int(metadata['tracknumber']))
-                    base = "%s/%s - %s" % (base, metadata['tracknumber'], metadata['title'])
-                else:
-                    base = "%s/%s" % (base, metadata['title'])
-            else:
-                base = "%s/%s" % (base, original_name)
-        else:
-            base = "%s/%s" % (storage_directory, original_name)
-
-        base = "%s%s" % (base, file_ext)
-
-        filepath = self.create_unique_filename(base)
-        self.ensure_dir(filepath)
-        shutil.move(imported_filepath, filepath)
-
-    def update_airtime(self, event):
-        self.logger.info("Updating Change to Airtime")
         try:
-            md5 = self.get_md5(event.pathname)
+            #get rid of file extention from original name, name might have more than 1 '.' in it.
-            md = {'filepath':event.pathname, 'md5':md5}
+            original_name = os.path.basename(imported_filepath)
+            #self.logger.info('original name: %s', original_name)
+            original_name = original_name.split(".")[0:-1]
+            #self.logger.info('original name: %s', original_name)
+            original_name = ''.join(original_name)
+            #self.logger.info('original name: %s', original_name)
 
-            file_info = mutagen.File(event.pathname, easy=True)
+            file_ext = os.path.splitext(imported_filepath)[1]
-            attrs = self.mutagen2airtime
+            file_info = mutagen.File(imported_filepath, easy=True)
-            for key in file_info.keys() :
-                if key in attrs :
-                    md[attrs[key]] = file_info[key][0]
 
-            data = {'md': md}
+            metadata = {'artist':None,
-            response = self.api_client.update_media_metadata(data)
+                        'album':None,
+                        'title':None,
+                        'tracknumber':None}
+
+            for key in metadata.keys():
+                if key in file_info:
+                    metadata[key] = file_info[key][0]
+
+            if metadata['artist'] is not None:
+                base = "%s/%s" % (storage_directory, metadata['artist'])
+                if metadata['album'] is not None:
+                    base = "%s/%s" % (base, metadata['album'])
+                if metadata['title'] is not None:
+                    if metadata['tracknumber'] is not None:
+                        metadata['tracknumber'] = "%02d" % (int(metadata['tracknumber']))
+                        base = "%s/%s - %s" % (base, metadata['tracknumber'], metadata['title'])
+                    else:
+                        base = "%s/%s" % (base, metadata['title'])
+                else:
+                    base = "%s/%s" % (base, original_name)
+            else:
+                base = "%s/%s" % (storage_directory, original_name)
+
+            base = "%s%s" % (base, file_ext)
+
+            filepath = self.create_unique_filename(base)
+            self.ensure_dir(filepath)
 
         except Exception, e:
-            self.logger.info("%s", e)
+            self.logger.error('Exception: %s', e)
 
-    def is_renamed_file(self, filename):
+        return filepath
-        if filename in self.imported_renamed_files:
-            del self.imported_renamed_files[filename]
-            return True
 
-        return False
+    def get_mutagen_info(self, filepath):
+        md = {}
+        md5 = self.get_md5(filepath)
+        md['MDATA_KEY_MD5'] = md5
+
+        file_info = mutagen.File(filepath, easy=True)
+        attrs = self.mutagen2airtime
+        for key in file_info.keys() :
+            if key in attrs :
+                md[attrs[key]] = file_info[key][0]
+
+        md['MDATA_KEY_BITRATE'] = file_info.info.bitrate
+        md['MDATA_KEY_SAMPLERATE'] = file_info.info.sample_rate
+        md['MDATA_KEY_DURATION'] = self.format_length(file_info.info.length)
+        md['MDATA_KEY_MIME'] = file_info.mime[0]
+
+        if "mp3" in md['MDATA_KEY_MIME']:
+            md['MDATA_KEY_FTYPE'] = "audioclip"
+        elif "vorbis" in md['MDATA_KEY_MIME']:
+            md['MDATA_KEY_FTYPE'] = "audioclip"
+
+        return md
+
+
+    def update_airtime(self, d):
+
+        filepath = d['filepath']
+        mode = d['mode']
+
+        data = None
+        md = {}
+        md['MDATA_KEY_FILEPATH'] = filepath
+
+        if (os.path.exists(filepath) and (mode == MODE_CREATE)):
+            mutagen = self.get_mutagen_info(filepath)
+            md.update(mutagen)
+            data = {'md': md}
+        elif (os.path.exists(filepath) and (mode == MODE_MODIFY)):
+            mutagen = self.get_mutagen_info(filepath)
+            md.update(mutagen)
+            data = {'md': md}
+        elif (mode == MODE_MOVED):
+            mutagen = self.get_mutagen_info(filepath)
+            md.update(mutagen)
+            data = {'md': md}
+        elif (mode == MODE_DELETE):
+            data = {'md': md}
+
+        if data is not None:
+            self.logger.info("Updating Change to Airtime")
+            response = None
+            while response is None:
+                response = self.api_client.update_media_metadata(data, mode)
+                time.sleep(5)
 
     def is_temp_file(self, filename):
         info = filename.split(".")
 
         if(info[-2] in self.supported_file_formats):
             return True
-        else :
+        else:
             return False
 
     def is_audio_file(self, filename):
@@ -245,63 +326,124 @@ class MediaMonitor(ProcessEvent):
 
         if(info[-1] in self.supported_file_formats):
             return True
-        else :
+        else:
             return False
 
-
     def process_IN_CREATE(self, event):
         if not event.dir:
-
+            self.logger.info("%s: %s", event.maskname, event.pathname)
             #file created is a tmp file which will be modified and then moved back to the original filename.
             if self.is_temp_file(event.name) :
                 self.temp_files[event.pathname] = None
             #This is a newly imported file.
             else :
-                #if not is_renamed_file(event.pathname):
+                global plupload_directory
-                self.create_file_path(event.pathname)
+                #files that have been added through plupload have a placeholder already put in Airtime's database.
+                if not self.is_parent_directory(event.pathname, plupload_directory):
+                    md5 = self.get_md5(event.pathname)
+                    response = self.api_client.check_media_status(md5)
 
-        self.logger.info("%s: %s", event.maskname, event.pathname)
+                    #this file is new, md5 does not exist in Airtime.
+                    if(response['airtime_status'] == 0):
+                        filepath = self.create_file_path(event.pathname)
+                        os.rename(event.pathname, filepath)
+                        self.file_events.append({'mode': MODE_CREATE, 'filepath': filepath})
 
     def process_IN_MODIFY(self, event):
-        if not event.dir :
+        if not event.dir:
-
+            self.logger.info("%s: %s", event.maskname, event.pathname)
-            if self.is_audio_file(event.name) :
+            global plupload_directory
-                self.update_airtime(event)
+            #files that have been added through plupload have a placeholder already put in Airtime's database.
-
+            if not self.is_parent_directory(event.pathname, plupload_directory):
-        self.logger.info("%s: %s", event.maskname, event.pathname)
+                if self.is_audio_file(event.name) :
+                    self.file_events.append({'filepath': event.pathname, 'mode': MODE_MODIFY})
 
     def process_IN_MOVED_FROM(self, event):
-        if event.pathname in self.temp_files :
+        self.logger.info("%s: %s", event.maskname, event.pathname)
+        if event.pathname in self.temp_files:
             del self.temp_files[event.pathname]
             self.temp_files[event.cookie] = event.pathname
-
+        else:
-        self.logger.info("%s: %s", event.maskname, event.pathname)
+            self.moved_files[event.cookie] = event.pathname
 
     def process_IN_MOVED_TO(self, event):
-        if event.cookie in self.temp_files :
-            del self.temp_files[event.cookie]
-            self.update_airtime(event)
-
         self.logger.info("%s: %s", event.maskname, event.pathname)
+        if event.cookie in self.temp_files:
+            del self.temp_files[event.cookie]
+            self.file_events.append({'filepath': event.pathname, 'mode': MODE_MODIFY})
+        elif event.cookie in self.moved_files:
+            old_filepath = self.moved_files[event.cookie]
+            del self.moved_files[event.cookie]
+
+            global plupload_directory
+            if self.is_parent_directory(old_filepath, plupload_directory):
+                #file renamed from /tmp/plupload does not have a path in our naming scheme yet.
+                md_filepath = self.create_file_path(event.pathname)
+                #move the file a second time to its correct Airtime naming schema.
+                os.rename(event.pathname, md_filepath)
+                self.file_events.append({'filepath': md_filepath, 'mode': MODE_MOVED})
+            else:
+                self.file_events.append({'filepath': event.pathname, 'mode': MODE_MOVED})
+
+        else:
+            #TODO need to pass in if md5 exists to this file creation function, identical files will just replace current files not have a (1) etc.
+            #file has been most likely dropped into stor folder from an unwatched location. (from gui, mv command not cp)
+            md_filepath = self.create_file_path(event.pathname)
+            os.rename(event.pathname, md_filepath)
+            self.file_events.append({'mode': MODE_CREATE, 'filepath': md_filepath})
+
+    def process_IN_DELETE(self, event):
+        if not event.dir:
+            self.logger.info("%s: %s", event.maskname, event.pathname)
+            self.file_events.append({'filepath': event.pathname, 'mode': MODE_DELETE})
 
     def process_default(self, event):
         self.logger.info("%s: %s", event.maskname, event.pathname)
 
+    def notifier_loop_callback(self, notifier):
+
+        while len(self.file_events) > 0:
+            file_info = self.file_events.popleft()
+            self.update_airtime(file_info)
+
+        try:
+            notifier.connection.drain_events(timeout=1)
+        except Exception, e:
+            self.logger.info("%s", e)
+
 if __name__ == '__main__':
 
     try:
-        # watched events
-        mask = pyinotify.IN_CREATE | pyinotify.IN_MODIFY | pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
-        #mask = pyinotify.ALL_EVENTS
-
-        wm = WatchManager()
-        wdd = wm.add_watch(storage_directory, mask, rec=True, auto_add=True)
-
         logger = logging.getLogger('root')
-        logger.info("Added watch to %s", storage_directory)
+        mm = MediaMonitor()
 
-        notifier = AirtimeNotifier(wm, MediaMonitor(), read_freq=int(config["check_filesystem_events"]), timeout=1)
+        response = None
+        while response is None:
+            response = mm.api_client.setup_media_monitor()
+            time.sleep(5)
+
+        storage_directory = response["stor"]
+        plupload_directory = response["plupload"]
+
+        wdd = mm.watch_directory(storage_directory)
+        logger.info("Added watch to %s", storage_directory)
+        logger.info("wdd result %s", wdd[storage_directory])
+        wdd = mm.watch_directory(plupload_directory)
+        logger.info("Added watch to %s", plupload_directory)
+        logger.info("wdd result %s", wdd[plupload_directory])
+
+        notifier = AirtimeNotifier(mm.wm, mm, read_freq=int(config["check_filesystem_events"]), timeout=1)
         notifier.coalesce_events()
-        notifier.loop(callback=checkRabbitMQ)
+
+        #notifier.loop(callback=mm.notifier_loop_callback)
+
+        while True:
+            if(notifier.check_events(1)):
+                notifier.read_events()
+                notifier.process_events()
+            mm.notifier_loop_callback(notifier)
+
     except KeyboardInterrupt:
         notifier.stop()
+    except Exception, e:
+        logger.error('Exception: %s', e)

python_apps/media-monitor/media-monitor.cfg

@@ -19,8 +19,14 @@ api_base = 'api'
 # URL to get the version number of the server API
 version_url = 'version/api_key/%%api_key%%'
 
+# URL to setup the media monitor
+media_setup_url = 'media-monitor-setup/format/json/api_key/%%api_key%%'
+
+# URL to check Airtime's status of a file
+media_status_url = 'media-item-status/format/json/api_key/%%api_key%%/md5/%%md5%%'
+
 # URL to tell Airtime to update file's meta data
-update_media_url = 'reload-metadata/format/json/api_key/%%api_key%%'
+update_media_url = 'reload-metadata/format/json/api_key/%%api_key%%/mode/%%mode%%'
 
 ############################################
 # RabbitMQ settings                        #