Massive refactor of the analyzer branch and sync it back up with the

cloud storage branch (for the last time)

* Backported all the bugfixes from cc-5709-airtime-analyzer-cloud-storage
* Backported missing FileStorageBackend.php
* Fixed CC-6001: Track titles and artist names with slashes break audio preview
* Refactored all the MediaController code, pulling out the logic into MediaService
* Fixed an API key leak to guests in the Media API
* Made this branch work without cloud_storage.conf (defaults to file storage)
* Made ApiController's getMediaAction use the MediaService code

airtime_mvc/application/cloud_storage/Amazon_S3StorageBackend.php

@@ -4,12 +4,12 @@ require_once 'StorageBackend.php';
 
 use Aws\S3\S3Client;
 
-class Amazon_S3 extends StorageBackend
+class Amazon_S3StorageBackend extends StorageBackend
 {
     
     private $s3Client;
     
-    public function Amazon_S3($securityCredentials)
+    public function Amazon_S3StorageBackend($securityCredentials)
     {
         $this->setBucket($securityCredentials['bucket']);
         $this->setAccessKey($securityCredentials['api_key']);

airtime_mvc/application/cloud_storage/FileStorageBackend.php

@@ -0,0 +1,41 @@
+<?php
+
+class FileStorageBackend extends StorageBackend
+{
+    //Stub class
+    public function FileStorageBackend()
+    {
+    }
+
+    public function getAbsoluteFilePath($resourceId)
+    {
+        //TODO
+        return $resourceId;
+    }
+
+    public function getSignedURL($resourceId)
+    {
+        return "";
+    }
+
+    public function getFileSize($resourceId)
+    {
+        //TODO
+        return filesize($resourceId);
+    }
+
+    public function deletePhysicalFile($resourceId)
+    {
+        //TODO
+    }
+
+    public function deleteAllCloudFileObjects()
+    {
+        return "";
+    }
+
+    public function getFilePrefix()
+    {
+        return "";
+    }
+}

airtime_mvc/application/cloud_storage/ProxyStorageBackend.php

@@ -1,7 +1,8 @@
 <?php
 
 require_once 'StorageBackend.php';
-require_once 'Amazon_S3.php';
+require_once 'FileStorageBackend.php';
+require_once 'Amazon_S3StorageBackend.php';
 
 /**
  *
@@ -13,7 +14,7 @@ class ProxyStorageBackend extends StorageBackend
     private $storageBackend;
 
     /**
-     * Receives the file's storage backend and instantiates the approriate
+     * Receives the file's storage backend and instantiates the appropriate
      * object.
      */
     public function ProxyStorageBackend($storageBackend)
@@ -23,8 +24,14 @@ class ProxyStorageBackend extends StorageBackend
         //The storage backend in the airtime.conf directly corresponds to
         //the name of the class that implements it (eg. Amazon_S3), so we 
         //can easily create the right backend object dynamically:
+        if ($storageBackend == "amazon_S3") {
+            $this->storageBackend = new Amazon_S3StorageBackend($CC_CONFIG["amazon_S3"]);
+        } else if ($storageBackend == "file") {
+            $this->storageBackend = new FileStorageBackend();
+        } else {
             $this->storageBackend = new $storageBackend($CC_CONFIG[$storageBackend]);
         }
+    }
 
     public function getAbsoluteFilePath($resourceId)
     {
@@ -46,4 +53,13 @@ class ProxyStorageBackend extends StorageBackend
         $this->storageBackend->deletePhysicalFile($resourceId);
     }
 
+    public function deleteAllCloudFileObjects()
+    {
+        $this->storageBackend->deleteAllCloudFileObjects();
+    }
+
+    public function getFilePrefix()
+    {
+        return $this->storageBackend->getFilePrefix();
+    }
 }

airtime_mvc/application/cloud_storage/StorageBackend.php

@@ -52,4 +52,14 @@ abstract class StorageBackend
     {
         $this->secretKey = $secretKey;
     }
+
+    public function deleteAllCloudFileObjects()
+    {
+        return false;
+    }
+
+    public function getFilePrefix()
+    {
+        return "";
+    }
 }

airtime_mvc/application/configs/conf.php

@@ -27,17 +27,22 @@ class Config {
         
         // Parse separate conf file for cloud storage values
         $cloudStorageConfig = isset($_SERVER['CLOUD_STORAGE_CONF']) ? $_SERVER['CLOUD_STORAGE_CONF'] : "/etc/airtime-saas/cloud_storage.conf";
-        $cloudStorageValues = parse_ini_file($cloudStorageConfig, true);
+        $cloudStorageValues = @parse_ini_file($cloudStorageConfig, true);
-        
+        if ($cloudStorageValues !== false) {
             $supportedStorageBackends = array('amazon_S3');
             foreach ($supportedStorageBackends as $backend) {
                 $CC_CONFIG[$backend] = $cloudStorageValues[$backend];
             }
-        
             // Tells us where file uploads will be uploaded to.
             // It will either be set to a cloud storage backend or local file storage.
             $CC_CONFIG["current_backend"] = $cloudStorageValues["current_backend"]["storage_backend"];
 
+        } else {
+            //Default to file storage if we didn't find a cloud_storage.conf
+            $CC_CONFIG["current_backend"] = "file";
+        }
+
+
         $values = parse_ini_file($filename, true);
 
         // Name of the web server user

airtime_mvc/application/controllers/ApiController.php

@@ -93,100 +93,12 @@ class ApiController extends Zend_Controller_Action
 
         $fileId = $this->_getParam("file");
 
-        $media = Application_Model_StoredFile::RecallById($fileId);
+        $inline = !($this->_getParam('download',false) == true);
-        if ($media != null) {
+        Application_Service_MediaService::streamFileDownload($fileId, $inline);
-            // Make sure we don't have some wrong result beecause of caching
-            clearstatcache();
-            
-            if ($media->getPropelOrm()->isValidPhysicalFile()) {
-                $filename = $media->getPropelOrm()->getFilename();
-
-                //Download user left clicks a track and selects Download.
-                if ("true" == $this->_getParam('download')) {
-                    //path_info breaks up a file path into seperate pieces of informaiton.
-                    //We just want the basename which is the file name with the path
-                    //information stripped away. We are using Content-Disposition to specify
-                    //to the browser what name the file should be saved as.
-                    header('Content-Disposition: attachment; filename="'.$filename.'"');
-                } else {
-                    //user clicks play button for track preview
-                    header('Content-Disposition: inline; filename="'.$filename.'"');
-                }
-
-                $this->smartReadFile($media);
-                exit;
-            } else {
-                header ("HTTP/1.1 404 Not Found");
-            }
-        }
 
         $this->_helper->json->sendJson(array());
     }
 
-    /**
-    * Reads the requested portion of a file and sends its contents to the client with the appropriate headers.
-    *
-    * This HTTP_RANGE compatible read file function is necessary for allowing streaming media to be skipped around in.
-    *
-    * @param string $location
-    * @param string $mimeType
-    * @return void
-    *
-    * @link https://groups.google.com/d/msg/jplayer/nSM2UmnSKKA/Hu76jDZS4xcJ
-    * @link http://php.net/manual/en/function.readfile.php#86244
-    */
-    public function smartReadFile($media)
-    {
-        $filepath = $media->getFilePath();
-        $size= $media->getFileSize();
-        $mimeType = $media->getPropelOrm()->getDbMime();
-
-        $fm = @fopen($filepath, 'rb');
-        if (!$fm) {
-            header ("HTTP/1.1 505 Internal server error");
-
-            return;
-        }
-
-        $begin = 0;
-        $end   = $size - 1;
-
-        if (isset($_SERVER['HTTP_RANGE'])) {
-            if (preg_match('/bytes=\h*(\d+)-(\d*)[\D.*]?/i', $_SERVER['HTTP_RANGE'], $matches)) {
-                $begin = intval($matches[1]);
-                if (!empty($matches[2])) {
-                    $end = intval($matches[2]);
-                }
-            }
-        }
-
-        if (isset($_SERVER['HTTP_RANGE'])) {
-            header('HTTP/1.1 206 Partial Content');
-        } else {
-            header('HTTP/1.1 200 OK');
-        }
-        header("Content-Type: $mimeType");
-        header('Cache-Control: public, must-revalidate, max-age=0');
-        header('Pragma: no-cache');
-        header('Accept-Ranges: bytes');
-        header('Content-Length:' . (($end - $begin) + 1));
-        if (isset($_SERVER['HTTP_RANGE'])) {
-            header("Content-Range: bytes $begin-$end/$size");
-        }
-        header("Content-Transfer-Encoding: binary");
-
-        //We can have multiple levels of output buffering. Need to
-        //keep looping until all have been disabled!!!
-        //http://www.php.net/manual/en/function.ob-end-flush.php
-        while (@ob_end_flush());
-
-        // NOTE: We can't use fseek here because it does not work with streams
-        // (a.k.a. Files stored in the cloud)
-        while(!feof($fm) && (connection_status() == 0)) {
-            echo fread($fm, 1024 * 8);
-        }
-        fclose($fm);
-    }
 
     //Used by the SaaS monitoring
     public function onAirLightAction()

airtime_mvc/application/controllers/AudiopreviewController.php

@@ -22,8 +22,6 @@ class AudiopreviewController extends Zend_Controller_Action
         $CC_CONFIG = Config::getConfig();
 
         $audioFileID     = $this->_getParam('audioFileID');
-        $audioFileArtist = $this->_getParam('audioFileArtist');
-        $audioFileTitle  = $this->_getParam('audioFileTitle');
         $type = $this->_getParam('type');
 
         $baseUrl = Application_Common_OsPath::getBaseDir();
@@ -60,10 +58,9 @@ class AudiopreviewController extends Zend_Controller_Action
         $this->view->uri             = $uri;
         $this->view->mime            = $mime;
         $this->view->audioFileID     = $audioFileID;
-        // We need to decode artist and title because it gets
+
-        // encoded twice in js
+        $this->view->audioFileArtist = htmlspecialchars($media->getPropelOrm()->getDbArtistName());
-        $this->view->audioFileArtist = htmlspecialchars(urldecode($audioFileArtist));
+        $this->view->audioFileTitle  = htmlspecialchars($media->getPropelOrm()->getDbTrackTitle());
-        $this->view->audioFileTitle  = htmlspecialchars(urldecode($audioFileTitle));
         $this->view->type            = $type;
 
         $this->_helper->viewRenderer->setRender('audio-preview');

airtime_mvc/application/models/Schedule.php

@@ -947,7 +947,6 @@ SQL;
                 
                 $baseUrl = Application_Common_OsPath::getBaseDir();
                 $filesize = $file->getFileSize();
-                
                 self::createFileScheduleEvent($data, $item, $media_id, $uri, $filesize);
             } 
             elseif (!is_null($item['stream_id'])) {

airtime_mvc/application/models/airtime/CcFiles.php

@@ -57,7 +57,7 @@ class CcFiles extends BaseCcFiles {
      * Retrieve a sanitized version of the file metadata, suitable for public access.
      * @param $fileId
      */
-    public static function getSantiziedFileById($fileId)
+    public static function getSanitizedFileById($fileId)
     {
         $file = CcFilesQuery::create()->findPk($fileId);
         if ($file) {
@@ -114,7 +114,7 @@ class CcFiles extends BaseCcFiles {
             $file->setDbHidden(true);
             $file->save();
 
-            $callbackUrl = Application_Common_HTTPHelper::getStationUrl() . "/rest/media/" . $file->getPrimaryKey();
+            $callbackUrl = Application_Common_HTTPHelper::getStationUrl() . "rest/media/" . $file->getPrimaryKey();
 
             Application_Service_MediaService::processUploadedFile($callbackUrl, $relativePath, self::getOwnerId());
             return CcFiles::sanitizeResponse($file);
@@ -138,14 +138,11 @@ class CcFiles extends BaseCcFiles {
     {
         $file = CcFilesQuery::create()->findPk($fileId);
 
-        // Since we check for this value when deleting files, set it first
-        $file->setDbDirectory(self::MUSIC_DIRS_STOR_PK);
-
         $fileArray = self::removeBlacklistedFields($fileArray);
         $fileArray = self::stripTimeStampFromYearTag($fileArray);
 
         self::validateFileArray($fileArray);
-        if ($file && isset($requestData["resource_id"])) {
+        if ($file && isset($fileArray["resource_id"])) {
 
             $file->fromArray($fileArray, BasePeer::TYPE_FIELDNAME);
 
@@ -155,9 +152,10 @@ class CcFiles extends BaseCcFiles {
             $fileSizeBytes = $fileArray["filesize"];
             if (!isset($fileSizeBytes) || $fileSizeBytes === false)
             {
-                $file->setDbImportStatus(2)->save();
+                $file->setDbImportStatus(self::IMPORT_STATUS_FAILED)->save();
                 throw new FileNotFoundException();
             }
+
             $cloudFile = new CloudFile();
             $cloudFile->setStorageBackend($fileArray["storage_backend"]);
             $cloudFile->setResourceId($fileArray["resource_id"]);
@@ -172,6 +170,9 @@ class CcFiles extends BaseCcFiles {
 
         } else if ($file) {
 
+            // Since we check for this value when deleting files, set it first
+            $file->setDbDirectory(self::MUSIC_DIRS_STOR_PK);
+
             $file->fromArray($fileArray, BasePeer::TYPE_FIELDNAME);
 
             //Our RESTful API takes "full_path" as a field, which we then split and translate to match
@@ -252,6 +253,7 @@ class CcFiles extends BaseCcFiles {
         }
     }
 
+
     private static function validateFileArray(&$fileArray)
     {
         // Sanitize any wildly incorrect metadata before it goes to be validated
@@ -342,6 +344,42 @@ class CcFiles extends BaseCcFiles {
         return $response;
     }
 
+    /**
+     * Returns the file size in bytes.
+     */
+    public function getFileSize()
+    {
+        return filesize($this->getAbsoluteFilePath());
+    }
+
+    public function getFilename()
+    {
+        $info = pathinfo($this->getAbsoluteFilePath());
+        return $info['filename'];
+    }
+
+    /**
+     * Returns the file's absolute file path stored on disk.
+     */
+    public function getURLForTrackPreviewOrDownload()
+    {
+        return $this->getAbsoluteFilePath();
+    }
+
+    /**
+     * Returns the file's absolute file path stored on disk.
+     */
+    public function getAbsoluteFilePath()
+    {
+        $music_dir = Application_Model_MusicDir::getDirByPK($this->getDbDirectory());
+        if (!$music_dir) {
+            throw new Exception("Invalid music_dir for file in database.");
+        }
+        $directory = $music_dir->getDirectory();
+        $filepath  = $this->getDbFilepath();
+        return Application_Common_OsPath::join($directory, $filepath);
+    }
+
     /**
      *
      * Strips out fields from incoming request data that should never be modified
@@ -421,42 +459,6 @@ class CcFiles extends BaseCcFiles {
         exec("find $path -empty -type d -delete");
     }
 
-    /**
-     * Returns the file size in bytes.
-     */
-    public function getFileSize()
-    {
-        return filesize($this->getAbsoluteFilePath());
-    }
-    
-    public function getFilename()
-    {
-        $info = pathinfo($this->getAbsoluteFilePath());
-        return $info['filename'];
-    }
-    
-    /**
-     * Returns the file's absolute file path stored on disk.
-     */
-    public function getURLForTrackPreviewOrDownload()
-    {
-        return $this->getAbsoluteFilePath();
-    }
-    
-    /**
-     * Returns the file's absolute file path stored on disk.
-     */
-    public function getAbsoluteFilePath()
-    {
-        $music_dir = Application_Model_MusicDir::getDirByPK($this->getDbDirectory());
-        if (!$music_dir) {
-            throw new Exception("Invalid music_dir for file in database.");
-        }
-        $directory = $music_dir->getDirectory();
-        $filepath  = $this->getDbFilepath();
-
-        return Application_Common_OsPath::join($directory, $filepath);
-    }
 
     /**
      * Checks if the file is a regular file that can be previewed and downloaded.

airtime_mvc/application/modules/rest/controllers/MediaController.php

@@ -41,8 +41,9 @@ class Rest_MediaController extends Zend_Rest_Controller
         try
         {
             $this->getResponse()
-                ->setHttpResponseCode(200)
+                ->setHttpResponseCode(200);
-                ->appendBody($this->_redirect(CcFiles::getDownloadUrl($id)));
+            $inline = false;
+            Application_Service_MediaService::streamFileDownload($id, $inline);
         }
         catch (FileNotFoundException $e) {
             $this->fileNotFoundResponse();
@@ -64,7 +65,7 @@ class Rest_MediaController extends Zend_Rest_Controller
         try {
             $this->getResponse()
                 ->setHttpResponseCode(200)
-                ->appendBody(json_encode(CcFiles::getSantiziedFileById($id)));
+                ->appendBody(json_encode(CcFiles::getSanitizedFileById($id)));
         }
         catch (FileNotFoundException $e) {
             $this->fileNotFoundResponse();

airtime_mvc/application/services/MediaService.php

@@ -1,5 +1,7 @@
 <?php
 
+require_once('ProxyStorageBackend.php');
+
 class Application_Service_MediaService
 {
     public static function processUploadedFile($callbackUrl, $originalFilename, $ownerId)
@@ -19,8 +21,11 @@ class Application_Service_MediaService
 
         //TODO: Remove uploadFileAction from ApiController.php **IMPORTANT** - It's used by the recorder daemon...
 
+        $importedStorageDirectory = "";
+        if ($CC_CONFIG["current_backend"] == "file") {
             $storDir = Application_Model_MusicDir::getStorDir();
             $importedStorageDirectory = $storDir->getDirectory() . "/imported/" . $ownerId;
+        }
 
         try {
             //Copy the temporary file over to the "organize" folder so that it's off our webserver
@@ -34,8 +39,121 @@ class Application_Service_MediaService
 
         //Dispatch a message to airtime_analyzer through RabbitMQ,
         //notifying it that there's a new upload to process!
+        $storageBackend = new ProxyStorageBackend($CC_CONFIG["current_backend"]);
         Application_Model_RabbitMq::SendMessageToAnalyzer($newTempFilePath,
             $importedStorageDirectory, basename($originalFilename),
-            $callbackUrl, $apiKey);
+            $callbackUrl, $apiKey,
+            $CC_CONFIG["current_backend"],
+            $storageBackend->getFilePrefix());
+    }
+
+
+    /**
+     * @param $fileId
+     * @param bool $inline Set the Content-Disposition header to inline to prevent a download dialog from popping up (or attachment if false)
+     * @throws Exception
+     * @throws FileNotFoundException
+     */
+    public static function streamFileDownload($fileId, $inline=false)
+    {
+        $media = Application_Model_StoredFile::RecallById($fileId);
+        if ($media == null) {
+            throw new FileNotFoundException();
+        }
+        $filepath = $media->getFilePath();
+        // Make sure we don't have some wrong result beecause of caching
+        clearstatcache();
+        $media = Application_Model_StoredFile::RecallById($fileId);
+        if ($media == null) {
+            throw new FileNotFoundException();
+        }
+
+        // Make sure we don't have some wrong result beecause of caching
+        clearstatcache();
+
+        if ($media->getPropelOrm()->isValidPhysicalFile()) {
+            $filename = $media->getPropelOrm()->getFilename();
+
+            //Download user left clicks a track and selects Download.
+            if (!$inline) {
+                //We are using Content-Disposition to specify
+                //to the browser what name the file should be saved as.
+                header('Content-Disposition: attachment; filename="' . $filename . '"');
+            } else {
+                //user clicks play button for track and downloads it.
+                header('Content-Disposition: inline; filename="' . $filename . '"');
+            }
+
+            self::smartReadFile($media);
+            exit;
+        } else {
+            throw new FileNotFoundException();
+        }
+    }
+
+
+    /**
+     * Reads the requested portion of a file and sends its contents to the client with the appropriate headers.
+     *
+     * This HTTP_RANGE compatible read file function is necessary for allowing streaming media to be skipped around in.
+     *
+     * @param CcFile $media
+     * @return void
+     *
+     * @link https://groups.google.com/d/msg/jplayer/nSM2UmnSKKA/Hu76jDZS4xcJ
+     * @link http://php.net/manual/en/function.readfile.php#86244
+     */
+    private static function smartReadFile($media)
+    {
+        $filepath = $media->getFilePath();
+        $size= $media->getFileSize();
+        $mimeType = $media->getPropelOrm()->getDbMime();
+
+        $fm = @fopen($filepath, 'rb');
+        if (!$fm) {
+            header ("HTTP/1.1 505 Internal server error");
+
+            return;
+        }
+
+        $begin = 0;
+        $end   = $size - 1;
+
+        if (isset($_SERVER['HTTP_RANGE'])) {
+            if (preg_match('/bytes=\h*(\d+)-(\d*)[\D.*]?/i', $_SERVER['HTTP_RANGE'], $matches)) {
+                $begin = intval($matches[1]);
+                if (!empty($matches[2])) {
+                    $end = intval($matches[2]);
+                }
+            }
+        }
+
+        if (isset($_SERVER['HTTP_RANGE'])) {
+            header('HTTP/1.1 206 Partial Content');
+        } else {
+            header('HTTP/1.1 200 OK');
+        }
+        header("Content-Type: $mimeType");
+        header('Cache-Control: public, must-revalidate, max-age=0');
+        header('Pragma: no-cache');
+        header('Accept-Ranges: bytes');
+        header('Content-Length:' . (($end - $begin) + 1));
+        if (isset($_SERVER['HTTP_RANGE'])) {
+            header("Content-Range: bytes $begin-$end/$size");
+        }
+        header("Content-Transfer-Encoding: binary");
+
+        //We can have multiple levels of output buffering. Need to
+        //keep looping until all have been disabled!!!
+        //http://www.php.net/manual/en/function.ob-end-flush.php
+        while (@ob_end_flush());
+
+        // NOTE: We can't use fseek here because it does not work with streams
+        // (a.k.a. Files stored in the cloud)
+        while(!feof($fm) && (connection_status() == 0)) {
+            echo fread($fm, 1024 * 8);
+        }
+        fclose($fm);
     }
 }
+

airtime_mvc/public/js/airtime/common/common.js

@@ -91,16 +91,11 @@ function openAudioPreview(p_event) {
     }
 }
 
-function open_audio_preview(type, id, audioFileTitle, audioFileArtist) {
+function open_audio_preview(type, id) {
-    // we need to remove soundcloud icon from audioFileTitle
-    var index = audioFileTitle.indexOf("<span class=");
-    if(index != -1){
-        audioFileTitle = audioFileTitle.substring(0,index);
-    }
     // The reason that we need to encode artist and title string is that
     // sometime they contain '/' or '\' and apache reject %2f or %5f
     // so the work around is to encode it twice.
-    openPreviewWindow(baseUrl+'audiopreview/audio-preview/audioFileID/'+id+'/audioFileArtist/'+encodeURIComponent(encodeURIComponent(audioFileArtist))+'/audioFileTitle/'+encodeURIComponent(encodeURIComponent(audioFileTitle))+'/type/'+type);
+    openPreviewWindow(baseUrl+'audiopreview/audio-preview/audioFileID/'+id+'/type/'+type);
     _preview_window.focus();
 }
 

airtime_mvc/public/js/airtime/library/library.js

@@ -643,11 +643,11 @@ var AIRTIME = (function(AIRTIME) {
                         open_playlist_preview(aData.audioFile, 0);
                     } else if (aData.ftype === 'audioclip') {
                         if (isAudioSupported(aData.mime)) {
-                            open_audio_preview(aData.ftype, aData.audioFile, aData.track_title, aData.artist_name);
+                            open_audio_preview(aData.ftype, aData.id);
                         }
                     } else if (aData.ftype == 'stream') {
                         if (isAudioSupported(aData.mime)) {
-                            open_audio_preview(aData.ftype, aData.audioFile, aData.track_title, aData.artist_name);
+                            open_audio_preview(aData.ftype, aData.id);
                         }
                     } else if (aData.ftype == 'block' && aData.bl_type == 'static') {
                         open_block_preview(aData.audioFile, 0);
@@ -955,7 +955,7 @@ var AIRTIME = (function(AIRTIME) {
                                                                                             // pl_
                                 open_playlist_preview(playlistIndex, 0);
                             } else if (data.ftype === 'audioclip' || data.ftype === 'stream') {
-                                open_audio_preview(data.ftype, data.audioFile, data.track_title, data.artist_name);
+                                open_audio_preview(data.ftype, data.id);
                             } else if (data.ftype === 'block') {
                                 blockIndex = $(this).parent().attr('id').substring(3); // remove
                                                                                         // the