From 2a89e4d5a098b7cb86de3c416d7017622499b433 Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Fri, 20 Feb 2015 14:01:06 -0500 Subject: [PATCH] Massive refactor of the analyzer branch and sync it back up with the cloud storage branch (for the last time) * Backported all the bugfixes from cc-5709-airtime-analyzer-cloud-storage * Backported missing FileStorageBackend.php * Fixed CC-6001: Track titles and artist names with slashes break audio preview * Refactored all the MediaController code, pulling out the logic into MediaService * Fixed an API key leak to guests in the Media API * Made this branch work without cloud_storage.conf (defaults to file storage) * Made ApiController's getMediaAction use the MediaService code --- ...zon_S3.php => Amazon_S3StorageBackend.php} | 4 +- .../cloud_storage/FileStorageBackend.php | 41 ++++++ .../cloud_storage/ProxyStorageBackend.php | 36 +++-- .../cloud_storage/StorageBackend.php | 10 ++ airtime_mvc/application/configs/conf.php | 25 ++-- .../application/controllers/ApiController.php | 92 +------------ .../controllers/AudiopreviewController.php | 9 +- airtime_mvc/application/models/Schedule.php | 1 - .../application/models/airtime/CcFiles.php | 88 ++++++------ .../rest/controllers/MediaController.php | 7 +- .../application/services/MediaService.php | 126 +++++++++++++++++- .../public/js/airtime/common/common.js | 9 +- .../public/js/airtime/library/library.js | 6 +- 13 files changed, 275 insertions(+), 179 deletions(-) rename airtime_mvc/application/cloud_storage/{Amazon_S3.php => Amazon_S3StorageBackend.php} (93%) create mode 100644 airtime_mvc/application/cloud_storage/FileStorageBackend.php diff --git a/airtime_mvc/application/cloud_storage/Amazon_S3.php b/airtime_mvc/application/cloud_storage/Amazon_S3StorageBackend.php similarity index 93% rename from airtime_mvc/application/cloud_storage/Amazon_S3.php rename to airtime_mvc/application/cloud_storage/Amazon_S3StorageBackend.php index 978e1ba07..24e398af0 100644 --- a/airtime_mvc/application/cloud_storage/Amazon_S3.php +++ b/airtime_mvc/application/cloud_storage/Amazon_S3StorageBackend.php @@ -4,12 +4,12 @@ require_once 'StorageBackend.php'; use Aws\S3\S3Client; -class Amazon_S3 extends StorageBackend +class Amazon_S3StorageBackend extends StorageBackend { private $s3Client; - public function Amazon_S3($securityCredentials) + public function Amazon_S3StorageBackend($securityCredentials) { $this->setBucket($securityCredentials['bucket']); $this->setAccessKey($securityCredentials['api_key']); diff --git a/airtime_mvc/application/cloud_storage/FileStorageBackend.php b/airtime_mvc/application/cloud_storage/FileStorageBackend.php new file mode 100644 index 000000000..65df4d55b --- /dev/null +++ b/airtime_mvc/application/cloud_storage/FileStorageBackend.php @@ -0,0 +1,41 @@ +storageBackend = new $storageBackend($CC_CONFIG[$storageBackend]); + if ($storageBackend == "amazon_S3") { + $this->storageBackend = new Amazon_S3StorageBackend($CC_CONFIG["amazon_S3"]); + } else if ($storageBackend == "file") { + $this->storageBackend = new FileStorageBackend(); + } else { + $this->storageBackend = new $storageBackend($CC_CONFIG[$storageBackend]); + } } - + public function getAbsoluteFilePath($resourceId) { return $this->storageBackend->getAbsoluteFilePath($resourceId); } - + public function getSignedURL($resourceId) { return $this->storageBackend->getSignedURL($resourceId); } - + public function getFileSize($resourceId) { return $this->storageBackend->getFileSize($resourceId); } - + public function deletePhysicalFile($resourceId) { $this->storageBackend->deletePhysicalFile($resourceId); } -} + public function deleteAllCloudFileObjects() + { + $this->storageBackend->deleteAllCloudFileObjects(); + } + + public function getFilePrefix() + { + return $this->storageBackend->getFilePrefix(); + } +} \ No newline at end of file diff --git a/airtime_mvc/application/cloud_storage/StorageBackend.php b/airtime_mvc/application/cloud_storage/StorageBackend.php index 84a9a8d72..a9547d3ee 100644 --- a/airtime_mvc/application/cloud_storage/StorageBackend.php +++ b/airtime_mvc/application/cloud_storage/StorageBackend.php @@ -52,4 +52,14 @@ abstract class StorageBackend { $this->secretKey = $secretKey; } + + public function deleteAllCloudFileObjects() + { + return false; + } + + public function getFilePrefix() + { + return ""; + } } diff --git a/airtime_mvc/application/configs/conf.php b/airtime_mvc/application/configs/conf.php index 6b6273a22..3bd092a1e 100644 --- a/airtime_mvc/application/configs/conf.php +++ b/airtime_mvc/application/configs/conf.php @@ -27,17 +27,22 @@ class Config { // Parse separate conf file for cloud storage values $cloudStorageConfig = isset($_SERVER['CLOUD_STORAGE_CONF']) ? $_SERVER['CLOUD_STORAGE_CONF'] : "/etc/airtime-saas/cloud_storage.conf"; - $cloudStorageValues = parse_ini_file($cloudStorageConfig, true); - - $supportedStorageBackends = array('amazon_S3'); - foreach ($supportedStorageBackends as $backend) { - $CC_CONFIG[$backend] = $cloudStorageValues[$backend]; + $cloudStorageValues = @parse_ini_file($cloudStorageConfig, true); + if ($cloudStorageValues !== false) { + $supportedStorageBackends = array('amazon_S3'); + foreach ($supportedStorageBackends as $backend) { + $CC_CONFIG[$backend] = $cloudStorageValues[$backend]; + } + // Tells us where file uploads will be uploaded to. + // It will either be set to a cloud storage backend or local file storage. + $CC_CONFIG["current_backend"] = $cloudStorageValues["current_backend"]["storage_backend"]; + + } else { + //Default to file storage if we didn't find a cloud_storage.conf + $CC_CONFIG["current_backend"] = "file"; } - - // Tells us where file uploads will be uploaded to. - // It will either be set to a cloud storage backend or local file storage. - $CC_CONFIG["current_backend"] = $cloudStorageValues["current_backend"]["storage_backend"]; - + + $values = parse_ini_file($filename, true); // Name of the web server user diff --git a/airtime_mvc/application/controllers/ApiController.php b/airtime_mvc/application/controllers/ApiController.php index c5d76cf06..9da4b4add 100644 --- a/airtime_mvc/application/controllers/ApiController.php +++ b/airtime_mvc/application/controllers/ApiController.php @@ -93,100 +93,12 @@ class ApiController extends Zend_Controller_Action $fileId = $this->_getParam("file"); - $media = Application_Model_StoredFile::RecallById($fileId); - if ($media != null) { - // Make sure we don't have some wrong result beecause of caching - clearstatcache(); - - if ($media->getPropelOrm()->isValidPhysicalFile()) { - $filename = $media->getPropelOrm()->getFilename(); - - //Download user left clicks a track and selects Download. - if ("true" == $this->_getParam('download')) { - //path_info breaks up a file path into seperate pieces of informaiton. - //We just want the basename which is the file name with the path - //information stripped away. We are using Content-Disposition to specify - //to the browser what name the file should be saved as. - header('Content-Disposition: attachment; filename="'.$filename.'"'); - } else { - //user clicks play button for track preview - header('Content-Disposition: inline; filename="'.$filename.'"'); - } - - $this->smartReadFile($media); - exit; - } else { - header ("HTTP/1.1 404 Not Found"); - } - } + $inline = !($this->_getParam('download',false) == true); + Application_Service_MediaService::streamFileDownload($fileId, $inline); $this->_helper->json->sendJson(array()); } - /** - * Reads the requested portion of a file and sends its contents to the client with the appropriate headers. - * - * This HTTP_RANGE compatible read file function is necessary for allowing streaming media to be skipped around in. - * - * @param string $location - * @param string $mimeType - * @return void - * - * @link https://groups.google.com/d/msg/jplayer/nSM2UmnSKKA/Hu76jDZS4xcJ - * @link http://php.net/manual/en/function.readfile.php#86244 - */ - public function smartReadFile($media) - { - $filepath = $media->getFilePath(); - $size= $media->getFileSize(); - $mimeType = $media->getPropelOrm()->getDbMime(); - - $fm = @fopen($filepath, 'rb'); - if (!$fm) { - header ("HTTP/1.1 505 Internal server error"); - - return; - } - - $begin = 0; - $end = $size - 1; - - if (isset($_SERVER['HTTP_RANGE'])) { - if (preg_match('/bytes=\h*(\d+)-(\d*)[\D.*]?/i', $_SERVER['HTTP_RANGE'], $matches)) { - $begin = intval($matches[1]); - if (!empty($matches[2])) { - $end = intval($matches[2]); - } - } - } - - if (isset($_SERVER['HTTP_RANGE'])) { - header('HTTP/1.1 206 Partial Content'); - } else { - header('HTTP/1.1 200 OK'); - } - header("Content-Type: $mimeType"); - header('Cache-Control: public, must-revalidate, max-age=0'); - header('Pragma: no-cache'); - header('Accept-Ranges: bytes'); - header('Content-Length:' . (($end - $begin) + 1)); - if (isset($_SERVER['HTTP_RANGE'])) { - header("Content-Range: bytes $begin-$end/$size"); - } - header("Content-Transfer-Encoding: binary"); - - //We can have multiple levels of output buffering. Need to - //keep looping until all have been disabled!!! - //http://www.php.net/manual/en/function.ob-end-flush.php - while (@ob_end_flush()); - - // NOTE: We can't use fseek here because it does not work with streams - // (a.k.a. Files stored in the cloud) - while(!feof($fm) && (connection_status() == 0)) { - echo fread($fm, 1024 * 8); - } - fclose($fm); - } //Used by the SaaS monitoring public function onAirLightAction() diff --git a/airtime_mvc/application/controllers/AudiopreviewController.php b/airtime_mvc/application/controllers/AudiopreviewController.php index e90c34a39..60c01ba8c 100644 --- a/airtime_mvc/application/controllers/AudiopreviewController.php +++ b/airtime_mvc/application/controllers/AudiopreviewController.php @@ -22,8 +22,6 @@ class AudiopreviewController extends Zend_Controller_Action $CC_CONFIG = Config::getConfig(); $audioFileID = $this->_getParam('audioFileID'); - $audioFileArtist = $this->_getParam('audioFileArtist'); - $audioFileTitle = $this->_getParam('audioFileTitle'); $type = $this->_getParam('type'); $baseUrl = Application_Common_OsPath::getBaseDir(); @@ -60,10 +58,9 @@ class AudiopreviewController extends Zend_Controller_Action $this->view->uri = $uri; $this->view->mime = $mime; $this->view->audioFileID = $audioFileID; - // We need to decode artist and title because it gets - // encoded twice in js - $this->view->audioFileArtist = htmlspecialchars(urldecode($audioFileArtist)); - $this->view->audioFileTitle = htmlspecialchars(urldecode($audioFileTitle)); + + $this->view->audioFileArtist = htmlspecialchars($media->getPropelOrm()->getDbArtistName()); + $this->view->audioFileTitle = htmlspecialchars($media->getPropelOrm()->getDbTrackTitle()); $this->view->type = $type; $this->_helper->viewRenderer->setRender('audio-preview'); diff --git a/airtime_mvc/application/models/Schedule.php b/airtime_mvc/application/models/Schedule.php index 53f156b24..a40823a5b 100644 --- a/airtime_mvc/application/models/Schedule.php +++ b/airtime_mvc/application/models/Schedule.php @@ -947,7 +947,6 @@ SQL; $baseUrl = Application_Common_OsPath::getBaseDir(); $filesize = $file->getFileSize(); - self::createFileScheduleEvent($data, $item, $media_id, $uri, $filesize); } elseif (!is_null($item['stream_id'])) { diff --git a/airtime_mvc/application/models/airtime/CcFiles.php b/airtime_mvc/application/models/airtime/CcFiles.php index 04a9e3b95..1e3a72dce 100644 --- a/airtime_mvc/application/models/airtime/CcFiles.php +++ b/airtime_mvc/application/models/airtime/CcFiles.php @@ -57,7 +57,7 @@ class CcFiles extends BaseCcFiles { * Retrieve a sanitized version of the file metadata, suitable for public access. * @param $fileId */ - public static function getSantiziedFileById($fileId) + public static function getSanitizedFileById($fileId) { $file = CcFilesQuery::create()->findPk($fileId); if ($file) { @@ -114,7 +114,7 @@ class CcFiles extends BaseCcFiles { $file->setDbHidden(true); $file->save(); - $callbackUrl = Application_Common_HTTPHelper::getStationUrl() . "/rest/media/" . $file->getPrimaryKey(); + $callbackUrl = Application_Common_HTTPHelper::getStationUrl() . "rest/media/" . $file->getPrimaryKey(); Application_Service_MediaService::processUploadedFile($callbackUrl, $relativePath, self::getOwnerId()); return CcFiles::sanitizeResponse($file); @@ -138,14 +138,11 @@ class CcFiles extends BaseCcFiles { { $file = CcFilesQuery::create()->findPk($fileId); - // Since we check for this value when deleting files, set it first - $file->setDbDirectory(self::MUSIC_DIRS_STOR_PK); - $fileArray = self::removeBlacklistedFields($fileArray); $fileArray = self::stripTimeStampFromYearTag($fileArray); self::validateFileArray($fileArray); - if ($file && isset($requestData["resource_id"])) { + if ($file && isset($fileArray["resource_id"])) { $file->fromArray($fileArray, BasePeer::TYPE_FIELDNAME); @@ -155,9 +152,10 @@ class CcFiles extends BaseCcFiles { $fileSizeBytes = $fileArray["filesize"]; if (!isset($fileSizeBytes) || $fileSizeBytes === false) { - $file->setDbImportStatus(2)->save(); + $file->setDbImportStatus(self::IMPORT_STATUS_FAILED)->save(); throw new FileNotFoundException(); } + $cloudFile = new CloudFile(); $cloudFile->setStorageBackend($fileArray["storage_backend"]); $cloudFile->setResourceId($fileArray["resource_id"]); @@ -172,6 +170,9 @@ class CcFiles extends BaseCcFiles { } else if ($file) { + // Since we check for this value when deleting files, set it first + $file->setDbDirectory(self::MUSIC_DIRS_STOR_PK); + $file->fromArray($fileArray, BasePeer::TYPE_FIELDNAME); //Our RESTful API takes "full_path" as a field, which we then split and translate to match @@ -252,6 +253,7 @@ class CcFiles extends BaseCcFiles { } } + private static function validateFileArray(&$fileArray) { // Sanitize any wildly incorrect metadata before it goes to be validated @@ -342,6 +344,42 @@ class CcFiles extends BaseCcFiles { return $response; } + /** + * Returns the file size in bytes. + */ + public function getFileSize() + { + return filesize($this->getAbsoluteFilePath()); + } + + public function getFilename() + { + $info = pathinfo($this->getAbsoluteFilePath()); + return $info['filename']; + } + + /** + * Returns the file's absolute file path stored on disk. + */ + public function getURLForTrackPreviewOrDownload() + { + return $this->getAbsoluteFilePath(); + } + + /** + * Returns the file's absolute file path stored on disk. + */ + public function getAbsoluteFilePath() + { + $music_dir = Application_Model_MusicDir::getDirByPK($this->getDbDirectory()); + if (!$music_dir) { + throw new Exception("Invalid music_dir for file in database."); + } + $directory = $music_dir->getDirectory(); + $filepath = $this->getDbFilepath(); + return Application_Common_OsPath::join($directory, $filepath); + } + /** * * Strips out fields from incoming request data that should never be modified @@ -421,43 +459,7 @@ class CcFiles extends BaseCcFiles { exec("find $path -empty -type d -delete"); } - /** - * Returns the file size in bytes. - */ - public function getFileSize() - { - return filesize($this->getAbsoluteFilePath()); - } - - public function getFilename() - { - $info = pathinfo($this->getAbsoluteFilePath()); - return $info['filename']; - } - - /** - * Returns the file's absolute file path stored on disk. - */ - public function getURLForTrackPreviewOrDownload() - { - return $this->getAbsoluteFilePath(); - } - - /** - * Returns the file's absolute file path stored on disk. - */ - public function getAbsoluteFilePath() - { - $music_dir = Application_Model_MusicDir::getDirByPK($this->getDbDirectory()); - if (!$music_dir) { - throw new Exception("Invalid music_dir for file in database."); - } - $directory = $music_dir->getDirectory(); - $filepath = $this->getDbFilepath(); - return Application_Common_OsPath::join($directory, $filepath); - } - /** * Checks if the file is a regular file that can be previewed and downloaded. */ diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index c6e26b6f9..0df6b6b3e 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -41,8 +41,9 @@ class Rest_MediaController extends Zend_Rest_Controller try { $this->getResponse() - ->setHttpResponseCode(200) - ->appendBody($this->_redirect(CcFiles::getDownloadUrl($id))); + ->setHttpResponseCode(200); + $inline = false; + Application_Service_MediaService::streamFileDownload($id, $inline); } catch (FileNotFoundException $e) { $this->fileNotFoundResponse(); @@ -64,7 +65,7 @@ class Rest_MediaController extends Zend_Rest_Controller try { $this->getResponse() ->setHttpResponseCode(200) - ->appendBody(json_encode(CcFiles::getSantiziedFileById($id))); + ->appendBody(json_encode(CcFiles::getSanitizedFileById($id))); } catch (FileNotFoundException $e) { $this->fileNotFoundResponse(); diff --git a/airtime_mvc/application/services/MediaService.php b/airtime_mvc/application/services/MediaService.php index 74b2609d0..81f199ae7 100644 --- a/airtime_mvc/application/services/MediaService.php +++ b/airtime_mvc/application/services/MediaService.php @@ -1,5 +1,7 @@ getDirectory() . "/imported/" . $ownerId; + $importedStorageDirectory = ""; + if ($CC_CONFIG["current_backend"] == "file") { + $storDir = Application_Model_MusicDir::getStorDir(); + $importedStorageDirectory = $storDir->getDirectory() . "/imported/" . $ownerId; + } try { //Copy the temporary file over to the "organize" folder so that it's off our webserver @@ -34,8 +39,121 @@ class Application_Service_MediaService //Dispatch a message to airtime_analyzer through RabbitMQ, //notifying it that there's a new upload to process! + $storageBackend = new ProxyStorageBackend($CC_CONFIG["current_backend"]); Application_Model_RabbitMq::SendMessageToAnalyzer($newTempFilePath, $importedStorageDirectory, basename($originalFilename), - $callbackUrl, $apiKey); + $callbackUrl, $apiKey, + $CC_CONFIG["current_backend"], + $storageBackend->getFilePrefix()); } -} \ No newline at end of file + + + /** + * @param $fileId + * @param bool $inline Set the Content-Disposition header to inline to prevent a download dialog from popping up (or attachment if false) + * @throws Exception + * @throws FileNotFoundException + */ + public static function streamFileDownload($fileId, $inline=false) + { + $media = Application_Model_StoredFile::RecallById($fileId); + if ($media == null) { + throw new FileNotFoundException(); + } + $filepath = $media->getFilePath(); + // Make sure we don't have some wrong result beecause of caching + clearstatcache(); + $media = Application_Model_StoredFile::RecallById($fileId); + if ($media == null) { + throw new FileNotFoundException(); + } + + // Make sure we don't have some wrong result beecause of caching + clearstatcache(); + + if ($media->getPropelOrm()->isValidPhysicalFile()) { + $filename = $media->getPropelOrm()->getFilename(); + + //Download user left clicks a track and selects Download. + if (!$inline) { + //We are using Content-Disposition to specify + //to the browser what name the file should be saved as. + header('Content-Disposition: attachment; filename="' . $filename . '"'); + } else { + //user clicks play button for track and downloads it. + header('Content-Disposition: inline; filename="' . $filename . '"'); + } + + self::smartReadFile($media); + exit; + } else { + throw new FileNotFoundException(); + } + } + + + /** + * Reads the requested portion of a file and sends its contents to the client with the appropriate headers. + * + * This HTTP_RANGE compatible read file function is necessary for allowing streaming media to be skipped around in. + * + * @param CcFile $media + * @return void + * + * @link https://groups.google.com/d/msg/jplayer/nSM2UmnSKKA/Hu76jDZS4xcJ + * @link http://php.net/manual/en/function.readfile.php#86244 + */ + private static function smartReadFile($media) + { + $filepath = $media->getFilePath(); + $size= $media->getFileSize(); + $mimeType = $media->getPropelOrm()->getDbMime(); + + $fm = @fopen($filepath, 'rb'); + if (!$fm) { + header ("HTTP/1.1 505 Internal server error"); + + return; + } + + $begin = 0; + $end = $size - 1; + + if (isset($_SERVER['HTTP_RANGE'])) { + if (preg_match('/bytes=\h*(\d+)-(\d*)[\D.*]?/i', $_SERVER['HTTP_RANGE'], $matches)) { + $begin = intval($matches[1]); + if (!empty($matches[2])) { + $end = intval($matches[2]); + } + } + } + + if (isset($_SERVER['HTTP_RANGE'])) { + header('HTTP/1.1 206 Partial Content'); + } else { + header('HTTP/1.1 200 OK'); + } + header("Content-Type: $mimeType"); + header('Cache-Control: public, must-revalidate, max-age=0'); + header('Pragma: no-cache'); + header('Accept-Ranges: bytes'); + header('Content-Length:' . (($end - $begin) + 1)); + if (isset($_SERVER['HTTP_RANGE'])) { + header("Content-Range: bytes $begin-$end/$size"); + } + header("Content-Transfer-Encoding: binary"); + + //We can have multiple levels of output buffering. Need to + //keep looping until all have been disabled!!! + //http://www.php.net/manual/en/function.ob-end-flush.php + while (@ob_end_flush()); + + // NOTE: We can't use fseek here because it does not work with streams + // (a.k.a. Files stored in the cloud) + while(!feof($fm) && (connection_status() == 0)) { + echo fread($fm, 1024 * 8); + } + fclose($fm); + } +} + diff --git a/airtime_mvc/public/js/airtime/common/common.js b/airtime_mvc/public/js/airtime/common/common.js index 7a043c286..476b5e77d 100644 --- a/airtime_mvc/public/js/airtime/common/common.js +++ b/airtime_mvc/public/js/airtime/common/common.js @@ -91,16 +91,11 @@ function openAudioPreview(p_event) { } } -function open_audio_preview(type, id, audioFileTitle, audioFileArtist) { - // we need to remove soundcloud icon from audioFileTitle - var index = audioFileTitle.indexOf("