From 0177e400836a235e6ce525a99efe56b8e9fc70fd Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Thu, 19 Feb 2015 15:10:01 -0500 Subject: [PATCH 1/3] Pull the logic for Media REST API out of the controller --- airtime_mvc/application/common/HTTPHelper.php | 18 + .../application/models/airtime/CcFiles.php | 366 +++++++++++++++- .../rest/controllers/MediaController.php | 389 +++--------------- .../application/services/MediaService.php | 41 ++ airtime_mvc/public/index.php | 3 + 5 files changed, 482 insertions(+), 335 deletions(-) create mode 100644 airtime_mvc/application/services/MediaService.php diff --git a/airtime_mvc/application/common/HTTPHelper.php b/airtime_mvc/application/common/HTTPHelper.php index db314bb0b..505befd7e 100644 --- a/airtime_mvc/application/common/HTTPHelper.php +++ b/airtime_mvc/application/common/HTTPHelper.php @@ -17,4 +17,22 @@ class Application_Common_HTTPHelper $request->getParam("timezone", null) ); } + + public static function getStationUrl() + { + $scheme = "http"; + if (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') { + $scheme = "https"; + } + $CC_CONFIG = Config::getConfig(); + $baseUrl = $CC_CONFIG['baseUrl']; + $baseDir = $CC_CONFIG['baseDir']; + $basePort = $CC_CONFIG['basePort']; + if (empty($baseDir)) { + $baseDir = "/"; + } + $stationUrl = "$scheme://${baseUrl}:${basePort}${baseDir}"; + + return $stationUrl; + } } diff --git a/airtime_mvc/application/models/airtime/CcFiles.php b/airtime_mvc/application/models/airtime/CcFiles.php index 93ef491a8..554139e40 100644 --- a/airtime_mvc/application/models/airtime/CcFiles.php +++ b/airtime_mvc/application/models/airtime/CcFiles.php @@ -11,17 +11,288 @@ * * @package propel.generator.campcaster */ + +class InvalidMetadataException extends Exception +{ +} + +class FileNotFoundException extends Exception +{ +} + +class OverDiskQuotaException extends Exception +{ + +} + class CcFiles extends BaseCcFiles { - + + const MUSIC_DIRS_STOR_PK = 1; + + + //fields that are not modifiable via our RESTful API + private static $blackList = array( + 'id', + 'directory', + 'filepath', + 'file_exists', + 'mtime', + 'utime', + 'lptime', + 'silan_check', + 'soundcloud_id', + 'is_scheduled', + 'is_playlist' + ); + //fields we should never expose through our RESTful API private static $privateFields = array( - 'file_exists', - 'silan_check', - 'is_scheduled', - 'is_playlist' + 'file_exists', + 'silan_check', + 'is_scheduled', + 'is_playlist' ); - - public function getCueLength() + + /** + * Retrieve a sanitized version of the file metadata, suitable for public access. + * @param $fileId + */ + public static function getSantiziedFileById($fileId) + { + $file = CcFilesQuery::create()->findPk($fileId); + if ($file) { + return CcFiles::sanitizeResponse($file); + } else { + throw new FileNotFoundException(); + } + } + + /** Used to create a CcFiles object from an array containing metadata and a file uploaded by POST. + * This is used by our Media REST API! + * @param $fileArray An array containing metadata for a CcFiles object. + * @throws Exception + */ + public static function createFromUpload($fileArray) + { + if (Application_Model_Systemstatus::isDiskOverQuota()) { + throw new OverDiskQuotaException(); + } + + $file = new CcFiles(); + + try{ + $fileArray = self::removeBlacklistedFields($fileArray); + + /*if (!self::validateFileArray($fileArray)) + { + $file->setDbTrackTitle($_FILES["file"]["name"]); + $file->setDbUtime(new DateTime("now", new DateTimeZone("UTC"))); + $file->save(); + return CcFiles::sanitizeResponse($file);*/ + self::validateFileArray($fileArray); + + /* If full_path is set, the post request came from ftp. + * Users are allowed to upload folders via ftp. If this is the case + * we need to include the folder name with the file name, otherwise + * files won't get removed from the organize folder. + */ + if (isset($fileArray["full_path"])) { + $fullPath = $fileArray["full_path"]; + $basePath = isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/organize/" : "/srv/airtime/stor/organize/"; + //$relativePath is the folder name(if one) + track name, that was uploaded via ftp + $relativePath = substr($fullPath, strlen($basePath)-1); + } else { + $relativePath = $_FILES["file"]["name"]; + } + + + $file->fromArray($fileArray); + $file->setDbOwnerId(self::getOwnerId()); + $now = new DateTime("now", new DateTimeZone("UTC")); + $file->setDbTrackTitle($_FILES["file"]["name"]); + $file->setDbUtime($now); + $file->setDbHidden(true); + $file->save(); + + $callbackUrl = Application_Common_HTTPHelper::getStationUrl() . "/rest/media/" . $file->getPrimaryKey(); + + Application_Service_MediaService::processUploadedFile($callbackUrl, $relativePath, self::getOwnerId()); + return CcFiles::sanitizeResponse($file); + + } catch (Exception $e) { + $file->setDbImportStatus(2); + $file->setDbHidden(true); + throw $e; + } + } + + /** Update a file with metadata specified in an array. + * @param $fileId The ID of the file to update in the DB. + * @param $fileArray An associative array containing metadata. Replaces those fields if they exist. + * @return array A sanitized version of the file metadata array. + * @throws Exception + * @throws FileNotFoundException + * @throws PropelException + */ + public static function updateFromArray($fileId, $fileArray) + { + $file = CcFilesQuery::create()->findPk($fileId); + + // Since we check for this value when deleting files, set it first + $file->setDbDirectory(self::MUSIC_DIRS_STOR_PK); + + $fileArray = self::removeBlacklistedFields($fileArray); + $fileArray = self::stripTimeStampFromYearTag($fileArray); + + self::validateFileArray($fileArray); + if ($file && isset($requestData["resource_id"])) { + + $file->fromArray($fileArray, BasePeer::TYPE_FIELDNAME); + + //store the original filename + $file->setDbFilepath($fileArray["filename"]); + + $fileSizeBytes = $fileArray["filesize"]; + if (!isset($fileSizeBytes) || $fileSizeBytes === false) + { + $file->setDbImportStatus(2)->save(); + throw new FileNotFoundException(); + } + $cloudFile = new CloudFile(); + $cloudFile->setStorageBackend($fileArray["storage_backend"]); + $cloudFile->setResourceId($fileArray["resource_id"]); + $cloudFile->setCcFiles($file); + $cloudFile->save(); + + Application_Model_Preference::updateDiskUsage($fileSizeBytes); + + $now = new DateTime("now", new DateTimeZone("UTC")); + $file->setDbMtime($now); + $file->save(); + + } else if ($file) { + + $file->fromArray($fileArray, BasePeer::TYPE_FIELDNAME); + + //Our RESTful API takes "full_path" as a field, which we then split and translate to match + //our internal schema. Internally, file path is stored relative to a directory, with the directory + //as a foreign key to cc_music_dirs. + if (isset($fileArray["full_path"])) { + $fileSizeBytes = filesize($fileArray["full_path"]); + if (!isset($fileSizeBytes) || $fileSizeBytes === false) + { + $file->setDbImportStatus(self::IMPORT_STATUS_FAILED)->save(); + throw new FileNotFoundException(); + } + Application_Model_Preference::updateDiskUsage($fileSizeBytes); + + $fullPath = $fileArray["full_path"]; + $storDir = Application_Model_MusicDir::getStorDir()->getDirectory(); + $pos = strpos($fullPath, $storDir); + + if ($pos !== FALSE) + { + assert($pos == 0); //Path must start with the stor directory path + + $filePathRelativeToStor = substr($fullPath, strlen($storDir)); + $file->setDbFilepath($filePathRelativeToStor); + } + } + + $now = new DateTime("now", new DateTimeZone("UTC")); + $file->setDbMtime($now); + $file->save(); + + /* $this->removeEmptySubFolders( + isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/organize/" : "/srv/airtime/stor/organize/"); */ + } else { + $file->setDbImportStatus(self::IMPORT_STATUS_FAILED)->save(); + throw new FileNotFoundException(); + } + + return CcFiles::sanitizeResponse($file); + } + + /** Delete a file from the database and disk (or cloud). + * @param $id The file ID + * @throws DeleteScheduledFileException + * @throws Exception + * @throws FileNoPermissionException + * @throws FileNotFoundException + * @throws PropelException + */ + public static function deleteById($id) + { + $file = CcFilesQuery::create()->findPk($id); + if ($file) { + $con = Propel::getConnection(); + $storedFile = new Application_Model_StoredFile($file, $con); + if ($storedFile->existsOnDisk()) { + $storedFile->delete(); //TODO: This checks your session permissions... Make it work without a session? + } + $file->delete(); + } else { + throw new FileNotFoundException(); + } + + } + + public static function getDownloadUrl($id) + { + $file = CcFilesQuery::create()->findPk($id); + if ($file) { + $con = Propel::getConnection(); + $storedFile = new Application_Model_StoredFile($file, $con); + $baseDir = Application_Common_OsPath::getBaseDir(); + + return $storedFile->getRelativeFileUrl($baseDir) . '/download/true'; + } + else { + throw new FileNotFoundException(); + } + } + + private static function validateFileArray(&$fileArray) + { + // Sanitize any wildly incorrect metadata before it goes to be validated + FileDataHelper::sanitizeData($fileArray); + + // EditAudioMD form is used here for validation + $fileForm = new Application_Form_EditAudioMD(); + $fileForm->startForm(0); //The file ID doesn't matter here + $fileForm->populate($fileArray); + + /* + * Here we are truncating metadata of any characters greater than the + * max string length set in the database. In the rare case a track's + * genre is more than 64 chars, for example, we don't want to reject + * tracks for that reason + */ + foreach($fileArray as $tag => &$value) { + if ($fileForm->getElement($tag)) { + $stringLengthValidator = $fileForm->getElement($tag)->getValidator('StringLength'); + //$stringLengthValidator will be false if the StringLength validator doesn't exist on the current element + //in which case we don't have to truncate the extra characters + if ($stringLengthValidator) { + $value = substr($value, 0, $stringLengthValidator->getMax()); + } + + $value = self::stripInvalidUtf8Characters($value); + } + } + + if (!$fileForm->isValidPartial($fileArray)) { + $errors = $fileForm->getErrors(); + $messages = $fileForm->getMessages(); + Logging::error($messages); + throw new Exception("Data validation failed: $errors - $messages"); + } + + return true; + } + + + public function getCueLength() { $cuein = $this->getDbCuein(); $cueout = $this->getDbCueout(); @@ -70,4 +341,83 @@ class CcFiles extends BaseCcFiles { return $response; } -} // CcFiles + + /** + * + * Strips out fields from incoming request data that should never be modified + * from outside of Airtime + * @param array $data + */ + private static function removeBlacklistedFields($data) + { + foreach (self::$blackList as $key) { + unset($data[$key]); + } + + return $data; + } + + + private static function getOwnerId() + { + try { + if (Zend_Auth::getInstance()->hasIdentity()) { + $service_user = new Application_Service_UserService(); + return $service_user->getCurrentUser()->getDbId(); + } else { + $defaultOwner = CcSubjsQuery::create() + ->filterByDbType('A') + ->orderByDbId() + ->findOne(); + if (!$defaultOwner) { + // what to do if there is no admin user? + // should we handle this case? + return null; + } + return $defaultOwner->getDbId(); + } + } catch(Exception $e) { + Logging::info($e->getMessage()); + } + } + /* + * It's possible that the year tag will be a timestamp but Airtime doesn't support this. + * The year field in cc_files can only be 16 chars max. + * + * This functions strips the year field of it's timestamp, if one, and leaves just the year + */ + private static function stripTimeStampFromYearTag($metadata) + { + if (isset($metadata["year"])) { + if (preg_match("/^(\d{4})-(\d{2})-(\d{2})(?:\s+(\d{2}):(\d{2}):(\d{2}))?$/", $metadata["year"])) { + $metadata["year"] = substr($metadata["year"], 0, 4); + } + } + return $metadata; + } + + private static function stripInvalidUtf8Characters($string) + { + //Remove invalid UTF-8 characters + //reject overly long 2 byte sequences, as well as characters above U+10000 and replace with ? + $string = preg_replace('/[\x00-\x08\x10\x0B\x0C\x0E-\x19\x7F]'. + '|[\x00-\x7F][\x80-\xBF]+'. + '|([\xC0\xC1]|[\xF0-\xFF])[\x80-\xBF]*'. + '|[\xC2-\xDF]((?![\x80-\xBF])|[\x80-\xBF]{2,})'. + '|[\xE0-\xEF](([\x80-\xBF](?![\x80-\xBF]))|(?![\x80-\xBF]{2})|[\x80-\xBF]{3,})/S', + '?', $string ); + + //reject overly long 3 byte sequences and UTF-16 surrogates and replace with ? + $string = preg_replace('/\xE0[\x80-\x9F][\x80-\xBF]'. + '|\xED[\xA0-\xBF][\x80-\xBF]/S','?', $string ); + + //Do a final encoding conversion to + $string = mb_convert_encoding($string, 'UTF-8', 'UTF-8'); + return $string; + } + + private function removeEmptySubFolders($path) + { + exec("find $path -empty -type d -delete"); + } +} diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index 90587d480..08240dc80 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -2,26 +2,10 @@ class Rest_MediaController extends Zend_Rest_Controller { - const MUSIC_DIRS_STOR_PK = 1; - const IMPORT_STATUS_SUCCESS = 0; const IMPORT_STATUS_PENDING = 1; const IMPORT_STATUS_FAILED = 2; - //fields that are not modifiable via our RESTful API - private static $blackList = array( - 'id', - 'directory', - 'filepath', - 'file_exists', - 'mtime', - 'utime', - 'lptime', - 'silan_check', - 'soundcloud_id', - 'is_scheduled', - 'is_playlist' - ); public function init() { @@ -54,17 +38,19 @@ class Rest_MediaController extends Zend_Rest_Controller return; } - $file = CcFilesQuery::create()->findPk($id); - if ($file) { - $con = Propel::getConnection(); - $storedFile = new Application_Model_StoredFile($file, $con); - $baseUrl = Application_Common_OsPath::getBaseDir(); - + try + { $this->getResponse() ->setHttpResponseCode(200) - ->appendBody($this->_redirect($storedFile->getRelativeFileUrl($baseUrl).'/download/true')); - } else { + ->appendBody($this->_redirect(CcFiles::getDownloadUrl($id))); + } + catch (FileNotFoundException $e) { $this->fileNotFoundResponse(); + Logging::error($e->getMessage()); + } + catch (Exception $e) { + $this->unknownErrorResponse(); + Logging::error($e->getMessage()); } } @@ -75,14 +61,18 @@ class Rest_MediaController extends Zend_Rest_Controller return; } - $file = CcFilesQuery::create()->findPk($id); - if ($file) { - + try { $this->getResponse() ->setHttpResponseCode(200) - ->appendBody(json_encode(CcFiles::sanitizeResponse($file))); - } else { + ->appendBody(json_encode(CcFiles::getSantiziedFileById($id))); + } + catch (FileNotFoundException $e) { $this->fileNotFoundResponse(); + Logging::error($e->getMessage()); + } + catch (Exception $e) { + $this->unknownErrorResponse(); + Logging::error($e->getMessage()); } } @@ -97,52 +87,24 @@ class Rest_MediaController extends Zend_Rest_Controller return; } - if (Application_Model_Systemstatus::isDiskOverQuota()) { + try { + $sanitizedFile = CcFiles::createFromUpload($this->getRequest()->getPost()); + $this->getResponse() + ->setHttpResponseCode(201) + ->appendBody(json_encode($sanitizedFile)); + } + catch (InvalidMetadataException $e) { + $this->invalidDataResponse(); + Logging::error($e->getMessage()); + } + catch (OverDiskQuotaException $e) { $this->getResponse() ->setHttpResponseCode(400) ->appendBody("ERROR: Disk Quota reached."); - return; } - - $file = new CcFiles(); - $whiteList = $this->removeBlacklistedFieldsFromRequestData($this->getRequest()->getPost()); - - if (!$this->validateRequestData($file, $whiteList)) { - $file->setDbTrackTitle($_FILES["file"]["name"]); - $file->setDbUtime(new DateTime("now", new DateTimeZone("UTC"))); - $file->save(); - return; - } else { - /* If full_path is set, the post request came from ftp. - * Users are allowed to upload folders via ftp. If this is the case - * we need to include the folder name with the file name, otherwise - * files won't get removed from the organize folder. - */ - if (isset($whiteList["full_path"])) { - $fullPath = $whiteList["full_path"]; - $basePath = isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/organize/" : "/srv/airtime/stor/organize/"; - //$relativePath is the folder name(if one) + track name, that was uploaded via ftp - $relativePath = substr($fullPath, strlen($basePath)-1); - } else { - $relativePath = $_FILES["file"]["name"]; - } - - - $file->fromArray($whiteList); - $file->setDbOwnerId($this->getOwnerId()); - $now = new DateTime("now", new DateTimeZone("UTC")); - $file->setDbTrackTitle($_FILES["file"]["name"]); - $file->setDbUtime($now); - $file->setDbHidden(true); - $file->save(); - - $callbackUrl = $this->getRequest()->getScheme() . '://' . $this->getRequest()->getHttpHost() . $this->getRequest()->getRequestUri() . "/" . $file->getPrimaryKey(); - - $this->processUploadedFile($callbackUrl, $relativePath, $this->getOwnerId()); - - $this->getResponse() - ->setHttpResponseCode(201) - ->appendBody(json_encode(CcFiles::sanitizeResponse($file))); + catch (Exception $e) { + $this->unknownErrorResponse(); + Logging::error($e->getMessage()); } } @@ -152,90 +114,25 @@ class Rest_MediaController extends Zend_Rest_Controller if (!$id) { return; } - - $file = CcFilesQuery::create()->findPk($id); - // Since we check for this value when deleting files, set it first - $file->setDbDirectory(self::MUSIC_DIRS_STOR_PK); - $requestData = json_decode($this->getRequest()->getRawBody(), true); - $whiteList = $this->removeBlacklistedFieldsFromRequestData($requestData); - $whiteList = $this->stripTimeStampFromYearTag($whiteList); - - if (!$this->validateRequestData($file, $whiteList)) { - $file->save(); - return; - } else if ($file && isset($requestData["resource_id"])) { - - $file->fromArray($whiteList, BasePeer::TYPE_FIELDNAME); - - //store the original filename - $file->setDbFilepath($requestData["filename"]); - - $fileSizeBytes = $requestData["filesize"]; - if (!isset($fileSizeBytes) || $fileSizeBytes === false) - { - $file->setDbImportStatus(2)->save(); - $this->fileNotFoundResponse(); - return; - } - $cloudFile = new CloudFile(); - $cloudFile->setStorageBackend($requestData["storage_backend"]); - $cloudFile->setResourceId($requestData["resource_id"]); - $cloudFile->setCcFiles($file); - $cloudFile->save(); - - Application_Model_Preference::updateDiskUsage($fileSizeBytes); - - $now = new DateTime("now", new DateTimeZone("UTC")); - $file->setDbMtime($now); - $file->save(); - + try { + $requestData = json_decode($this->getRequest()->getRawBody(), true); + $sanitizedFile = CcFiles::updateFromArray($id, $requestData); $this->getResponse() - ->setHttpResponseCode(200) - ->appendBody(json_encode(CcFiles::sanitizeResponse($file))); - } else if ($file) { - - $file->fromArray($whiteList, BasePeer::TYPE_FIELDNAME); - - //Our RESTful API takes "full_path" as a field, which we then split and translate to match - //our internal schema. Internally, file path is stored relative to a directory, with the directory - //as a foreign key to cc_music_dirs. - if (isset($requestData["full_path"])) { - $fileSizeBytes = filesize($requestData["full_path"]); - if (!isset($fileSizeBytes) || $fileSizeBytes === false) - { - $file->setDbImportStatus(self::IMPORT_STATUS_FAILED)->save(); - $this->fileNotFoundResponse(); - return; - } - Application_Model_Preference::updateDiskUsage($fileSizeBytes); - - $fullPath = $requestData["full_path"]; - $storDir = Application_Model_MusicDir::getStorDir()->getDirectory(); - $pos = strpos($fullPath, $storDir); - - if ($pos !== FALSE) - { - assert($pos == 0); //Path must start with the stor directory path - - $filePathRelativeToStor = substr($fullPath, strlen($storDir)); - $file->setDbFilepath($filePathRelativeToStor); - } - } - - $now = new DateTime("now", new DateTimeZone("UTC")); - $file->setDbMtime($now); - $file->save(); - - /* $this->removeEmptySubFolders( - isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/organize/" : "/srv/airtime/stor/organize/"); */ - - $this->getResponse() - ->setHttpResponseCode(200) - ->appendBody(json_encode(CcFiles::sanitizeResponse($file))); - } else { - $file->setDbImportStatus(self::IMPORT_STATUS_FAILED)->save(); + ->setHttpResponseCode(201) + ->appendBody(json_encode($sanitizedFile)); + } + catch (InvalidMetadataException $e) { + $this->invalidDataResponse(); + Logging::error($e->getMessage()); + } + catch (FileNotFoundException $e) { $this->fileNotFoundResponse(); + Logging::error($e->getMessage()); + } + catch (Exception $e) { + $this->unknownErrorResponse(); + Logging::error($e->getMessage()); } } @@ -245,18 +142,19 @@ class Rest_MediaController extends Zend_Rest_Controller if (!$id) { return; } - $file = CcFilesQuery::create()->findPk($id); - if ($file) { - $con = Propel::getConnection(); - $storedFile = new Application_Model_StoredFile($file, $con); - if ($storedFile->existsOnDisk()) { - $storedFile->delete(); //TODO: This checks your session permissions... Make it work without a session? - } - $file->delete(); + + try { + CcFiles::deleteById($id); $this->getResponse() ->setHttpResponseCode(204); - } else { + } + catch (FileNotFoundException $e) { $this->fileNotFoundResponse(); + Logging::error($e->getMessage()); + } + catch (Exception $e) { + $this->unknownErrorResponse(); + Logging::error($e->getMessage()); } } @@ -278,174 +176,11 @@ class Rest_MediaController extends Zend_Rest_Controller $resp->appendBody("ERROR: Media not found."); } - private function invalidDataResponse() + private function unknownErrorResponse() { $resp = $this->getResponse(); - $resp->setHttpResponseCode(422); - $resp->appendBody("ERROR: Invalid data"); - } - - private function validateRequestData($file, &$whiteList) - { - // Sanitize any wildly incorrect metadata before it goes to be validated - FileDataHelper::sanitizeData($whiteList); - - try { - // EditAudioMD form is used here for validation - $fileForm = new Application_Form_EditAudioMD(); - $fileForm->startForm($file->getDbId()); - $fileForm->populate($whiteList); - - /* - * Here we are truncating metadata of any characters greater than the - * max string length set in the database. In the rare case a track's - * genre is more than 64 chars, for example, we don't want to reject - * tracks for that reason - */ - foreach($whiteList as $tag => &$value) { - if ($fileForm->getElement($tag)) { - $stringLengthValidator = $fileForm->getElement($tag)->getValidator('StringLength'); - //$stringLengthValidator will be false if the StringLength validator doesn't exist on the current element - //in which case we don't have to truncate the extra characters - if ($stringLengthValidator) { - $value = substr($value, 0, $stringLengthValidator->getMax()); - } - - $value = $this->stripInvalidUtf8Characters($value); - } - } - - if (!$fileForm->isValidPartial($whiteList)) { - throw new Exception("Data validation failed"); - } - } catch (Exception $e) { - $errors = $fileForm->getErrors(); - $messages = $fileForm->getMessages(); - Logging::error($messages); - $file->setDbImportStatus(2); - $file->setDbHidden(true); - $this->invalidDataResponse(); - return false; - } - return true; - } - - private function processUploadedFile($callbackUrl, $originalFilename, $ownerId) - { - $CC_CONFIG = Config::getConfig(); - $apiKey = $CC_CONFIG["apiKey"][0]; - - $tempFilePath = $_FILES['file']['tmp_name']; - $tempFileName = basename($tempFilePath); - - //Only accept files with a file extension that we support. - $fileExtension = pathinfo($originalFilename, PATHINFO_EXTENSION); - if (!in_array(strtolower($fileExtension), explode(",", "ogg,mp3,oga,flac,wav,m4a,mp4,opus"))) - { - @unlink($tempFilePath); - throw new Exception("Bad file extension."); - } - - //TODO: Remove uploadFileAction from ApiController.php **IMPORTANT** - It's used by the recorder daemon... - - $storDir = Application_Model_MusicDir::getStorDir(); - $importedStorageDirectory = $storDir->getDirectory() . "/imported/" . $ownerId; - - try { - //Copy the temporary file over to the "organize" folder so that it's off our webserver - //and accessible by airtime_analyzer which could be running on a different machine. - $newTempFilePath = Application_Model_StoredFile::copyFileToStor($tempFilePath, $originalFilename); - } catch (Exception $e) { - @unlink($tempFilePath); - Logging::error($e->getMessage()); - return; - } - - //Dispatch a message to airtime_analyzer through RabbitMQ, - //notifying it that there's a new upload to process! - Application_Model_RabbitMq::SendMessageToAnalyzer($newTempFilePath, - $importedStorageDirectory, basename($originalFilename), - $callbackUrl, $apiKey); - } - - private function getOwnerId() - { - try { - if (Zend_Auth::getInstance()->hasIdentity()) { - $service_user = new Application_Service_UserService(); - return $service_user->getCurrentUser()->getDbId(); - } else { - $defaultOwner = CcSubjsQuery::create() - ->filterByDbType('A') - ->orderByDbId() - ->findOne(); - if (!$defaultOwner) { - // what to do if there is no admin user? - // should we handle this case? - return null; - } - return $defaultOwner->getDbId(); - } - } catch(Exception $e) { - Logging::info($e->getMessage()); - } - } - - /** - * - * Strips out fields from incoming request data that should never be modified - * from outside of Airtime - * @param array $data - */ - private static function removeBlacklistedFieldsFromRequestData($data) - { - foreach (self::$blackList as $key) { - unset($data[$key]); - } - - return $data; - } - - - private function removeEmptySubFolders($path) - { - exec("find $path -empty -type d -delete"); - } - - /* - * It's possible that the year tag will be a timestamp but Airtime doesn't support this. - * The year field in cc_files can only be 16 chars max. - * - * This functions strips the year field of it's timestamp, if one, and leaves just the year - */ - private function stripTimeStampFromYearTag($metadata) - { - if (isset($metadata["year"])) { - if (preg_match("/^(\d{4})-(\d{2})-(\d{2})(?:\s+(\d{2}):(\d{2}):(\d{2}))?$/", $metadata["year"])) { - $metadata["year"] = substr($metadata["year"], 0, 4); - } - } - return $metadata; - } - - private function stripInvalidUtf8Characters($string) - { - //Remove invalid UTF-8 characters - //reject overly long 2 byte sequences, as well as characters above U+10000 and replace with ? - $string = preg_replace('/[\x00-\x08\x10\x0B\x0C\x0E-\x19\x7F]'. - '|[\x00-\x7F][\x80-\xBF]+'. - '|([\xC0\xC1]|[\xF0-\xFF])[\x80-\xBF]*'. - '|[\xC2-\xDF]((?![\x80-\xBF])|[\x80-\xBF]{2,})'. - '|[\xE0-\xEF](([\x80-\xBF](?![\x80-\xBF]))|(?![\x80-\xBF]{2})|[\x80-\xBF]{3,})/S', - '?', $string ); - - //reject overly long 3 byte sequences and UTF-16 surrogates and replace with ? - $string = preg_replace('/\xE0[\x80-\x9F][\x80-\xBF]'. - '|\xED[\xA0-\xBF][\x80-\xBF]/S','?', $string ); - - //Do a final encoding conversion to - $string = mb_convert_encoding($string, 'UTF-8', 'UTF-8'); - return $string; + $resp->setHttpResponseCode(400); + $resp->appendBody("An unknown error occurred."); } } diff --git a/airtime_mvc/application/services/MediaService.php b/airtime_mvc/application/services/MediaService.php new file mode 100644 index 000000000..74b2609d0 --- /dev/null +++ b/airtime_mvc/application/services/MediaService.php @@ -0,0 +1,41 @@ +getDirectory() . "/imported/" . $ownerId; + + try { + //Copy the temporary file over to the "organize" folder so that it's off our webserver + //and accessible by airtime_analyzer which could be running on a different machine. + $newTempFilePath = Application_Model_StoredFile::copyFileToStor($tempFilePath, $originalFilename); + } catch (Exception $e) { + @unlink($tempFilePath); + Logging::error($e->getMessage()); + return; + } + + //Dispatch a message to airtime_analyzer through RabbitMQ, + //notifying it that there's a new upload to process! + Application_Model_RabbitMq::SendMessageToAnalyzer($newTempFilePath, + $importedStorageDirectory, basename($originalFilename), + $callbackUrl, $apiKey); + } +} \ No newline at end of file diff --git a/airtime_mvc/public/index.php b/airtime_mvc/public/index.php index f9c691f51..c45ac3b1d 100644 --- a/airtime_mvc/public/index.php +++ b/airtime_mvc/public/index.php @@ -49,6 +49,9 @@ set_include_path(APPLICATION_PATH . '/models' . PATH_SEPARATOR . get_include_pat //Controller plugins. set_include_path(APPLICATION_PATH . '/controllers/plugins' . PATH_SEPARATOR . get_include_path()); +//Services. +set_include_path(APPLICATION_PATH . '/services/' . PATH_SEPARATOR . get_include_path()); + //Zend framework if (file_exists('/usr/share/php/libzend-framework-php')) { set_include_path('/usr/share/php/libzend-framework-php' . PATH_SEPARATOR . get_include_path()); From 2a89e4d5a098b7cb86de3c416d7017622499b433 Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Fri, 20 Feb 2015 14:01:06 -0500 Subject: [PATCH 2/3] Massive refactor of the analyzer branch and sync it back up with the cloud storage branch (for the last time) * Backported all the bugfixes from cc-5709-airtime-analyzer-cloud-storage * Backported missing FileStorageBackend.php * Fixed CC-6001: Track titles and artist names with slashes break audio preview * Refactored all the MediaController code, pulling out the logic into MediaService * Fixed an API key leak to guests in the Media API * Made this branch work without cloud_storage.conf (defaults to file storage) * Made ApiController's getMediaAction use the MediaService code --- ...zon_S3.php => Amazon_S3StorageBackend.php} | 4 +- .../cloud_storage/FileStorageBackend.php | 41 ++++++ .../cloud_storage/ProxyStorageBackend.php | 36 +++-- .../cloud_storage/StorageBackend.php | 10 ++ airtime_mvc/application/configs/conf.php | 25 ++-- .../application/controllers/ApiController.php | 92 +------------ .../controllers/AudiopreviewController.php | 9 +- airtime_mvc/application/models/Schedule.php | 1 - .../application/models/airtime/CcFiles.php | 88 ++++++------ .../rest/controllers/MediaController.php | 7 +- .../application/services/MediaService.php | 126 +++++++++++++++++- .../public/js/airtime/common/common.js | 9 +- .../public/js/airtime/library/library.js | 6 +- 13 files changed, 275 insertions(+), 179 deletions(-) rename airtime_mvc/application/cloud_storage/{Amazon_S3.php => Amazon_S3StorageBackend.php} (93%) create mode 100644 airtime_mvc/application/cloud_storage/FileStorageBackend.php diff --git a/airtime_mvc/application/cloud_storage/Amazon_S3.php b/airtime_mvc/application/cloud_storage/Amazon_S3StorageBackend.php similarity index 93% rename from airtime_mvc/application/cloud_storage/Amazon_S3.php rename to airtime_mvc/application/cloud_storage/Amazon_S3StorageBackend.php index 978e1ba07..24e398af0 100644 --- a/airtime_mvc/application/cloud_storage/Amazon_S3.php +++ b/airtime_mvc/application/cloud_storage/Amazon_S3StorageBackend.php @@ -4,12 +4,12 @@ require_once 'StorageBackend.php'; use Aws\S3\S3Client; -class Amazon_S3 extends StorageBackend +class Amazon_S3StorageBackend extends StorageBackend { private $s3Client; - public function Amazon_S3($securityCredentials) + public function Amazon_S3StorageBackend($securityCredentials) { $this->setBucket($securityCredentials['bucket']); $this->setAccessKey($securityCredentials['api_key']); diff --git a/airtime_mvc/application/cloud_storage/FileStorageBackend.php b/airtime_mvc/application/cloud_storage/FileStorageBackend.php new file mode 100644 index 000000000..65df4d55b --- /dev/null +++ b/airtime_mvc/application/cloud_storage/FileStorageBackend.php @@ -0,0 +1,41 @@ +storageBackend = new $storageBackend($CC_CONFIG[$storageBackend]); + if ($storageBackend == "amazon_S3") { + $this->storageBackend = new Amazon_S3StorageBackend($CC_CONFIG["amazon_S3"]); + } else if ($storageBackend == "file") { + $this->storageBackend = new FileStorageBackend(); + } else { + $this->storageBackend = new $storageBackend($CC_CONFIG[$storageBackend]); + } } - + public function getAbsoluteFilePath($resourceId) { return $this->storageBackend->getAbsoluteFilePath($resourceId); } - + public function getSignedURL($resourceId) { return $this->storageBackend->getSignedURL($resourceId); } - + public function getFileSize($resourceId) { return $this->storageBackend->getFileSize($resourceId); } - + public function deletePhysicalFile($resourceId) { $this->storageBackend->deletePhysicalFile($resourceId); } -} + public function deleteAllCloudFileObjects() + { + $this->storageBackend->deleteAllCloudFileObjects(); + } + + public function getFilePrefix() + { + return $this->storageBackend->getFilePrefix(); + } +} \ No newline at end of file diff --git a/airtime_mvc/application/cloud_storage/StorageBackend.php b/airtime_mvc/application/cloud_storage/StorageBackend.php index 84a9a8d72..a9547d3ee 100644 --- a/airtime_mvc/application/cloud_storage/StorageBackend.php +++ b/airtime_mvc/application/cloud_storage/StorageBackend.php @@ -52,4 +52,14 @@ abstract class StorageBackend { $this->secretKey = $secretKey; } + + public function deleteAllCloudFileObjects() + { + return false; + } + + public function getFilePrefix() + { + return ""; + } } diff --git a/airtime_mvc/application/configs/conf.php b/airtime_mvc/application/configs/conf.php index 6b6273a22..3bd092a1e 100644 --- a/airtime_mvc/application/configs/conf.php +++ b/airtime_mvc/application/configs/conf.php @@ -27,17 +27,22 @@ class Config { // Parse separate conf file for cloud storage values $cloudStorageConfig = isset($_SERVER['CLOUD_STORAGE_CONF']) ? $_SERVER['CLOUD_STORAGE_CONF'] : "/etc/airtime-saas/cloud_storage.conf"; - $cloudStorageValues = parse_ini_file($cloudStorageConfig, true); - - $supportedStorageBackends = array('amazon_S3'); - foreach ($supportedStorageBackends as $backend) { - $CC_CONFIG[$backend] = $cloudStorageValues[$backend]; + $cloudStorageValues = @parse_ini_file($cloudStorageConfig, true); + if ($cloudStorageValues !== false) { + $supportedStorageBackends = array('amazon_S3'); + foreach ($supportedStorageBackends as $backend) { + $CC_CONFIG[$backend] = $cloudStorageValues[$backend]; + } + // Tells us where file uploads will be uploaded to. + // It will either be set to a cloud storage backend or local file storage. + $CC_CONFIG["current_backend"] = $cloudStorageValues["current_backend"]["storage_backend"]; + + } else { + //Default to file storage if we didn't find a cloud_storage.conf + $CC_CONFIG["current_backend"] = "file"; } - - // Tells us where file uploads will be uploaded to. - // It will either be set to a cloud storage backend or local file storage. - $CC_CONFIG["current_backend"] = $cloudStorageValues["current_backend"]["storage_backend"]; - + + $values = parse_ini_file($filename, true); // Name of the web server user diff --git a/airtime_mvc/application/controllers/ApiController.php b/airtime_mvc/application/controllers/ApiController.php index c5d76cf06..9da4b4add 100644 --- a/airtime_mvc/application/controllers/ApiController.php +++ b/airtime_mvc/application/controllers/ApiController.php @@ -93,100 +93,12 @@ class ApiController extends Zend_Controller_Action $fileId = $this->_getParam("file"); - $media = Application_Model_StoredFile::RecallById($fileId); - if ($media != null) { - // Make sure we don't have some wrong result beecause of caching - clearstatcache(); - - if ($media->getPropelOrm()->isValidPhysicalFile()) { - $filename = $media->getPropelOrm()->getFilename(); - - //Download user left clicks a track and selects Download. - if ("true" == $this->_getParam('download')) { - //path_info breaks up a file path into seperate pieces of informaiton. - //We just want the basename which is the file name with the path - //information stripped away. We are using Content-Disposition to specify - //to the browser what name the file should be saved as. - header('Content-Disposition: attachment; filename="'.$filename.'"'); - } else { - //user clicks play button for track preview - header('Content-Disposition: inline; filename="'.$filename.'"'); - } - - $this->smartReadFile($media); - exit; - } else { - header ("HTTP/1.1 404 Not Found"); - } - } + $inline = !($this->_getParam('download',false) == true); + Application_Service_MediaService::streamFileDownload($fileId, $inline); $this->_helper->json->sendJson(array()); } - /** - * Reads the requested portion of a file and sends its contents to the client with the appropriate headers. - * - * This HTTP_RANGE compatible read file function is necessary for allowing streaming media to be skipped around in. - * - * @param string $location - * @param string $mimeType - * @return void - * - * @link https://groups.google.com/d/msg/jplayer/nSM2UmnSKKA/Hu76jDZS4xcJ - * @link http://php.net/manual/en/function.readfile.php#86244 - */ - public function smartReadFile($media) - { - $filepath = $media->getFilePath(); - $size= $media->getFileSize(); - $mimeType = $media->getPropelOrm()->getDbMime(); - - $fm = @fopen($filepath, 'rb'); - if (!$fm) { - header ("HTTP/1.1 505 Internal server error"); - - return; - } - - $begin = 0; - $end = $size - 1; - - if (isset($_SERVER['HTTP_RANGE'])) { - if (preg_match('/bytes=\h*(\d+)-(\d*)[\D.*]?/i', $_SERVER['HTTP_RANGE'], $matches)) { - $begin = intval($matches[1]); - if (!empty($matches[2])) { - $end = intval($matches[2]); - } - } - } - - if (isset($_SERVER['HTTP_RANGE'])) { - header('HTTP/1.1 206 Partial Content'); - } else { - header('HTTP/1.1 200 OK'); - } - header("Content-Type: $mimeType"); - header('Cache-Control: public, must-revalidate, max-age=0'); - header('Pragma: no-cache'); - header('Accept-Ranges: bytes'); - header('Content-Length:' . (($end - $begin) + 1)); - if (isset($_SERVER['HTTP_RANGE'])) { - header("Content-Range: bytes $begin-$end/$size"); - } - header("Content-Transfer-Encoding: binary"); - - //We can have multiple levels of output buffering. Need to - //keep looping until all have been disabled!!! - //http://www.php.net/manual/en/function.ob-end-flush.php - while (@ob_end_flush()); - - // NOTE: We can't use fseek here because it does not work with streams - // (a.k.a. Files stored in the cloud) - while(!feof($fm) && (connection_status() == 0)) { - echo fread($fm, 1024 * 8); - } - fclose($fm); - } //Used by the SaaS monitoring public function onAirLightAction() diff --git a/airtime_mvc/application/controllers/AudiopreviewController.php b/airtime_mvc/application/controllers/AudiopreviewController.php index e90c34a39..60c01ba8c 100644 --- a/airtime_mvc/application/controllers/AudiopreviewController.php +++ b/airtime_mvc/application/controllers/AudiopreviewController.php @@ -22,8 +22,6 @@ class AudiopreviewController extends Zend_Controller_Action $CC_CONFIG = Config::getConfig(); $audioFileID = $this->_getParam('audioFileID'); - $audioFileArtist = $this->_getParam('audioFileArtist'); - $audioFileTitle = $this->_getParam('audioFileTitle'); $type = $this->_getParam('type'); $baseUrl = Application_Common_OsPath::getBaseDir(); @@ -60,10 +58,9 @@ class AudiopreviewController extends Zend_Controller_Action $this->view->uri = $uri; $this->view->mime = $mime; $this->view->audioFileID = $audioFileID; - // We need to decode artist and title because it gets - // encoded twice in js - $this->view->audioFileArtist = htmlspecialchars(urldecode($audioFileArtist)); - $this->view->audioFileTitle = htmlspecialchars(urldecode($audioFileTitle)); + + $this->view->audioFileArtist = htmlspecialchars($media->getPropelOrm()->getDbArtistName()); + $this->view->audioFileTitle = htmlspecialchars($media->getPropelOrm()->getDbTrackTitle()); $this->view->type = $type; $this->_helper->viewRenderer->setRender('audio-preview'); diff --git a/airtime_mvc/application/models/Schedule.php b/airtime_mvc/application/models/Schedule.php index 53f156b24..a40823a5b 100644 --- a/airtime_mvc/application/models/Schedule.php +++ b/airtime_mvc/application/models/Schedule.php @@ -947,7 +947,6 @@ SQL; $baseUrl = Application_Common_OsPath::getBaseDir(); $filesize = $file->getFileSize(); - self::createFileScheduleEvent($data, $item, $media_id, $uri, $filesize); } elseif (!is_null($item['stream_id'])) { diff --git a/airtime_mvc/application/models/airtime/CcFiles.php b/airtime_mvc/application/models/airtime/CcFiles.php index 04a9e3b95..1e3a72dce 100644 --- a/airtime_mvc/application/models/airtime/CcFiles.php +++ b/airtime_mvc/application/models/airtime/CcFiles.php @@ -57,7 +57,7 @@ class CcFiles extends BaseCcFiles { * Retrieve a sanitized version of the file metadata, suitable for public access. * @param $fileId */ - public static function getSantiziedFileById($fileId) + public static function getSanitizedFileById($fileId) { $file = CcFilesQuery::create()->findPk($fileId); if ($file) { @@ -114,7 +114,7 @@ class CcFiles extends BaseCcFiles { $file->setDbHidden(true); $file->save(); - $callbackUrl = Application_Common_HTTPHelper::getStationUrl() . "/rest/media/" . $file->getPrimaryKey(); + $callbackUrl = Application_Common_HTTPHelper::getStationUrl() . "rest/media/" . $file->getPrimaryKey(); Application_Service_MediaService::processUploadedFile($callbackUrl, $relativePath, self::getOwnerId()); return CcFiles::sanitizeResponse($file); @@ -138,14 +138,11 @@ class CcFiles extends BaseCcFiles { { $file = CcFilesQuery::create()->findPk($fileId); - // Since we check for this value when deleting files, set it first - $file->setDbDirectory(self::MUSIC_DIRS_STOR_PK); - $fileArray = self::removeBlacklistedFields($fileArray); $fileArray = self::stripTimeStampFromYearTag($fileArray); self::validateFileArray($fileArray); - if ($file && isset($requestData["resource_id"])) { + if ($file && isset($fileArray["resource_id"])) { $file->fromArray($fileArray, BasePeer::TYPE_FIELDNAME); @@ -155,9 +152,10 @@ class CcFiles extends BaseCcFiles { $fileSizeBytes = $fileArray["filesize"]; if (!isset($fileSizeBytes) || $fileSizeBytes === false) { - $file->setDbImportStatus(2)->save(); + $file->setDbImportStatus(self::IMPORT_STATUS_FAILED)->save(); throw new FileNotFoundException(); } + $cloudFile = new CloudFile(); $cloudFile->setStorageBackend($fileArray["storage_backend"]); $cloudFile->setResourceId($fileArray["resource_id"]); @@ -172,6 +170,9 @@ class CcFiles extends BaseCcFiles { } else if ($file) { + // Since we check for this value when deleting files, set it first + $file->setDbDirectory(self::MUSIC_DIRS_STOR_PK); + $file->fromArray($fileArray, BasePeer::TYPE_FIELDNAME); //Our RESTful API takes "full_path" as a field, which we then split and translate to match @@ -252,6 +253,7 @@ class CcFiles extends BaseCcFiles { } } + private static function validateFileArray(&$fileArray) { // Sanitize any wildly incorrect metadata before it goes to be validated @@ -342,6 +344,42 @@ class CcFiles extends BaseCcFiles { return $response; } + /** + * Returns the file size in bytes. + */ + public function getFileSize() + { + return filesize($this->getAbsoluteFilePath()); + } + + public function getFilename() + { + $info = pathinfo($this->getAbsoluteFilePath()); + return $info['filename']; + } + + /** + * Returns the file's absolute file path stored on disk. + */ + public function getURLForTrackPreviewOrDownload() + { + return $this->getAbsoluteFilePath(); + } + + /** + * Returns the file's absolute file path stored on disk. + */ + public function getAbsoluteFilePath() + { + $music_dir = Application_Model_MusicDir::getDirByPK($this->getDbDirectory()); + if (!$music_dir) { + throw new Exception("Invalid music_dir for file in database."); + } + $directory = $music_dir->getDirectory(); + $filepath = $this->getDbFilepath(); + return Application_Common_OsPath::join($directory, $filepath); + } + /** * * Strips out fields from incoming request data that should never be modified @@ -421,43 +459,7 @@ class CcFiles extends BaseCcFiles { exec("find $path -empty -type d -delete"); } - /** - * Returns the file size in bytes. - */ - public function getFileSize() - { - return filesize($this->getAbsoluteFilePath()); - } - - public function getFilename() - { - $info = pathinfo($this->getAbsoluteFilePath()); - return $info['filename']; - } - - /** - * Returns the file's absolute file path stored on disk. - */ - public function getURLForTrackPreviewOrDownload() - { - return $this->getAbsoluteFilePath(); - } - - /** - * Returns the file's absolute file path stored on disk. - */ - public function getAbsoluteFilePath() - { - $music_dir = Application_Model_MusicDir::getDirByPK($this->getDbDirectory()); - if (!$music_dir) { - throw new Exception("Invalid music_dir for file in database."); - } - $directory = $music_dir->getDirectory(); - $filepath = $this->getDbFilepath(); - return Application_Common_OsPath::join($directory, $filepath); - } - /** * Checks if the file is a regular file that can be previewed and downloaded. */ diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index c6e26b6f9..0df6b6b3e 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -41,8 +41,9 @@ class Rest_MediaController extends Zend_Rest_Controller try { $this->getResponse() - ->setHttpResponseCode(200) - ->appendBody($this->_redirect(CcFiles::getDownloadUrl($id))); + ->setHttpResponseCode(200); + $inline = false; + Application_Service_MediaService::streamFileDownload($id, $inline); } catch (FileNotFoundException $e) { $this->fileNotFoundResponse(); @@ -64,7 +65,7 @@ class Rest_MediaController extends Zend_Rest_Controller try { $this->getResponse() ->setHttpResponseCode(200) - ->appendBody(json_encode(CcFiles::getSantiziedFileById($id))); + ->appendBody(json_encode(CcFiles::getSanitizedFileById($id))); } catch (FileNotFoundException $e) { $this->fileNotFoundResponse(); diff --git a/airtime_mvc/application/services/MediaService.php b/airtime_mvc/application/services/MediaService.php index 74b2609d0..81f199ae7 100644 --- a/airtime_mvc/application/services/MediaService.php +++ b/airtime_mvc/application/services/MediaService.php @@ -1,5 +1,7 @@ getDirectory() . "/imported/" . $ownerId; + $importedStorageDirectory = ""; + if ($CC_CONFIG["current_backend"] == "file") { + $storDir = Application_Model_MusicDir::getStorDir(); + $importedStorageDirectory = $storDir->getDirectory() . "/imported/" . $ownerId; + } try { //Copy the temporary file over to the "organize" folder so that it's off our webserver @@ -34,8 +39,121 @@ class Application_Service_MediaService //Dispatch a message to airtime_analyzer through RabbitMQ, //notifying it that there's a new upload to process! + $storageBackend = new ProxyStorageBackend($CC_CONFIG["current_backend"]); Application_Model_RabbitMq::SendMessageToAnalyzer($newTempFilePath, $importedStorageDirectory, basename($originalFilename), - $callbackUrl, $apiKey); + $callbackUrl, $apiKey, + $CC_CONFIG["current_backend"], + $storageBackend->getFilePrefix()); } -} \ No newline at end of file + + + /** + * @param $fileId + * @param bool $inline Set the Content-Disposition header to inline to prevent a download dialog from popping up (or attachment if false) + * @throws Exception + * @throws FileNotFoundException + */ + public static function streamFileDownload($fileId, $inline=false) + { + $media = Application_Model_StoredFile::RecallById($fileId); + if ($media == null) { + throw new FileNotFoundException(); + } + $filepath = $media->getFilePath(); + // Make sure we don't have some wrong result beecause of caching + clearstatcache(); + $media = Application_Model_StoredFile::RecallById($fileId); + if ($media == null) { + throw new FileNotFoundException(); + } + + // Make sure we don't have some wrong result beecause of caching + clearstatcache(); + + if ($media->getPropelOrm()->isValidPhysicalFile()) { + $filename = $media->getPropelOrm()->getFilename(); + + //Download user left clicks a track and selects Download. + if (!$inline) { + //We are using Content-Disposition to specify + //to the browser what name the file should be saved as. + header('Content-Disposition: attachment; filename="' . $filename . '"'); + } else { + //user clicks play button for track and downloads it. + header('Content-Disposition: inline; filename="' . $filename . '"'); + } + + self::smartReadFile($media); + exit; + } else { + throw new FileNotFoundException(); + } + } + + + /** + * Reads the requested portion of a file and sends its contents to the client with the appropriate headers. + * + * This HTTP_RANGE compatible read file function is necessary for allowing streaming media to be skipped around in. + * + * @param CcFile $media + * @return void + * + * @link https://groups.google.com/d/msg/jplayer/nSM2UmnSKKA/Hu76jDZS4xcJ + * @link http://php.net/manual/en/function.readfile.php#86244 + */ + private static function smartReadFile($media) + { + $filepath = $media->getFilePath(); + $size= $media->getFileSize(); + $mimeType = $media->getPropelOrm()->getDbMime(); + + $fm = @fopen($filepath, 'rb'); + if (!$fm) { + header ("HTTP/1.1 505 Internal server error"); + + return; + } + + $begin = 0; + $end = $size - 1; + + if (isset($_SERVER['HTTP_RANGE'])) { + if (preg_match('/bytes=\h*(\d+)-(\d*)[\D.*]?/i', $_SERVER['HTTP_RANGE'], $matches)) { + $begin = intval($matches[1]); + if (!empty($matches[2])) { + $end = intval($matches[2]); + } + } + } + + if (isset($_SERVER['HTTP_RANGE'])) { + header('HTTP/1.1 206 Partial Content'); + } else { + header('HTTP/1.1 200 OK'); + } + header("Content-Type: $mimeType"); + header('Cache-Control: public, must-revalidate, max-age=0'); + header('Pragma: no-cache'); + header('Accept-Ranges: bytes'); + header('Content-Length:' . (($end - $begin) + 1)); + if (isset($_SERVER['HTTP_RANGE'])) { + header("Content-Range: bytes $begin-$end/$size"); + } + header("Content-Transfer-Encoding: binary"); + + //We can have multiple levels of output buffering. Need to + //keep looping until all have been disabled!!! + //http://www.php.net/manual/en/function.ob-end-flush.php + while (@ob_end_flush()); + + // NOTE: We can't use fseek here because it does not work with streams + // (a.k.a. Files stored in the cloud) + while(!feof($fm) && (connection_status() == 0)) { + echo fread($fm, 1024 * 8); + } + fclose($fm); + } +} + diff --git a/airtime_mvc/public/js/airtime/common/common.js b/airtime_mvc/public/js/airtime/common/common.js index 7a043c286..476b5e77d 100644 --- a/airtime_mvc/public/js/airtime/common/common.js +++ b/airtime_mvc/public/js/airtime/common/common.js @@ -91,16 +91,11 @@ function openAudioPreview(p_event) { } } -function open_audio_preview(type, id, audioFileTitle, audioFileArtist) { - // we need to remove soundcloud icon from audioFileTitle - var index = audioFileTitle.indexOf(" */ - public static function copyFileToStor($tempFilePath, $originalFilename) + public static function moveFileToStor($tempFilePath, $originalFilename, $copyFile=false) { $audio_file = $tempFilePath; - Logging::info('copyFileToStor: moving file '.$audio_file); - + $storDir = Application_Model_MusicDir::getStorDir(); $stor = $storDir->getDirectory(); // check if "organize" dir exists and if not create one @@ -966,55 +964,35 @@ SQL; Logging::info("Warning: couldn't change permissions of $audio_file to 0644"); } - // Check if liquidsoap can play this file - // TODO: Move this to airtime_analyzer - /* - if (!self::liquidsoapFilePlayabilityTest($audio_file)) { - return array( - "code" => 110, - "message" => _("This file appears to be corrupted and will not " - ."be added to media library.")); - }*/ - - // Did all the checks for real, now trying to copy $audio_stor = Application_Common_OsPath::join($stor, "organize", $originalFilename); - $user = Application_Model_User::getCurrentUser(); - if (is_null($user)) { - $uid = Application_Model_User::getFirstAdminId(); - } else { - $uid = $user->getId(); - } - /* - $id_file = "$audio_stor.identifier"; - if (file_put_contents($id_file, $uid) === false) { - Logging::info("Could not write file to identify user: '$uid'"); - Logging::info("Id file path: '$id_file'"); - Logging::info("Defaulting to admin (no identification file was - written)"); - } else { - Logging::info("Successfully written identification file for - uploaded '$audio_stor'"); - }*/ - + //if the uploaded file is not UTF-8 encoded, let's encode it. Assuming source //encoding is ISO-8859-1 $audio_stor = mb_detect_encoding($audio_stor, "UTF-8") == "UTF-8" ? $audio_stor : utf8_encode($audio_stor); - Logging::info("copyFileToStor: moving file $audio_file to $audio_stor"); - // Martin K.: changed to rename: Much less load + quicker since this is - // an atomic operation - if (@rename($audio_file, $audio_stor) === false) { - //something went wrong likely there wasn't enough space in . - //the audio_stor to move the file too warn the user that . - //the file wasn't uploaded and they should check if there . - //is enough disk space . - unlink($audio_file); //remove the file after failed rename - //unlink($id_file); // Also remove the identifier file - - throw new Exception("The file was not uploaded, this error can occur if the computer " - ."hard drive does not have enough disk space or the stor " - ."directory does not have correct write permissions."); + if ($copyFile) { + Logging::info("Copying file $audio_file to $audio_stor"); + if (@copy($audio_file, $audio_stor) === false) { + throw new Exception("Failed to copy $audio_file to $audio_stor"); + } + } else { + Logging::info("Moving file $audio_file to $audio_stor"); + + // Martin K.: changed to rename: Much less load + quicker since this is + // an atomic operation + if (@rename($audio_file, $audio_stor) === false) { + //something went wrong likely there wasn't enough space in . + //the audio_stor to move the file too warn the user that . + //the file wasn't uploaded and they should check if there . + //is enough disk space . + unlink($audio_file); //remove the file after failed rename + //unlink($id_file); // Also remove the identifier file + + throw new Exception("The file was not uploaded, this error can occur if the computer " + . "hard drive does not have enough disk space or the stor " + . "directory does not have correct write permissions."); + } } return $audio_stor; } diff --git a/airtime_mvc/application/models/airtime/CcFiles.php b/airtime_mvc/application/models/airtime/CcFiles.php index 1e3a72dce..8463e2542 100644 --- a/airtime_mvc/application/models/airtime/CcFiles.php +++ b/airtime_mvc/application/models/airtime/CcFiles.php @@ -29,6 +29,10 @@ class CcFiles extends BaseCcFiles { const MUSIC_DIRS_STOR_PK = 1; + const IMPORT_STATUS_SUCCESS = 0; + const IMPORT_STATUS_PENDING = 1; + const IMPORT_STATUS_FAILED = 2; + //fields that are not modifiable via our RESTful API private static $blackList = array( @@ -78,57 +82,104 @@ class CcFiles extends BaseCcFiles { throw new OverDiskQuotaException(); } - $file = new CcFiles(); + /* If full_path is set, the post request came from ftp. + * Users are allowed to upload folders via ftp. If this is the case + * we need to include the folder name with the file name, otherwise + * files won't get removed from the organize folder. + */ - try{ - $fileArray = self::removeBlacklistedFields($fileArray); + //Extract the relative path to the temporary uploaded file on disk. + if (isset($fileArray["full_path"])) { + $fullPath = $fileArray["full_path"]; + $basePath = isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/organize/" : "/srv/airtime/stor/organize/"; + //$relativePath is the folder name(if one) + track name, that was uploaded via ftp + $filePathRelativeToOrganize = substr($fullPath, strlen($basePath)-1); + $originalFilename = $filePathRelativeToOrganize; + } else { + //Extract the original filename, which we set as the temporary title for the track + //until it's finished being processed by the analyzer. + $originalFilename = $_FILES["file"]["name"]; + } - /*if (!self::validateFileArray($fileArray)) - { - $file->setDbTrackTitle($_FILES["file"]["name"]); - $file->setDbUtime(new DateTime("now", new DateTimeZone("UTC"))); - $file->save(); - return CcFiles::sanitizeResponse($file);*/ - self::validateFileArray($fileArray); + $tempFilePath = $_FILES['file']['tmp_name']; - /* If full_path is set, the post request came from ftp. - * Users are allowed to upload folders via ftp. If this is the case - * we need to include the folder name with the file name, otherwise - * files won't get removed from the organize folder. - */ - if (isset($fileArray["full_path"])) { - $fullPath = $fileArray["full_path"]; - $basePath = isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/organize/" : "/srv/airtime/stor/organize/"; - //$relativePath is the folder name(if one) + track name, that was uploaded via ftp - $relativePath = substr($fullPath, strlen($basePath)-1); - } else { - $relativePath = $_FILES["file"]["name"]; - } - - - $file->fromArray($fileArray); - $file->setDbOwnerId(self::getOwnerId()); - $now = new DateTime("now", new DateTimeZone("UTC")); - $file->setDbTrackTitle($_FILES["file"]["name"]); - $file->setDbUtime($now); - $file->setDbHidden(true); - $file->save(); - - $callbackUrl = Application_Common_HTTPHelper::getStationUrl() . "rest/media/" . $file->getPrimaryKey(); - - Application_Service_MediaService::processUploadedFile($callbackUrl, $relativePath, self::getOwnerId()); - return CcFiles::sanitizeResponse($file); - - } catch (Exception $e) { - $file->setDbImportStatus(2); - $file->setDbHidden(true); + try { + self::createAndImport($fileArray, $tempFilePath, $originalFilename); + } catch (Exception $e) + { + @unlink($tempFilePath); throw $e; } } + /** Import a music file to the library from a local file on disk (something pre-existing). + * This function allows you to copy a file rather than move it, which is useful for importing + * static music files (like sample tracks). + * @param string $filePath The full path to the audio file to import. + * @param bool $copyFile True if you want to just copy the false, false if you want to move it (default false) + * @throws Exception + */ + public static function createFromLocalFile($filePath, $copyFile=false) + { + $fileArray = array(); + $info = pathinfo($filePath); + $fileName = basename($filePath).'.'.$info['extension']; + self::createAndImport($fileArray, $filePath, $fileName, $copyFile); + } + + /** Create a new CcFiles object/row and import a file for it. + * You shouldn't call this directly. Either use createFromUpload() or createFromLocalFile(). + * @param array $fileArray Any metadata to pre-fill for the audio file + * @param string $filePath The full path to the audio file to import + * @param string $originalFilename + * @param bool $copyFile + * @return mixed + * @throws Exception + * @throws PropelException + */ + private static function createAndImport($fileArray, $filePath, $originalFilename, $copyFile=false) + { + $file = new CcFiles(); + + try + { + $fileArray = self::removeBlacklistedFields($fileArray); + + self::validateFileArray($fileArray); + + $file->fromArray($fileArray); + $file->setDbOwnerId(self::getOwnerId()); + $now = new DateTime("now", new DateTimeZone("UTC")); + $file->setDbTrackTitle($originalFilename); + $file->setDbUtime($now); + $file->setDbHidden(true); + $file->save(); + + //Only accept files with a file extension that we support. + $fileExtension = pathinfo($originalFilename, PATHINFO_EXTENSION); + if (!in_array(strtolower($fileExtension), explode(",", "ogg,mp3,oga,flac,wav,m4a,mp4,opus"))) { + throw new Exception("Bad file extension."); + } + + $callbackUrl = Application_Common_HTTPHelper::getStationUrl() . "rest/media/" . $file->getPrimaryKey(); + + Application_Service_MediaService::importFileToLibrary($callbackUrl, $filePath, + $originalFilename, self::getOwnerId(), $copyFile); + + return CcFiles::sanitizeResponse($file); + + } catch (Exception $e) { + $file->setDbImportStatus(self::IMPORT_STATUS_FAILED); + $file->setDbHidden(true); + $file->save(); + throw $e; + } + } + + /** Update a file with metadata specified in an array. - * @param $fileId The ID of the file to update in the DB. - * @param $fileArray An associative array containing metadata. Replaces those fields if they exist. + * @param $fileId string The ID of the file to update in the DB. + * @param $fileArray array An associative array containing metadata. Replaces those fields if they exist. * @return array A sanitized version of the file metadata array. * @throws Exception * @throws FileNotFoundException @@ -238,22 +289,6 @@ class CcFiles extends BaseCcFiles { } - public static function getDownloadUrl($id) - { - $file = CcFilesQuery::create()->findPk($id); - if ($file) { - $con = Propel::getConnection(); - $storedFile = new Application_Model_StoredFile($file, $con); - $baseDir = Application_Common_OsPath::getBaseDir(); - - return $storedFile->getRelativeFileUrl($baseDir) . '/download/true'; - } - else { - throw new FileNotFoundException(); - } - } - - private static function validateFileArray(&$fileArray) { // Sanitize any wildly incorrect metadata before it goes to be validated @@ -330,7 +365,7 @@ class CcFiles extends BaseCcFiles { /** * * Strips out the private fields we do not want to send back in API responses - * @param $file a CcFiles object + * @param $file string a CcFiles object */ //TODO: rename this function? public static function sanitizeResponse($file) diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index 0df6b6b3e..1735751dc 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -2,10 +2,6 @@ class Rest_MediaController extends Zend_Rest_Controller { - const IMPORT_STATUS_SUCCESS = 0; - const IMPORT_STATUS_PENDING = 1; - const IMPORT_STATUS_FAILED = 2; - public function init() { diff --git a/airtime_mvc/application/services/MediaService.php b/airtime_mvc/application/services/MediaService.php index 81f199ae7..037b88de2 100644 --- a/airtime_mvc/application/services/MediaService.php +++ b/airtime_mvc/application/services/MediaService.php @@ -4,38 +4,30 @@ require_once('ProxyStorageBackend.php'); class Application_Service_MediaService { - public static function processUploadedFile($callbackUrl, $originalFilename, $ownerId) + /** Move (or copy) a file to the stor/organize directory and send it off to the + analyzer to be processed. + * @param $callbackUrl + * @param $filePath string Path to the local file to import to the library + * @param $originalFilename string The original filename, if you want it to be preserved after import. + * @param $ownerId string The ID of the user that will own the file inside Airtime. + * @param $copyFile bool True if you want to copy the file to the "organize" directory, false if you want to move it (default) + * @return Ambigous + * @throws Exception + */ + public static function importFileToLibrary($callbackUrl, $filePath, $originalFilename, $ownerId, $copyFile) { $CC_CONFIG = Config::getConfig(); $apiKey = $CC_CONFIG["apiKey"][0]; - $tempFilePath = $_FILES['file']['tmp_name']; - $tempFileName = basename($tempFilePath); - - //Only accept files with a file extension that we support. - $fileExtension = pathinfo($originalFilename, PATHINFO_EXTENSION); - if (!in_array(strtolower($fileExtension), explode(",", "ogg,mp3,oga,flac,wav,m4a,mp4,opus"))) { - @unlink($tempFilePath); - throw new Exception("Bad file extension."); - } - - //TODO: Remove uploadFileAction from ApiController.php **IMPORTANT** - It's used by the recorder daemon... - $importedStorageDirectory = ""; if ($CC_CONFIG["current_backend"] == "file") { $storDir = Application_Model_MusicDir::getStorDir(); $importedStorageDirectory = $storDir->getDirectory() . "/imported/" . $ownerId; } - try { - //Copy the temporary file over to the "organize" folder so that it's off our webserver - //and accessible by airtime_analyzer which could be running on a different machine. - $newTempFilePath = Application_Model_StoredFile::copyFileToStor($tempFilePath, $originalFilename); - } catch (Exception $e) { - @unlink($tempFilePath); - Logging::error($e->getMessage()); - return; - } + //Copy the temporary file over to the "organize" folder so that it's off our webserver + //and accessible by airtime_analyzer which could be running on a different machine. + $newTempFilePath = Application_Model_StoredFile::moveFileToStor($filePath, $originalFilename, $copyFile); //Dispatch a message to airtime_analyzer through RabbitMQ, //notifying it that there's a new upload to process! @@ -45,6 +37,8 @@ class Application_Service_MediaService $callbackUrl, $apiKey, $CC_CONFIG["current_backend"], $storageBackend->getFilePrefix()); + + return $newTempFilePath; }