From ff0a685243e684460200a518c36b3fd2eb26ca46 Mon Sep 17 00:00:00 2001 From: drigato Date: Tue, 16 Dec 2014 14:20:28 -0500 Subject: [PATCH 1/4] Updated propel models --- .../application/configs/classmap-airtime-conf.php | 7 ------- airtime_mvc/build/sql/schema.sql | 11 ----------- 2 files changed, 18 deletions(-) diff --git a/airtime_mvc/application/configs/classmap-airtime-conf.php b/airtime_mvc/application/configs/classmap-airtime-conf.php index 0c279aad1..814429f76 100644 --- a/airtime_mvc/application/configs/classmap-airtime-conf.php +++ b/airtime_mvc/application/configs/classmap-airtime-conf.php @@ -22,9 +22,6 @@ return array ( 'BaseCcLiveLog' => 'airtime/om/BaseCcLiveLog.php', 'BaseCcLiveLogPeer' => 'airtime/om/BaseCcLiveLogPeer.php', 'BaseCcLiveLogQuery' => 'airtime/om/BaseCcLiveLogQuery.php', - 'BaseCcLocale' => 'airtime/om/BaseCcLocale.php', - 'BaseCcLocalePeer' => 'airtime/om/BaseCcLocalePeer.php', - 'BaseCcLocaleQuery' => 'airtime/om/BaseCcLocaleQuery.php', 'BaseCcLoginAttempts' => 'airtime/om/BaseCcLoginAttempts.php', 'BaseCcLoginAttemptsPeer' => 'airtime/om/BaseCcLoginAttemptsPeer.php', 'BaseCcLoginAttemptsQuery' => 'airtime/om/BaseCcLoginAttemptsQuery.php', @@ -134,10 +131,6 @@ return array ( 'CcLiveLogPeer' => 'airtime/CcLiveLogPeer.php', 'CcLiveLogQuery' => 'airtime/CcLiveLogQuery.php', 'CcLiveLogTableMap' => 'airtime/map/CcLiveLogTableMap.php', - 'CcLocale' => 'airtime/CcLocale.php', - 'CcLocalePeer' => 'airtime/CcLocalePeer.php', - 'CcLocaleQuery' => 'airtime/CcLocaleQuery.php', - 'CcLocaleTableMap' => 'airtime/map/CcLocaleTableMap.php', 'CcLoginAttempts' => 'airtime/CcLoginAttempts.php', 'CcLoginAttemptsPeer' => 'airtime/CcLoginAttemptsPeer.php', 'CcLoginAttemptsQuery' => 'airtime/CcLoginAttemptsQuery.php', diff --git a/airtime_mvc/build/sql/schema.sql b/airtime_mvc/build/sql/schema.sql index 1efbc0768..b5be33147 100644 --- a/airtime_mvc/build/sql/schema.sql +++ b/airtime_mvc/build/sql/schema.sql @@ -605,17 +605,6 @@ CREATE TABLE "cc_listener_count" ); ----------------------------------------------------------------------- - -DROP TABLE IF EXISTS "cc_locale" CASCADE; - -CREATE TABLE "cc_locale" -( - "id" serial NOT NULL, - "locale_code" VARCHAR(16) NOT NULL, - "locale_lang" VARCHAR(128) NOT NULL, - PRIMARY KEY ("id") -); - -- cc_playout_history ----------------------------------------------------------------------- From 7059820ca01df839b4c23a02d76709edb7f4ab92 Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Tue, 16 Dec 2014 14:54:53 -0500 Subject: [PATCH 2/4] Add a comment about the proxy pattern --- .../application/cloud_storage/ProxyStorageBackend.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airtime_mvc/application/cloud_storage/ProxyStorageBackend.php b/airtime_mvc/application/cloud_storage/ProxyStorageBackend.php index 78aeb1b35..651063367 100644 --- a/airtime_mvc/application/cloud_storage/ProxyStorageBackend.php +++ b/airtime_mvc/application/cloud_storage/ProxyStorageBackend.php @@ -19,7 +19,10 @@ class ProxyStorageBackend extends StorageBackend public function ProxyStorageBackend($storageBackend) { $CC_CONFIG = Config::getConfig(); - + + //The storage backend in the airtime.conf directly corresponds to + //the name of the class that implements it (eg. Amazon_S3), so we + //can easily create the right backend object dynamically: $this->storageBackend = new $storageBackend($CC_CONFIG[$storageBackend]); } From 1de326283ea5229a994b2b5a29cc2589b5a3ed2e Mon Sep 17 00:00:00 2001 From: drigato Date: Tue, 16 Dec 2014 18:47:42 -0500 Subject: [PATCH 3/4] SAAS-527: Allow files to be uploaded to either the cloud or on local file storage Not quite done. --- airtime_mvc/application/configs/conf.php | 4 ++++ airtime_mvc/application/models/RabbitMq.php | 3 ++- .../modules/rest/controllers/MediaController.php | 13 +++++++++---- .../airtime_analyzer/analyzer_pipeline.py | 12 ++++++++---- .../airtime_analyzer/message_listener.py | 9 ++++++--- 5 files changed, 29 insertions(+), 12 deletions(-) diff --git a/airtime_mvc/application/configs/conf.php b/airtime_mvc/application/configs/conf.php index 932834fe3..6b6273a22 100644 --- a/airtime_mvc/application/configs/conf.php +++ b/airtime_mvc/application/configs/conf.php @@ -34,6 +34,10 @@ class Config { $CC_CONFIG[$backend] = $cloudStorageValues[$backend]; } + // Tells us where file uploads will be uploaded to. + // It will either be set to a cloud storage backend or local file storage. + $CC_CONFIG["current_backend"] = $cloudStorageValues["current_backend"]["storage_backend"]; + $values = parse_ini_file($filename, true); // Name of the web server user diff --git a/airtime_mvc/application/models/RabbitMq.php b/airtime_mvc/application/models/RabbitMq.php index b20beca2e..6918abc06 100644 --- a/airtime_mvc/application/models/RabbitMq.php +++ b/airtime_mvc/application/models/RabbitMq.php @@ -80,11 +80,12 @@ class Application_Model_RabbitMq } public static function SendMessageToAnalyzer($tmpFilePath, $importedStorageDirectory, $originalFilename, - $callbackUrl, $apiKey) + $callbackUrl, $apiKey, $currentStorageBackend) { $exchange = 'airtime-uploads'; $data['tmp_file_path'] = $tmpFilePath; + $data['current_storage_backend'] = $currentStorageBackend; $data['import_directory'] = $importedStorageDirectory; $data['original_filename'] = $originalFilename; $data['callback_url'] = $callbackUrl; diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index 8064c7f08..f19700ff1 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -413,9 +413,12 @@ class Rest_MediaController extends Zend_Rest_Controller } //TODO: Remove uploadFileAction from ApiController.php **IMPORTANT** - It's used by the recorder daemon... - - $storDir = Application_Model_MusicDir::getStorDir(); - $importedStorageDirectory = $storDir->getDirectory() . "/imported/" . $ownerId; + + $importedStorageDirectory = ""; + if ($CC_CONFIG["current_backend"] == "file") { + $storDir = Application_Model_MusicDir::getStorDir(); + $importedStorageDirectory = $storDir->getDirectory() . "/imported/" . $ownerId; + } try { //Copy the temporary file over to the "organize" folder so that it's off our webserver @@ -426,12 +429,14 @@ class Rest_MediaController extends Zend_Rest_Controller Logging::error($e->getMessage()); return; } + + Logging::info($importedStorageDirectory); //Dispatch a message to airtime_analyzer through RabbitMQ, //notifying it that there's a new upload to process! Application_Model_RabbitMq::SendMessageToAnalyzer($newTempFilePath, $importedStorageDirectory, basename($originalFilename), - $callbackUrl, $apiKey); + $callbackUrl, $apiKey, $CC_CONFIG["current_backend"]); } private function getOwnerId() diff --git a/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py b/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py index 2dd81c677..a11248066 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py @@ -21,7 +21,7 @@ class AnalyzerPipeline: """ @staticmethod - def run_analysis(queue, audio_file_path, import_directory, original_filename, station_domain): + def run_analysis(queue, audio_file_path, import_directory, original_filename, station_domain, current_storage_backend): """Analyze and import an audio file, and put all extracted metadata into queue. Keyword arguments: @@ -55,15 +55,19 @@ class AnalyzerPipeline: # Analyze the audio file we were told to analyze: # First, we extract the ID3 tags and other metadata: metadata = dict() - metadata = MetadataAnalyzer.analyze(audio_file_path, metadata) metadata["station_domain"] = station_domain + metadata = MetadataAnalyzer.analyze(audio_file_path, metadata) metadata = CuePointAnalyzer.analyze(audio_file_path, metadata) metadata = ReplayGainAnalyzer.analyze(audio_file_path, metadata) metadata = PlayabilityAnalyzer.analyze(audio_file_path, metadata) - csu = CloudStorageUploader() - metadata = csu.upload_obj(audio_file_path, metadata) + if current_storage_backend == "file": + metadata = FileMoverAnalyzer.move(audio_file_path, import_directory, original_filename, metadata) + else: + csu = CloudStorageUploader() + metadata = csu.upload_obj(audio_file_path, metadata) + metadata["import_status"] = 0 # Successfully imported # Note that the queue we're putting the results into is our interprocess communication diff --git a/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py b/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py index b61c2133e..245a74118 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py @@ -150,6 +150,8 @@ class MessageListener: original_filename = "" callback_url = "" api_key = "" + station_domain = "" + current_storage_backend = "" ''' Spin up a worker process. We use the multiprocessing module and multiprocessing.Queue to pass objects between the processes so that if the analyzer process crashes, it does not @@ -166,8 +168,9 @@ class MessageListener: audio_file_path = msg_dict["tmp_file_path"] import_directory = msg_dict["import_directory"] original_filename = msg_dict["original_filename"] + current_storage_backend = msg_dict["current_storage_backend"] - audio_metadata = MessageListener.spawn_analyzer_process(audio_file_path, import_directory, original_filename, station_domain) + audio_metadata = MessageListener.spawn_analyzer_process(audio_file_path, import_directory, original_filename, station_domain, current_storage_backend) StatusReporter.report_success_to_callback_url(callback_url, api_key, audio_metadata) except KeyError as e: @@ -206,11 +209,11 @@ class MessageListener: channel.basic_ack(delivery_tag=method_frame.delivery_tag) @staticmethod - def spawn_analyzer_process(audio_file_path, import_directory, original_filename, station_domain): + def spawn_analyzer_process(audio_file_path, import_directory, original_filename, station_domain, current_storage_backend): ''' Spawn a child process to analyze and import a new audio file. ''' q = multiprocessing.Queue() p = multiprocessing.Process(target=AnalyzerPipeline.run_analysis, - args=(q, audio_file_path, import_directory, original_filename, station_domain)) + args=(q, audio_file_path, import_directory, original_filename, station_domain, current_storage_backend)) p.start() p.join() if p.exitcode == 0: From cdabbc6648c505fa649bc6b88e2ec63176196790 Mon Sep 17 00:00:00 2001 From: drigato Date: Wed, 17 Dec 2014 13:11:11 -0500 Subject: [PATCH 4/4] SAAS-527: Allow files to be uploaded to either the cloud or on local file storage Done. --- .../rest/controllers/MediaController.php | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index f19700ff1..492d7140a 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -215,6 +215,42 @@ class Rest_MediaController extends Zend_Rest_Controller $file->setDbMtime($now); $file->save(); + $this->getResponse() + ->setHttpResponseCode(200) + ->appendBody(json_encode(CcFiles::sanitizeResponse($file))); + } else if ($file) { + //local file storage + $file->setDbDirectory(self::MUSIC_DIRS_STOR_PK); + $file->fromArray($whiteList, BasePeer::TYPE_FIELDNAME); + //Our RESTful API takes "full_path" as a field, which we then split and translate to match + //our internal schema. Internally, file path is stored relative to a directory, with the directory + //as a foreign key to cc_music_dirs. + if (isset($requestData["full_path"])) { + $fileSizeBytes = filesize($requestData["full_path"]); + if (!isset($fileSizeBytes) || $fileSizeBytes === false) + { + $file->setDbImportStatus(self::IMPORT_STATUS_FAILED)->save(); + $this->fileNotFoundResponse(); + return; + } + Application_Model_Preference::updateDiskUsage($fileSizeBytes); + $fullPath = $requestData["full_path"]; + $storDir = Application_Model_MusicDir::getStorDir()->getDirectory(); + $pos = strpos($fullPath, $storDir); + + if ($pos !== FALSE) + { + assert($pos == 0); //Path must start with the stor directory path + + $filePathRelativeToStor = substr($fullPath, strlen($storDir)); + $file->setDbFilepath($filePathRelativeToStor); + } + } + + $now = new DateTime("now", new DateTimeZone("UTC")); + $file->setDbMtime($now); + $file->save(); + $this->getResponse() ->setHttpResponseCode(200) ->appendBody(json_encode(CcFiles::sanitizeResponse($file))); @@ -430,8 +466,6 @@ class Rest_MediaController extends Zend_Rest_Controller return; } - - Logging::info($importedStorageDirectory); //Dispatch a message to airtime_analyzer through RabbitMQ, //notifying it that there's a new upload to process! Application_Model_RabbitMq::SendMessageToAnalyzer($newTempFilePath,