diff --git a/airtime_mvc/application/controllers/AudiopreviewController.php b/airtime_mvc/application/controllers/AudiopreviewController.php index 7f7b7b68d..e90c34a39 100644 --- a/airtime_mvc/application/controllers/AudiopreviewController.php +++ b/airtime_mvc/application/controllers/AudiopreviewController.php @@ -48,7 +48,6 @@ class AudiopreviewController extends Zend_Controller_Action if ($type == "audioclip") { $media = Application_Model_StoredFile::RecallById($audioFileID); $uri = $baseUrl."api/get-media/file/".$audioFileID; - //$uri = $media->getPropelOrm()->downloadFile(); $mime = $media->getPropelOrm()->getDbMime(); } elseif ($type == "stream") { $webstream = CcWebstreamQuery::create()->findPk($audioFileID); diff --git a/airtime_mvc/application/models/StoredFile.php b/airtime_mvc/application/models/StoredFile.php index a45e63e99..c3b72b3a2 100644 --- a/airtime_mvc/application/models/StoredFile.php +++ b/airtime_mvc/application/models/StoredFile.php @@ -368,8 +368,6 @@ SQL; */ public function delete() { - $filepath = $this->getFilePath(); - // Check if the file is scheduled to be played in the future if (Application_Model_Schedule::IsFileScheduledInTheFuture($this->getId())) { throw new DeleteScheduledFileException(); @@ -387,6 +385,8 @@ SQL; //try { //Delete the physical file from either the local stor directory //or from the cloud + // TODO: don't have deletePhysicalFile return the filesize. + // Instead, fetch that value before deleting the file. $filesize = $this->_file->deletePhysicalFile(); //Update the user's disk usage diff --git a/airtime_mvc/application/models/airtime/CloudFile.php b/airtime_mvc/application/models/airtime/CloudFile.php index ab7dfadaa..b815d4ed3 100644 --- a/airtime_mvc/application/models/airtime/CloudFile.php +++ b/airtime_mvc/application/models/airtime/CloudFile.php @@ -28,21 +28,22 @@ class CloudFile extends BaseCloudFile { //should be longer than track length $expires = 120; - $resource = $this->getResourceId(); + $resource_id = $this->getResourceId(); $expires = time()+$expires; - $string_to_sign = "GET\n\n\n{$expires}\n/{$bucket}/{$resource}"; - $signature = base64_encode((hash_hmac("sha1", utf8_encode($string_to_sign), $s3_key_secret, TRUE))); + $string_to_sign = utf8_encode("GET\n\n\n$expires\n/$bucket/$resource_id"); + // We need to urlencode the entire signature in case the hashed signature + // has spaces. (NOTE: utf8_encode() does not work here because it turns + // spaces into non-breaking spaces) + $signature = urlencode(base64_encode((hash_hmac("sha1", $string_to_sign, $s3_key_secret, true)))); - $authentication_params = "AWSAccessKeyId={$s3_key}&Expires={$expires}&Signature={$signature}"; + $authentication_params = "AWSAccessKeyId=$s3_key&Expires=$expires&Signature=$signature"; $s3 = new Zend_Service_Amazon_S3($s3_key, $s3_key_secret); $endpoint = $s3->getEndpoint(); $scheme = $endpoint->getScheme(); $host = $endpoint->getHost(); - - $url = "{$scheme}://{$host}/{$bucket}/".urlencode($resource)."?{$authentication_params}"; - Logging::info($url); + $url = "$scheme://$host/$bucket/".utf8_encode($resource_id)."?$authentication_params"; return $url; } @@ -78,33 +79,35 @@ class CloudFile extends BaseCloudFile /** * - * Deletes the file from cloud storage by executing a python script - * that uses Apache Libcloud to connect with the cloud storage service + * Deletes the file from Amazon S3 * * If the file was successfully deleted the filesize of that file is returned */ public function deletePhysicalFile() { $CC_CONFIG = Config::getConfig(); - - $provider = escapeshellarg($CC_CONFIG["cloud_storage"]["provider"]); - $bucket = escapeshellarg($CC_CONFIG["cloud_storage"]["bucket"]); - $apiKey = escapeshellarg($CC_CONFIG["cloud_storage"]["api_key"]); - $apiSecret = escapeshellarg($CC_CONFIG["cloud_storage"]["api_key_secret"]); - $objName = escapeshellarg($this->getResourceId()); - $command = "/usr/lib/airtime/pypo/bin/cloud_storage_deleter.py $provider $bucket $apiKey $apiSecret $objName 2>&1 echo $?"; + $s3 = new Zend_Service_Amazon_S3( + $CC_CONFIG['cloud_storage']['api_key'], + $CC_CONFIG['cloud_storage']['api_key_secret']); - $output = shell_exec($command); - if ($output != "") { - if (stripos($output, 'filesize') === false) { - Logging::info($output); - throw new Exception("Could not delete file from cloud storage"); - } + $bucket = $CC_CONFIG['cloud_storage']['bucket']; + $resource_id = $this->getResourceId(); + $amz_resource = utf8_encode("$bucket/$resource_id"); + + if ($s3->isObjectAvailable($amz_resource)) { + $obj_info = $s3->getInfo($amz_resource); + $filesize = $obj_info["size"]; + + // removeObject() returns true even if the object was not deleted (bug?) + // so that is not a good way to do error handling. isObjectAvailable() + // does however return the correct value; We have to assume that if the + // object is available the removeObject() function will work. + $s3->removeObject($amz_resource); + return $filesize; + } else { + throw new Exception("ERROR: Could not locate object on Amazon S3"); } - - $outputArr = json_decode($output, true); - return $outputArr["filesize"]; } /** @@ -116,16 +119,4 @@ class CloudFile extends BaseCloudFile CcFilesQuery::create()->findPk($this->getCcFileId())->delete(); parent::delete(); } - - public function downloadFile() - { - $CC_CONFIG = Config::getConfig(); - - $s3 = new Zend_Service_Amazon_S3($CC_CONFIG['cloud_storage']['api_key'], $CC_CONFIG['cloud_storage']['api_key_secret']); - //$fileObj = $s3->getObject($CC_CONFIG['cloud_storage']['bucket']."/".$this->getResourceId()); - - $response_stream = $s3->getObjectStream($CC_CONFIG['cloud_storage']['bucket']."/".$this->getResourceId()); - copy($response_stream->getStreamName(), "/tmp/".$this->getResourceId()); - Logging::info($response_stream); - } } diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index 15c8cb451..4962e4056 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -230,6 +230,12 @@ class Rest_MediaController extends Zend_Rest_Controller $file->setDbFilepath($requestData["filename"]); $fileSizeBytes = $requestData["filesize"]; + if ($fileSizeBytes === false) + { + $file->setDbImportStatus(2)->save(); + $this->fileNotFoundResponse(); + return; + } $cloudFile = new CloudFile(); $cloudFile->setResourceId($requestData["resource_id"]); $cloudFile->setCcFiles($file); diff --git a/airtime_mvc/build/airtime.conf b/airtime_mvc/build/airtime.conf index 517a81122..5ba766ad3 100644 --- a/airtime_mvc/build/airtime.conf +++ b/airtime_mvc/build/airtime.conf @@ -31,9 +31,7 @@ monit_password = airtime connection_retries = 3 time_between_retries = 60 -[cloud_storage] -provider = -endpoint = +[amazon] bucket = -api_key = -api_key_secret = \ No newline at end of file +access_key = +secret_key = \ No newline at end of file diff --git a/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py b/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py index d331b116a..49c508527 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py @@ -15,6 +15,13 @@ class CloudStorageUploader: def upload_obj(self, audio_file_path, metadata): file_base_name = os.path.basename(audio_file_path) file_name, extension = os.path.splitext(file_base_name) + + ''' + With Amazon S3 you cannot create a signed url if there are spaces + in the object name. URL encoding the object name doesn't solve the + problem. As a solution we will replace spaces with dashes. + ''' + file_name = file_name.replace(" ", "-") object_name = "%s_%s%s" % (file_name, str(uuid.uuid4()), extension) cls = get_driver(getattr(Provider, self._provider)) @@ -25,8 +32,7 @@ class CloudStorageUploader: except ContainerDoesNotExistError: container = driver.create_container(self._bucket) - extra = {'meta_data': {'filename': file_base_name}, - 'acl': 'public-read-write'} + extra = {'meta_data': {'filename': file_base_name}} obj = driver.upload_object(file_path=audio_file_path, container=container, diff --git a/python_apps/pypo/cloud_storage_deleter.py b/python_apps/pypo/cloud_storage_deleter.py deleted file mode 100755 index 93bccd76f..000000000 --- a/python_apps/pypo/cloud_storage_deleter.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/python - -import sys -import simplejson - -from libcloud.storage.providers import get_driver -from libcloud.storage.types import Provider, ObjectDoesNotExistError - -provider = str(sys.argv[1]) -bucket = str(sys.argv[2]) -api_key = str(sys.argv[3]) -api_key_secret = str(sys.argv[4]) -obj_name = str(sys.argv[5]) - -cls = get_driver(getattr(Provider, provider)) -driver = cls(api_key, api_key_secret) - -try: - cloud_obj = driver.get_object(container_name=bucket, - object_name=obj_name) - filesize = getattr(cloud_obj, 'size') - driver.delete_object(obj=cloud_obj) - - data = simplejson.dumps({"filesize": filesize}) - print data -except ObjectDoesNotExistError: - raise Exception("Could not find object on %s in bucket: %s and object: %s" % (provider, bucket, obj_name)) -