From 644e6b00d80012113a8b8e95d0321a1b082803d8 Mon Sep 17 00:00:00 2001 From: drigato Date: Tue, 9 Dec 2014 17:58:36 -0500 Subject: [PATCH 1/2] SAAS-516: Replace Zend_Service_Amazon_S3 with AWS SDK for PHP --- airtime_mvc/application/Bootstrap.php | 3 + .../application/cloud_storage/Amazon_S3.php | 71 +++++++--------- composer.json | 3 +- composer.lock | 81 +++++++++++-------- 4 files changed, 84 insertions(+), 74 deletions(-) diff --git a/airtime_mvc/application/Bootstrap.php b/airtime_mvc/application/Bootstrap.php index 286761395..09aacb052 100644 --- a/airtime_mvc/application/Bootstrap.php +++ b/airtime_mvc/application/Bootstrap.php @@ -10,6 +10,9 @@ if (!@include_once('propel/propel1/runtime/lib/Propel.php')) Propel::init(__DIR__."/configs/airtime-conf-production.php"); +//Composer's autoloader +require_once 'autoload.php'; + require_once __DIR__."/configs/constants.php"; require_once 'Preference.php'; require_once 'Locale.php'; diff --git a/airtime_mvc/application/cloud_storage/Amazon_S3.php b/airtime_mvc/application/cloud_storage/Amazon_S3.php index 527440c0a..f9de30d49 100644 --- a/airtime_mvc/application/cloud_storage/Amazon_S3.php +++ b/airtime_mvc/application/cloud_storage/Amazon_S3.php @@ -2,71 +2,60 @@ require_once 'StorageBackend.php'; +use Aws\S3\S3Client; + class Amazon_S3 extends StorageBackend { - private $zendServiceAmazonS3; + + private $s3Client; public function Amazon_S3($securityCredentials) { $this->setBucket($securityCredentials['bucket']); $this->setAccessKey($securityCredentials['api_key']); $this->setSecretKey($securityCredentials['api_key_secret']); - - $this->zendServiceAmazonS3 = new Zend_Service_Amazon_S3( - $this->getAccessKey(), - $this->getSecretKey()); + + $this->s3Client = S3Client::factory(array( + 'key' => $securityCredentials['api_key'], + 'secret' => $securityCredentials['api_key_secret'], + )); } public function getAbsoluteFilePath($resourceId) { - $endpoint = $this->zendServiceAmazonS3->getEndpoint(); - $scheme = $endpoint->getScheme(); - $host = $endpoint->getHost(); - $bucket = $this->getBucket(); - return "$scheme://$bucket.$host/".utf8_encode($resourceId); + return $this->s3Client->getObjectUrl($this->getBucket(), $resourceId); } public function getSignedURL($resourceId) { - //URL will be active for 30 minutes - $expires = time()+1800; - - $bucket = $this->getBucket(); - $secretKey = $this->getSecretKey(); - $accessKey = $this->getAccessKey(); - - $string_to_sign = utf8_encode("GET\n\n\n$expires\n/$bucket/$resourceId"); - // We need to urlencode the entire signature in case the hashed signature - // has spaces. (NOTE: utf8_encode() does not work here because it turns - // spaces into non-breaking spaces) - $signature = urlencode(base64_encode((hash_hmac("sha1", $string_to_sign, $secretKey, true)))); - - $resourceURL = $this->getAbsoluteFilePath($resourceId); - return $resourceURL."?AWSAccessKeyId=$accessKey&Expires=$expires&Signature=$signature"; + return $this->s3Client->getObjectUrl($this->getBucket(), $resourceId, '+60 minutes'); } public function getFileSize($resourceId) { - $bucket = $this->getBucket(); - - $amz_resource = utf8_encode("$bucket/$resourceId"); - $amz_resource_info = $this->zendServiceAmazonS3->getInfo($amz_resource); - return (int)$amz_resource_info["size"]; + $obj = $this->s3Client->getObject(array( + 'Bucket' => $this->getBucket(), + 'Key' => $resourceId, + )); + if (isset($obj["ContentLength"])) { + return (int)$obj["ContentLength"]; + } else { + return 0; + } } public function deletePhysicalFile($resourceId) { $bucket = $this->getBucket(); - $amz_resource = utf8_encode("$bucket/$resourceId"); - - if ($this->zendServiceAmazonS3->isObjectAvailable($amz_resource)) { - // removeObject() returns true even if the object was not deleted (bug?) - // so that is not a good way to do error handling. isObjectAvailable() - // does however return the correct value; We have to assume that if the - // object is available the removeObject() function will work. - $this->zendServiceAmazonS3->removeObject($amz_resource); - } else { - throw new Exception("ERROR: Could not locate object on Amazon S3"); - } + + if ($this->s3Client->doesObjectExist($bucket, $resourceId)) { + + $result = $this->s3Client->deleteObject(array( + 'Bucket' => $bucket, + 'Key' => $resourceId, + )); + } else { + throw new Exception("ERROR: Could not locate file to delete."); + } } } diff --git a/composer.json b/composer.json index 7a2989d96..f389308e4 100644 --- a/composer.json +++ b/composer.json @@ -1,5 +1,6 @@ { "require": { - "propel/propel1": "1.7.0-stable" + "propel/propel1": "1.7.0-stable", + "aws/aws-sdk-php": "2.7.9" } } diff --git a/composer.lock b/composer.lock index 7e6984eb5..17879613a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,32 +4,32 @@ "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "5776cc789514d71faac7021474c96d58", + "hash": "311983cf9bb84cfacbfcc6c5dfc9e841", "packages": [ { "name": "aws/aws-sdk-php", - "version": "dev-master", + "version": "2.7.9", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "c540fa2c66daf91383a2cd7fc044765f2da2aa52" + "reference": "f39354df58eec97f0ef22ccf3caf753607a47dca" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/c540fa2c66daf91383a2cd7fc044765f2da2aa52", - "reference": "c540fa2c66daf91383a2cd7fc044765f2da2aa52", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/f39354df58eec97f0ef22ccf3caf753607a47dca", + "reference": "f39354df58eec97f0ef22ccf3caf753607a47dca", "shasum": "" }, "require": { - "guzzle/guzzle": ">=3.7,<4", + "guzzle/guzzle": "~3.7", "php": ">=5.3.3" }, "require-dev": { "doctrine/cache": "~1.0", "ext-openssl": "*", - "monolog/monolog": "1.4.*", - "phpunit/phpunit": "4.*", - "symfony/yaml": "2.*" + "monolog/monolog": "~1.4", + "phpunit/phpunit": "~4.0", + "symfony/yaml": "~2.1" }, "suggest": { "doctrine/cache": "Adds support for caching of credentials and responses", @@ -41,7 +41,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "2.7.x-dev" + "dev-master": "2.7-dev" } }, "autoload": { @@ -71,7 +71,7 @@ "s3", "sdk" ], - "time": "2014-10-08 19:18:30" + "time": "2014-12-08 21:56:46" }, { "name": "guzzle/guzzle", @@ -167,24 +167,38 @@ }, { "name": "phing/phing", - "version": "2.8.2", + "version": "2.9.1", "source": { "type": "git", "url": "https://github.com/phingofficial/phing.git", - "reference": "345e8716122cf6c6b59c4894701d8b736f27fca9" + "reference": "393edeffa8a85d43636ce0c9b4deb1ff9ac60a5c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phingofficial/phing/zipball/345e8716122cf6c6b59c4894701d8b736f27fca9", - "reference": "345e8716122cf6c6b59c4894701d8b736f27fca9", + "url": "https://api.github.com/repos/phingofficial/phing/zipball/393edeffa8a85d43636ce0c9b4deb1ff9ac60a5c", + "reference": "393edeffa8a85d43636ce0c9b4deb1ff9ac60a5c", "shasum": "" }, "require": { "php": ">=5.2.0" }, "require-dev": { + "ext-pdo_sqlite": "*", + "lastcraft/simpletest": "@dev", + "pdepend/pdepend": "1.x", + "pear-pear.php.net/http_request2": "2.2.x", + "pear-pear.php.net/net_growl": "2.7.x", + "pear-pear.php.net/pear_packagefilemanager": "1.7.x", + "pear-pear.php.net/pear_packagefilemanager2": "1.0.x", + "pear-pear.php.net/xml_serializer": "0.20.x", + "pear/pear_exception": "@dev", + "pear/versioncontrol_git": "@dev", + "pear/versioncontrol_svn": "@dev", "phpdocumentor/phpdocumentor": "2.x", - "phpunit/phpunit": ">=3.7" + "phploc/phploc": "2.x", + "phpunit/phpunit": ">=3.7", + "sebastian/phpcpd": "2.x", + "squizlabs/php_codesniffer": "1.5.x" }, "suggest": { "pdepend/pdepend": "PHP version of JDepend", @@ -203,6 +217,11 @@ "bin/phing" ], "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.9.x-dev" + } + }, "autoload": { "classmap": [ "classes/phing/" @@ -216,14 +235,13 @@ "LGPL-3.0" ], "authors": [ - { - "name": "Michiel Rook", - "email": "mrook@php.net", - "role": "Lead" - }, { "name": "Phing Community", "homepage": "http://www.phing.info/trac/wiki/Development/Contributors" + }, + { + "name": "Michiel Rook", + "email": "mrook@php.net" } ], "description": "PHing Is Not GNU make; it's a PHP project build system or build tool based on Apache Ant.", @@ -234,7 +252,7 @@ "task", "tool" ], - "time": "2014-07-18 10:23:54" + "time": "2014-12-03 09:18:46" }, { "name": "propel/propel1", @@ -301,17 +319,17 @@ }, { "name": "symfony/event-dispatcher", - "version": "v2.5.5", + "version": "v2.6.1", "target-dir": "Symfony/Component/EventDispatcher", "source": { "type": "git", "url": "https://github.com/symfony/EventDispatcher.git", - "reference": "f6281337bf5f985f585d1db6a83adb05ce531f46" + "reference": "720fe9bca893df7ad1b4546649473b5afddf0216" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/f6281337bf5f985f585d1db6a83adb05ce531f46", - "reference": "f6281337bf5f985f585d1db6a83adb05ce531f46", + "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/720fe9bca893df7ad1b4546649473b5afddf0216", + "reference": "720fe9bca893df7ad1b4546649473b5afddf0216", "shasum": "" }, "require": { @@ -320,7 +338,8 @@ "require-dev": { "psr/log": "~1.0", "symfony/config": "~2.0", - "symfony/dependency-injection": "~2.0,<2.6.0", + "symfony/dependency-injection": "~2.6", + "symfony/expression-language": "~2.6", "symfony/stopwatch": "~2.2" }, "suggest": { @@ -330,7 +349,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "2.5-dev" + "dev-master": "2.6-dev" } }, "autoload": { @@ -354,15 +373,13 @@ ], "description": "Symfony EventDispatcher Component", "homepage": "http://symfony.com", - "time": "2014-09-28 15:56:11" + "time": "2014-12-02 20:19:20" } ], "packages-dev": [], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "aws/aws-sdk-php": 20 - }, + "stability-flags": [], "prefer-stable": false, "platform": [], "platform-dev": [] From d73b331376cde2b0db75a2c9c9ea8ce6c837b88b Mon Sep 17 00:00:00 2001 From: drigato Date: Wed, 10 Dec 2014 12:40:08 -0500 Subject: [PATCH 2/2] SAAS-515: Prefix object names with station name --- airtime_mvc/application/models/RabbitMq.php | 6 +++--- .../airtime_analyzer/cloud_storage_uploader.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airtime_mvc/application/models/RabbitMq.php b/airtime_mvc/application/models/RabbitMq.php index a9f699841..b20beca2e 100644 --- a/airtime_mvc/application/models/RabbitMq.php +++ b/airtime_mvc/application/models/RabbitMq.php @@ -90,9 +90,9 @@ class Application_Model_RabbitMq $data['callback_url'] = $callbackUrl; $data['api_key'] = $apiKey; // Pass station name to the analyzer so we can set it with the file's metadata - // before uploading it to the cloud. This isn't a requirement for cloud storage, - // but put there as a safeguard, since all Airtime Pro stations will share the - // same bucket. + // and prefix the object name with it before uploading it to the cloud. This + // isn't a requirement for cloud storage, but put there as a safeguard, since + // all Airtime Pro stations will share the same bucket. $data['station_domain'] = $stationDomain = Application_Model_Preference::GetStationName(); $jsonData = json_encode($data); diff --git a/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py b/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py index d060f3bcc..b9178709c 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py @@ -61,7 +61,7 @@ class CloudStorageUploader: # in the object name. URL encoding the object name doesn't solve the # problem. As a solution we will replace spaces with dashes. file_name = file_name.replace(" ", "-") - object_name = "%s_%s%s" % (file_name, str(uuid.uuid4()), extension) + object_name = "%s/%s_%s%s" % (metadata["station_domain"], file_name, str(uuid.uuid4()), extension) provider_driver_class = get_driver(getattr(Provider, self._provider)) driver = provider_driver_class(self._api_key, self._api_key_secret)