From 63b357f0ee0ce43b65b4e518825f81062cf10610 Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Tue, 11 Nov 2014 11:18:55 -0500 Subject: [PATCH 1/4] Updated requirements for airtime_analyzer to make python-requests support SNI properly --- python_apps/airtime_analyzer/setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python_apps/airtime_analyzer/setup.py b/python_apps/airtime_analyzer/setup.py index c47b05167..3694e6fe5 100644 --- a/python_apps/airtime_analyzer/setup.py +++ b/python_apps/airtime_analyzer/setup.py @@ -29,6 +29,10 @@ setup(name='airtime_analyzer', 'mock', 'python-daemon', 'requests', + # These next 3 are required for requests to support SSL with SNI. This is extremely important. Learned this the hard way... + 'ndg-httpsclient', + 'pyasn1', + 'pyopenssl' ], zip_safe=False, data_files=data_files) From b324031ee683005be0307e3b323c4709ce3a01eb Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Tue, 11 Nov 2014 18:41:09 -0500 Subject: [PATCH 2/4] Disable those new requirements because pip requires gcc to install them --- python_apps/airtime_analyzer/setup.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python_apps/airtime_analyzer/setup.py b/python_apps/airtime_analyzer/setup.py index 3694e6fe5..0816f8d14 100644 --- a/python_apps/airtime_analyzer/setup.py +++ b/python_apps/airtime_analyzer/setup.py @@ -29,10 +29,11 @@ setup(name='airtime_analyzer', 'mock', 'python-daemon', 'requests', - # These next 3 are required for requests to support SSL with SNI. This is extremely important. Learned this the hard way... - 'ndg-httpsclient', - 'pyasn1', - 'pyopenssl' + # These next 3 are required for requests to support SSL with SNI. Learned this the hard way... + # What sucks is that GCC is required to pip install these. + #'ndg-httpsclient', + #'pyasn1', + #'pyopenssl' ], zip_safe=False, data_files=data_files) From c132cac43d6daea5cd9a56c58f6f320bffb05213 Mon Sep 17 00:00:00 2001 From: drigato Date: Wed, 12 Nov 2014 14:42:34 -0500 Subject: [PATCH 3/4] Code cleanup --- .../application/controllers/ApiController.php | 14 +++-------- airtime_mvc/application/models/Schedule.php | 25 +++++++++++++------ .../application/models/airtime/CcFiles.php | 12 ++++++++- .../application/models/airtime/CloudFile.php | 2 +- python_apps/pypo/pypofile.py | 4 +-- 5 files changed, 35 insertions(+), 22 deletions(-) diff --git a/airtime_mvc/application/controllers/ApiController.php b/airtime_mvc/application/controllers/ApiController.php index 9f228bd97..9ea2d34fd 100644 --- a/airtime_mvc/application/controllers/ApiController.php +++ b/airtime_mvc/application/controllers/ApiController.php @@ -85,7 +85,7 @@ class ApiController extends Zend_Controller_Action // Make sure we don't have some wrong result beecause of caching clearstatcache(); - if ($media->getPropelOrm()->isValidFile()) { + if ($media->getPropelOrm()->isValidPhysicalFile()) { $filename = $media->getPropelOrm()->getFilename(); //Download user left clicks a track and selects Download. @@ -168,15 +168,9 @@ class ApiController extends Zend_Controller_Action //http://www.php.net/manual/en/function.ob-end-flush.php while (@ob_end_flush()); - /*$cur = $begin; - fseek($fm, $begin, 0); - - while (!feof($fm) && $cur <= $end && (connection_status() == 0)) { - echo fread($fm, min(1024 * 16, ($end - $cur) + 1)); - $cur += 1024 * 16; - }*/ - - while(!feof($fm)) { + // NOTE: We can't use fseek here because it does not work with streams + // (a.k.a. Files stored on Amazon S3) + while(!feof($fm) && (connection_status() == 0)) { echo fread($fm, 1024 * 8); } } diff --git a/airtime_mvc/application/models/Schedule.php b/airtime_mvc/application/models/Schedule.php index 4a1bcb8b0..2483a8b54 100644 --- a/airtime_mvc/application/models/Schedule.php +++ b/airtime_mvc/application/models/Schedule.php @@ -761,7 +761,19 @@ SQL; } } - private static function createFileScheduleEvent(&$data, $item, $media_id, $uri, $object_name=null) + /** + * + * Appends schedule "events" to an array of schedule events that gets + * sent to PYPO. Each schedule event contains information PYPO and + * Liquidsoap need for playout. + * + * @param Array $data array to be filled with schedule info - $item(s) + * @param Array $item schedule info about one track + * @param Integer $media_id scheduled item's cc_files id + * @param String $uri path to the scheduled item's physical location + * @param String $amazonS3ResourceId scheduled item's Amazon S3 resource id, if applicable + */ + private static function createFileScheduleEvent(&$data, $item, $media_id, $uri, $amazonS3ResourceId) { $start = self::AirtimeTimeToPypoTime($item["start"]); $end = self::AirtimeTimeToPypoTime($item["end"]); @@ -797,8 +809,8 @@ SQL; 'replay_gain' => $replay_gain, 'independent_event' => $independent_event ); - if (!is_null($object_name)) { - $schedule_item["object_name"] = $object_name; + if (!is_null($amazonS3ResourceId)) { + $schedule_item["amazonS3_resource_id"] = $amazonS3ResourceId; } if ($schedule_item['cue_in'] > $schedule_item['cue_out']) { @@ -934,11 +946,8 @@ SQL; $file = $storedFile->getPropelOrm(); $uri = $file->getAbsoluteFilePath(); - $object_name = null; - if ($file instanceof CloudFile) { - $object_name = $storedFile->getResourceId(); - } - self::createFileScheduleEvent($data, $item, $media_id, $uri, $object_name); + $amazonS3ResourceId = $file->getResourceId(); + self::createFileScheduleEvent($data, $item, $media_id, $uri, $amazonS3ResourceId); } elseif (!is_null($item['stream_id'])) { //row is type "webstream" diff --git a/airtime_mvc/application/models/airtime/CcFiles.php b/airtime_mvc/application/models/airtime/CcFiles.php index 01eed8d55..b13eb05ce 100644 --- a/airtime_mvc/application/models/airtime/CcFiles.php +++ b/airtime_mvc/application/models/airtime/CcFiles.php @@ -111,7 +111,7 @@ class CcFiles extends BaseCcFiles { /** * Checks if the file is a regular file that can be previewed and downloaded. */ - public function isValidFile() + public function isValidPhysicalFile() { return is_file($this->getAbsoluteFilePath()); } @@ -130,4 +130,14 @@ class CcFiles extends BaseCcFiles { } } + /** + * + * This function refers to the file's Amazon S3 resource id. + * Returns null because cc_files are stored on local disk. + */ + public function getResourceId() + { + return null; + } + } // CcFiles diff --git a/airtime_mvc/application/models/airtime/CloudFile.php b/airtime_mvc/application/models/airtime/CloudFile.php index a30eedc6c..c1d5b3b7a 100644 --- a/airtime_mvc/application/models/airtime/CloudFile.php +++ b/airtime_mvc/application/models/airtime/CloudFile.php @@ -93,7 +93,7 @@ class CloudFile extends BaseCloudFile /** * Checks if the file is a regular file that can be previewed and downloaded. */ - public function isValidFile() + public function isValidPhysicalFile() { $ch = curl_init(); curl_setopt_array($ch, array( diff --git a/python_apps/pypo/pypofile.py b/python_apps/pypo/pypofile.py index d0667a58f..2c14bbbf5 100644 --- a/python_apps/pypo/pypofile.py +++ b/python_apps/pypo/pypofile.py @@ -136,9 +136,9 @@ class PypoFile(Thread): """ If an object_name exists the file is stored on Amazon S3 """ - if 'object_name' in media_item: + if 'amazonS3_resource_id' in media_item: csd = CloudStorageDownloader() - csd.download_obj(media_item['dst'], media_item['object_name']) + csd.download_obj(media_item['dst'], media_item['amazonS3_resource_id']) media_item['file_ready'] = True else: self.copy_file(media_item) From 95517cac874aa807c836fa7aa53a3776a9b95ed5 Mon Sep 17 00:00:00 2001 From: drigato Date: Wed, 12 Nov 2014 14:45:50 -0500 Subject: [PATCH 4/4] Fix exception thrown when the analyzer doesn't return filesize with the metadata --- .../application/modules/rest/controllers/MediaController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index dbbc346ba..87af946bb 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -226,7 +226,7 @@ class Rest_MediaController extends Zend_Rest_Controller //as a foreign key to cc_music_dirs. if (isset($requestData["full_path"])) { $fileSizeBytes = filesize($requestData["full_path"]); - if ($fileSizeBytes === false) + if (!isset($fileSizeBytes) || $fileSizeBytes === false) { $file->setDbImportStatus(2)->save(); $this->fileNotFoundResponse();