From ca51dcf3aee8652a3e2bdb47617e7e473ed3d23a Mon Sep 17 00:00:00 2001 From: Duncan Sommerville Date: Thu, 24 Sep 2015 12:58:02 -0400 Subject: [PATCH] SAAS-1071 - more work on backend podcast implementation --- .../application/models/airtime/Podcast.php | 38 +++++++++---------- .../rest/controllers/PodcastController.php | 8 ++++ .../application/services/PodcastService.php | 33 +++++++++++----- .../views/scripts/podcast/podcast.phtml | 26 +++++++++---- .../public/js/airtime/library/podcast.js | 34 +++++++++++------ composer.lock | 22 ++++++----- .../airtime-celery/airtime-celery/tasks.py | 20 ++++++++-- 7 files changed, 119 insertions(+), 62 deletions(-) diff --git a/airtime_mvc/application/models/airtime/Podcast.php b/airtime_mvc/application/models/airtime/Podcast.php index 5a52bdcb5..74a2a2c41 100644 --- a/airtime_mvc/application/models/airtime/Podcast.php +++ b/airtime_mvc/application/models/airtime/Podcast.php @@ -38,7 +38,6 @@ class Podcast extends BasePodcast * @throws InvalidPodcastException * @throws PodcastLimitReachedException */ - public static function create($data) { if (Application_Service_PodcastService::podcastLimitReached()) { @@ -98,24 +97,11 @@ class Podcast extends BasePodcast $podcast->setDbType(IMPORTED_PODCAST); $podcast->save(); - $podcastArray = $podcast->toArray(BasePeer::TYPE_FIELDNAME); - - $podcastArray["episodes"] = array(); - foreach ($rss->get_items() as $item) { - array_push($podcastArray["episodes"], array( - "title" => $item->get_title(), - "author" => $item->get_author()->get_name(), - "description" => $item->get_description(), - "pubDate" => $item->get_date("Y-m-d H:i:s"), - "link" => $item->get_enclosure()->get_link() - )); - } - return $podcastArray; + return self::_generatePodcastArray($podcast, $rss); } catch(Exception $e) { $podcast->delete(); throw $e; } - } /** @@ -125,6 +111,7 @@ class Podcast extends BasePodcast * @param $podcastId * * @throws PodcastNotFoundException + * @throws InvalidPodcastException * @return array - Podcast Array with a full list of episodes */ public static function getPodcastById($podcastId) @@ -136,10 +123,22 @@ class Podcast extends BasePodcast $rss = Application_Service_PodcastService::getPodcastFeed($podcast->getDbUrl()); if (!$rss) { - throw new PodcastNotFoundException(); + throw new InvalidPodcastException(); } - // FIXME: Get rid of this duplication and move into a new function (serializer/deserializer) + return self::_generatePodcastArray($podcast, $rss); + } + + /** + * Given a podcast object and a SimplePie feed object, + * generate a data array to pass back to the front-end + * + * @param Podcast $podcast Podcast model object + * @param SimplePie $rss SimplePie feed object + * + * @return array + */ + private static function _generatePodcastArray($podcast, $rss) { $podcastArray = $podcast->toArray(BasePeer::TYPE_FIELDNAME); $podcastArray["episodes"] = array(); @@ -148,8 +147,9 @@ class Podcast extends BasePodcast "title" => $item->get_title(), "author" => $item->get_author()->get_name(), "description" => $item->get_description(), - "pubDate" => $item->get_date("Y-m-d H:i:s"), - "link" => $item->get_enclosure()->get_link() + "pub_date" => $item->get_date("Y-m-d H:i:s"), + "link" => $item->get_link(), + "enclosure" => $item->get_enclosure() )); } diff --git a/airtime_mvc/application/modules/rest/controllers/PodcastController.php b/airtime_mvc/application/modules/rest/controllers/PodcastController.php index 2913684b3..1000f7475 100644 --- a/airtime_mvc/application/modules/rest/controllers/PodcastController.php +++ b/airtime_mvc/application/modules/rest/controllers/PodcastController.php @@ -2,6 +2,12 @@ class Rest_PodcastController extends Zend_Rest_Controller { + + /** + * @var Application_Service_PodcastService + */ + protected $_service; + public function init() { $this->view->layout()->disableLayout(); @@ -9,6 +15,7 @@ class Rest_PodcastController extends Zend_Rest_Controller // Remove reliance on .phtml files to render requests $this->_helper->viewRenderer->setNoRender(true); $this->view->setScriptPath(APPLICATION_PATH . 'views/scripts/'); + $this->_service = new Application_Service_PodcastService(); } public function indexAction() @@ -119,6 +126,7 @@ class Rest_PodcastController extends Zend_Rest_Controller try { $requestData = json_decode($this->getRequest()->getRawBody(), true); + $this->_service->downloadEpisodes($requestData["podcast"]["episodes"]); $podcast = Podcast::updateFromArray($id, $requestData); $this->getResponse() diff --git a/airtime_mvc/application/services/PodcastService.php b/airtime_mvc/application/services/PodcastService.php index fe8006670..d34eb5cb2 100644 --- a/airtime_mvc/application/services/PodcastService.php +++ b/airtime_mvc/application/services/PodcastService.php @@ -56,7 +56,7 @@ class Application_Service_PodcastService extends Application_Service_ThirdPartyC $feed->enable_cache(false); $feed->init(); return $feed; - } catch (FeedException $e) { + } catch (Exception $e) { return false; } } @@ -66,15 +66,28 @@ class Application_Service_PodcastService extends Application_Service_ThirdPartyC } /** - * Given an array of track identifiers, download RSS feed tracks + * Given an array of episodes, extract the download URLs and send them to Celery * - * @param array $trackIds array of track identifiers to download + * @param array $episodes array of podcast episodes + */ + public function downloadEpisodes($episodes) { + $episodeUrls = array(); + foreach($episodes as $episode) { + array_push($episodeUrls, $episode["enclosure"]["link"]); + } + $this->_download($episodeUrls); + } + + /** + * Given an array of download URLs, download RSS feed tracks + * + * @param array $downloadUrls array of download URLs to send to Celery * TODO: do we need other parameters here...? */ - public function download($trackIds) { + private function _download($downloadUrls) { $CC_CONFIG = Config::getConfig(); $data = array( - // 'download_urls' => , TODO: get download urls to send to Celery + 'download_urls' => $downloadUrls, 'callback_url' => Application_Common_HTTPHelper::getStationUrl() . '/rest/media', 'api_key' => $apiKey = $CC_CONFIG["apiKey"][0], ); @@ -87,8 +100,8 @@ class Application_Service_PodcastService extends Application_Service_ThirdPartyC * Update a ThirdPartyTrackReferences object for a completed upload * * @param $task CeleryTasks the completed CeleryTasks object - * @param $trackId int ThirdPartyTrackReferences identifier - * @param $track object third-party service track object + * @param $episodeId int PodcastEpisodes identifier + * @param $episode object object containing Podcast episode information * @param $status string Celery task status * * @return ThirdPartyTrackReferences the updated ThirdPartyTrackReferences object @@ -96,14 +109,14 @@ class Application_Service_PodcastService extends Application_Service_ThirdPartyC * @throws Exception * @throws PropelException */ - function updateTrackReference($task, $trackId, $track, $status) { - $ref = parent::updateTrackReference($task, $trackId, $track, $status); + public function updateTrackReference($task, $episodeId, $episode, $status) { + $ref = parent::updateTrackReference($task, $episodeId, $episode, $status); if ($status == CELERY_SUCCESS_STATUS) { // TODO: handle successful download // $ref->setDbForeignId(); // FIXME: we need the file ID here, but 'track' is too arbitrary... - $ref->setDbFileId($track->fileId); + $ref->setDbFileId($episode->fileId); } $ref->save(); diff --git a/airtime_mvc/application/views/scripts/podcast/podcast.phtml b/airtime_mvc/application/views/scripts/podcast/podcast.phtml index eb99f4241..8ccf2412c 100644 --- a/airtime_mvc/application/views/scripts/podcast/podcast.phtml +++ b/airtime_mvc/application/views/scripts/podcast/podcast.phtml @@ -6,14 +6,24 @@
- - +

+ + +

+

+ + +

+

+ + +

diff --git a/airtime_mvc/public/js/airtime/library/podcast.js b/airtime_mvc/public/js/airtime/library/podcast.js index ec22d67dc..52e77224a 100644 --- a/airtime_mvc/public/js/airtime/library/podcast.js +++ b/airtime_mvc/public/js/airtime/library/podcast.js @@ -11,7 +11,8 @@ var AIRTIME = (function (AIRTIME) { //AngularJS app var podcastApp = angular.module('podcast', []) - .controller('RestController', function($scope, $http, podcast, tab) { + .controller('RestController', function($scope, $http, podcast, tab, episodeTable) { + // We need to pass in the tab object and the episodes table object so we can reference them //We take a podcast object in as a parameter rather fetching the podcast by ID here because //when you're creating a new podcast, we already have the object from the result of the POST. We're saving @@ -20,7 +21,9 @@ var AIRTIME = (function (AIRTIME) { tab.setName($scope.podcast.title); $scope.savePodcast = function() { - $http.put(endpoint + $scope.podcast.id, { csrf_token: jQuery("#csrf").val(), podcast: $scope.podcast }) + var podcastData = $scope.podcast; // Copy the podcast in scope so we can modify it + podcastData.episodes = episodeTable.getSelectedRows(); + $http.put(endpoint + $scope.podcast.id, { csrf_token: jQuery("#csrf").val(), podcast: podcastData }) .success(function() { // TODO }); @@ -44,9 +47,10 @@ var AIRTIME = (function (AIRTIME) { $.post(endpoint + "bulk", { csrf_token: $("#csrf").val(), method: method, ids: ids }, callback); } - function _bootstrapAngularApp(podcast, tab) { + function _bootstrapAngularApp(podcast, tab, table) { podcastApp.value('podcast', podcast); podcastApp.value('tab', tab); + podcastApp.value('episodeTable', table); var wrapper = tab.contents.find(".editor_pane_wrapper"); wrapper.attr("ng-controller", "RestController"); angular.bootstrap(wrapper.get(0), ["podcast"]); @@ -70,9 +74,9 @@ var AIRTIME = (function (AIRTIME) { var podcast = JSON.parse(json.podcast); var uid = AIRTIME.library.MediaTypeStringEnum.PODCAST+"_"+podcast.id, tab = AIRTIME.tabs.openTab(json, uid, null); - _bootstrapAngularApp(podcast, tab); + var table = mod.initPodcastEpisodeDatatable(podcast.episodes); + _bootstrapAngularApp(podcast, tab, table); $("#podcast_url_dialog").dialog("close"); - mod.initPodcastEpisodeDatatable(podcast.episodes); }); }; @@ -82,8 +86,8 @@ var AIRTIME = (function (AIRTIME) { var podcast = JSON.parse(el.podcast); var uid = AIRTIME.library.MediaTypeStringEnum.PODCAST+"_"+podcast.id, tab = AIRTIME.tabs.openTab(el, uid, null); - _bootstrapAngularApp(podcast, tab); - mod.initPodcastEpisodeDatatable(podcast.episodes); + var table = mod.initPodcastEpisodeDatatable(podcast.episodes); + _bootstrapAngularApp(podcast, tab, table); }); }); }; @@ -102,13 +106,13 @@ var AIRTIME = (function (AIRTIME) { /* Author */ { "sTitle" : $.i18n._("Author") , "mDataProp" : "author" , "sClass" : "podcast_episodes_author" , "sWidth" : "170px" }, /* Description */ { "sTitle" : $.i18n._("Description") , "mDataProp" : "description" , "sClass" : "podcast_episodes_description" , "sWidth" : "300px" }, /* Link */ { "sTitle" : $.i18n._("Link") , "mDataProp" : "link" , "sClass" : "podcast_episodes_link" , "sWidth" : "170px" }, - /* Publication Date */ { "sTitle" : $.i18n._("Publication Date") , "mDataProp" : "pubDate" , "sClass" : "podcast_episodes_pub_date" , "sWidth" : "170px" } + /* Publication Date */ { "sTitle" : $.i18n._("Publication Date") , "mDataProp" : "pub_date" , "sClass" : "podcast_episodes_pub_date" , "sWidth" : "170px" } ]; var podcastToolbarButtons = AIRTIME.widgets.Table.getStandardToolbarButtons(); // Set up the div with id "podcast_table" as a datatable. - mod.podcastEpisodesTableWidget = new AIRTIME.widgets.Table( + var podcastEpisodesTableWidget = new AIRTIME.widgets.Table( AIRTIME.tabs.getActiveTab().contents.find('#podcast_episodes'), // DOM node to create the table inside. true, // Enable item selection podcastToolbarButtons, // Toolbar buttons @@ -116,11 +120,17 @@ var AIRTIME = (function (AIRTIME) { 'aoColumns' : aoColumns, 'bServerSide': false, 'sAjaxSource' : null, - 'aaData' : episodes + 'aaData' : episodes, + "oColVis": { + "sAlign": "right", + "aiExclude": [0, 1], + "buttonText": $.i18n._("Columns"), + "iOverlayFade": 0 + } }); - mod.podcastEpisodesDatatable = mod.podcastEpisodesTableWidget.getDatatable(); - mod.podcastEpisodesDatatable.textScroll("td"); + podcastEpisodesTableWidget.getDatatable().textScroll("td"); + return podcastEpisodesTableWidget; }; return AIRTIME; diff --git a/composer.lock b/composer.lock index 5b61be35b..3a0c8db54 100644 --- a/composer.lock +++ b/composer.lock @@ -1,7 +1,7 @@ { "_readme": [ "This file locks the dependencies of your project to a known state", - "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], "hash": "8d8a51740ad37127ff6618f80861ccfc", @@ -231,7 +231,7 @@ "shasum": "" }, "require": { - "predis/predis": "0.8.5", + "predis/predis": ">=0.8.5", "videlalvaro/php-amqplib": ">=2.4.0" }, "type": "library", @@ -363,29 +363,32 @@ }, { "name": "predis/predis", - "version": "v0.8.5", + "version": "v1.0.3", "source": { "type": "git", "url": "https://github.com/nrk/predis.git", - "reference": "5f2eea628eb465d866ad2771927d83769c8f956c" + "reference": "84060b9034d756b4d79641667d7f9efe1aeb8e04" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nrk/predis/zipball/5f2eea628eb465d866ad2771927d83769c8f956c", - "reference": "5f2eea628eb465d866ad2771927d83769c8f956c", + "url": "https://api.github.com/repos/nrk/predis/zipball/84060b9034d756b4d79641667d7f9efe1aeb8e04", + "reference": "84060b9034d756b4d79641667d7f9efe1aeb8e04", "shasum": "" }, "require": { "php": ">=5.3.2" }, + "require-dev": { + "phpunit/phpunit": "~4.0" + }, "suggest": { "ext-curl": "Allows access to Webdis when paired with phpiredis", "ext-phpiredis": "Allows faster serialization and deserialization of the Redis protocol" }, "type": "library", "autoload": { - "psr-0": { - "Predis": "lib/" + "psr-4": { + "Predis\\": "src/" } }, "notification-url": "https://packagist.org/downloads/", @@ -406,7 +409,7 @@ "predis", "redis" ], - "time": "2014-01-16 14:10:29" + "time": "2015-07-30 18:34:15" }, { "name": "propel/propel1", @@ -700,6 +703,7 @@ "simplepie/simplepie": 20 }, "prefer-stable": false, + "prefer-lowest": false, "platform": [], "platform-dev": [] } diff --git a/python_apps/airtime-celery/airtime-celery/tasks.py b/python_apps/airtime-celery/airtime-celery/tasks.py index 7d28282fa..96ef1b7a1 100644 --- a/python_apps/airtime-celery/airtime-celery/tasks.py +++ b/python_apps/airtime-celery/airtime-celery/tasks.py @@ -3,8 +3,12 @@ import json import urllib2 import requests import soundcloud +import cgi +import urlparse +import posixpath from celery import Celery from celery.utils.log import get_task_logger +from contextlib import closing celery = Celery() logger = get_task_logger(__name__) @@ -93,10 +97,18 @@ def podcast_download(download_urls, callback_url, api_key): """ try: for url in download_urls: - r = requests.get(url, stream=True) - r.raise_for_status() - with r as f: - requests.post(callback_url, data=f, auth=requests.auth.HTTPBasicAuth(api_key, '')) + with closing(requests.get(url, stream=True)) as r: + # Try to get the filename from the content disposition + d = r.headers.get('Content-Disposition') + if d: + _, params = cgi.parse_header(d) + filename = params['filename'] + else: + # Since we don't necessarily get the filename back in the response headers, + # parse the URL and get the filename and extension + path = urlparse.urlsplit(r.url).path + filename = posixpath.basename(path) + requests.post(callback_url, files={'file': (filename, r.content)}, auth=requests.auth.HTTPBasicAuth(api_key, '')) except Exception as e: logger.info('Error during file download: {0}'.format(e.message)) logger.info(str(e))