From e3feb17f0c05bb038c44def198ada502613ac6a7 Mon Sep 17 00:00:00 2001 From: Duncan Sommerville Date: Thu, 29 Oct 2015 17:53:45 -0400 Subject: [PATCH] * Change the Celery timeout to 15 minutes (from 10) to better accommodate automatic ingest * Fix various small bugs in auto ingestion and tab implementation * Update TaskManager run conditions to piggyback on API calls - guarantees a certain frequency of requests and greatly reduces chances of lock contention --- airtime_mvc/application/common/CeleryManager.php | 2 +- airtime_mvc/application/common/PodcastManager.php | 13 +++++++++---- airtime_mvc/application/common/TaskManager.php | 1 + .../controllers/plugins/PageLayoutInitPlugin.php | 6 +++++- .../modules/rest/controllers/PodcastController.php | 2 +- .../application/services/PodcastEpisodeService.php | 11 +++++++++-- .../airtime/library/events/library_showbuilder.js | 4 ++-- airtime_mvc/public/js/airtime/library/library.js | 4 ++-- airtime_mvc/public/js/airtime/library/podcast.js | 2 +- airtime_mvc/public/js/airtime/library/publish.js | 2 +- airtime_mvc/public/js/airtime/library/spl.js | 3 +-- python_apps/airtime-celery/airtime-celery/tasks.py | 1 + 12 files changed, 34 insertions(+), 17 deletions(-) diff --git a/airtime_mvc/application/common/CeleryManager.php b/airtime_mvc/application/common/CeleryManager.php index 6e0c900d0..2f27dd3d4 100644 --- a/airtime_mvc/application/common/CeleryManager.php +++ b/airtime_mvc/application/common/CeleryManager.php @@ -5,7 +5,7 @@ class CeleryManager { /** * @var int milliseconds (for compatibility with celery) until we consider a message to have timed out */ - private static $_CELERY_MESSAGE_TIMEOUT = 600000; // 10 minutes + private static $_CELERY_MESSAGE_TIMEOUT = 900000; // 15 minutes /** * We have to use celeryresults (the default results exchange) because php-celery diff --git a/airtime_mvc/application/common/PodcastManager.php b/airtime_mvc/application/common/PodcastManager.php index 2af85f97c..29a68699c 100644 --- a/airtime_mvc/application/common/PodcastManager.php +++ b/airtime_mvc/application/common/PodcastManager.php @@ -50,12 +50,17 @@ class PodcastManager { $podcastArray = Application_Service_PodcastService::getPodcastById($podcast->getDbPodcastId()); $episodeList = $podcastArray["episodes"]; $episodes = array(); - // Sort the episodes by publication date to get the most recent - // usort($episodeList, array(static::class, "_sortByEpisodePubDate")); + usort($episodeList, array(static::class, "_sortByEpisodePubDate")); for ($i = 0; $i < sizeof($episodeList); $i++) { $episodeData = $episodeList[$i]; + $ts = $podcast->getDbAutoIngestTimestamp(); + // If the timestamp for this podcast is empty (no previous episodes have been ingested) and there are no + // episodes in the list of episodes to ingest, don't skip this episode - we should try to ingest the + // most recent episode when the user first sets the podcast to automatic ingest. // If the publication date of this episode is before the ingest timestamp, we don't need to ingest it - if (strtotime($episodeData["pub_date"]) < strtotime($podcast->getDbAutoIngestTimestamp())) continue; + if ((empty($ts) && !empty($episodes)) || strtotime($episodeData["pub_date"]) < strtotime($ts)) { + continue; + } $episode = PodcastEpisodesQuery::create()->findOneByDbEpisodeGuid($episodeData["guid"]); // Make sure there's no existing episode placeholder or import, and that the data is non-empty if (empty($episode) && !empty($episodeData)) { @@ -87,7 +92,7 @@ class PodcastManager { */ protected static function _sortByEpisodePubDate($a, $b) { if ($a["pub_date"] == $b["pub_date"]) return 0; - return ($a["pub_date"] < $b["pub_date"]) ? 1 : -1; // Descending order + return (strtotime($a["pub_date"]) < strtotime($b["pub_date"])) ? 1 : -1; // Descending order } } \ No newline at end of file diff --git a/airtime_mvc/application/common/TaskManager.php b/airtime_mvc/application/common/TaskManager.php index 0425f340c..45268da0c 100644 --- a/airtime_mvc/application/common/TaskManager.php +++ b/airtime_mvc/application/common/TaskManager.php @@ -82,6 +82,7 @@ final class TaskManager { if ($this->_isUserSessionRequest()) { return; } + Logging::info("Running the tasks!"); $this->_con = Propel::getConnection(CcPrefPeer::DATABASE_NAME); $this->_con->beginTransaction(); try { diff --git a/airtime_mvc/application/controllers/plugins/PageLayoutInitPlugin.php b/airtime_mvc/application/controllers/plugins/PageLayoutInitPlugin.php index 069ee6ef0..5934b152b 100644 --- a/airtime_mvc/application/controllers/plugins/PageLayoutInitPlugin.php +++ b/airtime_mvc/application/controllers/plugins/PageLayoutInitPlugin.php @@ -48,13 +48,17 @@ class PageLayoutInitPlugin extends Zend_Controller_Plugin_Abstract $this->_initGlobals(); $this->_initCsrfNamespace(); - $this->_initTasks(); $this->_initHeadLink(); $this->_initHeadScript(); $this->_initTitle(); $this->_initTranslationGlobals(); $this->_initViewHelpers(); } + + // Piggyback the TaskManager onto API calls + if ($controller == "api") { + $this->_initTasks(); + } } /** diff --git a/airtime_mvc/application/modules/rest/controllers/PodcastController.php b/airtime_mvc/application/modules/rest/controllers/PodcastController.php index 1d972f1c7..241ea3e13 100644 --- a/airtime_mvc/application/modules/rest/controllers/PodcastController.php +++ b/airtime_mvc/application/modules/rest/controllers/PodcastController.php @@ -179,7 +179,7 @@ class Rest_PodcastController extends Zend_Rest_Controller case HttpRequestType::GET: foreach($ids as $id) { $responseBody[] = array( - "podcast" => Application_Service_PodcastService::getPodcastById($id), + "podcast" => json_encode(Application_Service_PodcastService::getPodcastById($id)), "html" => $this->view->render('podcast/podcast.phtml') ); } diff --git a/airtime_mvc/application/services/PodcastEpisodeService.php b/airtime_mvc/application/services/PodcastEpisodeService.php index c4371a0d7..de7550d59 100644 --- a/airtime_mvc/application/services/PodcastEpisodeService.php +++ b/airtime_mvc/application/services/PodcastEpisodeService.php @@ -151,11 +151,18 @@ class Application_Service_PodcastEpisodeService extends Application_Service_Thir $dbEpisode = PodcastEpisodesQuery::create() ->findOneByDbId($episode->episodeid); + + // If the placeholder for the episode is somehow removed, return with a warning + if (!$dbEpisode) { + Logging::warn("Celery task $task episode $episode->episodeid unsuccessful: episode placeholder removed"); + return $ref; + } + // Even if the task itself succeeds, the download could have failed, so check the status - if ($status == CELERY_SUCCESS_STATUS && $episode->status) { + if ($status == CELERY_SUCCESS_STATUS && $episode->status == 1) { $dbEpisode->setDbFileId($episode->fileid)->save(); } else { - Logging::warn("Celery task $task episode $episode->episodeid unsuccessful with status $episode->status"); + Logging::warn("Celery task $task episode $episode->episodeid unsuccessful with message $episode->error"); $dbEpisode->delete(); } diff --git a/airtime_mvc/public/js/airtime/library/events/library_showbuilder.js b/airtime_mvc/public/js/airtime/library/events/library_showbuilder.js index 2e7db95c4..9a9934160 100644 --- a/airtime_mvc/public/js/airtime/library/events/library_showbuilder.js +++ b/airtime_mvc/public/js/airtime/library/events/library_showbuilder.js @@ -374,10 +374,10 @@ var AIRTIME = (function(AIRTIME) { //buildEditMetadataDialog(json); }); } else if (data.ftype === "playlist" || data.ftype === "block") { - AIRTIME.playlist.fnEdit(data.id, data.ftype, baseUrl + 'playlist/edit'); + AIRTIME.playlist.fnEdit(data.id, data.tr_id, baseUrl + 'playlist/edit'); AIRTIME.playlist.validatePlaylistElements(); } else if (data.ftype === "stream") { - AIRTIME.playlist.fnEdit(data.id, data.ftype, baseUrl + 'webstream/edit'); + AIRTIME.playlist.fnEdit(data.id, data.tr_id, baseUrl + 'webstream/edit'); } }); }); diff --git a/airtime_mvc/public/js/airtime/library/library.js b/airtime_mvc/public/js/airtime/library/library.js index 42622208d..aaec55222 100644 --- a/airtime_mvc/public/js/airtime/library/library.js +++ b/airtime_mvc/public/js/airtime/library/library.js @@ -1080,12 +1080,12 @@ var AIRTIME = (function(AIRTIME) { }; } else if (data.ftype === "playlist" || data.ftype === "block") { callback = function() { - AIRTIME.playlist.fnEdit(data.id, data.ftype, baseUrl+'playlist/edit'); + AIRTIME.playlist.fnEdit(data.id, data.tr_id, baseUrl+'playlist/edit'); AIRTIME.playlist.validatePlaylistElements(); }; } else if (data.ftype === "stream") { callback = function() { - AIRTIME.playlist.fnEdit(data.id, data.ftype, baseUrl + 'webstream/edit'); + AIRTIME.playlist.fnEdit(data.id, data.tr_id, baseUrl + 'webstream/edit'); } } else { throw new Exception($.i18n._("Unknown type: ") + data.ftype); diff --git a/airtime_mvc/public/js/airtime/library/podcast.js b/airtime_mvc/public/js/airtime/library/podcast.js index 3fc482487..4fbd15fda 100644 --- a/airtime_mvc/public/js/airtime/library/podcast.js +++ b/airtime_mvc/public/js/airtime/library/podcast.js @@ -137,7 +137,7 @@ var AIRTIME = (function (AIRTIME) { * Called when editing one or more podcasts. * * @param data JSON data returned from the server. - * Contains stringified podcast object JSON and tab + * Contains a JSON encoded podcast object and tab * content HTML and has the following form: * { * podcast: '{ diff --git a/airtime_mvc/public/js/airtime/library/publish.js b/airtime_mvc/public/js/airtime/library/publish.js index 9e777021b..e1956c286 100644 --- a/airtime_mvc/public/js/airtime/library/publish.js +++ b/airtime_mvc/public/js/airtime/library/publish.js @@ -119,7 +119,7 @@ var AIRTIME = (function (AIRTIME) { jQuery.get(dialogUrl, { csrf_token: jQuery("#csrf").val() }) .success(function(html) { - var tab = AIRTIME.tabs.openTab(html, mediaId, null); + var tab = AIRTIME.tabs.openTab(html, PUBLISH_APP_NAME+"_"+mediaId, null); _bootstrapAngularApp(mediaId, tab); }); diff --git a/airtime_mvc/public/js/airtime/library/spl.js b/airtime_mvc/public/js/airtime/library/spl.js index 9c7e8d3bd..e9a5f895f 100644 --- a/airtime_mvc/public/js/airtime/library/spl.js +++ b/airtime_mvc/public/js/airtime/library/spl.js @@ -1117,14 +1117,13 @@ var AIRTIME = (function(AIRTIME){ AIRTIME.tabs.openTab(json.html, uid, AIRTIME.playlist._initFileMdEvents); }; - mod.fnEdit = function(id, type, url) { + mod.fnEdit = function(id, uid, url) { //openPlaylistPanel(); stopAudioPreview(); $.post(url, {format: "json", id: id, type: type}, function(json) { - var uid = AIRTIME.library.MediaTypeFullToStringEnum.type+"_"+id; AIRTIME.tabs.openTab(json.html, uid, AIRTIME.playlist._initPlaylistTabEvents); redrawLib(); }); diff --git a/python_apps/airtime-celery/airtime-celery/tasks.py b/python_apps/airtime-celery/airtime-celery/tasks.py index dcd671808..4c751e26d 100644 --- a/python_apps/airtime-celery/airtime-celery/tasks.py +++ b/python_apps/airtime-celery/airtime-celery/tasks.py @@ -109,6 +109,7 @@ def podcast_download(id, url, callback_url, api_key): obj['fileid'] = f['id'] obj['status'] = 1 except Exception as e: + obj['error'] = e.message logger.info('Error during file download: {0}'.format(e.message)) obj['status'] = 0 return json.dumps(obj)