* Change the Celery timeout to 15 minutes (from 10) to better accommodate automatic ingest

* Fix various small bugs in auto ingestion and tab implementation
* Update TaskManager run conditions to piggyback on API calls - guarantees a certain frequency of requests and greatly reduces chances of lock contention
This commit is contained in:
Duncan Sommerville 2015-10-29 17:53:45 -04:00
parent c92eceb60d
commit e3feb17f0c
12 changed files with 34 additions and 17 deletions

View File

@ -5,7 +5,7 @@ class CeleryManager {
/**
* @var int milliseconds (for compatibility with celery) until we consider a message to have timed out
*/
private static $_CELERY_MESSAGE_TIMEOUT = 600000; // 10 minutes
private static $_CELERY_MESSAGE_TIMEOUT = 900000; // 15 minutes
/**
* We have to use celeryresults (the default results exchange) because php-celery

View File

@ -50,12 +50,17 @@ class PodcastManager {
$podcastArray = Application_Service_PodcastService::getPodcastById($podcast->getDbPodcastId());
$episodeList = $podcastArray["episodes"];
$episodes = array();
// Sort the episodes by publication date to get the most recent
// usort($episodeList, array(static::class, "_sortByEpisodePubDate"));
usort($episodeList, array(static::class, "_sortByEpisodePubDate"));
for ($i = 0; $i < sizeof($episodeList); $i++) {
$episodeData = $episodeList[$i];
$ts = $podcast->getDbAutoIngestTimestamp();
// If the timestamp for this podcast is empty (no previous episodes have been ingested) and there are no
// episodes in the list of episodes to ingest, don't skip this episode - we should try to ingest the
// most recent episode when the user first sets the podcast to automatic ingest.
// If the publication date of this episode is before the ingest timestamp, we don't need to ingest it
if (strtotime($episodeData["pub_date"]) < strtotime($podcast->getDbAutoIngestTimestamp())) continue;
if ((empty($ts) && !empty($episodes)) || strtotime($episodeData["pub_date"]) < strtotime($ts)) {
continue;
}
$episode = PodcastEpisodesQuery::create()->findOneByDbEpisodeGuid($episodeData["guid"]);
// Make sure there's no existing episode placeholder or import, and that the data is non-empty
if (empty($episode) && !empty($episodeData)) {
@ -87,7 +92,7 @@ class PodcastManager {
*/
protected static function _sortByEpisodePubDate($a, $b) {
if ($a["pub_date"] == $b["pub_date"]) return 0;
return ($a["pub_date"] < $b["pub_date"]) ? 1 : -1; // Descending order
return (strtotime($a["pub_date"]) < strtotime($b["pub_date"])) ? 1 : -1; // Descending order
}
}

View File

@ -82,6 +82,7 @@ final class TaskManager {
if ($this->_isUserSessionRequest()) {
return;
}
Logging::info("Running the tasks!");
$this->_con = Propel::getConnection(CcPrefPeer::DATABASE_NAME);
$this->_con->beginTransaction();
try {

View File

@ -48,13 +48,17 @@ class PageLayoutInitPlugin extends Zend_Controller_Plugin_Abstract
$this->_initGlobals();
$this->_initCsrfNamespace();
$this->_initTasks();
$this->_initHeadLink();
$this->_initHeadScript();
$this->_initTitle();
$this->_initTranslationGlobals();
$this->_initViewHelpers();
}
// Piggyback the TaskManager onto API calls
if ($controller == "api") {
$this->_initTasks();
}
}
/**

View File

@ -179,7 +179,7 @@ class Rest_PodcastController extends Zend_Rest_Controller
case HttpRequestType::GET:
foreach($ids as $id) {
$responseBody[] = array(
"podcast" => Application_Service_PodcastService::getPodcastById($id),
"podcast" => json_encode(Application_Service_PodcastService::getPodcastById($id)),
"html" => $this->view->render('podcast/podcast.phtml')
);
}

View File

@ -151,11 +151,18 @@ class Application_Service_PodcastEpisodeService extends Application_Service_Thir
$dbEpisode = PodcastEpisodesQuery::create()
->findOneByDbId($episode->episodeid);
// If the placeholder for the episode is somehow removed, return with a warning
if (!$dbEpisode) {
Logging::warn("Celery task $task episode $episode->episodeid unsuccessful: episode placeholder removed");
return $ref;
}
// Even if the task itself succeeds, the download could have failed, so check the status
if ($status == CELERY_SUCCESS_STATUS && $episode->status) {
if ($status == CELERY_SUCCESS_STATUS && $episode->status == 1) {
$dbEpisode->setDbFileId($episode->fileid)->save();
} else {
Logging::warn("Celery task $task episode $episode->episodeid unsuccessful with status $episode->status");
Logging::warn("Celery task $task episode $episode->episodeid unsuccessful with message $episode->error");
$dbEpisode->delete();
}

View File

@ -374,10 +374,10 @@ var AIRTIME = (function(AIRTIME) {
//buildEditMetadataDialog(json);
});
} else if (data.ftype === "playlist" || data.ftype === "block") {
AIRTIME.playlist.fnEdit(data.id, data.ftype, baseUrl + 'playlist/edit');
AIRTIME.playlist.fnEdit(data.id, data.tr_id, baseUrl + 'playlist/edit');
AIRTIME.playlist.validatePlaylistElements();
} else if (data.ftype === "stream") {
AIRTIME.playlist.fnEdit(data.id, data.ftype, baseUrl + 'webstream/edit');
AIRTIME.playlist.fnEdit(data.id, data.tr_id, baseUrl + 'webstream/edit');
}
});
});

View File

@ -1080,12 +1080,12 @@ var AIRTIME = (function(AIRTIME) {
};
} else if (data.ftype === "playlist" || data.ftype === "block") {
callback = function() {
AIRTIME.playlist.fnEdit(data.id, data.ftype, baseUrl+'playlist/edit');
AIRTIME.playlist.fnEdit(data.id, data.tr_id, baseUrl+'playlist/edit');
AIRTIME.playlist.validatePlaylistElements();
};
} else if (data.ftype === "stream") {
callback = function() {
AIRTIME.playlist.fnEdit(data.id, data.ftype, baseUrl + 'webstream/edit');
AIRTIME.playlist.fnEdit(data.id, data.tr_id, baseUrl + 'webstream/edit');
}
} else {
throw new Exception($.i18n._("Unknown type: ") + data.ftype);

View File

@ -137,7 +137,7 @@ var AIRTIME = (function (AIRTIME) {
* Called when editing one or more podcasts.
*
* @param data JSON data returned from the server.
* Contains stringified podcast object JSON and tab
* Contains a JSON encoded podcast object and tab
* content HTML and has the following form:
* {
* podcast: '{

View File

@ -119,7 +119,7 @@ var AIRTIME = (function (AIRTIME) {
jQuery.get(dialogUrl, { csrf_token: jQuery("#csrf").val() })
.success(function(html) {
var tab = AIRTIME.tabs.openTab(html, mediaId, null);
var tab = AIRTIME.tabs.openTab(html, PUBLISH_APP_NAME+"_"+mediaId, null);
_bootstrapAngularApp(mediaId, tab);
});

View File

@ -1117,14 +1117,13 @@ var AIRTIME = (function(AIRTIME){
AIRTIME.tabs.openTab(json.html, uid, AIRTIME.playlist._initFileMdEvents);
};
mod.fnEdit = function(id, type, url) {
mod.fnEdit = function(id, uid, url) {
//openPlaylistPanel();
stopAudioPreview();
$.post(url,
{format: "json", id: id, type: type},
function(json) {
var uid = AIRTIME.library.MediaTypeFullToStringEnum.type+"_"+id;
AIRTIME.tabs.openTab(json.html, uid, AIRTIME.playlist._initPlaylistTabEvents);
redrawLib();
});

View File

@ -109,6 +109,7 @@ def podcast_download(id, url, callback_url, api_key):
obj['fileid'] = f['id']
obj['status'] = 1
except Exception as e:
obj['error'] = e.message
logger.info('Error during file download: {0}'.format(e.message))
obj['status'] = 0
return json.dumps(obj)