From bcbd8a8050ce16e4cd6ea9ed86b00b567c4fb3d0 Mon Sep 17 00:00:00 2001 From: Duncan Sommerville Date: Mon, 19 Oct 2015 15:36:34 -0400 Subject: [PATCH] Fix bug in auto ingest, immediately retry on failure --- .../application/services/PodcastEpisodeService.php | 10 +++++++--- python_apps/airtime-celery/airtime-celery/tasks.py | 13 +++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/airtime_mvc/application/services/PodcastEpisodeService.php b/airtime_mvc/application/services/PodcastEpisodeService.php index 45ce09fe3..6e8276ed5 100644 --- a/airtime_mvc/application/services/PodcastEpisodeService.php +++ b/airtime_mvc/application/services/PodcastEpisodeService.php @@ -117,17 +117,21 @@ class Application_Service_PodcastEpisodeService extends Application_Service_Thir $ref = parent::updateTrackReference($task, $episodeId, $episodes, $status); if ($status == CELERY_SUCCESS_STATUS) { - foreach($episodes as $episode) { + foreach ($episodes as $episode) { // Since we process episode downloads as a batch, individual downloads can fail // even if the task itself succeeds + $dbEpisode = PodcastEpisodesQuery::create() + ->findOneByDbId($episode->episodeid); if ($episode->status) { - $dbEpisode = PodcastEpisodesQuery::create() - ->findOneByDbId($episode->episodeid); $dbEpisode->setDbFileId($episode->fileid) ->save(); + } else { + Logging::warn("Celery task $task episode $episode->episodeid unsuccessful with status $episode->status"); + $dbEpisode->delete(); } } } + // TODO: do we need a broader fail condition here? return $ref; } diff --git a/python_apps/airtime-celery/airtime-celery/tasks.py b/python_apps/airtime-celery/airtime-celery/tasks.py index 83a7faefc..a3904c95f 100644 --- a/python_apps/airtime-celery/airtime-celery/tasks.py +++ b/python_apps/airtime-celery/airtime-celery/tasks.py @@ -110,10 +110,10 @@ def podcast_download(episodes, callback_url, api_key): f = json.loads(re.content) # Read the response from the media API to get the file id obj['fileid'] = f['id'] obj['status'] = 1 - response.append(obj) except Exception as e: logger.info('Error during file download: {0}'.format(e.message)) obj['status'] = 0 + response.append(obj) return json.dumps(response) @@ -127,10 +127,15 @@ def get_filename(r): """ # Try to get the filename from the content disposition d = r.headers.get('Content-Disposition') + filename = '' if d: - _, params = cgi.parse_header(d) - filename = params['filename'] - else: + try: + _, params = cgi.parse_header(d) + filename = params['filename'] + except Exception as e: + # We end up here if we get a Content-Disposition header with no filename + logger.warn("Couldn't find file name in Content-Disposition header, using url") + if not filename: # Since we don't necessarily get the filename back in the response headers, # parse the URL and get the filename and extension path = urlparse.urlsplit(r.url).path