Add SoundCloud update and download tasks to Celery backend; requires airtime-celery reinstall

airtime_mvc/application/common/TaskManager.php

@@ -82,7 +82,6 @@ final class TaskManager {
         if ($this->_isUserSessionRequest()) {
             return;
         }
-        Logging::info("Running the tasks!");
         $this->_con = Propel::getConnection(CcPrefPeer::DATABASE_NAME);
         $this->_con->beginTransaction();
         try {

airtime_mvc/application/controllers/LibraryController.php

@@ -139,33 +139,18 @@ class LibraryController extends Zend_Controller_Action
                 if ($type === "audioclip" && $soundcloudService->hasAccessToken()) {
                     $serviceId = $soundcloudService->getServiceId($id);
                     if (!is_null($file) && $serviceId != 0) {
+                        $trackRef = ThirdPartyTrackReferencesQuery::create()
+                            ->filterByDbService(SOUNDCLOUD_SERVICE_NAME)
+                            ->findOneByDbFileId($id);
+
                         //create a menu separator
                         $menu["sep1"] = "-----------";
 
                         //create a sub menu for Soundcloud actions.
                         $menu["soundcloud"] = array("name" => _(SOUNDCLOUD), "icon" => "soundcloud", "items" => array());
                         $menu["soundcloud"]["items"]["view"] = array("name" => _("View track"), "icon" => "soundcloud", "url" => $baseUrl . "soundcloud/view-on-sound-cloud/id/{$id}");
-                        // $menu["soundcloud"]["items"]["remove"] = array("name" => _("Remove track"), "icon" => "soundcloud", "url" => $baseUrl . "soundcloud/delete/id/{$id}");
+                        $menu["soundcloud"]["items"]["update"] = array("name" => _("Update track"), "icon" => "soundcloud", "url" => $baseUrl . "soundcloud/update/id/{$trackRef->getDbForeignId()}");
                     }
-                    /*
-                    Since we upload to SoundCloud from the Publish dialog now, this is unnecessary
-
-                    else {
-                        // If a reference exists for this file ID, that means the user has uploaded the track
-                        // but we haven't yet gotten a response from Celery, so disable the menu item
-                        if ($soundcloudService->referenceExists($id)) {
-                            $menu["soundcloud"]["items"]["upload"] = array(
-                                "name" => _("Upload track"), "icon" => "soundcloud",
-                                "url" => $baseUrl . "soundcloud/upload/id/{$id}", "disabled" => true
-                            );
-                        } else {
-                            $menu["soundcloud"]["items"]["upload"] = array(
-                                "name" => _("Upload track"), "icon" => "soundcloud",
-                                "url" => $baseUrl . "soundcloud/upload/id/{$id}"
-                            );
-                        }
-                    }
-                    */
                 }
             }
         } elseif ($type === "playlist" || $type === "block") {

airtime_mvc/application/controllers/SoundcloudController.php

@@ -28,11 +28,22 @@ class SoundcloudController extends ThirdPartyController implements OAuth2Control
      * @throws Zend_Controller_Response_Exception thrown if upload fails for any reason
      */
     public function uploadAction() {
-        $request = $this->getRequest();
+        $id = $this->getRequest()->getParam('id');
-        $id = $request->getParam('id');
         $this->_service->upload($id);
     }
 
+    /**
+     * Update the file with the given id on SoundCloud
+     *
+     * @return void
+     *
+     * @throws Zend_Controller_Response_Exception thrown if upload fails for any reason
+     */
+    public function updateAction() {
+        $id = $this->getRequest()->getParam('id');
+        $this->_service->update($id);
+    }
+
     /**
      * Download the file with the given id from SoundCloud
      *
@@ -41,8 +52,7 @@ class SoundcloudController extends ThirdPartyController implements OAuth2Control
      * @throws Zend_Controller_Response_Exception thrown if download fails for any reason
      */
     public function downloadAction() {
-        $request = $this->getRequest();
+        $id = $this->getRequest()->getParam('id');
-        $id = $request->getParam('id');
         $this->_service->download($id);
     }
 
@@ -54,8 +64,7 @@ class SoundcloudController extends ThirdPartyController implements OAuth2Control
      * @throws Zend_Controller_Response_Exception thrown if deletion fails for any reason
      */
     public function deleteAction() {
-        $request = $this->getRequest();
+        $id = $this->getRequest()->getParam('id');
-        $id = $request->getParam('id');
         $this->_service->delete($id);
     }
 
@@ -97,8 +106,7 @@ class SoundcloudController extends ThirdPartyController implements OAuth2Control
      * @return void
      */
     public function viewOnSoundCloudAction() {
-        $request = $this->getRequest();
+        $id = $this->getRequest()->getParam('id');
-        $id = $request->getParam('id');
         try {
             $soundcloudLink = $this->_service->getLinkToFile($id);
             header('Location: ' . $soundcloudLink);

airtime_mvc/application/services/PublishService.php

@@ -73,7 +73,7 @@ class Application_Service_PublishService {
         );
     }
 
-    /** @noinspection PhpUnusedPrivateMethodInspection
+    /**
      * Reflective accessor for SoundCloud publication status for the
      * file with the given ID
      *
@@ -84,10 +84,11 @@ class Application_Service_PublishService {
      */
     private static function getSoundCloudPublishStatus($fileId) {
         $soundcloudService = new Application_Service_SoundcloudService();
-        return ($soundcloudService->getServiceId($fileId) != 0);
+        return ($soundcloudService->referenceExists($fileId));
     }
 
-    /** @noinspection PhpUnusedPrivateMethodInspection
+    /**
+     *
      * Reflective accessor for Station podcast publication status for the
      * file with the given ID
      *

airtime_mvc/application/services/SoundcloudService.php

@@ -14,6 +14,7 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
      */
 
     const UPLOAD    = 'upload';
+    const UPDATE    = 'update';
     const DOWNLOAD  = 'download';
     const DELETE    = 'delete';
 
@@ -42,6 +43,7 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
      */
     protected static $_CELERY_TASKS = [
         self::UPLOAD      => 'soundcloud-upload',
+        self::UPDATE      => 'soundcloud-update',
         self::DOWNLOAD    => 'soundcloud-download',
         self::DELETE      => 'soundcloud-delete'
     ];
@@ -136,13 +138,28 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
     }
 
     /**
-     * Given a track identifier, download a track from SoundCloud
+     * Given a track identifier, update a track on SoundCloud
-     *
-     * If no identifier is given, download all the user's tracks
      *
      * @param int $trackId a track identifier
      */
-    public function download($trackId = null) {
+    public function update($trackId) {
+        $trackRef = ThirdPartyTrackReferencesQuery::create()
+            ->findOneByDbForeignId($trackId);
+        $file = Application_Model_StoredFile::RecallById($trackRef->getDbFileId());
+        $data = array(
+            'data' => $this->_getUploadData($file),
+            'token' => $this->_accessToken,
+            'track_id' => $trackId
+        );
+        $this->_executeTask(static::$_CELERY_TASKS[self::UPDATE], $data, $trackRef->getDbFileId());
+    }
+
+    /**
+     * Given a track identifier, download a track from SoundCloud
+     *
+     * @param int $trackId a track identifier
+     */
+    public function download($trackId) {
         $CC_CONFIG = Config::getConfig();
         $data = array(
             'callback_url'  => Application_Common_HTTPHelper::getStationUrl() . 'rest/media',
@@ -200,6 +217,9 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
                 return null;
             }
             $ref->setDbForeignId($track->id);  // SoundCloud identifier
+            if (isset($track->fileid)) {
+                $ref->setDbFileId($track->fileid);  // For downloads, set the cc_files ID
+            }
         }
         // TODO: set SoundCloud upload status?
         // $ref->setDbStatus($status);
@@ -288,6 +308,8 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
     }
 
     /**
+     * Publishing interface proxy
+     *
      * Publish the file with the given file ID to SoundCloud
      *
      * @param int $fileId ID of the file to be published
@@ -297,6 +319,8 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
     }
 
     /**
+     * Publishing interface proxy
+     *
      * Unpublish the file with the given file ID from SoundCloud
      *
      * @param int $fileId ID of the file to be unpublished

airtime_mvc/public/js/airtime/library/library.js

@@ -1178,33 +1178,15 @@ var AIRTIME = (function(AIRTIME) {
                     if (oItems.soundcloud !== undefined) {
                         var soundcloud = oItems.soundcloud.items;
 
-                        // define an upload to soundcloud callback.
+                        if (soundcloud.update !== undefined) {
-                        if (soundcloud.upload !== undefined) {
-
                             callback = function() {
-                                alert($.i18n._("Your track is being uploaded and will " +
+                                $.post(soundcloud.update.url, function () {});
-                                    "appear on SoundCloud in a couple of minutes"));
-                                $.post(soundcloud.upload.url, function(){});
                             };
-                            soundcloud.upload.callback = callback;
+                            soundcloud.update.callback = callback;
-                        }
-
-                        // define an upload to soundcloud callback.
-                        if (soundcloud.remove !== undefined) {
-
-                            callback = function() {
-                                if (confirm($.i18n._("Are you sure? SoundCloud stats and comments for this track will be permanently removed."))) {
-                                    alert($.i18n._("Your track is being deleted from SoundCloud"));
-                                    $.post(soundcloud.remove.url, function () {
-                                    });
-                                }
-                            };
-                            soundcloud.remove.callback = callback;
                         }
 
                         // define a view on soundcloud callback
                         if (soundcloud.view !== undefined) {
-
                             callback = function() {
                                 window.open(soundcloud.view.url);
                             };

airtime_mvc/public/js/airtime/library/publish.js

@@ -20,7 +20,7 @@ var AIRTIME = (function (AIRTIME) {
 
             function init () {
                 var csrfToken = jQuery("#csrf").val();
-                $http.get(endpoint + mediaId, { csrf_token: csrfToken })
+                $http.get(endpoint + mediaId, {csrf_token: csrfToken})
                     .success(function (json) {
                         $scope.media = json;
                         tab.setName($scope.media.track_title);
@@ -28,7 +28,7 @@ var AIRTIME = (function (AIRTIME) {
 
                 // Get an object containing all sources, their translated labels,
                 // and their publication state for the file with the given ID
-                $http.get(endpoint + mediaId + '/publish-sources', { csrf_token: csrfToken })
+                $http.get(endpoint + mediaId + '/publish-sources', {csrf_token: csrfToken})
                     .success(function (json) {
                         $scope.sources = json;
                         // Store the data (whether each source should be published to when publish is clicked)
@@ -49,14 +49,18 @@ var AIRTIME = (function (AIRTIME) {
 
             $scope.publish = function () {
                 var data = {};
-                jQuery.each($scope.publishData, function(k, v) {
+                jQuery.each($scope.publishData, function (k, v) {
                     if (v) {
+                        if (k == "soundcloud") {
+                            alert($.i18n._("Your track is being uploaded and will "
+                                + "appear on SoundCloud in a couple of minutes"));
+                        }
                         data[k] = 'publish';  // FIXME: should be more robust
                     }
                 });
 
                 if (Object.keys(data).length > 0) {
-                    $http.put(endpoint + mediaId + '/publish', { csrf_token: jQuery("#csrf").val(), sources: data})
+                    $http.put(endpoint + mediaId + '/publish', {csrf_token: jQuery("#csrf").val(), sources: data})
                         .success(function () {
                             init();
                         });
@@ -66,10 +70,13 @@ var AIRTIME = (function (AIRTIME) {
             $scope.remove = function (source) {
                 var data = {};
                 data[source] = 'unpublish';  // FIXME: should be more robust
-                $http.put(endpoint + mediaId + '/publish', { csrf_token: jQuery("#csrf").val(), sources: data })
+                if (source != "soundcloud" || confirm($.i18n._("Are you sure? SoundCloud stats and comments "
-                    .success(function () {
+                        + "for this track will be permanently removed."))) {
-                        init();
+                    $http.put(endpoint + mediaId + '/publish', {csrf_token: jQuery("#csrf").val(), sources: data})
-                    });
+                        .success(function () {
+                            init();
+                        });
+                }
             };
 
             $scope.discard = function () {

python_apps/airtime-celery/airtime-celery/celeryconfig.py

@@ -25,14 +25,14 @@ def parse_rmq_config(rmq_config):
 BROKER_URL = get_rmq_broker()
 CELERY_RESULT_BACKEND = 'amqp'            # Use RabbitMQ as the celery backend
 CELERY_RESULT_PERSISTENT = True           # Persist through a broker restart
-CELERY_TASK_RESULT_EXPIRES = 600          # Expire task results after 10 minutes
+CELERY_TASK_RESULT_EXPIRES = 900          # Expire task results after 15 minutes
 CELERY_RESULT_EXCHANGE = 'celeryresults'  # Default exchange - needed due to php-celery
 CELERY_QUEUES = (
     Queue('soundcloud', exchange=Exchange('soundcloud'), routing_key='soundcloud'),
     Queue('podcast', exchange=Exchange('podcast'), routing_key='podcast'),
     Queue(exchange=Exchange('celeryresults'), auto_delete=True),
 )
-CELERY_EVENT_QUEUE_EXPIRES = 600          # RabbitMQ x-expire after 10 minutes
+CELERY_EVENT_QUEUE_EXPIRES = 900          # RabbitMQ x-expire after 15 minutes
 
 # Celery task settings
 CELERY_TASK_SERIALIZER = 'json'

python_apps/airtime-celery/airtime-celery/tasks.py

@@ -23,8 +23,8 @@ def soundcloud_upload(data, token, file_path):
     :param token:     OAuth2 client access token
     :param file_path: path to the file being uploaded
 
-    :return: the SoundCloud response object
+    :return: JSON formatted string of the SoundCloud response object
-    :rtype: dict
+    :rtype: string
     """
     client = soundcloud.Client(access_token=token)
     # Open the file with urllib2 if it's a cloud file
@@ -40,28 +40,65 @@ def soundcloud_upload(data, token, file_path):
 
 
 @celery.task(name='soundcloud-download', acks_late=True)
-def soundcloud_download(token, callback_url, api_key, track_id=None):
+def soundcloud_download(token, callback_url, api_key, track_id):
     """
-    This is in stasis 
+    Download a file from SoundCloud
 
     :param token:        OAuth2 client access token
     :param callback_url: callback URL to send the downloaded file to
     :param api_key:      API key for callback authentication
     :param track_id:     SoundCloud track identifier
-    :rtype: None
+
+    :return: JSON formatted string of file identifiers for the downloaded tracks
+    :rtype: string
+    """
+    client = soundcloud.Client(access_token=token)
+    obj = {}
+    try:
+        track = client.get('/tracks/%s' % track_id)
+        obj.update(track.fields())
+        if track.downloadable:
+            re = None
+            with closing(requests.get('%s?oauth_token=%s' % (track.download_url, client.access_token), verify=True, stream=True)) as r:
+                filename = get_filename(r)
+                re = requests.post(callback_url, files={'file': (filename, r.content)}, auth=requests.auth.HTTPBasicAuth(api_key, ''))
+            re.raise_for_status()
+            f = json.loads(re.content)  # Read the response from the media API to get the file id
+            obj['fileid'] = f['id']
+        else:
+            # manually update the task state
+            self.update_state(
+                state = states.FAILURE,
+                meta = 'Track %s is not flagged as downloadable!' % track.title
+            )
+            # ignore the task so no other state is recorded
+            raise Ignore()
+    except Exception as e:
+        logger.info('Error during file download: {0}'.format(e.message))
+        raise e
+    return json.dumps(obj)
+
+
+@celery.task(name='soundcloud-update', acks_late=True)
+def soundcloud_update(data, token, track_id):
+    """
+    Update a file on SoundCloud
+
+    :param data:      associative array containing SoundCloud metadata
+    :param token:     OAuth2 client access token
+    :param track_id:  SoundCloud ID of the track to be updated
+
+    :return: JSON formatted string of the SoundCloud response object
+    :rtype: string
     """
     client = soundcloud.Client(access_token=token)
     try:
-        tracks = client.get('/me/tracks') if track_id is None else {client.get('/tracks/%s' % track_id)}
+        logger.info('Updating track {title}'.format(**data))
-        for track in tracks:
+        track = client.put('/tracks/%s' % track_id, track=data)
-            if track.downloadable:
-                track_file = client.get(track.download_url)
-                with track_file as f:
-                    requests.post(callback_url, data=f, auth=requests.auth.HTTPBasicAuth(api_key, ''))
     except Exception as e:
-        logger.info('Error during file download: {0}'.format(e.message))
+        logger.info('Error updating track {title}: {0}'.format(e.message, **data))
-        logger.info(str(e))
         raise e
+    return json.dumps(track.fields())
 
 
 @celery.task(name='soundcloud-delete', acks_late=True)
@@ -72,8 +109,8 @@ def soundcloud_delete(token, track_id):
     :param token:       OAuth2 client access token
     :param track_id:    SoundCloud track identifier
 
-    :return: the SoundCloud response object
+    :return: JSON formatted string of the SoundCloud response object
-    :rtype: dict
+    :rtype: string
     """
     client = soundcloud.Client(access_token=token)
     try:
@@ -94,7 +131,10 @@ def podcast_download(id, url, callback_url, api_key):
     :param url:             download url for the episode
     :param callback_url:    callback URL to send the downloaded file to
     :param api_key:         API key for callback authentication
-    :rtype: None
+
+    :return: JSON formatted string of a dictionary of download statuses
+             and file identifiers (for successful uploads)
+    :rtype: string
     """
     # Object to store file IDs, episode IDs, and download status
     # (important if there's an error before the file is posted)
@@ -121,6 +161,8 @@ def get_filename(r):
     by parsing either the content disposition or the request URL
 
     :param r: request object
+
+    :return: the file name
     :rtype: string
     """
     # Try to get the filename from the content disposition