Add SoundCloud delete functionality and fixes; implement TaskManager to run background jobs

airtime_mvc/application/Bootstrap.php

@@ -27,6 +27,7 @@ require_once "ProvisioningHelper.php";
 require_once "GoogleAnalytics.php";
 require_once "Timezone.php";
 require_once "Auth.php";
+require_once "TaskManager.php";
 require_once __DIR__.'/services/SoundcloudService.php';
 require_once __DIR__.'/forms/helpers/ValidationTypes.php';
 require_once __DIR__.'/forms/helpers/CustomDecorators.php';
@@ -123,15 +124,12 @@ class Bootstrap extends Zend_Application_Bootstrap_Bootstrap
         $view->headScript()->appendScript("var COMPANY_NAME = '" . COMPANY_NAME . "';");
     }
     
-    protected function _initUpgrade() {
+    protected function _initTasks() {
         /* We need to wrap this here so that we aren't checking when we're running the unit test suite
          */
         if (getenv("AIRTIME_UNIT_TEST") != 1) {
             //This will do the upgrade too if it's needed...
-            if (UpgradeManager::checkIfUpgradeIsNeeded()) {
+            TaskManager::getInstance()->runTasks();
-                $upgradeManager = new UpgradeManager();
-                $upgradeManager->doUpgrade();
-            }
         }
     }
 

airtime_mvc/application/common/TaskManager.php

@@ -0,0 +1,160 @@
+<?php
+
+/**
+ * Class TaskManager
+ */
+final class TaskManager {
+
+    /**
+     * @var array tasks to be run
+     */
+    protected $_taskList = [
+        AirtimeTask::SOUNDCLOUD,
+        AirtimeTask::UPGRADE
+    ];
+
+    /**
+     * @var TaskManager singleton instance object
+     */
+    protected static $_instance;
+
+    /**
+     * Private constructor so class is uninstantiable
+     */
+    private function __construct() {
+    }
+
+    /**
+     * Get the singleton instance of this class
+     *
+     * @return TaskManager the TaskManager instance
+     */
+    public static function getInstance() {
+        if (!self::$_instance) {
+            self::$_instance = new TaskManager();
+        }
+        return self::$_instance;
+    }
+
+    /**
+     * Run all tasks that need to be run
+     */
+    public function runTasks() {
+        foreach ($this->_taskList as $task) {
+            $task = TaskFactory::getTask($task);
+            assert(is_subclass_of($task, 'AirtimeTask'));  // Sanity check
+            /** @var $task AirtimeTask */
+            if ($task && $task->shouldBeRun()) $task->run();
+        }
+    }
+
+}
+
+/**
+ * Interface AirtimeTask Interface for task operations - also acts as task type ENUM
+ */
+interface AirtimeTask {
+
+    /**
+     * PHP doesn't have ENUMs so declare them as interface constants
+     * Task types - values don't really matter as long as they're unique
+     */
+
+    const SOUNDCLOUD = "soundcloud";
+    const UPGRADE = "upgrade";
+
+    /**
+     * Check whether the task should be run
+     *
+     * @return bool true if the task needs to be run, otherwise false
+     */
+    public function shouldBeRun();
+
+    /**
+     * Run the task
+     *
+     * @return void
+     */
+    public function run();
+
+}
+
+/**
+ * Class TaskFactory Factory class to abstract task instantiation
+ */
+class TaskFactory {
+
+    /**
+     * Get an AirtimeTask based on a task type
+     *
+     * @param $task string the task type; uses AirtimeTask constants as an ENUM
+     *
+     * @return AirtimeTask|null return a task of the given type or null if no corresponding
+     *                          task exists or is implemented
+     */
+    public static function getTask($task) {
+        switch($task) {
+            case AirtimeTask::SOUNDCLOUD:
+                return new SoundcloudUploadTask();
+            case AirtimeTask::UPGRADE:
+                return new UpgradeTask();
+        }
+        return null;
+    }
+
+}
+
+/**
+ * Class UpgradeTask
+ */
+class UpgradeTask implements AirtimeTask {
+
+    /**
+     * Check the current Airtime schema version to see if an upgrade should be run
+     *
+     * @return bool true if an upgrade is needed
+     */
+    public function shouldBeRun() {
+        return UpgradeManager::checkIfUpgradeIsNeeded();
+    }
+
+    /**
+     * Run all upgrades above the current schema version
+     */
+    public function run() {
+        UpgradeManager::doUpgrade();
+    }
+
+}
+
+/**
+ * Class SoundcloudUploadTask
+ */
+class SoundcloudUploadTask implements AirtimeTask {
+
+    /**
+     * @var SoundcloudService
+     */
+    protected $_service;
+
+    public function __construct() {
+        $this->_service = new SoundcloudService();
+    }
+
+    /**
+     * Check the ThirdPartyTrackReferences table to see if there are any pending SoundCloud tasks
+     *
+     * @return bool true if there are pending tasks in ThirdPartyTrackReferences
+     */
+    public function shouldBeRun() {
+        return !$this->_service->isBrokerTaskQueueEmpty();
+    }
+
+    /**
+     * Poll the task queue for any completed Celery tasks
+     */
+    public function run() {
+        $this->_service->pollBrokerTaskQueue();
+    }
+
+}

airtime_mvc/application/configs/constants.php

@@ -83,9 +83,20 @@ define('UI_BLOCK_SESSNAME', 'BLOCK');*/
 
 
 // Soundcloud contants
-define('SOUNDCLOUD_NOT_UPLOADED_YET' , -1);
+/**
-define('SOUNDCLOUD_PROGRESS'         , -2);
+ * @var string status string for pending Celery tasks
-define('SOUNDCLOUD_ERROR'            , -3);
+ */
+define('CELERY_PENDING_STATUS', 'PENDING');
+
+/**
+ * @var string status string for successful Celery tasks
+ */
+define('CELERY_SUCCESS_STATUS', 'SUCCESS');
+
+/**
+ * @var string status string for failed Celery tasks
+ */
+define('CELERY_FAILED_STATUS', 'FAILED');
 
 
 //WHMCS integration

airtime_mvc/application/controllers/ErrorController.php

@@ -74,7 +74,6 @@ class ErrorController extends Zend_Controller_Action {
      * 404 error - route or controller
      */
     public function error404Action() {
-        Logging::info("404!");
         $this->_helper->viewRenderer('error-404');
         $this->getResponse()->setHttpResponseCode(404);
         $this->view->message = _('Page not found.');

airtime_mvc/application/controllers/LibraryController.php

@@ -277,13 +277,11 @@ class LibraryController extends Zend_Controller_Action
 
             $serviceId = $soundcloudService->getServiceId($id);
             if (!is_null($file) && $serviceId != 0) {
-                $menu["soundcloud"]["items"]["view"] = array("name" => _("View on Soundcloud"), "icon" => "soundcloud", "url" => $baseUrl."soundcloud/view-on-sound-cloud/id/{$id}");
+                $menu["soundcloud"]["items"]["view"] = array("name" => _("View track"), "icon" => "soundcloud", "url" => $baseUrl."soundcloud/view-on-sound-cloud/id/{$id}");
-                $text = _("Re-upload to SoundCloud");
+                $menu["soundcloud"]["items"]["upload"] = array("name" => _("Remove track"), "icon" => "soundcloud", "url" => $baseUrl."soundcloud/delete/id/{$id}");
             } else {
-                $text = _("Upload to SoundCloud");
+                $menu["soundcloud"]["items"]["upload"] = array("name" => _("Upload track"), "icon" => "soundcloud", "url" => $baseUrl."soundcloud/upload/id/{$id}");
             }
-
-            $menu["soundcloud"]["items"]["upload"] = array("name" => $text, "icon" => "soundcloud", "url" => $baseUrl."soundcloud/upload/id/{$id}");
         }
 
         if (empty($menu)) {

airtime_mvc/application/controllers/SoundcloudController.php

@@ -33,13 +33,8 @@ class SoundcloudController extends ThirdPartyController {
             $soundcloudLink = $this->_service->getLinkToFile($id);
             header('Location: ' . $soundcloudLink);
         } catch (Soundcloud\Exception\InvalidHttpResponseCodeException $e) {
-            // If we end up here it means the track was removed from SoundCloud
-            // or the foreign id in our database is incorrect, so we should just
-            // get rid of the database record
-            Logging::warn("Error retrieving track data from SoundCloud: " . $e->getMessage());
-            $this->_service->removeTrackReference($id);
             // Redirect to a 404 so the user knows something went wrong
-            header('Location: ' . $this->_baseUrl . 'error/error-404'); // Redirect back to the Preference page
+            header('Location: ' . $this->_baseUrl . 'error/error-404');
         }
     }
 

airtime_mvc/application/controllers/ThirdPartyController.php

@@ -41,6 +41,17 @@ abstract class ThirdPartyController extends Zend_Controller_Action {
         header('Location: ' . $auth_url);
     }
 
+    /**
+     * Clear the previously saved request token from the preferences
+     *
+     * @return void
+     */
+    public function deauthorizeAction() {
+        $function = $this->_SERVICE_TOKEN_ACCESSOR;
+        Application_Model_Preference::$function("");
+        header('Location: ' . $this->_baseUrl . 'Preference');  // Redirect back to the Preference page
+    }
+
     /**
      * Called when user successfully completes third-party authorization
      * Store the returned request token for future requests
@@ -67,25 +78,16 @@ abstract class ThirdPartyController extends Zend_Controller_Action {
     }
 
     /**
-     * Clear the previously saved request token from the preferences
+     * Delete the file with the given id from a third-party service
      *
      * @return void
-     */
-    public function deauthorizeAction() {
-        Application_Model_Preference::$this->_SERVICE_TOKEN_ACCESSOR("");
-        header('Location: ' . $this->_baseUrl . 'Preference');  // Redirect back to the Preference page
-    }
-
-    /**
-     * Poll the task queue for completed tasks associated with this service
-     * Optionally accepts a specific task name as a parameter
      *
-     * @return void
+     * @throws Zend_Controller_Response_Exception thrown if deletion fails for any reason
      */
-    public function pollBrokerTaskQueueAction() {
+    public function deleteAction() {
         $request = $this->getRequest();
-        $taskName = $request->getParam('task');
+        $id = $request->getParam('id');
-        $this->_service->pollBrokerTaskQueue($taskName);
+        $this->_service->delete($id);
     }
 
 }

airtime_mvc/application/controllers/UpgradeController.php

@@ -14,8 +14,7 @@ class UpgradeController extends Zend_Controller_Action
         }
 
         try {
-            $upgradeManager = new UpgradeManager();
+            $didWePerformAnUpgrade = UpgradeManager::doUpgrade();
-            $didWePerformAnUpgrade = $upgradeManager->doUpgrade();
 
             if (!$didWePerformAnUpgrade) {
                 $this->getResponse()

airtime_mvc/application/models/RabbitMq.php

@@ -9,12 +9,15 @@ class Application_Model_RabbitMq
     /**
      * @var int milliseconds (for compatibility with celery) until we consider a message to have timed out
      */
-    public static $_CELERY_MESSAGE_TIMEOUT = 300000;  // 5 minutes
+    public static $_CELERY_MESSAGE_TIMEOUT = 600000;  // 10 minutes
 
     /**
+     * We have to use celeryresults (the default results exchange) because php-celery doesn't support
+     * named results exchanges.
+     *
      * @var string exchange for celery task results
      */
-    public static $_CELERY_RESULTS_EXCHANGE = 'airtime-results';
+    public static $_CELERY_RESULTS_EXCHANGE = 'celeryresults';
 
     /**
      * Sets a flag to push the schedule at the end of the request.
@@ -90,7 +93,7 @@ class Application_Model_RabbitMq
      * @throws CeleryException when no message is found
      */
     public static function sendCeleryMessage($task, $exchange, $data) {
-        $config = parse_ini_file($this->_getRmqConfigPath(), true);
+        $config = parse_ini_file(self::_getRmqConfigPath(), true);
         $queue = $routingKey = $exchange;
         $c = self::_setupCeleryExchange($config, $exchange, $queue);  // Use the exchange name for the queue
         $result = $c->PostTask($task, $data, true, $routingKey);      // and routing key
@@ -109,8 +112,8 @@ class Application_Model_RabbitMq
      * @throws CeleryException when no message is found
      */
     public static function getAsyncResultMessage($task, $id) {
-        $config = parse_ini_file($this->_getRmqConfigPath(), true);
+        $config = parse_ini_file(self::_getRmqConfigPath(), true);
-        $queue = self::$_CELERY_RESULTS_EXCHANGE . "." . $config["stationId"];
+        $queue = self::$_CELERY_RESULTS_EXCHANGE . "." . $task;
         $c = self::_setupCeleryExchange($config, self::$_CELERY_RESULTS_EXCHANGE, $queue);
         $message = $c->getAsyncResultMessage($task, $id);
 
@@ -157,7 +160,7 @@ class Application_Model_RabbitMq
         self::sendMessage($exchange, 'direct', true, $data);
     }
 
-    private function _getRmqConfigPath() {
+    private static function _getRmqConfigPath() {
         //Hack for Airtime Pro. The RabbitMQ settings for communicating with airtime_analyzer are global
         //and shared between all instances on Airtime Pro.
         $CC_CONFIG = Config::getConfig();
@@ -177,7 +180,7 @@ class Application_Model_RabbitMq
     public static function SendMessageToAnalyzer($tmpFilePath, $importedStorageDirectory, $originalFilename,
                                                 $callbackUrl, $apiKey, $storageBackend, $filePrefix)
     {
-        $config = parse_ini_file($this->_getRmqConfigPath(), true);
+        $config = parse_ini_file(self::_getRmqConfigPath(), true);
         $conn = new AMQPConnection($config["rabbitmq"]["host"],
                 $config["rabbitmq"]["port"],
                 $config["rabbitmq"]["user"],

airtime_mvc/application/services/SoundcloudService.php

@@ -17,28 +17,28 @@ class SoundcloudService extends ThirdPartyService {
     /**
      * @var string service name to store in ThirdPartyTrackReferences database
      */
-    protected $_SERVICE_NAME = 'SoundCloud';
+    protected static $_SERVICE_NAME = 'SoundCloud';
-
-    /**
-     * @var string base URI for SoundCloud tracks
-     */
-    protected $_THIRD_PARTY_TRACK_URI = 'http://api.soundcloud.com/tracks/';
 
     /**
      * @var string exchange name for SoundCloud tasks
      */
-    protected $_CELERY_EXCHANGE_NAME = 'soundcloud-uploads';
+    protected static $_CELERY_EXCHANGE_NAME = 'soundcloud';
 
     /**
      * @var string celery task name for third party uploads
      */
-    protected $_CELERY_UPLOAD_TASK_NAME = 'upload-to-soundcloud';
+    protected static $_CELERY_UPLOAD_TASK_NAME = 'soundcloud-upload';
+
+    /**
+     * @var string celery task name for third party deletions
+     */
+    protected static $_CELERY_DELETE_TASK_NAME = 'soundcloud-delete';
 
     /**
      * @var array Application_Model_Preference functions for SoundCloud and their
      *            associated API parameter keys so that we can call them dynamically
      */
-    private $_SOUNDCLOUD_PREF_FUNCTIONS = array(
+    private static $_SOUNDCLOUD_PREF_FUNCTIONS = array(
         "getDefaultSoundCloudLicenseType" => "license",
         "getDefaultSoundCloudSharingType" => "sharing"
     );
@@ -71,7 +71,7 @@ class SoundcloudService extends ThirdPartyService {
         $trackArray = array(
             'title' => $file->getName(),
         );
-        foreach ($this->_SOUNDCLOUD_PREF_FUNCTIONS as $func => $param) {
+        foreach (self::$_SOUNDCLOUD_PREF_FUNCTIONS as $func => $param) {
             $val = Application_Model_Preference::$func();
             if (!empty($val)) {
                 $trackArray[$param] = $val;
@@ -84,6 +84,7 @@ class SoundcloudService extends ThirdPartyService {
     /**
      * Update a ThirdPartyTrackReferences object for a completed upload
      * TODO: should we have a database layer class to handle Propel operations?
+     * TODO: break this function up, it's a bit of a beast
      *
      * @param $fileId int    local CcFiles identifier
      * @param $track  object third-party service track object
@@ -94,14 +95,18 @@ class SoundcloudService extends ThirdPartyService {
      */
     protected function _addOrUpdateTrackReference($fileId, $track, $status) {
         $ref = ThirdPartyTrackReferencesQuery::create()
-            ->filterByDbService($this->_SERVICE_NAME)
+            ->filterByDbService(static::$_SERVICE_NAME)
             ->findOneByDbFileId($fileId);
         if (is_null($ref)) {
             $ref = new ThirdPartyTrackReferences();
+        }  // If this was a delete task, just remove the record and return
+        else if ($ref->getDbBrokerTaskName() == static::$_CELERY_DELETE_TASK_NAME) {
+            $ref->delete();
+            return;
         }
-        $ref->setDbService($this->_SERVICE_NAME);
+        $ref->setDbService(static::$_SERVICE_NAME);
         // Only set the SoundCloud fields if the task was successful
-        if ($status == $this->_SUCCESS_STATUS) {
+        if ($status == CELERY_SUCCESS_STATUS) {
             // TODO: fetch any additional SoundCloud parameters we want to store
             $ref->setDbForeignId($track->id);  // SoundCloud identifier
         }
@@ -124,12 +129,23 @@ class SoundcloudService extends ThirdPartyService {
      * @param int $fileId the local CcFiles identifier
      *
      * @return string the link to the remote file
+     *
+     * @throws Soundcloud\Exception\InvalidHttpResponseCodeException when SoundCloud returns a 4xx/5xx response
      */
     public function getLinkToFile($fileId) {
         $serviceId = $this->getServiceId($fileId);
         // If we don't find a record for the file we'll get 0 back for the id
         if ($serviceId == 0) { return ''; }
-        $track = json_decode($this->_client->get('tracks/' . $serviceId));
+        try {
+            $track = json_decode($this->_client->get('tracks/' . $serviceId));
+        } catch (Soundcloud\Exception\InvalidHttpResponseCodeException $e) {
+            // If we end up here it means the track was removed from SoundCloud
+            // or the foreign id in our database is incorrect, so we should just
+            // get rid of the database record
+            Logging::warn("Error retrieving track data from SoundCloud: " . $e->getMessage());
+            $this->removeTrackReference($fileId);
+            throw $e;  // Throw the exception up to the controller so we can redirect to a 404
+        }
         return $track->permalink_url;
     }
 
@@ -152,7 +168,7 @@ class SoundcloudService extends ThirdPartyService {
         // in the redirect. This allows us to create a singular script to redirect
         // back to any station the request comes from.
         $url = urlencode('http'.(empty($_SERVER['HTTPS'])?'':'s').'://'.$_SERVER['HTTP_HOST'].'/soundcloud/redirect');
-        return $this->_client->getAuthorizeUrl(array("state" => $url));
+        return $this->_client->getAuthorizeUrl(array("state" => $url, "scope" => "non-expiring"));
     }
 
     /**
@@ -162,7 +178,7 @@ class SoundcloudService extends ThirdPartyService {
      */
     public function requestNewAccessToken($code) {
         // Get a non-expiring access token
-        $response = $this->_client->accessToken($code, $postData = array('scope' => 'non-expiring'));
+        $response = $this->_client->accessToken($code);
         $accessToken = $response['access_token'];
         Application_Model_Preference::setSoundCloudRequestToken($accessToken);
         $this->_accessToken = $accessToken;

airtime_mvc/application/services/ThirdPartyService.php

@@ -1,7 +1,13 @@
 <?php
 
+/**
+ * Class ServiceNotFoundException
+ */
+class ServiceNotFoundException extends Exception {}
+
 /**
  * Class ThirdPartyService generic superclass for third-party services
+ * TODO: decouple the media/track-specific functions into ThirdPartyMediaService class?
  */
 abstract class ThirdPartyService {
 
@@ -13,44 +19,32 @@ abstract class ThirdPartyService {
     /**
      * @var string service name to store in ThirdPartyTrackReferences database
      */
-    protected $_SERVICE_NAME;
+    protected static $_SERVICE_NAME;
 
     /**
      * @var string base URI for third-party tracks
      */
-    protected $_THIRD_PARTY_TRACK_URI;
+    protected static $_THIRD_PARTY_TRACK_URI;
 
     /**
      * @var string broker exchange name for third party tasks
      */
-    protected $_CELERY_EXCHANGE_NAME = 'default';
+    protected static $_CELERY_EXCHANGE_NAME;
 
     /**
      * @var string celery task name for third party uploads
      */
-    protected $_CELERY_UPLOAD_TASK_NAME = 'upload';
+    protected static $_CELERY_UPLOAD_TASK_NAME;
 
     /**
-     * @var string status string for pending tasks
+     * @var string celery task name for third party deletion
      */
-    protected $_PENDING_STATUS = 'PENDING';
+    protected static $_CELERY_DELETE_TASK_NAME;
-
-    /**
-     * @var string status string for successful tasks
-     */
-    protected $_SUCCESS_STATUS = 'SUCCESS';
-
-    /**
-     * @var string status string for failed tasks
-     */
-    protected $_FAILED_STATUS = 'FAILED';
 
     /**
      * Upload the file with the given identifier to a third-party service
      *
      * @param int $fileId the local CcFiles identifier
-     *
-     * @throws Exception thrown when the upload fails for any reason
      */
     public function upload($fileId) {
         $file = Application_Model_StoredFile::RecallById($fileId);
@@ -60,11 +54,40 @@ abstract class ThirdPartyService {
             'file_path' => $file->getFilePaths()[0]
         );
         try {
-            $brokerTaskId = Application_Model_RabbitMq::sendCeleryMessage($this->_CELERY_UPLOAD_TASK_NAME,
+            $brokerTaskId = Application_Model_RabbitMq::sendCeleryMessage(static::$_CELERY_UPLOAD_TASK_NAME,
-                                                                          $this->_CELERY_EXCHANGE_NAME,
+                                                                          static::$_CELERY_EXCHANGE_NAME,
                                                                           $data);
-            $this->_createTaskReference($fileId, $brokerTaskId, $this->_CELERY_UPLOAD_TASK_NAME);
+            $this->_createTaskReference($fileId, $brokerTaskId, static::$_CELERY_UPLOAD_TASK_NAME);
-        } catch(Exception $e) {
+        } catch (Exception $e) {
+            Logging::info("Invalid request: " . $e->getMessage());
+            // We should only get here if we have an access token, so attempt to refresh
+            $this->accessTokenRefresh();
+        }
+    }
+
+    /**
+     * Delete the file with the given identifier from a third-party service
+     *
+     * @param int $fileId the local CcFiles identifier
+     *
+     * @throws ServiceNotFoundException when a $fileId with no corresponding
+     *                                  service identifier is given
+     */
+    public function delete($fileId) {
+        $serviceId = $this->getServiceId($fileId);
+        if ($serviceId == 0) {
+            throw new ServiceNotFoundException("No service found for file with ID $fileId");
+        }
+        $data = array(
+            'token' => $this->_accessToken,
+            'track_id' => $serviceId
+        );
+        try {
+            $brokerTaskId = Application_Model_RabbitMq::sendCeleryMessage(static::$_CELERY_DELETE_TASK_NAME,
+                                                                          static::$_CELERY_EXCHANGE_NAME,
+                                                                          $data);
+            $this->_createTaskReference($fileId, $brokerTaskId, static::$_CELERY_DELETE_TASK_NAME);
+        } catch (Exception $e) {
             Logging::info("Invalid request: " . $e->getMessage());
             // We should only get here if we have an access token, so attempt to refresh
             $this->accessTokenRefresh();
@@ -86,18 +109,18 @@ abstract class ThirdPartyService {
     protected function _createTaskReference($fileId, $brokerTaskId, $taskName) {
         // First, check if the track already has an entry in the database
         $ref = ThirdPartyTrackReferencesQuery::create()
-            ->filterByDbService($this->_SERVICE_NAME)
+            ->filterByDbService(static::$_SERVICE_NAME)
             ->findOneByDbFileId($fileId);
         if (is_null($ref)) {
             $ref = new ThirdPartyTrackReferences();
         }
-        $ref->setDbService($this->_SERVICE_NAME);
+        $ref->setDbService(static::$_SERVICE_NAME);
         $ref->setDbBrokerTaskId($brokerTaskId);
         $ref->setDbBrokerTaskName($taskName);
         $utc = new DateTimeZone("UTC");
         $ref->setDbBrokerTaskDispatchTime(new DateTime("now", $utc));
         $ref->setDbFileId($fileId);
-        $ref->setDbStatus($this->_PENDING_STATUS);
+        $ref->setDbStatus(CELERY_PENDING_STATUS);
         $ref->save();
     }
 
@@ -113,7 +136,7 @@ abstract class ThirdPartyService {
      */
     public function removeTrackReference($fileId) {
         $ref = ThirdPartyTrackReferencesQuery::create()
-            ->filterByDbService($this->_SERVICE_NAME)
+            ->filterByDbService(static::$_SERVICE_NAME)
             ->findOneByDbFileId($fileId);
         $ref->delete();
     }
@@ -128,9 +151,9 @@ abstract class ThirdPartyService {
      */
     public function getServiceId($fileId) {
         $ref = ThirdPartyTrackReferencesQuery::create()
-            ->filterByDbService($this->_SERVICE_NAME)
+            ->filterByDbService(static::$_SERVICE_NAME)
-            ->findOneByDbFileId($fileId); // There shouldn't be duplicates!
+            ->findOneByDbFileId($fileId);  // There shouldn't be duplicates!
-        return is_null($ref) ? 0 : $ref->getDbForeignId();
+        return empty($ref) ? 0 : $ref->getDbForeignId();
     }
 
     /**
@@ -143,7 +166,24 @@ abstract class ThirdPartyService {
      */
     public function getLinkToFile($fileId) {
         $serviceId = $this->getServiceId($fileId);
-        return $serviceId > 0 ? $this->_THIRD_PARTY_TRACK_URI . $serviceId : '';
+        return $serviceId > 0 ? static::$_THIRD_PARTY_TRACK_URI . $serviceId : '';
+    }
+
+    /**
+     * Check to see if there are any pending tasks for this service
+     *
+     * @param string $taskName
+     *
+     * @return bool true if there are any pending tasks, otherwise false
+     */
+    public function isBrokerTaskQueueEmpty($taskName="") {
+        $query = ThirdPartyTrackReferencesQuery::create()
+            ->filterByDbService(static::$_SERVICE_NAME);
+        if (!empty($taskName)) {
+            $query->filterByDbBrokerTaskName($taskName);
+        }
+        $result = $query->findOneByDbStatus(CELERY_PENDING_STATUS);
+        return empty($result);
     }
 
     /**
@@ -154,18 +194,22 @@ abstract class ThirdPartyService {
      * @param string $taskName the name of the task to poll for
      */
     public function pollBrokerTaskQueue($taskName="") {
-        $pendingTasks = $this->_getPendingTasks($taskName);
+        $pendingTasks = static::_getPendingTasks($taskName);
         foreach ($pendingTasks as $task) {
             try {
-                $message = $this->_getTaskMessage($task);
+                $message = static::_getTaskMessage($task);
-                $this->_addOrUpdateTrackReference($task->getDbFileId(), json_decode($message->result), $message->status);
+                static::_addOrUpdateTrackReference($task->getDbFileId(), json_decode($message->result), $message->status);
-            } catch(CeleryException $e) {
+            } catch (CeleryException $e) {
-                Logging::info("Couldn't retrieve task message for task " . $task->getDbBrokerTaskName()
+                // Fail silently unless the message has timed out; often we end up here when
-                              . " with ID " . $task->getDbBrokerTaskId() . ": " . $e->getMessage());
+                // the Celery task takes a while to execute
-                if ($this->_checkMessageTimeout($task)) {
+                if (static::_checkMessageTimeout($task)) {
-                    $task->setDbStatus($this->_FAILED_STATUS);
+                    Logging::info($e->getMessage());
+                    $task->setDbStatus(CELERY_FAILED_STATUS);
                     $task->save();
                 }
+            } catch (Exception $e) {
+                // Sometimes we might catch a json_decode error and end up here
+                Logging::info($e->getMessage());
             }
         }
     }
@@ -180,8 +224,8 @@ abstract class ThirdPartyService {
      */
     protected function _getPendingTasks($taskName) {
         $query = ThirdPartyTrackReferencesQuery::create()
-            ->filterByDbService($this->_SERVICE_NAME)
+            ->filterByDbService(static::$_SERVICE_NAME)
-            ->filterByDbStatus($this->_PENDING_STATUS)
+            ->filterByDbStatus(CELERY_PENDING_STATUS)
             ->filterByDbBrokerTaskId('', Criteria::NOT_EQUAL);
         if (!empty($taskName)) {
             $query->filterByDbBrokerTaskName($taskName);
@@ -198,7 +242,7 @@ abstract class ThirdPartyService {
      *
      * @throws CeleryException when the result message for this task no longer exists
      */
-    protected function _getTaskMessage($task) {
+    protected static function _getTaskMessage($task) {
         $message =  Application_Model_RabbitMq::getAsyncResultMessage($task->getDbBrokerTaskName(),
                                                                       $task->getDbBrokerTaskId());
         return json_decode($message['body']);
@@ -212,7 +256,7 @@ abstract class ThirdPartyService {
      * @return bool true if the dispatch time is empty or it's been more than our timeout time
      *              since the message was dispatched, otherwise false
      */
-    protected function _checkMessageTimeout($task) {
+    protected static function _checkMessageTimeout($task) {
         $utc = new DateTimeZone("UTC");
         $dispatchTime = new DateTime($task->getDbBrokerTaskDispatchTime(), $utc);
         $now = new DateTime("now", $utc);

airtime_mvc/application/upgrade/Upgrades.php

@@ -49,7 +49,7 @@ class UpgradeManager
      *
      * @return boolean whether or not an upgrade was performed
      */
-    public function doUpgrade()
+    public static function doUpgrade()
     {
         // Get all upgrades dynamically (in declaration order!) so we don't have to add them explicitly each time
         // TODO: explicitly sort classnames by ascending version suffix for safety
@@ -58,7 +58,7 @@ class UpgradeManager
         $upgradePerformed = false;
 
         foreach ($upgraders as $upgrader) {
-            $upgradePerformed = $this->_runUpgrade(new $upgrader($dir)) ? true : $upgradePerformed;
+            $upgradePerformed = self::_runUpgrade(new $upgrader($dir)) ? true : $upgradePerformed;
         }
 
         return $upgradePerformed;
@@ -71,7 +71,7 @@ class UpgradeManager
      *
      * @return bool true if the upgrade was successful, otherwise false
      */
-    private function _runUpgrade(AirtimeUpgrader $upgrader) {
+    private static function _runUpgrade(AirtimeUpgrader $upgrader) {
         return $upgrader->checkIfUpgradeSupported() && $upgrader->upgrade();
     }
 
@@ -327,6 +327,22 @@ class AirtimeUpgrader2512 extends AirtimeUpgrader
     }
 }
 
+/**
+ * Class AirtimeUpgrader2513 - Celery and SoundCloud upgrade
+ *
+ * Adds third_party_track_references table for third party service
+ * authentication and task architecture.
+ *
+ * Schema:
+ *      id                          -> int          PK
+ *      service                     -> string       internal service name
+ *      foreign_id                  -> int          external unique service id
+ *      broker_task_id              -> int          external unique amqp results identifier
+ *      broker_task_name            -> string       external Celery task name
+ *      broker_task_dispatch_time   -> timestamp    internal message dispatch time
+ *      file_id                     -> int          internal FK->cc_files track id
+ *      status                      -> string       external Celery task status - PENDING, SUCCESS, or FAILED
+ */
 class AirtimeUpgrader2513 extends AirtimeUpgrader
 {
     protected function getSupportedSchemaVersions() {

airtime_mvc/public/js/airtime/common/common.js

@@ -9,7 +9,6 @@ $(document).ready(function() {
 
     //this statement tells the browser to fade out any success message after 5 seconds
     setTimeout(function(){$(".success").fadeOut("slow", function(){$(this).empty()});}, 5000);
-    pollTaskQueues();
 });
 
 /*
@@ -156,9 +155,4 @@ function removeSuccessMsg() {
     var $status = $('.success');
     
     $status.fadeOut("slow", function(){$status.empty()});
-}
-
-function pollTaskQueues() {
-    console.log("Polling broker queues...");
-    $.get(baseUrl + 'soundcloud/poll-broker-task-queue');
 }

python_apps/airtime-celery/airtime-celery/celeryconfig.py

@@ -23,23 +23,15 @@ def parse_rmq_config(rmq_config):
 
 # Celery amqp settings
 BROKER_URL = get_rmq_broker()
-CELERY_RESULT_BACKEND = 'amqp'     # Use RabbitMQ as the celery backend
+CELERY_RESULT_BACKEND = 'amqp'            # Use RabbitMQ as the celery backend
-CELERY_RESULT_PERSISTENT = True    # Persist through a broker restart
+CELERY_RESULT_PERSISTENT = True           # Persist through a broker restart
-CELERY_TASK_RESULT_EXPIRES = 300   # Expire task results after 5 minutes
+CELERY_TASK_RESULT_EXPIRES = 600          # Expire task results after 10 minutes
-CELERY_TRACK_STARTED = False
+CELERY_RESULT_EXCHANGE = 'celeryresults'  # Default exchange - needed due to php-celery
-CELERY_RESULT_EXCHANGE = 'airtime-results'
 CELERY_QUEUES = (
-    Queue('soundcloud-uploads', exchange=Exchange('soundcloud-uploads'), routing_key='soundcloud-uploads'),
+    Queue('soundcloud', exchange=Exchange('soundcloud'), routing_key='soundcloud'),
-    Queue('airtime-results.soundcloud-uploads', exchange=Exchange('airtime-results')),
+    Queue(exchange=Exchange('celeryresults'), auto_delete=True),
-)
-CELERY_ROUTES = (
-    {
-        'soundcloud_uploads.tasks.upload_to_soundcloud': {
-            'exchange': 'airtime-results',
-            'queue': 'airtime-results.soundcloud-uploads',
-        }
-    },
 )
+CELERY_EVENT_QUEUE_EXPIRES = 600          # RabbitMQ x-expire after 10 minutes
 
 # Celery task settings
 CELERY_TASK_SERIALIZER = 'json'

python_apps/airtime-celery/airtime-celery/tasks.py

@@ -9,8 +9,8 @@ celery = Celery()
 logger = get_task_logger(__name__)
 
 
-@celery.task(name='upload-to-soundcloud')
+@celery.task(name='soundcloud-upload')
-def upload_to_soundcloud(data, token, file_path):
+def soundcloud_upload(data, token, file_path):
     """
     Upload a file to SoundCloud
 
@@ -32,3 +32,22 @@ def upload_to_soundcloud(data, token, file_path):
         raise e
     data['asset_data'].close()
     return json.dumps(track.fields())
+
+@celery.task(name='soundcloud-delete')
+def soundcloud_delete(token, track_id):
+    """
+    Delete a file from SoundCloud
+
+    :param token: OAuth2 client access token
+
+    :return: the SoundCloud response object
+    :rtype: dict
+    """
+    client = soundcloud.Client(access_token=token)
+    try:
+        logger.info('Deleting track with ID {0}'.format(track_id))
+        track = client.delete('/tracks/%s' % track_id)
+    except Exception as e:
+        logger.info('Error deleting track!')
+        raise e
+    return json.dumps(track.fields())