Congegni
/
sintonia
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add SoundCloud delete functionality and fixes; implement TaskManager to run background jobs

This commit is contained in:
Duncan Sommerville 2015-06-15 15:12:37 -04:00
parent 706d7db2b2
commit 3902c8c746
15 changed files with 373 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
airtime_mvc/application/Bootstrap.php
View File

@ -27,6 +27,7 @@ require_once "ProvisioningHelper.php";
require_once "GoogleAnalytics.php";
require_once "Timezone.php";
require_once "Auth.php";
require_once "TaskManager.php";
require_once __DIR__.'/services/SoundcloudService.php';
require_once __DIR__.'/forms/helpers/ValidationTypes.php';
require_once __DIR__.'/forms/helpers/CustomDecorators.php';
@ -123,15 +124,12 @@ class Bootstrap extends Zend_Application_Bootstrap_Bootstrap
$view->headScript()->appendScript("var COMPANY_NAME = '" . COMPANY_NAME . "';");
}
protected function _initUpgrade() {
protected function _initTasks() {
/* We need to wrap this here so that we aren't checking when we're running the unit test suite
*/
if (getenv("AIRTIME_UNIT_TEST") != 1) {
//This will do the upgrade too if it's needed...
if (UpgradeManager::checkIfUpgradeIsNeeded()) {
$upgradeManager = new UpgradeManager();
$upgradeManager->doUpgrade();
}
TaskManager::getInstance()->runTasks();
}
}

160
airtime_mvc/application/common/TaskManager.php Normal file
View File

@ -0,0 +1,160 @@
<?php
/**
* Class TaskManager
*/
final class TaskManager {
/**
* @var array tasks to be run
*/
protected $_taskList = [
AirtimeTask::SOUNDCLOUD,
AirtimeTask::UPGRADE
];
/**
* @var TaskManager singleton instance object
*/
protected static $_instance;
/**
* Private constructor so class is uninstantiable
*/
private function __construct() {
}
/**
* Get the singleton instance of this class
*
* @return TaskManager the TaskManager instance
*/
public static function getInstance() {
if (!self::$_instance) {
self::$_instance = new TaskManager();
}
return self::$_instance;
}
/**
* Run all tasks that need to be run
*/
public function runTasks() {
foreach ($this->_taskList as $task) {
$task = TaskFactory::getTask($task);
assert(is_subclass_of($task, 'AirtimeTask')); // Sanity check
/** @var $task AirtimeTask */
if ($task && $task->shouldBeRun()) $task->run();
}
}
}
/**
* Interface AirtimeTask Interface for task operations - also acts as task type ENUM
*/
interface AirtimeTask {
/**
* PHP doesn't have ENUMs so declare them as interface constants
* Task types - values don't really matter as long as they're unique
*/
const SOUNDCLOUD = "soundcloud";
const UPGRADE = "upgrade";
/**
* Check whether the task should be run
*
* @return bool true if the task needs to be run, otherwise false
*/
public function shouldBeRun();
/**
* Run the task
*
* @return void
*/
public function run();
}
/**
* Class TaskFactory Factory class to abstract task instantiation
*/
class TaskFactory {
/**
* Get an AirtimeTask based on a task type
*
* @param $task string the task type; uses AirtimeTask constants as an ENUM
*
* @return AirtimeTask|null return a task of the given type or null if no corresponding
* task exists or is implemented
*/
public static function getTask($task) {
switch($task) {
case AirtimeTask::SOUNDCLOUD:
return new SoundcloudUploadTask();
case AirtimeTask::UPGRADE:
return new UpgradeTask();
}
return null;
}
}
/**
* Class UpgradeTask
*/
class UpgradeTask implements AirtimeTask {
/**
* Check the current Airtime schema version to see if an upgrade should be run
*
* @return bool true if an upgrade is needed
*/
public function shouldBeRun() {
return UpgradeManager::checkIfUpgradeIsNeeded();
}
/**
* Run all upgrades above the current schema version
*/
public function run() {
UpgradeManager::doUpgrade();
}
}
/**
* Class SoundcloudUploadTask
*/
class SoundcloudUploadTask implements AirtimeTask {
/**
* @var SoundcloudService
*/
protected $_service;
public function __construct() {
$this->_service = new SoundcloudService();
}
/**
* Check the ThirdPartyTrackReferences table to see if there are any pending SoundCloud tasks
*
* @return bool true if there are pending tasks in ThirdPartyTrackReferences
*/
public function shouldBeRun() {
return !$this->_service->isBrokerTaskQueueEmpty();
}
/**
* Poll the task queue for any completed Celery tasks
*/
public function run() {
$this->_service->pollBrokerTaskQueue();
}
}

17
airtime_mvc/application/configs/constants.php
View File

@ -83,9 +83,20 @@ define('UI_BLOCK_SESSNAME', 'BLOCK');*/
// Soundcloud contants
define('SOUNDCLOUD_NOT_UPLOADED_YET' , -1);
define('SOUNDCLOUD_PROGRESS' , -2);
define('SOUNDCLOUD_ERROR' , -3);
/**
* @var string status string for pending Celery tasks
*/
define('CELERY_PENDING_STATUS', 'PENDING');
/**
* @var string status string for successful Celery tasks
*/
define('CELERY_SUCCESS_STATUS', 'SUCCESS');
/**
* @var string status string for failed Celery tasks
*/
define('CELERY_FAILED_STATUS', 'FAILED');
//WHMCS integration

1
airtime_mvc/application/controllers/ErrorController.php
View File

@ -74,7 +74,6 @@ class ErrorController extends Zend_Controller_Action {
* 404 error - route or controller
*/
public function error404Action() {
Logging::info("404!");
$this->_helper->viewRenderer('error-404');
$this->getResponse()->setHttpResponseCode(404);
$this->view->message = _('Page not found.');

8
airtime_mvc/application/controllers/LibraryController.php
View File

@ -277,13 +277,11 @@ class LibraryController extends Zend_Controller_Action
$serviceId = $soundcloudService->getServiceId($id);
if (!is_null($file) && $serviceId != 0) {
$menu["soundcloud"]["items"]["view"] = array("name" => _("View on Soundcloud"), "icon" => "soundcloud", "url" => $baseUrl."soundcloud/view-on-sound-cloud/id/{$id}");
$text = _("Re-upload to SoundCloud");
$menu["soundcloud"]["items"]["view"] = array("name" => _("View track"), "icon" => "soundcloud", "url" => $baseUrl."soundcloud/view-on-sound-cloud/id/{$id}");
$menu["soundcloud"]["items"]["upload"] = array("name" => _("Remove track"), "icon" => "soundcloud", "url" => $baseUrl."soundcloud/delete/id/{$id}");
} else {
$text = _("Upload to SoundCloud");
$menu["soundcloud"]["items"]["upload"] = array("name" => _("Upload track"), "icon" => "soundcloud", "url" => $baseUrl."soundcloud/upload/id/{$id}");
}
$menu["soundcloud"]["items"]["upload"] = array("name" => $text, "icon" => "soundcloud", "url" => $baseUrl."soundcloud/upload/id/{$id}");
}
if (empty($menu)) {

7
airtime_mvc/application/controllers/SoundcloudController.php
View File

@ -33,13 +33,8 @@ class SoundcloudController extends ThirdPartyController {
$soundcloudLink = $this->_service->getLinkToFile($id);
header('Location: ' . $soundcloudLink);
} catch (Soundcloud\Exception\InvalidHttpResponseCodeException $e) {
// If we end up here it means the track was removed from SoundCloud
// or the foreign id in our database is incorrect, so we should just
// get rid of the database record
Logging::warn("Error retrieving track data from SoundCloud: " . $e->getMessage());
$this->_service->removeTrackReference($id);
// Redirect to a 404 so the user knows something went wrong
header('Location: ' . $this->_baseUrl . 'error/error-404'); // Redirect back to the Preference page
header('Location: ' . $this->_baseUrl . 'error/error-404');
}
}

30
airtime_mvc/application/controllers/ThirdPartyController.php
View File

@ -41,6 +41,17 @@ abstract class ThirdPartyController extends Zend_Controller_Action {
header('Location: ' . $auth_url);
}
/**
* Clear the previously saved request token from the preferences
*
* @return void
*/
public function deauthorizeAction() {
$function = $this->_SERVICE_TOKEN_ACCESSOR;
Application_Model_Preference::$function("");
header('Location: ' . $this->_baseUrl . 'Preference'); // Redirect back to the Preference page
}
/**
* Called when user successfully completes third-party authorization
* Store the returned request token for future requests
@ -67,25 +78,16 @@ abstract class ThirdPartyController extends Zend_Controller_Action {
}
/**
* Clear the previously saved request token from the preferences
* Delete the file with the given id from a third-party service
*
* @return void
*/
public function deauthorizeAction() {
Application_Model_Preference::$this->_SERVICE_TOKEN_ACCESSOR("");
header('Location: ' . $this->_baseUrl . 'Preference'); // Redirect back to the Preference page
}
/**
* Poll the task queue for completed tasks associated with this service
* Optionally accepts a specific task name as a parameter
*
* @return void
* @throws Zend_Controller_Response_Exception thrown if deletion fails for any reason
*/
public function pollBrokerTaskQueueAction() {
public function deleteAction() {
$request = $this->getRequest();
$taskName = $request->getParam('task');
$this->_service->pollBrokerTaskQueue($taskName);
$id = $request->getParam('id');
$this->_service->delete($id);
}
}

3
airtime_mvc/application/controllers/UpgradeController.php
View File

@ -14,8 +14,7 @@ class UpgradeController extends Zend_Controller_Action
}
try {
$upgradeManager = new UpgradeManager();
$didWePerformAnUpgrade = $upgradeManager->doUpgrade();
$didWePerformAnUpgrade = UpgradeManager::doUpgrade();
if (!$didWePerformAnUpgrade) {
$this->getResponse()

17
airtime_mvc/application/models/RabbitMq.php
View File

@ -9,12 +9,15 @@ class Application_Model_RabbitMq
/**
* @var int milliseconds (for compatibility with celery) until we consider a message to have timed out
*/
public static $_CELERY_MESSAGE_TIMEOUT = 300000; // 5 minutes
public static $_CELERY_MESSAGE_TIMEOUT = 600000; // 10 minutes
/**
* We have to use celeryresults (the default results exchange) because php-celery doesn't support
* named results exchanges.
*
* @var string exchange for celery task results
*/
public static $_CELERY_RESULTS_EXCHANGE = 'airtime-results';
public static $_CELERY_RESULTS_EXCHANGE = 'celeryresults';
/**
* Sets a flag to push the schedule at the end of the request.
@ -90,7 +93,7 @@ class Application_Model_RabbitMq
* @throws CeleryException when no message is found
*/
public static function sendCeleryMessage($task, $exchange, $data) {
$config = parse_ini_file($this->_getRmqConfigPath(), true);
$config = parse_ini_file(self::_getRmqConfigPath(), true);
$queue = $routingKey = $exchange;
$c = self::_setupCeleryExchange($config, $exchange, $queue); // Use the exchange name for the queue
$result = $c->PostTask($task, $data, true, $routingKey); // and routing key
@ -109,8 +112,8 @@ class Application_Model_RabbitMq
* @throws CeleryException when no message is found
*/
public static function getAsyncResultMessage($task, $id) {
$config = parse_ini_file($this->_getRmqConfigPath(), true);
$queue = self::$_CELERY_RESULTS_EXCHANGE . "." . $config["stationId"];
$config = parse_ini_file(self::_getRmqConfigPath(), true);
$queue = self::$_CELERY_RESULTS_EXCHANGE . "." . $task;
$c = self::_setupCeleryExchange($config, self::$_CELERY_RESULTS_EXCHANGE, $queue);
$message = $c->getAsyncResultMessage($task, $id);
@ -157,7 +160,7 @@ class Application_Model_RabbitMq
self::sendMessage($exchange, 'direct', true, $data);
}
private function _getRmqConfigPath() {
private static function _getRmqConfigPath() {
//Hack for Airtime Pro. The RabbitMQ settings for communicating with airtime_analyzer are global
//and shared between all instances on Airtime Pro.
$CC_CONFIG = Config::getConfig();
@ -177,7 +180,7 @@ class Application_Model_RabbitMq
public static function SendMessageToAnalyzer($tmpFilePath, $importedStorageDirectory, $originalFilename,
$callbackUrl, $apiKey, $storageBackend, $filePrefix)
{
$config = parse_ini_file($this->_getRmqConfigPath(), true);
$config = parse_ini_file(self::_getRmqConfigPath(), true);
$conn = new AMQPConnection($config["rabbitmq"]["host"],
$config["rabbitmq"]["port"],
$config["rabbitmq"]["user"],

48
airtime_mvc/application/services/SoundcloudService.php
View File

@ -17,28 +17,28 @@ class SoundcloudService extends ThirdPartyService {
/**
* @var string service name to store in ThirdPartyTrackReferences database
*/
protected $_SERVICE_NAME = 'SoundCloud';
/**
* @var string base URI for SoundCloud tracks
*/
protected $_THIRD_PARTY_TRACK_URI = 'http://api.soundcloud.com/tracks/';
protected static $_SERVICE_NAME = 'SoundCloud';
/**
* @var string exchange name for SoundCloud tasks
*/
protected $_CELERY_EXCHANGE_NAME = 'soundcloud-uploads';
protected static $_CELERY_EXCHANGE_NAME = 'soundcloud';
/**
* @var string celery task name for third party uploads
*/
protected $_CELERY_UPLOAD_TASK_NAME = 'upload-to-soundcloud';
protected static $_CELERY_UPLOAD_TASK_NAME = 'soundcloud-upload';
/**
* @var string celery task name for third party deletions
*/
protected static $_CELERY_DELETE_TASK_NAME = 'soundcloud-delete';
/**
* @var array Application_Model_Preference functions for SoundCloud and their
* associated API parameter keys so that we can call them dynamically
*/
private $_SOUNDCLOUD_PREF_FUNCTIONS = array(
private static $_SOUNDCLOUD_PREF_FUNCTIONS = array(
"getDefaultSoundCloudLicenseType" => "license",
"getDefaultSoundCloudSharingType" => "sharing"
);
@ -71,7 +71,7 @@ class SoundcloudService extends ThirdPartyService {
$trackArray = array(
'title' => $file->getName(),
);
foreach ($this->_SOUNDCLOUD_PREF_FUNCTIONS as $func => $param) {
foreach (self::$_SOUNDCLOUD_PREF_FUNCTIONS as $func => $param) {
$val = Application_Model_Preference::$func();
if (!empty($val)) {
$trackArray[$param] = $val;
@ -84,6 +84,7 @@ class SoundcloudService extends ThirdPartyService {
/**
* Update a ThirdPartyTrackReferences object for a completed upload
* TODO: should we have a database layer class to handle Propel operations?
* TODO: break this function up, it's a bit of a beast
*
* @param $fileId int local CcFiles identifier
* @param $track object third-party service track object
@ -94,14 +95,18 @@ class SoundcloudService extends ThirdPartyService {
*/
protected function _addOrUpdateTrackReference($fileId, $track, $status) {
$ref = ThirdPartyTrackReferencesQuery::create()
->filterByDbService($this->_SERVICE_NAME)
->filterByDbService(static::$_SERVICE_NAME)
->findOneByDbFileId($fileId);
if (is_null($ref)) {
$ref = new ThirdPartyTrackReferences();
} // If this was a delete task, just remove the record and return
else if ($ref->getDbBrokerTaskName() == static::$_CELERY_DELETE_TASK_NAME) {
$ref->delete();
return;
}
$ref->setDbService($this->_SERVICE_NAME);
$ref->setDbService(static::$_SERVICE_NAME);
// Only set the SoundCloud fields if the task was successful
if ($status == $this->_SUCCESS_STATUS) {
if ($status == CELERY_SUCCESS_STATUS) {
// TODO: fetch any additional SoundCloud parameters we want to store
$ref->setDbForeignId($track->id); // SoundCloud identifier
}
@ -124,12 +129,23 @@ class SoundcloudService extends ThirdPartyService {
* @param int $fileId the local CcFiles identifier
*
* @return string the link to the remote file
*
* @throws Soundcloud\Exception\InvalidHttpResponseCodeException when SoundCloud returns a 4xx/5xx response
*/
public function getLinkToFile($fileId) {
$serviceId = $this->getServiceId($fileId);
// If we don't find a record for the file we'll get 0 back for the id
if ($serviceId == 0) { return ''; }
$track = json_decode($this->_client->get('tracks/' . $serviceId));
try {
$track = json_decode($this->_client->get('tracks/' . $serviceId));
} catch (Soundcloud\Exception\InvalidHttpResponseCodeException $e) {
// If we end up here it means the track was removed from SoundCloud
// or the foreign id in our database is incorrect, so we should just
// get rid of the database record
Logging::warn("Error retrieving track data from SoundCloud: " . $e->getMessage());
$this->removeTrackReference($fileId);
throw $e; // Throw the exception up to the controller so we can redirect to a 404
}
return $track->permalink_url;
}
@ -152,7 +168,7 @@ class SoundcloudService extends ThirdPartyService {
// in the redirect. This allows us to create a singular script to redirect
// back to any station the request comes from.
$url = urlencode('http'.(empty($_SERVER['HTTPS'])?'':'s').'://'.$_SERVER['HTTP_HOST'].'/soundcloud/redirect');
return $this->_client->getAuthorizeUrl(array("state" => $url));
return $this->_client->getAuthorizeUrl(array("state" => $url, "scope" => "non-expiring"));
}
/**
@ -162,7 +178,7 @@ class SoundcloudService extends ThirdPartyService {
*/
public function requestNewAccessToken($code) {
// Get a non-expiring access token
$response = $this->_client->accessToken($code, $postData = array('scope' => 'non-expiring'));
$response = $this->_client->accessToken($code);
$accessToken = $response['access_token'];
Application_Model_Preference::setSoundCloudRequestToken($accessToken);
$this->_accessToken = $accessToken;

128
airtime_mvc/application/services/ThirdPartyService.php
View File

@ -1,7 +1,13 @@
<?php
/**
* Class ServiceNotFoundException
*/
class ServiceNotFoundException extends Exception {}
/**
* Class ThirdPartyService generic superclass for third-party services
* TODO: decouple the media/track-specific functions into ThirdPartyMediaService class?
*/
abstract class ThirdPartyService {
@ -13,44 +19,32 @@ abstract class ThirdPartyService {
/**
* @var string service name to store in ThirdPartyTrackReferences database
*/
protected $_SERVICE_NAME;
protected static $_SERVICE_NAME;
/**
* @var string base URI for third-party tracks
*/
protected $_THIRD_PARTY_TRACK_URI;
protected static $_THIRD_PARTY_TRACK_URI;
/**
* @var string broker exchange name for third party tasks
*/
protected $_CELERY_EXCHANGE_NAME = 'default';
protected static $_CELERY_EXCHANGE_NAME;
/**
* @var string celery task name for third party uploads
*/
protected $_CELERY_UPLOAD_TASK_NAME = 'upload';
protected static $_CELERY_UPLOAD_TASK_NAME;
/**
* @var string status string for pending tasks
* @var string celery task name for third party deletion
*/
protected $_PENDING_STATUS = 'PENDING';
/**
* @var string status string for successful tasks
*/
protected $_SUCCESS_STATUS = 'SUCCESS';
/**
* @var string status string for failed tasks
*/
protected $_FAILED_STATUS = 'FAILED';
protected static $_CELERY_DELETE_TASK_NAME;
/**
* Upload the file with the given identifier to a third-party service
*
* @param int $fileId the local CcFiles identifier
*
* @throws Exception thrown when the upload fails for any reason
*/
public function upload($fileId) {
$file = Application_Model_StoredFile::RecallById($fileId);
@ -60,11 +54,40 @@ abstract class ThirdPartyService {
'file_path' => $file->getFilePaths()[0]
);
try {
$brokerTaskId = Application_Model_RabbitMq::sendCeleryMessage($this->_CELERY_UPLOAD_TASK_NAME,
$this->_CELERY_EXCHANGE_NAME,
$brokerTaskId = Application_Model_RabbitMq::sendCeleryMessage(static::$_CELERY_UPLOAD_TASK_NAME,
static::$_CELERY_EXCHANGE_NAME,
$data);
$this->_createTaskReference($fileId, $brokerTaskId, $this->_CELERY_UPLOAD_TASK_NAME);
} catch(Exception $e) {
$this->_createTaskReference($fileId, $brokerTaskId, static::$_CELERY_UPLOAD_TASK_NAME);
} catch (Exception $e) {
Logging::info("Invalid request: " . $e->getMessage());
// We should only get here if we have an access token, so attempt to refresh
$this->accessTokenRefresh();
}
}
/**
* Delete the file with the given identifier from a third-party service
*
* @param int $fileId the local CcFiles identifier
*
* @throws ServiceNotFoundException when a $fileId with no corresponding
* service identifier is given
*/
public function delete($fileId) {
$serviceId = $this->getServiceId($fileId);
if ($serviceId == 0) {
throw new ServiceNotFoundException("No service found for file with ID $fileId");
}
$data = array(
'token' => $this->_accessToken,
'track_id' => $serviceId
);
try {
$brokerTaskId = Application_Model_RabbitMq::sendCeleryMessage(static::$_CELERY_DELETE_TASK_NAME,
static::$_CELERY_EXCHANGE_NAME,
$data);
$this->_createTaskReference($fileId, $brokerTaskId, static::$_CELERY_DELETE_TASK_NAME);
} catch (Exception $e) {
Logging::info("Invalid request: " . $e->getMessage());
// We should only get here if we have an access token, so attempt to refresh
$this->accessTokenRefresh();
@ -86,18 +109,18 @@ abstract class ThirdPartyService {
protected function _createTaskReference($fileId, $brokerTaskId, $taskName) {
// First, check if the track already has an entry in the database
$ref = ThirdPartyTrackReferencesQuery::create()
->filterByDbService($this->_SERVICE_NAME)
->filterByDbService(static::$_SERVICE_NAME)
->findOneByDbFileId($fileId);
if (is_null($ref)) {
$ref = new ThirdPartyTrackReferences();
}
$ref->setDbService($this->_SERVICE_NAME);
$ref->setDbService(static::$_SERVICE_NAME);
$ref->setDbBrokerTaskId($brokerTaskId);
$ref->setDbBrokerTaskName($taskName);
$utc = new DateTimeZone("UTC");
$ref->setDbBrokerTaskDispatchTime(new DateTime("now", $utc));
$ref->setDbFileId($fileId);
$ref->setDbStatus($this->_PENDING_STATUS);
$ref->setDbStatus(CELERY_PENDING_STATUS);
$ref->save();
}
@ -113,7 +136,7 @@ abstract class ThirdPartyService {
*/
public function removeTrackReference($fileId) {
$ref = ThirdPartyTrackReferencesQuery::create()
->filterByDbService($this->_SERVICE_NAME)
->filterByDbService(static::$_SERVICE_NAME)
->findOneByDbFileId($fileId);
$ref->delete();
}
@ -128,9 +151,9 @@ abstract class ThirdPartyService {
*/
public function getServiceId($fileId) {
$ref = ThirdPartyTrackReferencesQuery::create()
->filterByDbService($this->_SERVICE_NAME)
->findOneByDbFileId($fileId); // There shouldn't be duplicates!
return is_null($ref) ? 0 : $ref->getDbForeignId();
->filterByDbService(static::$_SERVICE_NAME)
->findOneByDbFileId($fileId); // There shouldn't be duplicates!
return empty($ref) ? 0 : $ref->getDbForeignId();
}
/**
@ -143,7 +166,24 @@ abstract class ThirdPartyService {
*/
public function getLinkToFile($fileId) {
$serviceId = $this->getServiceId($fileId);
return $serviceId > 0 ? $this->_THIRD_PARTY_TRACK_URI . $serviceId : '';
return $serviceId > 0 ? static::$_THIRD_PARTY_TRACK_URI . $serviceId : '';
}
/**
* Check to see if there are any pending tasks for this service
*
* @param string $taskName
*
* @return bool true if there are any pending tasks, otherwise false
*/
public function isBrokerTaskQueueEmpty($taskName="") {
$query = ThirdPartyTrackReferencesQuery::create()
->filterByDbService(static::$_SERVICE_NAME);
if (!empty($taskName)) {
$query->filterByDbBrokerTaskName($taskName);
}
$result = $query->findOneByDbStatus(CELERY_PENDING_STATUS);
return empty($result);
}
/**
@ -154,18 +194,22 @@ abstract class ThirdPartyService {
* @param string $taskName the name of the task to poll for
*/
public function pollBrokerTaskQueue($taskName="") {
$pendingTasks = $this->_getPendingTasks($taskName);
$pendingTasks = static::_getPendingTasks($taskName);
foreach ($pendingTasks as $task) {
try {
$message = $this->_getTaskMessage($task);
$this->_addOrUpdateTrackReference($task->getDbFileId(), json_decode($message->result), $message->status);
} catch(CeleryException $e) {
Logging::info("Couldn't retrieve task message for task " . $task->getDbBrokerTaskName()
. " with ID " . $task->getDbBrokerTaskId() . ": " . $e->getMessage());
if ($this->_checkMessageTimeout($task)) {
$task->setDbStatus($this->_FAILED_STATUS);
$message = static::_getTaskMessage($task);
static::_addOrUpdateTrackReference($task->getDbFileId(), json_decode($message->result), $message->status);
} catch (CeleryException $e) {
// Fail silently unless the message has timed out; often we end up here when
// the Celery task takes a while to execute
if (static::_checkMessageTimeout($task)) {
Logging::info($e->getMessage());
$task->setDbStatus(CELERY_FAILED_STATUS);
$task->save();
}
} catch (Exception $e) {
// Sometimes we might catch a json_decode error and end up here
Logging::info($e->getMessage());
}
}
}
@ -180,8 +224,8 @@ abstract class ThirdPartyService {
*/
protected function _getPendingTasks($taskName) {
$query = ThirdPartyTrackReferencesQuery::create()
->filterByDbService($this->_SERVICE_NAME)
->filterByDbStatus($this->_PENDING_STATUS)
->filterByDbService(static::$_SERVICE_NAME)
->filterByDbStatus(CELERY_PENDING_STATUS)
->filterByDbBrokerTaskId('', Criteria::NOT_EQUAL);
if (!empty($taskName)) {
$query->filterByDbBrokerTaskName($taskName);
@ -198,7 +242,7 @@ abstract class ThirdPartyService {
*
* @throws CeleryException when the result message for this task no longer exists
*/
protected function _getTaskMessage($task) {
protected static function _getTaskMessage($task) {
$message = Application_Model_RabbitMq::getAsyncResultMessage($task->getDbBrokerTaskName(),
$task->getDbBrokerTaskId());
return json_decode($message['body']);
@ -212,7 +256,7 @@ abstract class ThirdPartyService {
* @return bool true if the dispatch time is empty or it's been more than our timeout time
* since the message was dispatched, otherwise false
*/
protected function _checkMessageTimeout($task) {
protected static function _checkMessageTimeout($task) {
$utc = new DateTimeZone("UTC");
$dispatchTime = new DateTime($task->getDbBrokerTaskDispatchTime(), $utc);
$now = new DateTime("now", $utc);

22
airtime_mvc/application/upgrade/Upgrades.php
View File

@ -49,7 +49,7 @@ class UpgradeManager
*
* @return boolean whether or not an upgrade was performed
*/
public function doUpgrade()
public static function doUpgrade()
{
// Get all upgrades dynamically (in declaration order!) so we don't have to add them explicitly each time
// TODO: explicitly sort classnames by ascending version suffix for safety
@ -58,7 +58,7 @@ class UpgradeManager
$upgradePerformed = false;
foreach ($upgraders as $upgrader) {
$upgradePerformed = $this->_runUpgrade(new $upgrader($dir)) ? true : $upgradePerformed;
$upgradePerformed = self::_runUpgrade(new $upgrader($dir)) ? true : $upgradePerformed;
}
return $upgradePerformed;
@ -71,7 +71,7 @@ class UpgradeManager
*
* @return bool true if the upgrade was successful, otherwise false
*/
private function _runUpgrade(AirtimeUpgrader $upgrader) {
private static function _runUpgrade(AirtimeUpgrader $upgrader) {
return $upgrader->checkIfUpgradeSupported() && $upgrader->upgrade();
}
@ -327,6 +327,22 @@ class AirtimeUpgrader2512 extends AirtimeUpgrader
}
}
/**
* Class AirtimeUpgrader2513 - Celery and SoundCloud upgrade
*
* Adds third_party_track_references table for third party service
* authentication and task architecture.
*
* Schema:
* id -> int PK
* service -> string internal service name
* foreign_id -> int external unique service id
* broker_task_id -> int external unique amqp results identifier
* broker_task_name -> string external Celery task name
* broker_task_dispatch_time -> timestamp internal message dispatch time
* file_id -> int internal FK->cc_files track id
* status -> string external Celery task status - PENDING, SUCCESS, or FAILED
*/
class AirtimeUpgrader2513 extends AirtimeUpgrader
{
protected function getSupportedSchemaVersions() {

6
airtime_mvc/public/js/airtime/common/common.js
View File

@ -9,7 +9,6 @@ $(document).ready(function() {
//this statement tells the browser to fade out any success message after 5 seconds
setTimeout(function(){$(".success").fadeOut("slow", function(){$(this).empty()});}, 5000);
pollTaskQueues();
});
/*
@ -156,9 +155,4 @@ function removeSuccessMsg() {
var $status = $('.success');
$status.fadeOut("slow", function(){$status.empty()});
}
function pollTaskQueues() {
console.log("Polling broker queues...");
$.get(baseUrl + 'soundcloud/poll-broker-task-queue');
}

22
python_apps/airtime-celery/airtime-celery/celeryconfig.py
View File

@ -23,23 +23,15 @@ def parse_rmq_config(rmq_config):
# Celery amqp settings
BROKER_URL = get_rmq_broker()
CELERY_RESULT_BACKEND = 'amqp' # Use RabbitMQ as the celery backend
CELERY_RESULT_PERSISTENT = True # Persist through a broker restart
CELERY_TASK_RESULT_EXPIRES = 300 # Expire task results after 5 minutes
CELERY_TRACK_STARTED = False
CELERY_RESULT_EXCHANGE = 'airtime-results'
CELERY_RESULT_BACKEND = 'amqp' # Use RabbitMQ as the celery backend
CELERY_RESULT_PERSISTENT = True # Persist through a broker restart
CELERY_TASK_RESULT_EXPIRES = 600 # Expire task results after 10 minutes
CELERY_RESULT_EXCHANGE = 'celeryresults' # Default exchange - needed due to php-celery
CELERY_QUEUES = (
Queue('soundcloud-uploads', exchange=Exchange('soundcloud-uploads'), routing_key='soundcloud-uploads'),
Queue('airtime-results.soundcloud-uploads', exchange=Exchange('airtime-results')),
)
CELERY_ROUTES = (
{
'soundcloud_uploads.tasks.upload_to_soundcloud': {
'exchange': 'airtime-results',
'queue': 'airtime-results.soundcloud-uploads',
}
},
Queue('soundcloud', exchange=Exchange('soundcloud'), routing_key='soundcloud'),
Queue(exchange=Exchange('celeryresults'), auto_delete=True),
)
CELERY_EVENT_QUEUE_EXPIRES = 600 # RabbitMQ x-expire after 10 minutes
# Celery task settings
CELERY_TASK_SERIALIZER = 'json'

23
python_apps/airtime-celery/airtime-celery/tasks.py
View File

@ -9,8 +9,8 @@ celery = Celery()
logger = get_task_logger(__name__)
@celery.task(name='upload-to-soundcloud')
def upload_to_soundcloud(data, token, file_path):
@celery.task(name='soundcloud-upload')
def soundcloud_upload(data, token, file_path):
"""
Upload a file to SoundCloud
@ -32,3 +32,22 @@ def upload_to_soundcloud(data, token, file_path):
raise e
data['asset_data'].close()
return json.dumps(track.fields())
@celery.task(name='soundcloud-delete')
def soundcloud_delete(token, track_id):
"""
Delete a file from SoundCloud
:param token: OAuth2 client access token
:return: the SoundCloud response object
:rtype: dict
"""
client = soundcloud.Client(access_token=token)
try:
logger.info('Deleting track with ID {0}'.format(track_id))
track = client.delete('/tracks/%s' % track_id)
except Exception as e:
logger.info('Error deleting track!')
raise e
return json.dumps(track.fields())