Congegni/sintonia
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add SoundCloud delete functionality and fixes; implement TaskManager to run background jobs

Browse source
This commit is contained in:
Duncan Sommerville 2015-06-15 15:12:37 -04:00
parent 706d7db2b2
commit 3902c8c746
15 changed files with 373 additions and 127 deletions
Download patch file Download diff file Expand all files Collapse all files

48
airtime_mvc/application/services/SoundcloudService.php
View file

@ -17,28 +17,28 @@ class SoundcloudService extends ThirdPartyService {
/**
* @var string service name to store in ThirdPartyTrackReferences database
*/
protected $_SERVICE_NAME = 'SoundCloud';
/**
* @var string base URI for SoundCloud tracks
*/
protected $_THIRD_PARTY_TRACK_URI = 'http://api.soundcloud.com/tracks/';
protected static $_SERVICE_NAME = 'SoundCloud';
/**
* @var string exchange name for SoundCloud tasks
*/
protected $_CELERY_EXCHANGE_NAME = 'soundcloud-uploads';
protected static $_CELERY_EXCHANGE_NAME = 'soundcloud';
/**
* @var string celery task name for third party uploads
*/
protected $_CELERY_UPLOAD_TASK_NAME = 'upload-to-soundcloud';
protected static $_CELERY_UPLOAD_TASK_NAME = 'soundcloud-upload';
/**
* @var string celery task name for third party deletions
*/
protected static $_CELERY_DELETE_TASK_NAME = 'soundcloud-delete';
/**
* @var array Application_Model_Preference functions for SoundCloud and their
* associated API parameter keys so that we can call them dynamically
*/
private $_SOUNDCLOUD_PREF_FUNCTIONS = array(
private static $_SOUNDCLOUD_PREF_FUNCTIONS = array(
"getDefaultSoundCloudLicenseType" => "license",
"getDefaultSoundCloudSharingType" => "sharing"
);
@ -71,7 +71,7 @@ class SoundcloudService extends ThirdPartyService {
$trackArray = array(
'title' => $file->getName(),
);
foreach ($this->_SOUNDCLOUD_PREF_FUNCTIONS as $func => $param) {
foreach (self::$_SOUNDCLOUD_PREF_FUNCTIONS as $func => $param) {
$val = Application_Model_Preference::$func();
if (!empty($val)) {
$trackArray[$param] = $val;
@ -84,6 +84,7 @@ class SoundcloudService extends ThirdPartyService {
/**
* Update a ThirdPartyTrackReferences object for a completed upload
* TODO: should we have a database layer class to handle Propel operations?
* TODO: break this function up, it's a bit of a beast
*
* @param $fileId int local CcFiles identifier
* @param $track object third-party service track object
@ -94,14 +95,18 @@ class SoundcloudService extends ThirdPartyService {
*/
protected function _addOrUpdateTrackReference($fileId, $track, $status) {
$ref = ThirdPartyTrackReferencesQuery::create()
->filterByDbService($this->_SERVICE_NAME)
->filterByDbService(static::$_SERVICE_NAME)
->findOneByDbFileId($fileId);
if (is_null($ref)) {
$ref = new ThirdPartyTrackReferences();
} // If this was a delete task, just remove the record and return
else if ($ref->getDbBrokerTaskName() == static::$_CELERY_DELETE_TASK_NAME) {
$ref->delete();
return;
}
$ref->setDbService($this->_SERVICE_NAME);
$ref->setDbService(static::$_SERVICE_NAME);
// Only set the SoundCloud fields if the task was successful
if ($status == $this->_SUCCESS_STATUS) {
if ($status == CELERY_SUCCESS_STATUS) {
// TODO: fetch any additional SoundCloud parameters we want to store
$ref->setDbForeignId($track->id); // SoundCloud identifier
}
@ -124,12 +129,23 @@ class SoundcloudService extends ThirdPartyService {
* @param int $fileId the local CcFiles identifier
*
* @return string the link to the remote file
*
* @throws Soundcloud\Exception\InvalidHttpResponseCodeException when SoundCloud returns a 4xx/5xx response
*/
public function getLinkToFile($fileId) {
$serviceId = $this->getServiceId($fileId);
// If we don't find a record for the file we'll get 0 back for the id
if ($serviceId == 0) { return ''; }
$track = json_decode($this->_client->get('tracks/' . $serviceId));
try {
$track = json_decode($this->_client->get('tracks/' . $serviceId));
} catch (Soundcloud\Exception\InvalidHttpResponseCodeException $e) {
// If we end up here it means the track was removed from SoundCloud
// or the foreign id in our database is incorrect, so we should just
// get rid of the database record
Logging::warn("Error retrieving track data from SoundCloud: " . $e->getMessage());
$this->removeTrackReference($fileId);
throw $e; // Throw the exception up to the controller so we can redirect to a 404
}
return $track->permalink_url;
}
@ -152,7 +168,7 @@ class SoundcloudService extends ThirdPartyService {
// in the redirect. This allows us to create a singular script to redirect
// back to any station the request comes from.
$url = urlencode('http'.(empty($_SERVER['HTTPS'])?'':'s').'://'.$_SERVER['HTTP_HOST'].'/soundcloud/redirect');
return $this->_client->getAuthorizeUrl(array("state" => $url));
return $this->_client->getAuthorizeUrl(array("state" => $url, "scope" => "non-expiring"));
}
/**
@ -162,7 +178,7 @@ class SoundcloudService extends ThirdPartyService {
*/
public function requestNewAccessToken($code) {
// Get a non-expiring access token
$response = $this->_client->accessToken($code, $postData = array('scope' => 'non-expiring'));
$response = $this->_client->accessToken($code);
$accessToken = $response['access_token'];
Application_Model_Preference::setSoundCloudRequestToken($accessToken);
$this->_accessToken = $accessToken;

128
airtime_mvc/application/services/ThirdPartyService.php
View file

@ -1,7 +1,13 @@
<?php
/**
* Class ServiceNotFoundException
*/
class ServiceNotFoundException extends Exception {}
/**
* Class ThirdPartyService generic superclass for third-party services
* TODO: decouple the media/track-specific functions into ThirdPartyMediaService class?
*/
abstract class ThirdPartyService {
@ -13,44 +19,32 @@ abstract class ThirdPartyService {
/**
* @var string service name to store in ThirdPartyTrackReferences database
*/
protected $_SERVICE_NAME;
protected static $_SERVICE_NAME;
/**
* @var string base URI for third-party tracks
*/
protected $_THIRD_PARTY_TRACK_URI;
protected static $_THIRD_PARTY_TRACK_URI;
/**
* @var string broker exchange name for third party tasks
*/
protected $_CELERY_EXCHANGE_NAME = 'default';
protected static $_CELERY_EXCHANGE_NAME;
/**
* @var string celery task name for third party uploads
*/
protected $_CELERY_UPLOAD_TASK_NAME = 'upload';
protected static $_CELERY_UPLOAD_TASK_NAME;
/**
* @var string status string for pending tasks
* @var string celery task name for third party deletion
*/
protected $_PENDING_STATUS = 'PENDING';
/**
* @var string status string for successful tasks
*/
protected $_SUCCESS_STATUS = 'SUCCESS';
/**
* @var string status string for failed tasks
*/
protected $_FAILED_STATUS = 'FAILED';
protected static $_CELERY_DELETE_TASK_NAME;
/**
* Upload the file with the given identifier to a third-party service
*
* @param int $fileId the local CcFiles identifier
*
* @throws Exception thrown when the upload fails for any reason
*/
public function upload($fileId) {
$file = Application_Model_StoredFile::RecallById($fileId);
@ -60,11 +54,40 @@ abstract class ThirdPartyService {
'file_path' => $file->getFilePaths()[0]
);
try {
$brokerTaskId = Application_Model_RabbitMq::sendCeleryMessage($this->_CELERY_UPLOAD_TASK_NAME,
$this->_CELERY_EXCHANGE_NAME,
$brokerTaskId = Application_Model_RabbitMq::sendCeleryMessage(static::$_CELERY_UPLOAD_TASK_NAME,
static::$_CELERY_EXCHANGE_NAME,
$data);
$this->_createTaskReference($fileId, $brokerTaskId, $this->_CELERY_UPLOAD_TASK_NAME);
} catch(Exception $e) {
$this->_createTaskReference($fileId, $brokerTaskId, static::$_CELERY_UPLOAD_TASK_NAME);
} catch (Exception $e) {
Logging::info("Invalid request: " . $e->getMessage());
// We should only get here if we have an access token, so attempt to refresh
$this->accessTokenRefresh();
}
}
/**
* Delete the file with the given identifier from a third-party service
*
* @param int $fileId the local CcFiles identifier
*
* @throws ServiceNotFoundException when a $fileId with no corresponding
* service identifier is given
*/
public function delete($fileId) {
$serviceId = $this->getServiceId($fileId);
if ($serviceId == 0) {
throw new ServiceNotFoundException("No service found for file with ID $fileId");
}
$data = array(
'token' => $this->_accessToken,
'track_id' => $serviceId
);
try {
$brokerTaskId = Application_Model_RabbitMq::sendCeleryMessage(static::$_CELERY_DELETE_TASK_NAME,
static::$_CELERY_EXCHANGE_NAME,
$data);
$this->_createTaskReference($fileId, $brokerTaskId, static::$_CELERY_DELETE_TASK_NAME);
} catch (Exception $e) {
Logging::info("Invalid request: " . $e->getMessage());
// We should only get here if we have an access token, so attempt to refresh
$this->accessTokenRefresh();
@ -86,18 +109,18 @@ abstract class ThirdPartyService {
protected function _createTaskReference($fileId, $brokerTaskId, $taskName) {
// First, check if the track already has an entry in the database
$ref = ThirdPartyTrackReferencesQuery::create()
->filterByDbService($this->_SERVICE_NAME)
->filterByDbService(static::$_SERVICE_NAME)
->findOneByDbFileId($fileId);
if (is_null($ref)) {
$ref = new ThirdPartyTrackReferences();
}
$ref->setDbService($this->_SERVICE_NAME);
$ref->setDbService(static::$_SERVICE_NAME);
$ref->setDbBrokerTaskId($brokerTaskId);
$ref->setDbBrokerTaskName($taskName);
$utc = new DateTimeZone("UTC");
$ref->setDbBrokerTaskDispatchTime(new DateTime("now", $utc));
$ref->setDbFileId($fileId);
$ref->setDbStatus($this->_PENDING_STATUS);
$ref->setDbStatus(CELERY_PENDING_STATUS);
$ref->save();
}
@ -113,7 +136,7 @@ abstract class ThirdPartyService {
*/
public function removeTrackReference($fileId) {
$ref = ThirdPartyTrackReferencesQuery::create()
->filterByDbService($this->_SERVICE_NAME)
->filterByDbService(static::$_SERVICE_NAME)
->findOneByDbFileId($fileId);
$ref->delete();
}
@ -128,9 +151,9 @@ abstract class ThirdPartyService {
*/
public function getServiceId($fileId) {
$ref = ThirdPartyTrackReferencesQuery::create()
->filterByDbService($this->_SERVICE_NAME)
->findOneByDbFileId($fileId); // There shouldn't be duplicates!
return is_null($ref) ? 0 : $ref->getDbForeignId();
->filterByDbService(static::$_SERVICE_NAME)
->findOneByDbFileId($fileId); // There shouldn't be duplicates!
return empty($ref) ? 0 : $ref->getDbForeignId();
}
/**
@ -143,7 +166,24 @@ abstract class ThirdPartyService {
*/
public function getLinkToFile($fileId) {
$serviceId = $this->getServiceId($fileId);
return $serviceId > 0 ? $this->_THIRD_PARTY_TRACK_URI . $serviceId : '';
return $serviceId > 0 ? static::$_THIRD_PARTY_TRACK_URI . $serviceId : '';
}
/**
* Check to see if there are any pending tasks for this service
*
* @param string $taskName
*
* @return bool true if there are any pending tasks, otherwise false
*/
public function isBrokerTaskQueueEmpty($taskName="") {
$query = ThirdPartyTrackReferencesQuery::create()
->filterByDbService(static::$_SERVICE_NAME);
if (!empty($taskName)) {
$query->filterByDbBrokerTaskName($taskName);
}
$result = $query->findOneByDbStatus(CELERY_PENDING_STATUS);
return empty($result);
}
/**
@ -154,18 +194,22 @@ abstract class ThirdPartyService {
* @param string $taskName the name of the task to poll for
*/
public function pollBrokerTaskQueue($taskName="") {
$pendingTasks = $this->_getPendingTasks($taskName);
$pendingTasks = static::_getPendingTasks($taskName);
foreach ($pendingTasks as $task) {
try {
$message = $this->_getTaskMessage($task);
$this->_addOrUpdateTrackReference($task->getDbFileId(), json_decode($message->result), $message->status);
} catch(CeleryException $e) {
Logging::info("Couldn't retrieve task message for task " . $task->getDbBrokerTaskName()
. " with ID " . $task->getDbBrokerTaskId() . ": " . $e->getMessage());
if ($this->_checkMessageTimeout($task)) {
$task->setDbStatus($this->_FAILED_STATUS);
$message = static::_getTaskMessage($task);
static::_addOrUpdateTrackReference($task->getDbFileId(), json_decode($message->result), $message->status);
} catch (CeleryException $e) {
// Fail silently unless the message has timed out; often we end up here when
// the Celery task takes a while to execute
if (static::_checkMessageTimeout($task)) {
Logging::info($e->getMessage());
$task->setDbStatus(CELERY_FAILED_STATUS);
$task->save();
}
} catch (Exception $e) {
// Sometimes we might catch a json_decode error and end up here
Logging::info($e->getMessage());
}
}
}
@ -180,8 +224,8 @@ abstract class ThirdPartyService {
*/
protected function _getPendingTasks($taskName) {
$query = ThirdPartyTrackReferencesQuery::create()
->filterByDbService($this->_SERVICE_NAME)
->filterByDbStatus($this->_PENDING_STATUS)
->filterByDbService(static::$_SERVICE_NAME)
->filterByDbStatus(CELERY_PENDING_STATUS)
->filterByDbBrokerTaskId('', Criteria::NOT_EQUAL);
if (!empty($taskName)) {
$query->filterByDbBrokerTaskName($taskName);
@ -198,7 +242,7 @@ abstract class ThirdPartyService {
*
* @throws CeleryException when the result message for this task no longer exists
*/
protected function _getTaskMessage($task) {
protected static function _getTaskMessage($task) {
$message = Application_Model_RabbitMq::getAsyncResultMessage($task->getDbBrokerTaskName(),
$task->getDbBrokerTaskId());
return json_decode($message['body']);
@ -212,7 +256,7 @@ abstract class ThirdPartyService {
* @return bool true if the dispatch time is empty or it's been more than our timeout time
* since the message was dispatched, otherwise false
*/
protected function _checkMessageTimeout($task) {
protected static function _checkMessageTimeout($task) {
$utc = new DateTimeZone("UTC");
$dispatchTime = new DateTime($task->getDbBrokerTaskDispatchTime(), $utc);
$now = new DateTime("now", $utc);