Add SoundCloud update and download tasks to Celery backend; requires airtime-celery reinstall
This commit is contained in:
parent
bf97c42167
commit
4f281a30ed
|
@ -82,7 +82,6 @@ final class TaskManager {
|
|||
if ($this->_isUserSessionRequest()) {
|
||||
return;
|
||||
}
|
||||
Logging::info("Running the tasks!");
|
||||
$this->_con = Propel::getConnection(CcPrefPeer::DATABASE_NAME);
|
||||
$this->_con->beginTransaction();
|
||||
try {
|
||||
|
|
|
@ -139,33 +139,18 @@ class LibraryController extends Zend_Controller_Action
|
|||
if ($type === "audioclip" && $soundcloudService->hasAccessToken()) {
|
||||
$serviceId = $soundcloudService->getServiceId($id);
|
||||
if (!is_null($file) && $serviceId != 0) {
|
||||
$trackRef = ThirdPartyTrackReferencesQuery::create()
|
||||
->filterByDbService(SOUNDCLOUD_SERVICE_NAME)
|
||||
->findOneByDbFileId($id);
|
||||
|
||||
//create a menu separator
|
||||
$menu["sep1"] = "-----------";
|
||||
|
||||
//create a sub menu for Soundcloud actions.
|
||||
$menu["soundcloud"] = array("name" => _(SOUNDCLOUD), "icon" => "soundcloud", "items" => array());
|
||||
$menu["soundcloud"]["items"]["view"] = array("name" => _("View track"), "icon" => "soundcloud", "url" => $baseUrl . "soundcloud/view-on-sound-cloud/id/{$id}");
|
||||
// $menu["soundcloud"]["items"]["remove"] = array("name" => _("Remove track"), "icon" => "soundcloud", "url" => $baseUrl . "soundcloud/delete/id/{$id}");
|
||||
$menu["soundcloud"]["items"]["update"] = array("name" => _("Update track"), "icon" => "soundcloud", "url" => $baseUrl . "soundcloud/update/id/{$trackRef->getDbForeignId()}");
|
||||
}
|
||||
/*
|
||||
Since we upload to SoundCloud from the Publish dialog now, this is unnecessary
|
||||
|
||||
else {
|
||||
// If a reference exists for this file ID, that means the user has uploaded the track
|
||||
// but we haven't yet gotten a response from Celery, so disable the menu item
|
||||
if ($soundcloudService->referenceExists($id)) {
|
||||
$menu["soundcloud"]["items"]["upload"] = array(
|
||||
"name" => _("Upload track"), "icon" => "soundcloud",
|
||||
"url" => $baseUrl . "soundcloud/upload/id/{$id}", "disabled" => true
|
||||
);
|
||||
} else {
|
||||
$menu["soundcloud"]["items"]["upload"] = array(
|
||||
"name" => _("Upload track"), "icon" => "soundcloud",
|
||||
"url" => $baseUrl . "soundcloud/upload/id/{$id}"
|
||||
);
|
||||
}
|
||||
}
|
||||
*/
|
||||
}
|
||||
}
|
||||
} elseif ($type === "playlist" || $type === "block") {
|
||||
|
|
|
@ -28,11 +28,22 @@ class SoundcloudController extends ThirdPartyController implements OAuth2Control
|
|||
* @throws Zend_Controller_Response_Exception thrown if upload fails for any reason
|
||||
*/
|
||||
public function uploadAction() {
|
||||
$request = $this->getRequest();
|
||||
$id = $request->getParam('id');
|
||||
$id = $this->getRequest()->getParam('id');
|
||||
$this->_service->upload($id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the file with the given id on SoundCloud
|
||||
*
|
||||
* @return void
|
||||
*
|
||||
* @throws Zend_Controller_Response_Exception thrown if upload fails for any reason
|
||||
*/
|
||||
public function updateAction() {
|
||||
$id = $this->getRequest()->getParam('id');
|
||||
$this->_service->update($id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Download the file with the given id from SoundCloud
|
||||
*
|
||||
|
@ -41,8 +52,7 @@ class SoundcloudController extends ThirdPartyController implements OAuth2Control
|
|||
* @throws Zend_Controller_Response_Exception thrown if download fails for any reason
|
||||
*/
|
||||
public function downloadAction() {
|
||||
$request = $this->getRequest();
|
||||
$id = $request->getParam('id');
|
||||
$id = $this->getRequest()->getParam('id');
|
||||
$this->_service->download($id);
|
||||
}
|
||||
|
||||
|
@ -54,8 +64,7 @@ class SoundcloudController extends ThirdPartyController implements OAuth2Control
|
|||
* @throws Zend_Controller_Response_Exception thrown if deletion fails for any reason
|
||||
*/
|
||||
public function deleteAction() {
|
||||
$request = $this->getRequest();
|
||||
$id = $request->getParam('id');
|
||||
$id = $this->getRequest()->getParam('id');
|
||||
$this->_service->delete($id);
|
||||
}
|
||||
|
||||
|
@ -97,8 +106,7 @@ class SoundcloudController extends ThirdPartyController implements OAuth2Control
|
|||
* @return void
|
||||
*/
|
||||
public function viewOnSoundCloudAction() {
|
||||
$request = $this->getRequest();
|
||||
$id = $request->getParam('id');
|
||||
$id = $this->getRequest()->getParam('id');
|
||||
try {
|
||||
$soundcloudLink = $this->_service->getLinkToFile($id);
|
||||
header('Location: ' . $soundcloudLink);
|
||||
|
|
|
@ -73,7 +73,7 @@ class Application_Service_PublishService {
|
|||
);
|
||||
}
|
||||
|
||||
/** @noinspection PhpUnusedPrivateMethodInspection
|
||||
/**
|
||||
* Reflective accessor for SoundCloud publication status for the
|
||||
* file with the given ID
|
||||
*
|
||||
|
@ -84,10 +84,11 @@ class Application_Service_PublishService {
|
|||
*/
|
||||
private static function getSoundCloudPublishStatus($fileId) {
|
||||
$soundcloudService = new Application_Service_SoundcloudService();
|
||||
return ($soundcloudService->getServiceId($fileId) != 0);
|
||||
return ($soundcloudService->referenceExists($fileId));
|
||||
}
|
||||
|
||||
/** @noinspection PhpUnusedPrivateMethodInspection
|
||||
/**
|
||||
*
|
||||
* Reflective accessor for Station podcast publication status for the
|
||||
* file with the given ID
|
||||
*
|
||||
|
|
|
@ -14,6 +14,7 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
|
|||
*/
|
||||
|
||||
const UPLOAD = 'upload';
|
||||
const UPDATE = 'update';
|
||||
const DOWNLOAD = 'download';
|
||||
const DELETE = 'delete';
|
||||
|
||||
|
@ -42,6 +43,7 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
|
|||
*/
|
||||
protected static $_CELERY_TASKS = [
|
||||
self::UPLOAD => 'soundcloud-upload',
|
||||
self::UPDATE => 'soundcloud-update',
|
||||
self::DOWNLOAD => 'soundcloud-download',
|
||||
self::DELETE => 'soundcloud-delete'
|
||||
];
|
||||
|
@ -136,13 +138,28 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
|
|||
}
|
||||
|
||||
/**
|
||||
* Given a track identifier, download a track from SoundCloud
|
||||
*
|
||||
* If no identifier is given, download all the user's tracks
|
||||
* Given a track identifier, update a track on SoundCloud
|
||||
*
|
||||
* @param int $trackId a track identifier
|
||||
*/
|
||||
public function download($trackId = null) {
|
||||
public function update($trackId) {
|
||||
$trackRef = ThirdPartyTrackReferencesQuery::create()
|
||||
->findOneByDbForeignId($trackId);
|
||||
$file = Application_Model_StoredFile::RecallById($trackRef->getDbFileId());
|
||||
$data = array(
|
||||
'data' => $this->_getUploadData($file),
|
||||
'token' => $this->_accessToken,
|
||||
'track_id' => $trackId
|
||||
);
|
||||
$this->_executeTask(static::$_CELERY_TASKS[self::UPDATE], $data, $trackRef->getDbFileId());
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a track identifier, download a track from SoundCloud
|
||||
*
|
||||
* @param int $trackId a track identifier
|
||||
*/
|
||||
public function download($trackId) {
|
||||
$CC_CONFIG = Config::getConfig();
|
||||
$data = array(
|
||||
'callback_url' => Application_Common_HTTPHelper::getStationUrl() . 'rest/media',
|
||||
|
@ -200,6 +217,9 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
|
|||
return null;
|
||||
}
|
||||
$ref->setDbForeignId($track->id); // SoundCloud identifier
|
||||
if (isset($track->fileid)) {
|
||||
$ref->setDbFileId($track->fileid); // For downloads, set the cc_files ID
|
||||
}
|
||||
}
|
||||
// TODO: set SoundCloud upload status?
|
||||
// $ref->setDbStatus($status);
|
||||
|
@ -288,6 +308,8 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
|
|||
}
|
||||
|
||||
/**
|
||||
* Publishing interface proxy
|
||||
*
|
||||
* Publish the file with the given file ID to SoundCloud
|
||||
*
|
||||
* @param int $fileId ID of the file to be published
|
||||
|
@ -297,6 +319,8 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
|
|||
}
|
||||
|
||||
/**
|
||||
* Publishing interface proxy
|
||||
*
|
||||
* Unpublish the file with the given file ID from SoundCloud
|
||||
*
|
||||
* @param int $fileId ID of the file to be unpublished
|
||||
|
|
|
@ -1178,33 +1178,15 @@ var AIRTIME = (function(AIRTIME) {
|
|||
if (oItems.soundcloud !== undefined) {
|
||||
var soundcloud = oItems.soundcloud.items;
|
||||
|
||||
// define an upload to soundcloud callback.
|
||||
if (soundcloud.upload !== undefined) {
|
||||
|
||||
if (soundcloud.update !== undefined) {
|
||||
callback = function() {
|
||||
alert($.i18n._("Your track is being uploaded and will " +
|
||||
"appear on SoundCloud in a couple of minutes"));
|
||||
$.post(soundcloud.upload.url, function(){});
|
||||
$.post(soundcloud.update.url, function () {});
|
||||
};
|
||||
soundcloud.upload.callback = callback;
|
||||
}
|
||||
|
||||
// define an upload to soundcloud callback.
|
||||
if (soundcloud.remove !== undefined) {
|
||||
|
||||
callback = function() {
|
||||
if (confirm($.i18n._("Are you sure? SoundCloud stats and comments for this track will be permanently removed."))) {
|
||||
alert($.i18n._("Your track is being deleted from SoundCloud"));
|
||||
$.post(soundcloud.remove.url, function () {
|
||||
});
|
||||
}
|
||||
};
|
||||
soundcloud.remove.callback = callback;
|
||||
soundcloud.update.callback = callback;
|
||||
}
|
||||
|
||||
// define a view on soundcloud callback
|
||||
if (soundcloud.view !== undefined) {
|
||||
|
||||
callback = function() {
|
||||
window.open(soundcloud.view.url);
|
||||
};
|
||||
|
|
|
@ -20,7 +20,7 @@ var AIRTIME = (function (AIRTIME) {
|
|||
|
||||
function init () {
|
||||
var csrfToken = jQuery("#csrf").val();
|
||||
$http.get(endpoint + mediaId, { csrf_token: csrfToken })
|
||||
$http.get(endpoint + mediaId, {csrf_token: csrfToken})
|
||||
.success(function (json) {
|
||||
$scope.media = json;
|
||||
tab.setName($scope.media.track_title);
|
||||
|
@ -28,7 +28,7 @@ var AIRTIME = (function (AIRTIME) {
|
|||
|
||||
// Get an object containing all sources, their translated labels,
|
||||
// and their publication state for the file with the given ID
|
||||
$http.get(endpoint + mediaId + '/publish-sources', { csrf_token: csrfToken })
|
||||
$http.get(endpoint + mediaId + '/publish-sources', {csrf_token: csrfToken})
|
||||
.success(function (json) {
|
||||
$scope.sources = json;
|
||||
// Store the data (whether each source should be published to when publish is clicked)
|
||||
|
@ -49,14 +49,18 @@ var AIRTIME = (function (AIRTIME) {
|
|||
|
||||
$scope.publish = function () {
|
||||
var data = {};
|
||||
jQuery.each($scope.publishData, function(k, v) {
|
||||
jQuery.each($scope.publishData, function (k, v) {
|
||||
if (v) {
|
||||
if (k == "soundcloud") {
|
||||
alert($.i18n._("Your track is being uploaded and will "
|
||||
+ "appear on SoundCloud in a couple of minutes"));
|
||||
}
|
||||
data[k] = 'publish'; // FIXME: should be more robust
|
||||
}
|
||||
});
|
||||
|
||||
if (Object.keys(data).length > 0) {
|
||||
$http.put(endpoint + mediaId + '/publish', { csrf_token: jQuery("#csrf").val(), sources: data})
|
||||
$http.put(endpoint + mediaId + '/publish', {csrf_token: jQuery("#csrf").val(), sources: data})
|
||||
.success(function () {
|
||||
init();
|
||||
});
|
||||
|
@ -66,10 +70,13 @@ var AIRTIME = (function (AIRTIME) {
|
|||
$scope.remove = function (source) {
|
||||
var data = {};
|
||||
data[source] = 'unpublish'; // FIXME: should be more robust
|
||||
$http.put(endpoint + mediaId + '/publish', { csrf_token: jQuery("#csrf").val(), sources: data })
|
||||
.success(function () {
|
||||
init();
|
||||
});
|
||||
if (source != "soundcloud" || confirm($.i18n._("Are you sure? SoundCloud stats and comments "
|
||||
+ "for this track will be permanently removed."))) {
|
||||
$http.put(endpoint + mediaId + '/publish', {csrf_token: jQuery("#csrf").val(), sources: data})
|
||||
.success(function () {
|
||||
init();
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
$scope.discard = function () {
|
||||
|
|
|
@ -25,14 +25,14 @@ def parse_rmq_config(rmq_config):
|
|||
BROKER_URL = get_rmq_broker()
|
||||
CELERY_RESULT_BACKEND = 'amqp' # Use RabbitMQ as the celery backend
|
||||
CELERY_RESULT_PERSISTENT = True # Persist through a broker restart
|
||||
CELERY_TASK_RESULT_EXPIRES = 600 # Expire task results after 10 minutes
|
||||
CELERY_TASK_RESULT_EXPIRES = 900 # Expire task results after 15 minutes
|
||||
CELERY_RESULT_EXCHANGE = 'celeryresults' # Default exchange - needed due to php-celery
|
||||
CELERY_QUEUES = (
|
||||
Queue('soundcloud', exchange=Exchange('soundcloud'), routing_key='soundcloud'),
|
||||
Queue('podcast', exchange=Exchange('podcast'), routing_key='podcast'),
|
||||
Queue(exchange=Exchange('celeryresults'), auto_delete=True),
|
||||
)
|
||||
CELERY_EVENT_QUEUE_EXPIRES = 600 # RabbitMQ x-expire after 10 minutes
|
||||
CELERY_EVENT_QUEUE_EXPIRES = 900 # RabbitMQ x-expire after 15 minutes
|
||||
|
||||
# Celery task settings
|
||||
CELERY_TASK_SERIALIZER = 'json'
|
||||
|
|
|
@ -23,8 +23,8 @@ def soundcloud_upload(data, token, file_path):
|
|||
:param token: OAuth2 client access token
|
||||
:param file_path: path to the file being uploaded
|
||||
|
||||
:return: the SoundCloud response object
|
||||
:rtype: dict
|
||||
:return: JSON formatted string of the SoundCloud response object
|
||||
:rtype: string
|
||||
"""
|
||||
client = soundcloud.Client(access_token=token)
|
||||
# Open the file with urllib2 if it's a cloud file
|
||||
|
@ -40,28 +40,65 @@ def soundcloud_upload(data, token, file_path):
|
|||
|
||||
|
||||
@celery.task(name='soundcloud-download', acks_late=True)
|
||||
def soundcloud_download(token, callback_url, api_key, track_id=None):
|
||||
def soundcloud_download(token, callback_url, api_key, track_id):
|
||||
"""
|
||||
This is in stasis
|
||||
Download a file from SoundCloud
|
||||
|
||||
:param token: OAuth2 client access token
|
||||
:param callback_url: callback URL to send the downloaded file to
|
||||
:param api_key: API key for callback authentication
|
||||
:param track_id: SoundCloud track identifier
|
||||
:rtype: None
|
||||
|
||||
:return: JSON formatted string of file identifiers for the downloaded tracks
|
||||
:rtype: string
|
||||
"""
|
||||
client = soundcloud.Client(access_token=token)
|
||||
obj = {}
|
||||
try:
|
||||
track = client.get('/tracks/%s' % track_id)
|
||||
obj.update(track.fields())
|
||||
if track.downloadable:
|
||||
re = None
|
||||
with closing(requests.get('%s?oauth_token=%s' % (track.download_url, client.access_token), verify=True, stream=True)) as r:
|
||||
filename = get_filename(r)
|
||||
re = requests.post(callback_url, files={'file': (filename, r.content)}, auth=requests.auth.HTTPBasicAuth(api_key, ''))
|
||||
re.raise_for_status()
|
||||
f = json.loads(re.content) # Read the response from the media API to get the file id
|
||||
obj['fileid'] = f['id']
|
||||
else:
|
||||
# manually update the task state
|
||||
self.update_state(
|
||||
state = states.FAILURE,
|
||||
meta = 'Track %s is not flagged as downloadable!' % track.title
|
||||
)
|
||||
# ignore the task so no other state is recorded
|
||||
raise Ignore()
|
||||
except Exception as e:
|
||||
logger.info('Error during file download: {0}'.format(e.message))
|
||||
raise e
|
||||
return json.dumps(obj)
|
||||
|
||||
|
||||
@celery.task(name='soundcloud-update', acks_late=True)
|
||||
def soundcloud_update(data, token, track_id):
|
||||
"""
|
||||
Update a file on SoundCloud
|
||||
|
||||
:param data: associative array containing SoundCloud metadata
|
||||
:param token: OAuth2 client access token
|
||||
:param track_id: SoundCloud ID of the track to be updated
|
||||
|
||||
:return: JSON formatted string of the SoundCloud response object
|
||||
:rtype: string
|
||||
"""
|
||||
client = soundcloud.Client(access_token=token)
|
||||
try:
|
||||
tracks = client.get('/me/tracks') if track_id is None else {client.get('/tracks/%s' % track_id)}
|
||||
for track in tracks:
|
||||
if track.downloadable:
|
||||
track_file = client.get(track.download_url)
|
||||
with track_file as f:
|
||||
requests.post(callback_url, data=f, auth=requests.auth.HTTPBasicAuth(api_key, ''))
|
||||
logger.info('Updating track {title}'.format(**data))
|
||||
track = client.put('/tracks/%s' % track_id, track=data)
|
||||
except Exception as e:
|
||||
logger.info('Error during file download: {0}'.format(e.message))
|
||||
logger.info(str(e))
|
||||
logger.info('Error updating track {title}: {0}'.format(e.message, **data))
|
||||
raise e
|
||||
return json.dumps(track.fields())
|
||||
|
||||
|
||||
@celery.task(name='soundcloud-delete', acks_late=True)
|
||||
|
@ -72,8 +109,8 @@ def soundcloud_delete(token, track_id):
|
|||
:param token: OAuth2 client access token
|
||||
:param track_id: SoundCloud track identifier
|
||||
|
||||
:return: the SoundCloud response object
|
||||
:rtype: dict
|
||||
:return: JSON formatted string of the SoundCloud response object
|
||||
:rtype: string
|
||||
"""
|
||||
client = soundcloud.Client(access_token=token)
|
||||
try:
|
||||
|
@ -94,7 +131,10 @@ def podcast_download(id, url, callback_url, api_key):
|
|||
:param url: download url for the episode
|
||||
:param callback_url: callback URL to send the downloaded file to
|
||||
:param api_key: API key for callback authentication
|
||||
:rtype: None
|
||||
|
||||
:return: JSON formatted string of a dictionary of download statuses
|
||||
and file identifiers (for successful uploads)
|
||||
:rtype: string
|
||||
"""
|
||||
# Object to store file IDs, episode IDs, and download status
|
||||
# (important if there's an error before the file is posted)
|
||||
|
@ -121,6 +161,8 @@ def get_filename(r):
|
|||
by parsing either the content disposition or the request URL
|
||||
|
||||
:param r: request object
|
||||
|
||||
:return: the file name
|
||||
:rtype: string
|
||||
"""
|
||||
# Try to get the filename from the content disposition
|
||||
|
|
Loading…
Reference in New Issue