Congegni
/
libretime
mirror of https://github.com/libretime/libretime.git
7
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add SoundCloud update and download tasks to Celery backend; requires airtime-celery reinstall

This commit is contained in:
Duncan Sommerville 2015-10-30 16:10:16 -04:00
parent bf97c42167
commit 4f281a30ed
9 changed files with 131 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
airtime_mvc/application/common/TaskManager.php
View File

@ -82,7 +82,6 @@ final class TaskManager {
if ($this->_isUserSessionRequest()) {
return;
}
Logging::info("Running the tasks!");
$this->_con = Propel::getConnection(CcPrefPeer::DATABASE_NAME);
$this->_con->beginTransaction();
try {

25
airtime_mvc/application/controllers/LibraryController.php
View File

@ -139,33 +139,18 @@ class LibraryController extends Zend_Controller_Action
if ($type === "audioclip" && $soundcloudService->hasAccessToken()) {
$serviceId = $soundcloudService->getServiceId($id);
if (!is_null($file) && $serviceId != 0) {
$trackRef = ThirdPartyTrackReferencesQuery::create()
->filterByDbService(SOUNDCLOUD_SERVICE_NAME)
->findOneByDbFileId($id);
//create a menu separator
$menu["sep1"] = "-----------";
//create a sub menu for Soundcloud actions.
$menu["soundcloud"] = array("name" => _(SOUNDCLOUD), "icon" => "soundcloud", "items" => array());
$menu["soundcloud"]["items"]["view"] = array("name" => _("View track"), "icon" => "soundcloud", "url" => $baseUrl . "soundcloud/view-on-sound-cloud/id/{$id}");
// $menu["soundcloud"]["items"]["remove"] = array("name" => _("Remove track"), "icon" => "soundcloud", "url" => $baseUrl . "soundcloud/delete/id/{$id}");
$menu["soundcloud"]["items"]["update"] = array("name" => _("Update track"), "icon" => "soundcloud", "url" => $baseUrl . "soundcloud/update/id/{$trackRef->getDbForeignId()}");
}
/*
Since we upload to SoundCloud from the Publish dialog now, this is unnecessary
else {
// If a reference exists for this file ID, that means the user has uploaded the track
// but we haven't yet gotten a response from Celery, so disable the menu item
if ($soundcloudService->referenceExists($id)) {
$menu["soundcloud"]["items"]["upload"] = array(
"name" => _("Upload track"), "icon" => "soundcloud",
"url" => $baseUrl . "soundcloud/upload/id/{$id}", "disabled" => true
);
} else {
$menu["soundcloud"]["items"]["upload"] = array(
"name" => _("Upload track"), "icon" => "soundcloud",
"url" => $baseUrl . "soundcloud/upload/id/{$id}"
);
}
}
*/
}
}
} elseif ($type === "playlist" || $type === "block") {

24
airtime_mvc/application/controllers/SoundcloudController.php
View File

@ -28,11 +28,22 @@ class SoundcloudController extends ThirdPartyController implements OAuth2Control
* @throws Zend_Controller_Response_Exception thrown if upload fails for any reason
*/
public function uploadAction() {
$request = $this->getRequest();
$id = $request->getParam('id');
$id = $this->getRequest()->getParam('id');
$this->_service->upload($id);
}
/**
* Update the file with the given id on SoundCloud
*
* @return void
*
* @throws Zend_Controller_Response_Exception thrown if upload fails for any reason
*/
public function updateAction() {
$id = $this->getRequest()->getParam('id');
$this->_service->update($id);
}
/**
* Download the file with the given id from SoundCloud
*
@ -41,8 +52,7 @@ class SoundcloudController extends ThirdPartyController implements OAuth2Control
* @throws Zend_Controller_Response_Exception thrown if download fails for any reason
*/
public function downloadAction() {
$request = $this->getRequest();
$id = $request->getParam('id');
$id = $this->getRequest()->getParam('id');
$this->_service->download($id);
}
@ -54,8 +64,7 @@ class SoundcloudController extends ThirdPartyController implements OAuth2Control
* @throws Zend_Controller_Response_Exception thrown if deletion fails for any reason
*/
public function deleteAction() {
$request = $this->getRequest();
$id = $request->getParam('id');
$id = $this->getRequest()->getParam('id');
$this->_service->delete($id);
}
@ -97,8 +106,7 @@ class SoundcloudController extends ThirdPartyController implements OAuth2Control
* @return void
*/
public function viewOnSoundCloudAction() {
$request = $this->getRequest();
$id = $request->getParam('id');
$id = $this->getRequest()->getParam('id');
try {
$soundcloudLink = $this->_service->getLinkToFile($id);
header('Location: ' . $soundcloudLink);

7
airtime_mvc/application/services/PublishService.php
View File

@ -73,7 +73,7 @@ class Application_Service_PublishService {
);
}
/** @noinspection PhpUnusedPrivateMethodInspection
/**
* Reflective accessor for SoundCloud publication status for the
* file with the given ID
*
@ -84,10 +84,11 @@ class Application_Service_PublishService {
*/
private static function getSoundCloudPublishStatus($fileId) {
$soundcloudService = new Application_Service_SoundcloudService();
return ($soundcloudService->getServiceId($fileId) != 0);
return ($soundcloudService->referenceExists($fileId));
}
/** @noinspection PhpUnusedPrivateMethodInspection
/**
*
* Reflective accessor for Station podcast publication status for the
* file with the given ID
*

32
airtime_mvc/application/services/SoundcloudService.php
View File

@ -14,6 +14,7 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
*/
const UPLOAD = 'upload';
const UPDATE = 'update';
const DOWNLOAD = 'download';
const DELETE = 'delete';
@ -42,6 +43,7 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
*/
protected static $_CELERY_TASKS = [
self::UPLOAD => 'soundcloud-upload',
self::UPDATE => 'soundcloud-update',
self::DOWNLOAD => 'soundcloud-download',
self::DELETE => 'soundcloud-delete'
];
@ -136,13 +138,28 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
}
/**
* Given a track identifier, download a track from SoundCloud
*
* If no identifier is given, download all the user's tracks
* Given a track identifier, update a track on SoundCloud
*
* @param int $trackId a track identifier
*/
public function download($trackId = null) {
public function update($trackId) {
$trackRef = ThirdPartyTrackReferencesQuery::create()
->findOneByDbForeignId($trackId);
$file = Application_Model_StoredFile::RecallById($trackRef->getDbFileId());
$data = array(
'data' => $this->_getUploadData($file),
'token' => $this->_accessToken,
'track_id' => $trackId
);
$this->_executeTask(static::$_CELERY_TASKS[self::UPDATE], $data, $trackRef->getDbFileId());
}
/**
* Given a track identifier, download a track from SoundCloud
*
* @param int $trackId a track identifier
*/
public function download($trackId) {
$CC_CONFIG = Config::getConfig();
$data = array(
'callback_url' => Application_Common_HTTPHelper::getStationUrl() . 'rest/media',
@ -200,6 +217,9 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
return null;
}
$ref->setDbForeignId($track->id); // SoundCloud identifier
if (isset($track->fileid)) {
$ref->setDbFileId($track->fileid); // For downloads, set the cc_files ID
}
}
// TODO: set SoundCloud upload status?
// $ref->setDbStatus($status);
@ -288,6 +308,8 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
}
/**
* Publishing interface proxy
*
* Publish the file with the given file ID to SoundCloud
*
* @param int $fileId ID of the file to be published
@ -297,6 +319,8 @@ class Application_Service_SoundcloudService extends Application_Service_ThirdPar
}
/**
* Publishing interface proxy
*
* Unpublish the file with the given file ID from SoundCloud
*
* @param int $fileId ID of the file to be unpublished

24
airtime_mvc/public/js/airtime/library/library.js
View File

@ -1178,33 +1178,15 @@ var AIRTIME = (function(AIRTIME) {
if (oItems.soundcloud !== undefined) {
var soundcloud = oItems.soundcloud.items;
// define an upload to soundcloud callback.
if (soundcloud.upload !== undefined) {
if (soundcloud.update !== undefined) {
callback = function() {
alert($.i18n._("Your track is being uploaded and will " +
"appear on SoundCloud in a couple of minutes"));
$.post(soundcloud.upload.url, function(){});
$.post(soundcloud.update.url, function () {});
};
soundcloud.upload.callback = callback;
}
// define an upload to soundcloud callback.
if (soundcloud.remove !== undefined) {
callback = function() {
if (confirm($.i18n._("Are you sure? SoundCloud stats and comments for this track will be permanently removed."))) {
alert($.i18n._("Your track is being deleted from SoundCloud"));
$.post(soundcloud.remove.url, function () {
});
}
};
soundcloud.remove.callback = callback;
soundcloud.update.callback = callback;
}
// define a view on soundcloud callback
if (soundcloud.view !== undefined) {
callback = function() {
window.open(soundcloud.view.url);
};

23
airtime_mvc/public/js/airtime/library/publish.js
View File

@ -20,7 +20,7 @@ var AIRTIME = (function (AIRTIME) {
function init () {
var csrfToken = jQuery("#csrf").val();
$http.get(endpoint + mediaId, { csrf_token: csrfToken })
$http.get(endpoint + mediaId, {csrf_token: csrfToken})
.success(function (json) {
$scope.media = json;
tab.setName($scope.media.track_title);
@ -28,7 +28,7 @@ var AIRTIME = (function (AIRTIME) {
// Get an object containing all sources, their translated labels,
// and their publication state for the file with the given ID
$http.get(endpoint + mediaId + '/publish-sources', { csrf_token: csrfToken })
$http.get(endpoint + mediaId + '/publish-sources', {csrf_token: csrfToken})
.success(function (json) {
$scope.sources = json;
// Store the data (whether each source should be published to when publish is clicked)
@ -49,14 +49,18 @@ var AIRTIME = (function (AIRTIME) {
$scope.publish = function () {
var data = {};
jQuery.each($scope.publishData, function(k, v) {
jQuery.each($scope.publishData, function (k, v) {
if (v) {
if (k == "soundcloud") {
alert($.i18n._("Your track is being uploaded and will "
+ "appear on SoundCloud in a couple of minutes"));
}
data[k] = 'publish'; // FIXME: should be more robust
}
});
if (Object.keys(data).length > 0) {
$http.put(endpoint + mediaId + '/publish', { csrf_token: jQuery("#csrf").val(), sources: data})
$http.put(endpoint + mediaId + '/publish', {csrf_token: jQuery("#csrf").val(), sources: data})
.success(function () {
init();
});
@ -66,10 +70,13 @@ var AIRTIME = (function (AIRTIME) {
$scope.remove = function (source) {
var data = {};
data[source] = 'unpublish'; // FIXME: should be more robust
$http.put(endpoint + mediaId + '/publish', { csrf_token: jQuery("#csrf").val(), sources: data })
.success(function () {
init();
});
if (source != "soundcloud" || confirm($.i18n._("Are you sure? SoundCloud stats and comments "
+ "for this track will be permanently removed."))) {
$http.put(endpoint + mediaId + '/publish', {csrf_token: jQuery("#csrf").val(), sources: data})
.success(function () {
init();
});
}
};
$scope.discard = function () {

4
python_apps/airtime-celery/airtime-celery/celeryconfig.py
View File

@ -25,14 +25,14 @@ def parse_rmq_config(rmq_config):
BROKER_URL = get_rmq_broker()
CELERY_RESULT_BACKEND = 'amqp' # Use RabbitMQ as the celery backend
CELERY_RESULT_PERSISTENT = True # Persist through a broker restart
CELERY_TASK_RESULT_EXPIRES = 600 # Expire task results after 10 minutes
CELERY_TASK_RESULT_EXPIRES = 900 # Expire task results after 15 minutes
CELERY_RESULT_EXCHANGE = 'celeryresults' # Default exchange - needed due to php-celery
CELERY_QUEUES = (
Queue('soundcloud', exchange=Exchange('soundcloud'), routing_key='soundcloud'),
Queue('podcast', exchange=Exchange('podcast'), routing_key='podcast'),
Queue(exchange=Exchange('celeryresults'), auto_delete=True),
)
CELERY_EVENT_QUEUE_EXPIRES = 600 # RabbitMQ x-expire after 10 minutes
CELERY_EVENT_QUEUE_EXPIRES = 900 # RabbitMQ x-expire after 15 minutes
# Celery task settings
CELERY_TASK_SERIALIZER = 'json'

74
python_apps/airtime-celery/airtime-celery/tasks.py
View File

@ -23,8 +23,8 @@ def soundcloud_upload(data, token, file_path):
:param token: OAuth2 client access token
:param file_path: path to the file being uploaded
:return: the SoundCloud response object
:rtype: dict
:return: JSON formatted string of the SoundCloud response object
:rtype: string
"""
client = soundcloud.Client(access_token=token)
# Open the file with urllib2 if it's a cloud file
@ -40,28 +40,65 @@ def soundcloud_upload(data, token, file_path):
@celery.task(name='soundcloud-download', acks_late=True)
def soundcloud_download(token, callback_url, api_key, track_id=None):
def soundcloud_download(token, callback_url, api_key, track_id):
"""
This is in stasis
Download a file from SoundCloud
:param token: OAuth2 client access token
:param callback_url: callback URL to send the downloaded file to
:param api_key: API key for callback authentication
:param track_id: SoundCloud track identifier
:rtype: None
:return: JSON formatted string of file identifiers for the downloaded tracks
:rtype: string
"""
client = soundcloud.Client(access_token=token)
obj = {}
try:
track = client.get('/tracks/%s' % track_id)
obj.update(track.fields())
if track.downloadable:
re = None
with closing(requests.get('%s?oauth_token=%s' % (track.download_url, client.access_token), verify=True, stream=True)) as r:
filename = get_filename(r)
re = requests.post(callback_url, files={'file': (filename, r.content)}, auth=requests.auth.HTTPBasicAuth(api_key, ''))
re.raise_for_status()
f = json.loads(re.content) # Read the response from the media API to get the file id
obj['fileid'] = f['id']
else:
# manually update the task state
self.update_state(
state = states.FAILURE,
meta = 'Track %s is not flagged as downloadable!' % track.title
)
# ignore the task so no other state is recorded
raise Ignore()
except Exception as e:
logger.info('Error during file download: {0}'.format(e.message))
raise e
return json.dumps(obj)
@celery.task(name='soundcloud-update', acks_late=True)
def soundcloud_update(data, token, track_id):
"""
Update a file on SoundCloud
:param data: associative array containing SoundCloud metadata
:param token: OAuth2 client access token
:param track_id: SoundCloud ID of the track to be updated
:return: JSON formatted string of the SoundCloud response object
:rtype: string
"""
client = soundcloud.Client(access_token=token)
try:
tracks = client.get('/me/tracks') if track_id is None else {client.get('/tracks/%s' % track_id)}
for track in tracks:
if track.downloadable:
track_file = client.get(track.download_url)
with track_file as f:
requests.post(callback_url, data=f, auth=requests.auth.HTTPBasicAuth(api_key, ''))
logger.info('Updating track {title}'.format(**data))
track = client.put('/tracks/%s' % track_id, track=data)
except Exception as e:
logger.info('Error during file download: {0}'.format(e.message))
logger.info(str(e))
logger.info('Error updating track {title}: {0}'.format(e.message, **data))
raise e
return json.dumps(track.fields())
@celery.task(name='soundcloud-delete', acks_late=True)
@ -72,8 +109,8 @@ def soundcloud_delete(token, track_id):
:param token: OAuth2 client access token
:param track_id: SoundCloud track identifier
:return: the SoundCloud response object
:rtype: dict
:return: JSON formatted string of the SoundCloud response object
:rtype: string
"""
client = soundcloud.Client(access_token=token)
try:
@ -94,7 +131,10 @@ def podcast_download(id, url, callback_url, api_key):
:param url: download url for the episode
:param callback_url: callback URL to send the downloaded file to
:param api_key: API key for callback authentication
:rtype: None
:return: JSON formatted string of a dictionary of download statuses
and file identifiers (for successful uploads)
:rtype: string
"""
# Object to store file IDs, episode IDs, and download status
# (important if there's an error before the file is posted)
@ -121,6 +161,8 @@ def get_filename(r):
by parsing either the content disposition or the request URL
:param r: request object
:return: the file name
:rtype: string
"""
# Try to get the filename from the content disposition