Congegni/sintonia
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Celery backend and support for dev-env worker parallelization

Browse source
This commit is contained in:
Duncan Sommerville 2015-06-12 12:31:55 -04:00
parent c1b5b53a16
commit 15c7ef5885
17 changed files with 1664 additions and 368 deletions
Download patch file Download diff file Expand all files Collapse all files

26
python_apps/airtime-celery/airtime-celery/celeryconfig.py
View file

@ -1,26 +1,26 @@
import os
from configobj import ConfigObj
from kombu import Exchange, Queue
# Get the broker string from airtime.conf
DEFAULT_RMQ_CONFIG_PATH = '/etc/airtime/airtime.conf'
RMQ_CONFIG_SECTION = "rabbitmq"
def get_rmq_broker():
rmq_config = ConfigObj(os.environ['RMQ_CONFIG_FILE'])
rmq_settings = parse_rmq_config(rmq_config)
return 'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(**rmq_settings)
def parse_rmq_config(rmq_config):
return {
'host' : rmq_config[RMQ_CONFIG_SECTION]['host'],
'port' : rmq_config[RMQ_CONFIG_SECTION]['port'],
'username' : rmq_config[RMQ_CONFIG_SECTION]['user'],
'password' : rmq_config[RMQ_CONFIG_SECTION]['password'],
'vhost' : rmq_config[RMQ_CONFIG_SECTION]['vhost']
'host' : rmq_config[RMQ_CONFIG_SECTION]['host'],
'port' : rmq_config[RMQ_CONFIG_SECTION]['port'],
'username': rmq_config[RMQ_CONFIG_SECTION]['user'],
'password': rmq_config[RMQ_CONFIG_SECTION]['password'],
'vhost' : rmq_config[RMQ_CONFIG_SECTION]['vhost']
}
def get_rmq_broker():
rmq_config = ConfigObj(DEFAULT_RMQ_CONFIG_PATH)
rmq_settings = parse_rmq_config(rmq_config)
return 'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(**rmq_settings)
# Celery amqp settings
BROKER_URL = get_rmq_broker()
CELERY_RESULT_BACKEND = 'amqp' # Use RabbitMQ as the celery backend
@ -34,7 +34,7 @@ CELERY_QUEUES = (
)
CELERY_ROUTES = (
{
'soundcloud_uploads.uploader.upload_to_soundcloud': {
'soundcloud_uploads.tasks.upload_to_soundcloud': {
'exchange': 'airtime-results',
'queue': 'airtime-results.soundcloud-uploads',
}

10
python_apps/airtime-celery/airtime-celery/uploader.py → python_apps/airtime-celery/airtime-celery/tasks.py
View file

@ -11,6 +11,16 @@ logger = get_task_logger(__name__)
@celery.task(name='upload-to-soundcloud')
def upload_to_soundcloud(data, token, file_path):
"""
Upload a file to SoundCloud
:param data: associative array containing SoundCloud metadata
:param token: OAuth2 client access token
:param file_path: path to the file being uploaded
:return: the SoundCloud response object
:rtype: dict
"""
client = soundcloud.Client(access_token=token)
# Open the file with urllib2 if it's a cloud file
data['asset_data'] = open(file_path, 'rb') if os.path.isfile(file_path) else urllib2.urlopen(file_path)