SAAS-853 - Celery backend for SoundCloud uploads

python_apps/airtime-celery/airtime-celery/__init__.py

@@ -0,0 +1,3 @@
+import os
+# Make the celeryconfig module visible to celery
+os.environ['CELERY_CONFIG_MODULE'] = 'airtime-celery.celeryconfig'

python_apps/airtime-celery/airtime-celery/celeryconfig.py

@@ -0,0 +1,49 @@
+from configobj import ConfigObj
+from kombu import Exchange, Queue
+
+# Get the broker string from airtime.conf
+DEFAULT_RMQ_CONFIG_PATH = '/etc/airtime/airtime.conf'
+RMQ_CONFIG_SECTION = "rabbitmq"
+
+
+def parse_rmq_config(rmq_config):
+    return {
+        'host'     : rmq_config[RMQ_CONFIG_SECTION]['host'],
+        'port'     : rmq_config[RMQ_CONFIG_SECTION]['port'],
+        'username' : rmq_config[RMQ_CONFIG_SECTION]['user'],
+        'password' : rmq_config[RMQ_CONFIG_SECTION]['password'],
+        'vhost'    : rmq_config[RMQ_CONFIG_SECTION]['vhost']
+    }
+
+
+def get_rmq_broker():
+    rmq_config = ConfigObj(DEFAULT_RMQ_CONFIG_PATH)
+    rmq_settings = parse_rmq_config(rmq_config)
+    return 'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(**rmq_settings)
+
+# Celery amqp settings
+BROKER_URL = get_rmq_broker()
+CELERY_RESULT_BACKEND = 'amqp'     # Use RabbitMQ as the celery backend
+CELERY_RESULT_PERSISTENT = True    # Persist through a broker restart
+CELERY_TASK_RESULT_EXPIRES = 300   # Expire task results after 5 minutes
+CELERY_TRACK_STARTED = False
+CELERY_RESULT_EXCHANGE = 'airtime-results'
+CELERY_QUEUES = (
+    Queue('soundcloud-uploads', exchange=Exchange('soundcloud-uploads'), routing_key='soundcloud-uploads'),
+    Queue('airtime-results.soundcloud-uploads', exchange=Exchange('airtime-results')),
+)
+CELERY_ROUTES = (
+    {
+        'soundcloud_uploads.uploader.upload_to_soundcloud': {
+            'exchange': 'airtime-results',
+            'queue': 'airtime-results.soundcloud-uploads',
+        }
+    },
+)
+
+# Celery task settings
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+CELERY_ACCEPT_CONTENT = ['json']
+CELERY_TIMEZONE = 'Europe/Berlin'
+CELERY_ENABLE_UTC = True

python_apps/airtime-celery/airtime-celery/uploader.py

@@ -0,0 +1,24 @@
+import os
+import json
+import urllib2
+import soundcloud
+from celery import Celery
+from celery.utils.log import get_task_logger
+
+celery = Celery()
+logger = get_task_logger(__name__)
+
+
+@celery.task(name='upload-to-soundcloud')
+def upload_to_soundcloud(data, token, file_path):
+    client = soundcloud.Client(access_token=token)
+    # Open the file with urllib2 if it's a cloud file
+    data['asset_data'] = open(file_path, 'rb') if os.path.isfile(file_path) else urllib2.urlopen(file_path)
+    try:
+        logger.info('Uploading track: {0}'.format(data))
+        track = client.post('/tracks', track=data)
+    except Exception as e:
+        logger.info('Error uploading track {title}: {0}'.format(e.message, **data))
+        raise e
+    data['asset_data'].close()
+    return json.dumps(track.fields())