SAAS-853 - Celery backend for SoundCloud uploads

python_apps/airtime-celery/airtime-celery/__init__.py

@@ -0,0 +1,3 @@
+import os
+# Make the celeryconfig module visible to celery
+os.environ['CELERY_CONFIG_MODULE'] = 'airtime-celery.celeryconfig'

python_apps/airtime-celery/airtime-celery/celeryconfig.py

@@ -24,15 +24,22 @@ def get_rmq_broker():
 # Celery amqp settings
 BROKER_URL = get_rmq_broker()
 CELERY_RESULT_BACKEND = 'amqp'     # Use RabbitMQ as the celery backend
-# CELERY_RESULT_EXCHANGE = 'upload-results'
 CELERY_RESULT_PERSISTENT = True    # Persist through a broker restart
-CELERY_TASK_RESULT_EXPIRES = None  # Don't expire tasks
+CELERY_TASK_RESULT_EXPIRES = 300   # Expire task results after 5 minutes
 CELERY_TRACK_STARTED = False
+CELERY_RESULT_EXCHANGE = 'airtime-results'
 CELERY_QUEUES = (
-    Queue('soundcloud-uploads', Exchange('soundcloud-uploads'), routing_key='celery'),
+    Queue('soundcloud-uploads', exchange=Exchange('soundcloud-uploads'), routing_key='soundcloud-uploads'),
+    Queue('airtime-results.soundcloud-uploads', exchange=Exchange('airtime-results')),
+)
+CELERY_ROUTES = (
+    {
+        'soundcloud_uploads.uploader.upload_to_soundcloud': {
+            'exchange': 'airtime-results',
+            'queue': 'airtime-results.soundcloud-uploads',
+        }
+    },
 )
-CELERY_DEFAULT_QUEUE = 'soundcloud-uploads'
-CELERY_DEFAULT_EXCHANGE_TYPE = 'topic'
 
 # Celery task settings
 CELERY_TASK_SERIALIZER = 'json'

python_apps/airtime-celery/airtime-celery/uploader.py

@@ -0,0 +1,24 @@
+import os
+import json
+import urllib2
+import soundcloud
+from celery import Celery
+from celery.utils.log import get_task_logger
+
+celery = Celery()
+logger = get_task_logger(__name__)
+
+
+@celery.task(name='upload-to-soundcloud')
+def upload_to_soundcloud(data, token, file_path):
+    client = soundcloud.Client(access_token=token)
+    # Open the file with urllib2 if it's a cloud file
+    data['asset_data'] = open(file_path, 'rb') if os.path.isfile(file_path) else urllib2.urlopen(file_path)
+    try:
+        logger.info('Uploading track: {0}'.format(data))
+        track = client.post('/tracks', track=data)
+    except Exception as e:
+        logger.info('Error uploading track {title}: {0}'.format(e.message, **data))
+        raise e
+    data['asset_data'].close()
+    return json.dumps(track.fields())

python_apps/airtime-celery/install/conf/airtime-celery

@@ -1,11 +1,11 @@
 # Names of nodes to start
-CELERYD_NODES="soundcloud_uploader"
+CELERYD_NODES="airtime-celery"
 
 # Absolute or relative path to the 'celery' command:
 CELERY_BIN="/usr/local/bin/celery"
 
 # App instance to use
-CELERY_APP="soundcloud_uploader.uploader:celery"
+CELERY_APP="airtime-celery.uploader:celery"
 
 # Extra command-line arguments to the worker
 CELERYD_OPTS="--time-limit=300 --concurrency=8 --config=celeryconfig"

python_apps/airtime-celery/setup.py

@@ -0,0 +1,45 @@
+from setuptools import setup
+from subprocess import call
+import os
+import sys
+
+install_args = ['install', 'install_data', 'develop']
+
+# Definitely not the best way of doing this...
+if sys.argv[1] in install_args:
+    data_files = [('/etc/default', ['install/conf/airtime-celery']),
+                  ('/etc/init.d', ['install/upstart/airtime-celery'])]
+else:
+    data_files = []
+
+
+def postinst():
+    print "Reloading initctl configuration"
+    call(['initctl', 'reload-configuration'])
+    # Make /etc/init.d file executable and set proper
+    # permissions for the defaults config file
+    os.chmod('/etc/init.d/airtime-celery', 0755)
+    os.chmod('/etc/default/airtime-celery', 0640)
+    print "Setting uploader to start on boot"
+    call(['update-rc.d', 'airtime-celery', 'defaults'])
+    print "Run \"sudo service airtime-celery restart\" now."
+
+setup(name='airtime-celery',
+      version='0.1',
+      description='Airtime Celery service',
+      url='http://github.com/sourcefabric/Airtime',
+      author='Sourcefabric',
+      author_email='duncan.sommerville@sourcefabric.org',
+      license='MIT',
+      packages=['airtime-celery'],
+      install_requires=[
+          'soundcloud',
+          'celery',
+          'kombu'
+      ],
+      zip_safe=False,
+      data_files=data_files)
+
+if data_files:
+    postinst()
+

python_apps/soundcloud_uploader/bin/soundcloud_uploader

@@ -1,8 +0,0 @@
-#!/usr/bin/env sh
-
-if [ "$(id -u)" != "0" ]; then
-    echo "Please run as root user."
-    exit 1
-fi
-
-service soundcloud_uploader restart

python_apps/soundcloud_uploader/setup.py

@@ -1,35 +0,0 @@
-from setuptools import setup
-from subprocess import call
-import os
-
-data_files = [('/etc/default', ['install/conf/soundcloud_uploader']),
-              ('/etc/init.d', ['install/upstart/soundcloud_uploader'])]
-print data_files
-
-setup(name='soundcloud_uploader',
-      version='0.1',
-      description='Celery SoundCloud upload worker',
-      url='http://github.com/sourcefabric/Airtime',
-      author='Sourcefabric',
-      author_email='duncan.sommerville@sourcefabric.org',
-      license='MIT',
-      packages=['soundcloud_uploader'],
-      scripts=['bin/soundcloud_uploader'],
-      install_requires=[
-          'soundcloud',
-          'celery',
-          'kombu'
-      ],
-      zip_safe=False,
-      data_files=data_files)
-
-if data_files:
-    print "Reloading initctl configuration"
-    call(['initctl', 'reload-configuration'])
-    # Make /etc/init.d file executable and set proper
-    # permissions for the defaults config file
-    os.chmod('/etc/init.d/soundcloud_uploader', 0755)
-    os.chmod('/etc/default/soundcloud_uploader', 0640)
-    print "Setting uploader to start on boot"
-    call(['update-rc.d', 'soundcloud_uploader', 'defaults'])
-    print "Run \"sudo service soundcloud_uploader restart\" now."

python_apps/soundcloud_uploader/soundcloud_uploader/__init__.py

@@ -1,3 +0,0 @@
-import os
-# Make the celeryconfig module visible to celery
-os.environ['CELERY_CONFIG_MODULE'] = 'soundcloud_uploader.celeryconfig'

python_apps/soundcloud_uploader/soundcloud_uploader/uploader.py

@@ -1,24 +0,0 @@
-import os
-import json
-import urllib2
-import soundcloud
-from celery import Celery
-from celery.utils.log import get_task_logger
-
-celery = Celery('uploader')
-logger = get_task_logger(__name__)
-
-
-@celery.task(queue='soundcloud-uploads', name='upload')
-def upload(track_data, token, file_path):
-    client = soundcloud.Client(access_token=token)
-    # Open the file with urllib2 if it's a cloud file
-    track_data['asset_data'] = open(file_path, 'rb') if os.path.isfile(file_path) else urllib2.urlopen(file_path)
-    try:
-        logger.info('Uploading track: {0}'.format(track_data))
-        track = client.post('/tracks', track=track_data)
-    except Exception as e:
-        logger.info('Error uploading track {name}: {0}'.format(e.message, **track_data))
-        raise e
-    track_data['asset_data'].close()
-    return json.dumps(track.fields())