Congegni
/
sintonia
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

♻️ (celery) python3 compat fixes

This commit is contained in:
Lucas Bickel 2019-08-18 17:45:48 +02:00
parent e232469551
commit 9bea08dc03
4 changed files with 146 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
python_apps/airtime-celery/airtime-celery/__init__.py
View File

@ -1,3 +1,4 @@
import os import os
# Make the celeryconfig module visible to celery # Make the celeryconfig module visible to celery
os.environ['CELERY_CONFIG_MODULE'] = 'airtime-celery.celeryconfig' os.environ["CELERY_CONFIG_MODULE"] = "airtime-celery.celeryconfig"

39
python_apps/airtime-celery/airtime-celery/celeryconfig.py
View File

@ -7,36 +7,37 @@ RMQ_CONFIG_SECTION = "rabbitmq"
def get_rmq_broker(): def get_rmq_broker():
rmq_config = ConfigObj(os.environ['RMQ_CONFIG_FILE']) rmq_config = ConfigObj(os.environ["RMQ_CONFIG_FILE"])
rmq_settings = parse_rmq_config(rmq_config) rmq_settings = parse_rmq_config(rmq_config)
return 'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(**rmq_settings) return "amqp://{username}:{password}@{host}:{port}/{vhost}".format(**rmq_settings)
def parse_rmq_config(rmq_config): def parse_rmq_config(rmq_config):
return { return {
'host' : rmq_config[RMQ_CONFIG_SECTION]['host'], "host": rmq_config[RMQ_CONFIG_SECTION]["host"],
'port' : rmq_config[RMQ_CONFIG_SECTION]['port'], "port": rmq_config[RMQ_CONFIG_SECTION]["port"],
'username': rmq_config[RMQ_CONFIG_SECTION]['user'], "username": rmq_config[RMQ_CONFIG_SECTION]["user"],
'password': rmq_config[RMQ_CONFIG_SECTION]['password'], "password": rmq_config[RMQ_CONFIG_SECTION]["password"],
'vhost' : rmq_config[RMQ_CONFIG_SECTION]['vhost'] "vhost": rmq_config[RMQ_CONFIG_SECTION]["vhost"],
} }
# Celery amqp settings # Celery amqp settings
BROKER_URL = get_rmq_broker() BROKER_URL = get_rmq_broker()
CELERY_RESULT_BACKEND = 'amqp' # Use RabbitMQ as the celery backend CELERY_RESULT_BACKEND = "amqp" # Use RabbitMQ as the celery backend
CELERY_RESULT_PERSISTENT = True # Persist through a broker restart CELERY_RESULT_PERSISTENT = True # Persist through a broker restart
CELERY_TASK_RESULT_EXPIRES = 900 # Expire task results after 15 minutes CELERY_TASK_RESULT_EXPIRES = 900 # Expire task results after 15 minutes
CELERY_RESULT_EXCHANGE = 'celeryresults' # Default exchange - needed due to php-celery CELERY_RESULT_EXCHANGE = "celeryresults" # Default exchange - needed due to php-celery
CELERY_QUEUES = ( CELERY_QUEUES = (
Queue('soundcloud', exchange=Exchange('soundcloud'), routing_key='soundcloud'), Queue("soundcloud", exchange=Exchange("soundcloud"), routing_key="soundcloud"),
Queue('podcast', exchange=Exchange('podcast'), routing_key='podcast'), Queue("podcast", exchange=Exchange("podcast"), routing_key="podcast"),
Queue(exchange=Exchange('celeryresults'), auto_delete=True), Queue(exchange=Exchange("celeryresults"), auto_delete=True),
) )
CELERY_EVENT_QUEUE_EXPIRES = 900 # RabbitMQ x-expire after 15 minutes CELERY_EVENT_QUEUE_EXPIRES = 900 # RabbitMQ x-expire after 15 minutes
# Celery task settings # Celery task settings
CELERY_TASK_SERIALIZER = 'json' CELERY_TASK_SERIALIZER = "json"
CELERY_RESULT_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = "json"
CELERY_ACCEPT_CONTENT = ['json'] CELERY_ACCEPT_CONTENT = ["json"]
CELERY_TIMEZONE = 'Europe/Berlin' CELERY_TIMEZONE = "Europe/Berlin"
CELERY_ENABLE_UTC = True CELERY_ENABLE_UTC = True

156
python_apps/airtime-celery/airtime-celery/tasks.py
View File

@ -1,25 +1,29 @@
from future.standard_library import install_aliases
install_aliases()
import os import os
import json import json
import urllib2
import requests import requests
import soundcloud import soundcloud
import cgi import cgi
import urlparse
import posixpath import posixpath
import shutil import shutil
import tempfile import tempfile
import traceback import traceback
import mutagen import mutagen
from StringIO import StringIO from io import StringIO
from celery import Celery from celery import Celery
from celery.utils.log import get_task_logger from celery.utils.log import get_task_logger
from contextlib import closing from contextlib import closing
from urllib.parse import urlsplit
celery = Celery() celery = Celery()
logger = get_task_logger(__name__) logger = get_task_logger(__name__)
@celery.task(name='soundcloud-upload', acks_late=True) @celery.task(name="soundcloud-upload", acks_late=True)
def soundcloud_upload(data, token, file_path): def soundcloud_upload(data, token, file_path):
""" """
Upload a file to SoundCloud Upload a file to SoundCloud
@ -32,19 +36,23 @@ def soundcloud_upload(data, token, file_path):
:rtype: string :rtype: string
""" """
client = soundcloud.Client(access_token=token) client = soundcloud.Client(access_token=token)
# Open the file with urllib2 if it's a cloud file # Open the file with requests if it's a cloud file
data['asset_data'] = open(file_path, 'rb') if os.path.isfile(file_path) else urllib2.urlopen(file_path) data["asset_data"] = (
open(file_path, "rb")
if os.path.isfile(file_path)
else requests.get(file_path).content
)
try: try:
logger.info('Uploading track: {0}'.format(data)) logger.info("Uploading track: {0}".format(data))
track = client.post('/tracks', track=data) track = client.post("/tracks", track=data)
except Exception as e: except Exception as e:
logger.info('Error uploading track {title}: {0}'.format(e.message, **data)) logger.info("Error uploading track {title}: {0}".format(e.message, **data))
raise e raise e
data['asset_data'].close() data["asset_data"].close()
return json.dumps(track.fields()) return json.dumps(track.fields())
@celery.task(name='soundcloud-download', acks_late=True) @celery.task(name="soundcloud-download", acks_late=True)
def soundcloud_download(token, callback_url, api_key, track_id): def soundcloud_download(token, callback_url, api_key, track_id):
""" """
Download a file from SoundCloud Download a file from SoundCloud
@ -60,31 +68,43 @@ def soundcloud_download(token, callback_url, api_key, track_id):
client = soundcloud.Client(access_token=token) client = soundcloud.Client(access_token=token)
obj = {} obj = {}
try: try:
track = client.get('/tracks/%s' % track_id) track = client.get("/tracks/%s" % track_id)
obj.update(track.fields()) obj.update(track.fields())
if track.downloadable: if track.downloadable:
re = None re = None
with closing(requests.get('%s?oauth_token=%s' % (track.download_url, client.access_token), verify=True, stream=True)) as r: with closing(
requests.get(
"%s?oauth_token=%s" % (track.download_url, client.access_token),
verify=True,
stream=True,
)
) as r:
filename = get_filename(r) filename = get_filename(r)
re = requests.post(callback_url, files={'file': (filename, r.content)}, auth=requests.auth.HTTPBasicAuth(api_key, '')) re = requests.post(
callback_url,
files={"file": (filename, r.content)},
auth=requests.auth.HTTPBasicAuth(api_key, ""),
)
re.raise_for_status() re.raise_for_status()
f = json.loads(re.content) # Read the response from the media API to get the file id f = json.loads(
obj['fileid'] = f['id'] re.content
) # Read the response from the media API to get the file id
obj["fileid"] = f["id"]
else: else:
# manually update the task state # manually update the task state
self.update_state( self.update_state(
state = states.FAILURE, state=states.FAILURE,
meta = 'Track %s is not flagged as downloadable!' % track.title meta="Track %s is not flagged as downloadable!" % track.title,
) )
# ignore the task so no other state is recorded # ignore the task so no other state is recorded
raise Ignore() raise Ignore()
except Exception as e: except Exception as e:
logger.info('Error during file download: {0}'.format(e.message)) logger.info("Error during file download: {0}".format(e.message))
raise e raise e
return json.dumps(obj) return json.dumps(obj)
@celery.task(name='soundcloud-update', acks_late=True) @celery.task(name="soundcloud-update", acks_late=True)
def soundcloud_update(data, token, track_id): def soundcloud_update(data, token, track_id):
""" """
Update a file on SoundCloud Update a file on SoundCloud
@ -98,15 +118,15 @@ def soundcloud_update(data, token, track_id):
""" """
client = soundcloud.Client(access_token=token) client = soundcloud.Client(access_token=token)
try: try:
logger.info('Updating track {title}'.format(**data)) logger.info("Updating track {title}".format(**data))
track = client.put('/tracks/%s' % track_id, track=data) track = client.put("/tracks/%s" % track_id, track=data)
except Exception as e: except Exception as e:
logger.info('Error updating track {title}: {0}'.format(e.message, **data)) logger.info("Error updating track {title}: {0}".format(e.message, **data))
raise e raise e
return json.dumps(track.fields()) return json.dumps(track.fields())
@celery.task(name='soundcloud-delete', acks_late=True) @celery.task(name="soundcloud-delete", acks_late=True)
def soundcloud_delete(token, track_id): def soundcloud_delete(token, track_id):
""" """
Delete a file from SoundCloud Delete a file from SoundCloud
@ -119,16 +139,18 @@ def soundcloud_delete(token, track_id):
""" """
client = soundcloud.Client(access_token=token) client = soundcloud.Client(access_token=token)
try: try:
logger.info('Deleting track with ID {0}'.format(track_id)) logger.info("Deleting track with ID {0}".format(track_id))
track = client.delete('/tracks/%s' % track_id) track = client.delete("/tracks/%s" % track_id)
except Exception as e: except Exception as e:
logger.info('Error deleting track!') logger.info("Error deleting track!")
raise e raise e
return json.dumps(track.fields()) return json.dumps(track.fields())
@celery.task(name='podcast-download', acks_late=True) @celery.task(name="podcast-download", acks_late=True)
def podcast_download(id, url, callback_url, api_key, podcast_name, album_override, track_title): def podcast_download(
id, url, callback_url, api_key, podcast_name, album_override, track_title
):
""" """
Download a podcast episode Download a podcast episode
@ -146,12 +168,12 @@ def podcast_download(id, url, callback_url, api_key, podcast_name, album_overrid
""" """
# Object to store file IDs, episode IDs, and download status # Object to store file IDs, episode IDs, and download status
# (important if there's an error before the file is posted) # (important if there's an error before the file is posted)
obj = { 'episodeid': id } obj = {"episodeid": id}
try: try:
re = None re = None
with closing(requests.get(url, stream=True)) as r: with closing(requests.get(url, stream=True)) as r:
filename = get_filename(r) filename = get_filename(r)
with tempfile.NamedTemporaryFile(mode ='wb+', delete=False) as audiofile: with tempfile.NamedTemporaryFile(mode="wb+", delete=False) as audiofile:
r.raw.decode_content = True r.raw.decode_content = True
shutil.copyfileobj(r.raw, audiofile) shutil.copyfileobj(r.raw, audiofile)
# mutagen should be able to guess the write file type # mutagen should be able to guess the write file type
@ -162,44 +184,66 @@ def podcast_download(id, url, callback_url, api_key, podcast_name, album_overrid
mp3suffix = ("mp3", "MP3", "Mp3", "mP3") mp3suffix = ("mp3", "MP3", "Mp3", "mP3")
# so we treat it like a mp3 if it has a mp3 file extension and hope for the best # so we treat it like a mp3 if it has a mp3 file extension and hope for the best
if filename.endswith(mp3suffix): if filename.endswith(mp3suffix):
metadata_audiofile = mutagen.mp3.MP3(audiofile.name, ID3=mutagen.easyid3.EasyID3) metadata_audiofile = mutagen.mp3.MP3(
#replace track metadata as indicated by album_override setting audiofile.name, ID3=mutagen.easyid3.EasyID3
)
# replace track metadata as indicated by album_override setting
# replace album title as needed # replace album title as needed
metadata_audiofile = podcast_override_metadata(metadata_audiofile, podcast_name, album_override, track_title) metadata_audiofile = podcast_override_metadata(
metadata_audiofile, podcast_name, album_override, track_title
)
metadata_audiofile.save() metadata_audiofile.save()
filetypeinfo = metadata_audiofile.pprint() filetypeinfo = metadata_audiofile.pprint()
logger.info('filetypeinfo is {0}'.format(filetypeinfo.encode('ascii', 'ignore'))) logger.info(
re = requests.post(callback_url, files={'file': (filename, open(audiofile.name, 'rb'))}, auth=requests.auth.HTTPBasicAuth(api_key, '')) "filetypeinfo is {0}".format(filetypeinfo.encode("ascii", "ignore"))
)
re = requests.post(
callback_url,
files={"file": (filename, open(audiofile.name, "rb"))},
auth=requests.auth.HTTPBasicAuth(api_key, ""),
)
re.raise_for_status() re.raise_for_status()
f = json.loads(re.content) # Read the response from the media API to get the file id f = json.loads(
obj['fileid'] = f['id'] re.content
obj['status'] = 1 ) # Read the response from the media API to get the file id
obj["fileid"] = f["id"]
obj["status"] = 1
except Exception as e: except Exception as e:
obj['error'] = e.message obj["error"] = e.message
logger.info('Error during file download: {0}'.format(e)) logger.info("Error during file download: {0}".format(e))
logger.debug('Original Traceback: %s' % (traceback.format_exc(e))) logger.debug("Original Traceback: %s" % (traceback.format_exc(e)))
obj['status'] = 0 obj["status"] = 0
return json.dumps(obj) return json.dumps(obj)
def podcast_override_metadata(m, podcast_name, override, track_title): def podcast_override_metadata(m, podcast_name, override, track_title):
""" """
Override m['album'] if empty or forced with override arg Override m['album'] if empty or forced with override arg
""" """
# if the album override option is enabled replace the album id3 tag with the podcast name even if the album tag contains data # if the album override option is enabled replace the album id3 tag with the podcast name even if the album tag contains data
if override is True: if override is True:
logger.debug('overriding album name to {0} in podcast'.format(podcast_name.encode('ascii', 'ignore'))) logger.debug(
m['album'] = podcast_name "overriding album name to {0} in podcast".format(
m['title'] = track_title podcast_name.encode("ascii", "ignore")
m['artist'] = podcast_name )
)
m["album"] = podcast_name
m["title"] = track_title
m["artist"] = podcast_name
else: else:
# replace the album id3 tag with the podcast name if the album tag is empty # replace the album id3 tag with the podcast name if the album tag is empty
try: try:
m['album'] m["album"]
except KeyError: except KeyError:
logger.debug('setting new album name to {0} in podcast'.format(podcast_name.encode('ascii', 'ignore'))) logger.debug(
m['album'] = podcast_name "setting new album name to {0} in podcast".format(
podcast_name.encode("ascii", "ignore")
)
)
m["album"] = podcast_name
return m return m
def get_filename(r): def get_filename(r):
""" """
Given a request object to a file resource, get the name of the file to be downloaded Given a request object to a file resource, get the name of the file to be downloaded
@ -211,18 +255,20 @@ def get_filename(r):
:rtype: string :rtype: string
""" """
# Try to get the filename from the content disposition # Try to get the filename from the content disposition
d = r.headers.get('Content-Disposition') d = r.headers.get("Content-Disposition")
filename = '' filename = ""
if d: if d:
try: try:
_, params = cgi.parse_header(d) _, params = cgi.parse_header(d)
filename = params['filename'] filename = params["filename"]
except Exception as e: except Exception as e:
# We end up here if we get a Content-Disposition header with no filename # We end up here if we get a Content-Disposition header with no filename
logger.warn("Couldn't find file name in Content-Disposition header, using url") logger.warn(
"Couldn't find file name in Content-Disposition header, using url"
)
if not filename: if not filename:
# Since we don't necessarily get the filename back in the response headers, # Since we don't necessarily get the filename back in the response headers,
# parse the URL and get the filename and extension # parse the URL and get the filename and extension
path = urlparse.urlsplit(r.url).path path = urlsplit(r.url).path
filename = posixpath.basename(path) filename = posixpath.basename(path)
return filename return filename

46
python_apps/airtime-celery/setup.py
View File

@ -5,18 +5,20 @@ import sys
# Change directory since setuptools uses relative paths # Change directory since setuptools uses relative paths
script_path = os.path.dirname(os.path.realpath(__file__)) script_path = os.path.dirname(os.path.realpath(__file__))
print script_path print(script_path)
os.chdir(script_path) os.chdir(script_path)
install_args = ['install', 'install_data', 'develop'] install_args = ["install", "install_data", "develop"]
no_init = False no_init = False
run_postinst = False run_postinst = False
# XXX Definitely not the best way of doing this... # XXX Definitely not the best way of doing this...
if sys.argv[1] in install_args and "--no-init-script" not in sys.argv: if sys.argv[1] in install_args and "--no-init-script" not in sys.argv:
run_postinst = True run_postinst = True
data_files = [('/etc/default', ['install/conf/airtime-celery']), data_files = [
('/etc/init.d', ['install/initd/airtime-celery'])] ("/etc/default", ["install/conf/airtime-celery"]),
("/etc/init.d", ["install/initd/airtime-celery"]),
]
else: else:
if "--no-init-script" in sys.argv: if "--no-init-script" in sys.argv:
no_init = True no_init = True
@ -29,26 +31,24 @@ def postinst():
if not no_init: if not no_init:
# Make /etc/init.d file executable and set proper # Make /etc/init.d file executable and set proper
# permissions for the defaults config file # permissions for the defaults config file
os.chmod('/etc/init.d/airtime-celery', 0755) os.chmod("/etc/init.d/airtime-celery", 0o755)
os.chmod('/etc/default/airtime-celery', 0640) os.chmod("/etc/default/airtime-celery", 0o640)
print "Run \"sudo service airtime-celery restart\" now." print('Run "sudo service airtime-celery restart" now.')
setup(name='airtime-celery',
version='0.1', setup(
description='Airtime Celery service', name="airtime-celery",
url='http://github.com/sourcefabric/Airtime', version="0.1",
author='Sourcefabric', description="Airtime Celery service",
author_email='duncan.sommerville@sourcefabric.org', url="http://github.com/sourcefabric/Airtime",
license='MIT', author="Sourcefabric",
packages=['airtime-celery'], author_email="duncan.sommerville@sourcefabric.org",
install_requires=[ license="MIT",
'soundcloud', packages=["airtime-celery"],
'celery < 4', install_requires=["soundcloud", "celery < 4", "kombu < 3.1", "configobj"],
'kombu < 3.1', zip_safe=False,
'configobj' data_files=data_files,
], )
zip_safe=False,
data_files=data_files)
if run_postinst: if run_postinst:
postinst() postinst()