From 632ba9acfe69da684c747473aae5cca52a48625b Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Thu, 16 Jan 2020 15:33:44 +0200 Subject: [PATCH 01/38] convert print statements to py3 --- .../airtime_analyzer/config_file.py | 7 ++++--- .../airtime_analyzer/bin/airtime_analyzer | 7 ++++--- python_apps/airtime_analyzer/setup.py | 15 ++++++++------- .../tests/metadata_analyzer_tests.py | 5 +++-- .../tests/replaygain_analyzer_tests.py | 3 ++- python_apps/api_clients/setup.py | 5 +++-- python_apps/icecast2/install/icecast2-install.py | 5 +++-- python_apps/pypo/bin/pyponotify | 16 ++++++++-------- python_apps/pypo/liquidsoap/__main__.py | 3 ++- .../pypo/liquidsoap/generate_liquidsoap_cfg.py | 3 ++- python_apps/pypo/liquidsoap/liquidsoap_auth.py | 5 +++-- python_apps/pypo/pypo/recorder.py | 3 ++- python_apps/pypo/pypo/telnetliquidsoap.py | 5 +++-- python_apps/pypo/pypo/testpypoliqqueue.py | 3 ++- python_apps/pypo/setup.py | 9 +++++---- 15 files changed, 54 insertions(+), 40 deletions(-) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/config_file.py b/python_apps/airtime_analyzer/airtime_analyzer/config_file.py index 7bd5a0b59..506af020a 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/config_file.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/config_file.py @@ -1,3 +1,4 @@ +from __future__ import print_function import ConfigParser def read_config_file(config_path): @@ -6,10 +7,10 @@ def read_config_file(config_path): try: config.readfp(open(config_path)) except IOError as e: - print "Failed to open config file at " + config_path + ": " + e.strerror + print("Failed to open config file at {}: {}".format(config_path, e.strerror)) exit(-1) except Exception as e: - print e.strerror + print(e.strerror) exit(-1) - return config \ No newline at end of file + return config diff --git a/python_apps/airtime_analyzer/bin/airtime_analyzer b/python_apps/airtime_analyzer/bin/airtime_analyzer index b3e4ab0aa..e81fb5085 100755 --- a/python_apps/airtime_analyzer/bin/airtime_analyzer +++ b/python_apps/airtime_analyzer/bin/airtime_analyzer @@ -2,6 +2,7 @@ """Runs the airtime_analyzer application. """ +from __future__ import print_function import daemon import argparse import os @@ -14,7 +15,7 @@ DEFAULT_HTTP_RETRY_PATH = '/tmp/airtime_analyzer_http_retries' def run(): '''Entry-point for this application''' - print "Airtime Analyzer " + VERSION + print("Airtime Analyzer {}".format(VERSION)) parser = argparse.ArgumentParser() parser.add_argument("-d", "--daemon", help="run as a daemon", action="store_true") parser.add_argument("--debug", help="log full debugging output", action="store_true") @@ -57,8 +58,8 @@ def check_if_media_monitor_is_running(): try: process_name = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read() if 'media_monitor.py' in process_name: - print "Error: This process conflicts with media_monitor, and media_monitor is running." - print " Please terminate the running media_monitor.py process and try again." + print("Error: This process conflicts with media_monitor, and media_monitor is running.") + print(" Please terminate the running media_monitor.py process and try again.") exit(1) except IOError: # proc has already terminated continue diff --git a/python_apps/airtime_analyzer/setup.py b/python_apps/airtime_analyzer/setup.py index eb697d941..d819a171a 100644 --- a/python_apps/airtime_analyzer/setup.py +++ b/python_apps/airtime_analyzer/setup.py @@ -1,3 +1,4 @@ +from __future__ import print_function from setuptools import setup from subprocess import call import sys @@ -5,7 +6,7 @@ import os # Change directory since setuptools uses relative paths script_path = os.path.dirname(os.path.realpath(__file__)) -print script_path +print(script_path) os.chdir(script_path) # Allows us to avoid installing the upstart init script when deploying airtime_analyzer @@ -16,7 +17,7 @@ if '--no-init-script' in sys.argv: else: data_files = [('/etc/init', ['install/upstart/airtime_analyzer.conf']), ('/etc/init.d', ['install/sysvinit/airtime_analyzer'])] - print data_files + print(data_files) setup(name='airtime_analyzer', version='0.1', @@ -49,8 +50,8 @@ setup(name='airtime_analyzer', # Remind users to reload the initctl config so that "service start airtime_analyzer" works if data_files: - print "Remember to reload the initctl configuration" - print "Run \"sudo initctl reload-configuration; sudo service airtime_analyzer restart\" now." - print "Or on Ubuntu Xenial (16.04)" - print "Remember to reload the systemd configuration" - print "Run \"sudo systemctl daemon-reload; sudo service airtime_analyzer restart\" now." + print("Remember to reload the initctl configuration") + print("Run \"sudo initctl reload-configuration; sudo service airtime_analyzer restart\" now.") + print("Or on Ubuntu Xenial (16.04)") + print("Remember to reload the systemd configuration") + print("Run \"sudo systemctl daemon-reload; sudo service airtime_analyzer restart\" now.") diff --git a/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py b/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py index 2af30326b..3c60244bc 100644 --- a/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py +++ b/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import print_function import datetime import mutagen import mock @@ -79,8 +80,8 @@ def test_ogg_stereo(): ''' faac and avconv can't seem to create a proper mono AAC file... ugh def test_aac_mono(): metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.m4a') - print "Mono AAC metadata:" - print metadata + print("Mono AAC metadata:") + print(metadata) check_default_metadata(metadata) assert metadata['channels'] == 1 assert metadata['bit_rate'] == 80000 diff --git a/python_apps/airtime_analyzer/tests/replaygain_analyzer_tests.py b/python_apps/airtime_analyzer/tests/replaygain_analyzer_tests.py index 4a4e8ca58..0739e3126 100644 --- a/python_apps/airtime_analyzer/tests/replaygain_analyzer_tests.py +++ b/python_apps/airtime_analyzer/tests/replaygain_analyzer_tests.py @@ -1,3 +1,4 @@ +from __future__ import print_function from nose.tools import * from airtime_analyzer.replaygain_analyzer import ReplayGainAnalyzer @@ -28,7 +29,7 @@ def check_default_metadata(metadata): ''' tolerance = 0.30 expected_replaygain = 5.0 - print metadata['replay_gain'] + print(metadata['replay_gain']) assert abs(metadata['replay_gain'] - expected_replaygain) < tolerance def test_missing_replaygain(): diff --git a/python_apps/api_clients/setup.py b/python_apps/api_clients/setup.py index b71f509a2..a615e6c0f 100644 --- a/python_apps/api_clients/setup.py +++ b/python_apps/api_clients/setup.py @@ -1,10 +1,11 @@ +from __future__ import print_function from setuptools import setup from subprocess import call import sys import os script_path = os.path.dirname(os.path.realpath(__file__)) -print script_path +print(script_path) os.chdir(script_path) setup(name='api_clients', @@ -30,4 +31,4 @@ setup(name='api_clients', # 'wsgiref' ], zip_safe=False, - data_files=[]) \ No newline at end of file + data_files=[]) diff --git a/python_apps/icecast2/install/icecast2-install.py b/python_apps/icecast2/install/icecast2-install.py index 0b411d8da..f1424f9bc 100644 --- a/python_apps/icecast2/install/icecast2-install.py +++ b/python_apps/icecast2/install/icecast2-install.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- +from __future__ import print_function import shutil import os import sys if os.geteuid() != 0: - print "Please run this as root." + print("Please run this as root.") sys.exit(1) def get_current_script_dir(): @@ -18,5 +19,5 @@ try: shutil.copy(current_script_dir+"/../airtime-icecast-status.xsl", "/usr/share/icecast2/web") except Exception, e: - print "exception: %s" % e + print("exception: {}".format(e)) sys.exit(1) diff --git a/python_apps/pypo/bin/pyponotify b/python_apps/pypo/bin/pyponotify index 9d8fe4ab8..b3ca44b4d 100755 --- a/python_apps/pypo/bin/pyponotify +++ b/python_apps/pypo/bin/pyponotify @@ -1,5 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import print_function import traceback """ @@ -133,21 +134,20 @@ class Notify: elif options.liquidsoap_started: self.notify_liquidsoap_started() else: - logger.debug("Unrecognized option in options(%s). Doing nothing" \ - % str(options)) + logger.debug("Unrecognized option in options({}). Doing nothing".format(options)) if __name__ == '__main__': - print - print '#########################################' - print '# *** pypo *** #' - print '# pypo notification gateway #' - print '#########################################' + print() + print('#########################################') + print('# *** pypo *** #') + print('# pypo notification gateway #') + print('#########################################') # initialize try: n = Notify() n.run_with_options(options) except Exception as e: - print( traceback.format_exc() ) + print(traceback.format_exc()) diff --git a/python_apps/pypo/liquidsoap/__main__.py b/python_apps/pypo/liquidsoap/__main__.py index 09e4abe5f..f696ef999 100644 --- a/python_apps/pypo/liquidsoap/__main__.py +++ b/python_apps/pypo/liquidsoap/__main__.py @@ -1,6 +1,7 @@ """ Runs Airtime liquidsoap """ +from __future__ import print_function import argparse import os import generate_liquidsoap_cfg @@ -11,7 +12,7 @@ PYPO_HOME = '/var/tmp/airtime/pypo/' def run(): '''Entry-point for this application''' - print "Airtime Liquidsoap" + print("Airtime Liquidsoap") parser = argparse.ArgumentParser() parser.add_argument("-d", "--debug", help="run in debug mode", action="store_true") args = parser.parse_args() diff --git a/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py b/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py index 4f104c62c..4043b83be 100644 --- a/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py +++ b/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py @@ -1,3 +1,4 @@ +from __future__ import print_function import logging import os import sys @@ -49,7 +50,7 @@ def run(): generate_liquidsoap_config(ss) successful = True except Exception, e: - print "Unable to connect to the Airtime server." + print("Unable to connect to the Airtime server.") logging.error(str(e)) logging.error("traceback: %s", traceback.format_exc()) if attempts == max_attempts: diff --git a/python_apps/pypo/liquidsoap/liquidsoap_auth.py b/python_apps/pypo/liquidsoap/liquidsoap_auth.py index fe4725305..fb07905bd 100644 --- a/python_apps/pypo/liquidsoap/liquidsoap_auth.py +++ b/python_apps/pypo/liquidsoap/liquidsoap_auth.py @@ -1,3 +1,4 @@ +from __future__ import print_function from api_clients import * import sys @@ -16,8 +17,8 @@ elif dj_type == '--dj': response = api_clients.check_live_stream_auth(username, password, source_type) if 'msg' in response and response['msg'] == True: - print response['msg'] + print(response['msg']) sys.exit(0) else: - print False + print(False) sys.exit(1) diff --git a/python_apps/pypo/pypo/recorder.py b/python_apps/pypo/pypo/recorder.py index bc74f3c10..b98f5d4ef 100644 --- a/python_apps/pypo/pypo/recorder.py +++ b/python_apps/pypo/pypo/recorder.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import print_function import logging import json import time @@ -36,7 +37,7 @@ def api_client(logger): try: config = ConfigObj('/etc/airtime/airtime.conf') except Exception, e: - print ('Error loading config file: %s', e) + print('Error loading config file: %s', e) sys.exit() # TODO : add docstrings everywhere in this module diff --git a/python_apps/pypo/pypo/telnetliquidsoap.py b/python_apps/pypo/pypo/telnetliquidsoap.py index c294a3b0f..d3a7d39d1 100644 --- a/python_apps/pypo/pypo/telnetliquidsoap.py +++ b/python_apps/pypo/pypo/telnetliquidsoap.py @@ -1,3 +1,4 @@ +from __future__ import print_function import telnetlib from timeout import ls_timeout @@ -302,7 +303,7 @@ class DummyTelnetLiquidsoap: self.logger.info("Pushing %s to queue %s" % (media_item, queue_id)) from datetime import datetime - print "Time now: %s" % datetime.utcnow() + print("Time now: {:s}".format(datetime.utcnow())) annotation = create_liquidsoap_annotation(media_item) self.liquidsoap_mock_queues[queue_id].append(annotation) @@ -318,7 +319,7 @@ class DummyTelnetLiquidsoap: self.logger.info("Purging queue %s" % queue_id) from datetime import datetime - print "Time now: %s" % datetime.utcnow() + print("Time now: {:s}".format(datetime.utcnow())) except Exception: raise diff --git a/python_apps/pypo/pypo/testpypoliqqueue.py b/python_apps/pypo/pypo/testpypoliqqueue.py index f1847b34f..d3fe2ead2 100644 --- a/python_apps/pypo/pypo/testpypoliqqueue.py +++ b/python_apps/pypo/pypo/testpypoliqqueue.py @@ -1,3 +1,4 @@ +from __future__ import print_function from pypoliqqueue import PypoLiqQueue from telnetliquidsoap import DummyTelnetLiquidsoap, TelnetLiquidsoap @@ -45,7 +46,7 @@ plq.daemon = True plq.start() -print "Time now: %s" % datetime.utcnow() +print("Time now: {:s}".format(datetime.utcnow())) media_schedule = {} diff --git a/python_apps/pypo/setup.py b/python_apps/pypo/setup.py index 618339435..325b0a916 100644 --- a/python_apps/pypo/setup.py +++ b/python_apps/pypo/setup.py @@ -1,10 +1,11 @@ +from __future__ import print_function from setuptools import setup from subprocess import call import sys import os script_path = os.path.dirname(os.path.realpath(__file__)) -print script_path +print(script_path) os.chdir(script_path) # Allows us to avoid installing the upstart init script when deploying on Airtime Pro: @@ -29,7 +30,7 @@ else: ('/var/tmp/airtime/pypo/files', []), ('/var/tmp/airtime/pypo/tmp', []), ] - print data_files + print(data_files) setup(name='airtime-playout', version='1.0', @@ -67,6 +68,6 @@ setup(name='airtime-playout', # Reload the initctl config so that playout services works if data_files: - print "Reloading initctl configuration" + print("Reloading initctl configuration") #call(['initctl', 'reload-configuration']) - print "Run \"sudo service airtime-playout start\" and \"sudo service airtime-liquidsoap start\"" + print("Run \"sudo service airtime-playout start\" and \"sudo service airtime-liquidsoap start\"") From 9298ea7e05974fcc14046993302156dd6124fc42 Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Thu, 16 Jan 2020 15:34:12 +0200 Subject: [PATCH 02/38] airtime_analyzer: depend on rgain3 for py3 --- python_apps/airtime_analyzer/setup.py | 40 +++++++++++++++------------ 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/python_apps/airtime_analyzer/setup.py b/python_apps/airtime_analyzer/setup.py index d819a171a..915e4cb38 100644 --- a/python_apps/airtime_analyzer/setup.py +++ b/python_apps/airtime_analyzer/setup.py @@ -19,6 +19,28 @@ else: ('/etc/init.d', ['install/sysvinit/airtime_analyzer'])] print(data_files) +dependencies = [ + 'mutagen~=1.43.0', + 'pika~=1.1.0', + 'daemon', + 'file-magic', + 'nose', + 'coverage', + 'mock', + 'python-daemon==1.6', + 'requests>=2.7.0', + # These next 3 are required for requests to support SSL with SNI. Learned this the hard way... + # What sucks is that GCC is required to pip install these. + #'ndg-httpsclient', + #'pyasn1', + #'pyopenssl' +] + +if sys.version_info >= (3, 0): + dependencies += ['rgain3'] +else: + dependencies += ['rgain'] + setup(name='airtime_analyzer', version='0.1', description='Airtime Analyzer Worker and File Importer', @@ -28,23 +50,7 @@ setup(name='airtime_analyzer', license='MIT', packages=['airtime_analyzer'], scripts=['bin/airtime_analyzer'], - install_requires=[ - 'mutagen~=1.43.0', # got rid of specific version requirement - 'pika', - 'daemon', - 'file-magic', - 'nose', - 'coverage', - 'mock', - 'python-daemon==1.6', - 'requests>=2.7.0', - 'rgain', - # These next 3 are required for requests to support SSL with SNI. Learned this the hard way... - # What sucks is that GCC is required to pip install these. - #'ndg-httpsclient', - #'pyasn1', - #'pyopenssl' - ], + install_requires=dependencies, zip_safe=False, data_files=data_files) From 81ad2fc7051978250bd46846eb36bca9d052c726 Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Thu, 16 Jan 2020 15:48:17 +0200 Subject: [PATCH 03/38] do not be backwards compatible --- python_apps/airtime_analyzer/setup.py | 40 ++++++++++++--------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/python_apps/airtime_analyzer/setup.py b/python_apps/airtime_analyzer/setup.py index 915e4cb38..b77861cf2 100644 --- a/python_apps/airtime_analyzer/setup.py +++ b/python_apps/airtime_analyzer/setup.py @@ -19,28 +19,6 @@ else: ('/etc/init.d', ['install/sysvinit/airtime_analyzer'])] print(data_files) -dependencies = [ - 'mutagen~=1.43.0', - 'pika~=1.1.0', - 'daemon', - 'file-magic', - 'nose', - 'coverage', - 'mock', - 'python-daemon==1.6', - 'requests>=2.7.0', - # These next 3 are required for requests to support SSL with SNI. Learned this the hard way... - # What sucks is that GCC is required to pip install these. - #'ndg-httpsclient', - #'pyasn1', - #'pyopenssl' -] - -if sys.version_info >= (3, 0): - dependencies += ['rgain3'] -else: - dependencies += ['rgain'] - setup(name='airtime_analyzer', version='0.1', description='Airtime Analyzer Worker and File Importer', @@ -50,7 +28,23 @@ setup(name='airtime_analyzer', license='MIT', packages=['airtime_analyzer'], scripts=['bin/airtime_analyzer'], - install_requires=dependencies, + install_requires=[ + 'mutagen>=1.41.1', # got rid of specific version requirement + 'pika', + 'daemon', + 'file-magic', + 'nose', + 'coverage', + 'mock', + 'python-daemon==1.6', + 'requests>=2.7.0', + 'rgain3', + # These next 3 are required for requests to support SSL with SNI. Learned this the hard way... + # What sucks is that GCC is required to pip install these. + #'ndg-httpsclient', + #'pyasn1', + #'pyopenssl' + ], zip_safe=False, data_files=data_files) From cf2dda45320167fb458628796db8a25fc13febf2 Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Thu, 16 Jan 2020 15:58:20 +0200 Subject: [PATCH 04/38] test travis against py3 --- .travis.yml | 11 +++++------ travis/python.sh | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index e9cd45305..98b47313d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,10 +58,9 @@ addons: - liquidsoap-plugin-faad - liquidsoap-plugin-vorbis - liquidsoap-plugin-opus - - python-nose - - python-rgain - - python-gst-1.0 - - python-magic + - python3-nose + - python3-gst-1.0 + - python3-magic - dos2unix install: - > @@ -70,9 +69,9 @@ install: fi - > if [[ "$PYTHON" == true ]]; then - pip install --user mkdocs + pip3 install --user mkdocs rgain3 pushd python_apps/airtime_analyzer - python setup.py install --dry-run --no-init-script + python3 setup.py install --dry-run --no-init-script popd fi before_script: diff --git a/travis/python.sh b/travis/python.sh index 0050005ec..477d277a0 100755 --- a/travis/python.sh +++ b/travis/python.sh @@ -5,8 +5,8 @@ set -xe [[ "$PYTHON" == false ]] && exit 0 pushd python_apps/airtime_analyzer -pyenv local 2.7 -pip install -e . +pyenv local 3.7 +pip3 install -e . nosetests -a '!rgain' echo "replaygain tests where skipped due to not having a reliable replaygain install on travis." popd From 8346e89e99b080e68e9ca4687e61acd85f46186a Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Thu, 16 Jan 2020 16:32:51 +0200 Subject: [PATCH 05/38] install successfully using py3 --- docs/scripts/install.sh | 6 +- install | 30 ++--- installer/lib/requirements-debian-buster.apt | 103 ++++++++---------- .../airtime_analyzer/airtime_analyzer.py | 12 +- .../airtime_analyzer/analyzer_pipeline.py | 26 ++--- .../airtime_analyzer/config_file.py | 6 +- .../airtime_analyzer/cuepoint_analyzer.py | 2 +- .../airtime_analyzer/filemover_analyzer.py | 12 +- .../airtime_analyzer/message_listener.py | 10 +- .../airtime_analyzer/metadata_analyzer.py | 14 +-- .../airtime_analyzer/playability_analyzer.py | 2 +- .../airtime_analyzer/replaygain_analyzer.py | 2 +- .../airtime_analyzer/status_reporter.py | 10 +- .../airtime_analyzer/bin/airtime_analyzer | 2 +- python_apps/airtime_analyzer/setup.py | 3 +- .../api_clients/api_clients/api_client.py | 66 +++++------ python_apps/api_clients/setup.py | 3 +- python_apps/api_clients/tests/test_apcurl.py | 8 +- .../api_clients/tests/test_apirequest.py | 6 +- .../api_clients/tests/test_requestprovider.py | 4 +- .../icecast2/install/icecast2-install.py | 4 +- python_apps/pypo/bin/pyponotify | 4 +- python_apps/pypo/liquidsoap/__main__.py | 4 +- .../liquidsoap/generate_liquidsoap_cfg.py | 6 +- .../pypo/liquidsoap/liquidsoap_auth.py | 2 +- .../liquidsoap_prepare_terminate.py | 4 +- python_apps/pypo/pypo/__main__.py | 9 +- python_apps/pypo/pypo/listenerstat.py | 10 +- python_apps/pypo/pypo/pypofetch.py | 40 +++---- python_apps/pypo/pypo/pypofile.py | 22 ++-- python_apps/pypo/pypo/pypoliqqueue.py | 6 +- python_apps/pypo/pypo/pypoliquidsoap.py | 17 ++- python_apps/pypo/pypo/pypomessagehandler.py | 6 +- python_apps/pypo/pypo/pypopush.py | 14 +-- python_apps/pypo/pypo/recorder.py | 32 +++--- python_apps/pypo/pypo/telnetliquidsoap.py | 18 +-- python_apps/pypo/pypo/testpypoliqqueue.py | 8 +- python_apps/pypo/pypo/timeout.py | 2 +- python_apps/pypo/setup.py | 5 +- python_apps/pypo/tests/test_modify_cue_in.py | 2 +- uninstall | 4 +- 41 files changed, 259 insertions(+), 287 deletions(-) diff --git a/docs/scripts/install.sh b/docs/scripts/install.sh index d7fb756f3..34bbb55fb 100755 --- a/docs/scripts/install.sh +++ b/docs/scripts/install.sh @@ -3,8 +3,8 @@ echo "Updating Apt." apt-get update > /dev/null echo "Ensuring Pip is installed." -DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python-pip > /dev/null +DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3-pip > /dev/null echo "Updating Pip." -pip install pip -q -q --upgrade > /dev/null +pip3 install pip -q -q --upgrade > /dev/null echo "Ensuring Mkdocs is installed." -pip install -q mkdocs > /dev/null +pip3 install mkdocs diff --git a/install b/install index 4d4a629d5..bbbe94860 100755 --- a/install +++ b/install @@ -465,7 +465,7 @@ while :; do ;; --no-rabbitmq) skip_rabbitmq=1 - ;; + ;; --) shift break @@ -923,22 +923,12 @@ loud "\n-----------------------------------------------------" loud " * Installing Airtime Services * " loud "-----------------------------------------------------" -verbose "\n * Installing necessary python services..." -loudCmd "pip install setuptools --upgrade" -loudCmd "pip install zipp==1.0.0" -verbose "...Done" - -# Ubuntu Trusty and Debian Wheezy needs a workaround for python version SSL downloads -# This affects all python installs where python < 2.7.9 -python_version=$(python --version 2>&1 | awk '{ print $2 }') +python_version=$(python3 --version 2>&1 | awk '{ print $2 }') verbose "Detected Python version: $python_version" -# Convert version so each segment is zero padded for easy comparison -python_version_formatted=$(awk 'BEGIN {FS = "."} {printf "%03d.%03d.%03d\n", $1,$2,$3}' <<< $python_version) -if [[ "$python_version_formatted" < "002.007.009" ]]; then - verbose "\n * Installing pyOpenSSL and ca db for SNI support..." - loudCmd "pip install pyOpenSSL cryptography idna certifi --upgrade" - verbose "...Done" -fi + +verbose "\n * Installing necessary python services..." +loudCmd "pip3 install setuptools --upgrade" +verbose "...Done" verbose "\n * Creating /run/airtime..." mkdir -p /run/airtime @@ -960,11 +950,11 @@ if [ ! -d /var/log/airtime ]; then fi verbose "\n * Installing API client..." -loudCmd "python ${AIRTIMEROOT}/python_apps/api_clients/setup.py install --install-scripts=/usr/bin" +loudCmd "python3 ${AIRTIMEROOT}/python_apps/api_clients/setup.py install --install-scripts=/usr/bin" verbose "...Done" verbose "\n * Installing pypo and liquidsoap..." -loudCmd "python ${AIRTIMEROOT}/python_apps/pypo/setup.py install --install-scripts=/usr/bin --no-init-script" +loudCmd "python3 ${AIRTIMEROOT}/python_apps/pypo/setup.py install --install-scripts=/usr/bin --no-init-script" loudCmd "mkdir -p /var/log/airtime/{pypo,pypo-liquidsoap} /var/tmp/airtime/pypo/{cache,files,tmp} /var/tmp/airtime/show-recorder/" loudCmd "chown -R ${web_user}:${web_user} /var/log/airtime/{pypo,pypo-liquidsoap} /var/tmp/airtime/pypo/{cache,files,tmp} /var/tmp/airtime/show-recorder/" systemInitInstall airtime-liquidsoap $web_user @@ -972,7 +962,7 @@ systemInitInstall airtime-playout $web_user verbose "...Done" verbose "\n * Installing airtime-celery..." -loudCmd "python ${AIRTIMEROOT}/python_apps/airtime-celery/setup.py install --no-init-script" +loudCmd "python3 ${AIRTIMEROOT}/python_apps/airtime-celery/setup.py install --no-init-script" # Create the Celery user if $is_centos_dist; then loudCmd "id celery 2>/dev/null || adduser --no-create-home -c 'LibreTime Celery' -r celery || true" @@ -988,7 +978,7 @@ systemInitInstall airtime-celery verbose "...Done" verbose "\n * Installing airtime_analyzer..." -loudCmd "python ${AIRTIMEROOT}/python_apps/airtime_analyzer/setup.py install --install-scripts=/usr/bin --no-init-script" +loudCmd "python3 ${AIRTIMEROOT}/python_apps/airtime_analyzer/setup.py install --install-scripts=/usr/bin --no-init-script" systemInitInstall airtime_analyzer $web_user verbose "...Done" diff --git a/installer/lib/requirements-debian-buster.apt b/installer/lib/requirements-debian-buster.apt index aceab8a25..a12d84728 100644 --- a/installer/lib/requirements-debian-buster.apt +++ b/installer/lib/requirements-debian-buster.apt @@ -1,71 +1,56 @@ apache2 +coreutils +curl +ecasound +flac git +gstreamer1.0-plugins-ugly +icecast2 +lame +libao-ocaml libapache2-mod-php7.3 -php7.3 -php7.3-dev -php7.3-bcmath -php7.3-mbstring -php-pear -php7.3-gd -php-amqplib - -lsb-release - -zip -unzip - -rabbitmq-server - -postgresql -postgresql-client -php7.3-pgsql - -python -python-virtualenv -python-pip - +libcairo2-dev +libcamomile-ocaml-data +libfaad2 +libmad-ocaml +libopus0 +libportaudio2 +libpulse0 +libsamplerate0 libsoundtouch-ocaml libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 libvo-aacenc0 - -python-rgain -python-gst-1.0 -gstreamer1.0-plugins-ugly -python-pika - -patch - -icecast2 - -curl -php7.3-curl -mpg123 - -libcamomile-ocaml-data -libpulse0 -vorbis-tools +liquidsoap +lsb-release lsb-release lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 +mpg123 +patch +php7.3 +php7.3-bcmath +php7.3-curl +php7.3-dev +php7.3-gd +php7.3-mbstring +php7.3-pgsql +php-amqplib php-apcu - -lame +php-pear +pkg-config +postgresql +postgresql-client +pwgen +python3 +python3-gst-1.0 +python3-pika +python3-pip +python3-virtualenv +rabbitmq-server silan -coreutils - -liquidsoap - -libopus0 - systemd-sysv - +unzip +vorbisgain +vorbis-tools +vorbis-tools xmlstarlet +zip diff --git a/python_apps/airtime_analyzer/airtime_analyzer/airtime_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/airtime_analyzer.py index eafa8f743..325b2df3d 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/airtime_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/airtime_analyzer.py @@ -5,12 +5,12 @@ import logging.handlers import sys import signal import traceback -import config_file +from . import config_file from functools import partial -from metadata_analyzer import MetadataAnalyzer -from replaygain_analyzer import ReplayGainAnalyzer -from status_reporter import StatusReporter -from message_listener import MessageListener +from .metadata_analyzer import MetadataAnalyzer +from .replaygain_analyzer import ReplayGainAnalyzer +from .status_reporter import StatusReporter +from .message_listener import MessageListener class AirtimeAnalyzerServer: @@ -76,7 +76,7 @@ class AirtimeAnalyzerServer: def dump_stacktrace(stack): ''' Dump a stacktrace for all threads ''' code = [] - for threadId, stack in sys._current_frames().items(): + for threadId, stack in list(sys._current_frames().items()): code.append("\n# ThreadID: %s" % threadId) for filename, lineno, name, line in traceback.extract_stack(stack): code.append('File: "%s", line %d, in %s' % (filename, lineno, name)) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py b/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py index 20973fdb4..54892838c 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py @@ -3,13 +3,13 @@ import logging import threading import multiprocessing -import Queue -import ConfigParser -from metadata_analyzer import MetadataAnalyzer -from filemover_analyzer import FileMoverAnalyzer -from cuepoint_analyzer import CuePointAnalyzer -from replaygain_analyzer import ReplayGainAnalyzer -from playability_analyzer import * +import queue +import configparser +from .metadata_analyzer import MetadataAnalyzer +from .filemover_analyzer import FileMoverAnalyzer +from .cuepoint_analyzer import CuePointAnalyzer +from .replaygain_analyzer import ReplayGainAnalyzer +from .playability_analyzer import * class AnalyzerPipeline: """ Analyzes and imports an audio file into the Airtime library. @@ -46,15 +46,15 @@ class AnalyzerPipeline: AnalyzerPipeline.python_logger_deadlock_workaround() try: - if not isinstance(queue, Queue.Queue): + if not isinstance(queue, queue.Queue): raise TypeError("queue must be a Queue.Queue()") - if not isinstance(audio_file_path, unicode): + if not isinstance(audio_file_path, str): raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__ + " instead.") - if not isinstance(import_directory, unicode): + if not isinstance(import_directory, str): raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__ + " instead.") - if not isinstance(original_filename, unicode): + if not isinstance(original_filename, str): raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__ + " instead.") - if not isinstance(file_prefix, unicode): + if not isinstance(file_prefix, str): raise TypeError("file_prefix must be unicode. Was of type " + type(file_prefix).__name__ + " instead.") @@ -91,7 +91,7 @@ class AnalyzerPipeline: @staticmethod def python_logger_deadlock_workaround(): # Workaround for: http://bugs.python.org/issue6721#msg140215 - logger_names = logging.Logger.manager.loggerDict.keys() + logger_names = list(logging.Logger.manager.loggerDict.keys()) logger_names.append(None) # Root logger for name in logger_names: for handler in logging.getLogger(name).handlers: diff --git a/python_apps/airtime_analyzer/airtime_analyzer/config_file.py b/python_apps/airtime_analyzer/airtime_analyzer/config_file.py index 506af020a..e98bd529d 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/config_file.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/config_file.py @@ -1,9 +1,9 @@ -from __future__ import print_function -import ConfigParser + +import configparser def read_config_file(config_path): """Parse the application's config file located at config_path.""" - config = ConfigParser.SafeConfigParser() + config = configparser.SafeConfigParser() try: config.readfp(open(config_path)) except IOError as e: diff --git a/python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py index 47a517b59..b10f573cb 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py @@ -3,7 +3,7 @@ import logging import traceback import json import datetime -from analyzer import Analyzer +from .analyzer import Analyzer class CuePointAnalyzer(Analyzer): diff --git a/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py index 65badaf50..f0e0e030e 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py @@ -6,7 +6,7 @@ import os, errno import time import uuid -from analyzer import Analyzer +from .analyzer import Analyzer class FileMoverAnalyzer(Analyzer): """This analyzer copies a file over from a temporary directory (stor/organize) @@ -29,11 +29,11 @@ class FileMoverAnalyzer(Analyzer): original_filename: The filename of the file when it was uploaded to Airtime. metadata: A dictionary where the "full_path" of where the file is moved to will be added. """ - if not isinstance(audio_file_path, unicode): + if not isinstance(audio_file_path, str): raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__) - if not isinstance(import_directory, unicode): + if not isinstance(import_directory, str): raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__) - if not isinstance(original_filename, unicode): + if not isinstance(original_filename, str): raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__) if not isinstance(metadata, dict): raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__) @@ -46,9 +46,9 @@ class FileMoverAnalyzer(Analyzer): max_file_len = 48 final_file_path = import_directory orig_file_basename, orig_file_extension = os.path.splitext(original_filename) - if metadata.has_key("artist_name"): + if "artist_name" in metadata: final_file_path += "/" + metadata["artist_name"][0:max_dir_len] # truncating with array slicing - if metadata.has_key("album_title"): + if "album_title" in metadata: final_file_path += "/" + metadata["album_title"][0:max_dir_len] # Note that orig_file_extension includes the "." already final_file_path += "/" + orig_file_basename[0:max_file_len] + orig_file_extension diff --git a/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py b/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py index c261f737b..48db243f2 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py @@ -6,9 +6,9 @@ import select import signal import logging import multiprocessing -import Queue -from analyzer_pipeline import AnalyzerPipeline -from status_reporter import StatusReporter +import queue +from .analyzer_pipeline import AnalyzerPipeline +from .status_reporter import StatusReporter EXCHANGE = "airtime-uploads" EXCHANGE_TYPE = "topic" @@ -198,7 +198,7 @@ class MessageListener: if callback_url: # If we got an invalid message, there might be no callback_url in the JSON # Report this as a failed upload to the File Upload REST API. StatusReporter.report_failure_to_callback_url(callback_url, api_key, import_status=2, - reason=u'An error occurred while importing this file') + reason='An error occurred while importing this file') else: @@ -224,7 +224,7 @@ class MessageListener: ''' metadata = {} - q = Queue.Queue() + q = queue.Queue() try: AnalyzerPipeline.run_analysis(q, audio_file_path, import_directory, original_filename, storage_backend, file_prefix) metadata = q.get() diff --git a/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py index 489e7ef86..d7a4210cb 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py @@ -6,7 +6,7 @@ import wave import logging import os import hashlib -from analyzer import Analyzer +from .analyzer import Analyzer class MetadataAnalyzer(Analyzer): @@ -18,7 +18,7 @@ class MetadataAnalyzer(Analyzer): filename: The path to the audio file to extract metadata from. metadata: A dictionary that the extracted metadata will be added to. ''' - if not isinstance(filename, unicode): + if not isinstance(filename, str): raise TypeError("filename must be unicode. Was of type " + type(filename).__name__) if not isinstance(metadata, dict): raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__) @@ -104,11 +104,11 @@ class MetadataAnalyzer(Analyzer): if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh track_number = track_number[0] track_number_tokens = track_number - if u'/' in track_number: - track_number_tokens = track_number.split(u'/') + if '/' in track_number: + track_number_tokens = track_number.split('/') track_number = track_number_tokens[0] - elif u'-' in track_number: - track_number_tokens = track_number.split(u'-') + elif '-' in track_number: + track_number_tokens = track_number.split('-') track_number = track_number_tokens[0] metadata["track_number"] = track_number track_total = track_number_tokens[1] @@ -146,7 +146,7 @@ class MetadataAnalyzer(Analyzer): #'mime_type': 'mime', } - for mutagen_tag, airtime_tag in mutagen_to_airtime_mapping.iteritems(): + for mutagen_tag, airtime_tag in mutagen_to_airtime_mapping.items(): try: metadata[airtime_tag] = audio_file[mutagen_tag] diff --git a/python_apps/airtime_analyzer/airtime_analyzer/playability_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/playability_analyzer.py index eb9062713..d7b2d546e 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/playability_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/playability_analyzer.py @@ -2,7 +2,7 @@ __author__ = 'asantoni' import subprocess import logging -from analyzer import Analyzer +from .analyzer import Analyzer class UnplayableFileError(Exception): pass diff --git a/python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py index 39ea2e439..4290ee059 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py @@ -1,6 +1,6 @@ import subprocess import logging -from analyzer import Analyzer +from .analyzer import Analyzer class ReplayGainAnalyzer(Analyzer): diff --git a/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py b/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py index bb1d03d34..ee3c78fac 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py @@ -2,12 +2,12 @@ import requests import json import logging import collections -import Queue +import queue import time import traceback import pickle import threading -from urlparse import urlparse +from urllib.parse import urlparse # Disable urllib3 warnings because these can cause a rare deadlock due to Python 2's crappy internal non-reentrant locking # around POSIX stuff. See SAAS-714. The hasattr() is for compatibility with older versions of requests. @@ -68,7 +68,7 @@ def process_http_requests(ipc_queue, http_retry_queue_path): break if not isinstance(request, PicklableHttpRequest): raise TypeError("request must be a PicklableHttpRequest. Was of type " + type(request).__name__) - except Queue.Empty: + except queue.Empty: request = None # If there's no new HTTP request we need to execute, let's check our "retry @@ -159,7 +159,7 @@ class StatusReporter(): ''' We use multiprocessing.Process again here because we need a thread for this stuff anyways, and Python gives us process isolation for free (crash safety). ''' - _ipc_queue = Queue.Queue() + _ipc_queue = queue.Queue() #_http_thread = multiprocessing.Process(target=process_http_requests, # args=(_ipc_queue,)) _http_thread = None @@ -222,7 +222,7 @@ class StatusReporter(): @classmethod def report_failure_to_callback_url(self, callback_url, api_key, import_status, reason): - if not isinstance(import_status, (int, long) ): + if not isinstance(import_status, int ): raise TypeError("import_status must be an integer. Was of type " + type(import_status).__name__) logging.debug("Reporting import failure to Airtime REST API...") diff --git a/python_apps/airtime_analyzer/bin/airtime_analyzer b/python_apps/airtime_analyzer/bin/airtime_analyzer index e81fb5085..8dfb99341 100755 --- a/python_apps/airtime_analyzer/bin/airtime_analyzer +++ b/python_apps/airtime_analyzer/bin/airtime_analyzer @@ -2,7 +2,7 @@ """Runs the airtime_analyzer application. """ -from __future__ import print_function + import daemon import argparse import os diff --git a/python_apps/airtime_analyzer/setup.py b/python_apps/airtime_analyzer/setup.py index b77861cf2..53b975b7e 100644 --- a/python_apps/airtime_analyzer/setup.py +++ b/python_apps/airtime_analyzer/setup.py @@ -31,12 +31,11 @@ setup(name='airtime_analyzer', install_requires=[ 'mutagen>=1.41.1', # got rid of specific version requirement 'pika', - 'daemon', 'file-magic', 'nose', 'coverage', 'mock', - 'python-daemon==1.6', + 'python-daemon', 'requests>=2.7.0', 'rgain3', # These next 3 are required for requests to support SSL with SNI. Learned this the hard way... diff --git a/python_apps/api_clients/api_clients/api_client.py b/python_apps/api_clients/api_clients/api_client.py index 49bac65e1..1fd9300c2 100644 --- a/python_apps/api_clients/api_clients/api_client.py +++ b/python_apps/api_clients/api_clients/api_client.py @@ -8,8 +8,8 @@ ############################################################################### import sys import time -import urllib -import urllib2 +import urllib.request, urllib.parse, urllib.error +import urllib.request, urllib.error, urllib.parse import requests import socket import logging @@ -26,19 +26,19 @@ AIRTIME_API_VERSION = "1.1" # instead of copy pasting them around def to_unicode(obj, encoding='utf-8'): - if isinstance(obj, basestring): - if not isinstance(obj, unicode): - obj = unicode(obj, encoding) + if isinstance(obj, str): + if not isinstance(obj, str): + obj = str(obj, encoding) return obj def encode_to(obj, encoding='utf-8'): - if isinstance(obj, unicode): + if isinstance(obj, str): obj = obj.encode(encoding) return obj def convert_dict_value_to_utf8(md): #list comprehension to convert all values of md to utf-8 - return dict([(item[0], encode_to(item[1], "utf-8")) for item in md.items()]) + return dict([(item[0], encode_to(item[1], "utf-8")) for item in list(md.items())]) api_config = {} @@ -114,7 +114,7 @@ class ApcUrl(object): def params(self, **params): temp_url = self.base_url - for k, v in params.iteritems(): + for k, v in params.items(): wrapped_param = "%%" + k + "%%" if wrapped_param in temp_url: temp_url = temp_url.replace(wrapped_param, str(v)) @@ -138,11 +138,11 @@ class ApiRequest(object): def __call__(self,_post_data=None, **kwargs): final_url = self.url.params(**kwargs).url() - if _post_data is not None: _post_data = urllib.urlencode(_post_data) + if _post_data is not None: _post_data = urllib.parse.urlencode(_post_data) self.logger.debug(final_url) try: - req = urllib2.Request(final_url, _post_data) - f = urllib2.urlopen(req, timeout=ApiRequest.API_HTTP_REQUEST_TIMEOUT) + req = urllib.request.Request(final_url, _post_data) + f = urllib.request.urlopen(req, timeout=ApiRequest.API_HTTP_REQUEST_TIMEOUT) content_type = f.info().getheader('Content-Type') response = f.read() #Everything that calls an ApiRequest should be catching URLError explicitly @@ -151,7 +151,7 @@ class ApiRequest(object): except socket.timeout: self.logger.error('HTTP request to %s timed out', final_url) raise - except Exception, e: + except Exception as e: #self.logger.error('Exception: %s', e) #self.logger.error("traceback: %s", traceback.format_exc()) raise @@ -193,13 +193,13 @@ class RequestProvider(object): self.config["general"]["base_dir"], self.config["api_base"], '%%action%%')) # Now we must discover the possible actions - actions = dict( (k,v) for k,v in cfg.iteritems() if '%%api_key%%' in v) - for action_name, action_value in actions.iteritems(): + actions = dict( (k,v) for k,v in cfg.items() if '%%api_key%%' in v) + for action_name, action_value in actions.items(): new_url = self.url.params(action=action_value).params( api_key=self.config["general"]['api_key']) self.requests[action_name] = ApiRequest(action_name, new_url) - def available_requests(self) : return self.requests.keys() + def available_requests(self) : return list(self.requests.keys()) def __contains__(self, request) : return request in self.requests def __getattr__(self, attr): @@ -217,17 +217,17 @@ class AirtimeApiClient(object): self.config = ConfigObj(config_path) self.config.update(api_config) self.services = RequestProvider(self.config) - except Exception, e: + except Exception as e: self.logger.error('Error loading config file: %s', config_path) self.logger.error("traceback: %s", traceback.format_exc()) sys.exit(1) def __get_airtime_version(self): - try: return self.services.version_url()[u'airtime_version'] + try: return self.services.version_url()['airtime_version'] except Exception: return -1 def __get_api_version(self): - try: return self.services.version_url()[u'api_version'] + try: return self.services.version_url()['api_version'] except Exception: return -1 def is_server_compatible(self, verbose=True): @@ -259,7 +259,7 @@ class AirtimeApiClient(object): def notify_liquidsoap_started(self): try: self.services.notify_liquidsoap_started() - except Exception, e: + except Exception as e: self.logger.error(str(e)) def notify_media_item_start_playing(self, media_id): @@ -268,14 +268,14 @@ class AirtimeApiClient(object): which we handed to liquidsoap in get_liquidsoap_data(). """ try: return self.services.update_start_playing_url(media_id=media_id) - except Exception, e: + except Exception as e: self.logger.error(str(e)) return None def get_shows_to_record(self): try: return self.services.show_schedule_url() - except Exception, e: + except Exception as e: self.logger.error(str(e)) return None @@ -321,13 +321,13 @@ class AirtimeApiClient(object): """ break - except requests.exceptions.HTTPError, e: + except requests.exceptions.HTTPError as e: logger.error("Http error code: %s", e.code) logger.error("traceback: %s", traceback.format_exc()) - except requests.exceptions.ConnectionError, e: + except requests.exceptions.ConnectionError as e: logger.error("Server is down: %s", e.args) logger.error("traceback: %s", traceback.format_exc()) - except Exception, e: + except Exception as e: logger.error("Exception: %s", e) logger.error("traceback: %s", traceback.format_exc()) @@ -340,7 +340,7 @@ class AirtimeApiClient(object): try: return self.services.check_live_stream_auth( username=username, password=password, djtype=dj_type) - except Exception, e: + except Exception as e: self.logger.error(str(e)) return {} @@ -422,10 +422,10 @@ class AirtimeApiClient(object): def list_all_db_files(self, dir_id, all_files=True): logger = self.logger try: - all_files = u"1" if all_files else u"0" + all_files = "1" if all_files else "0" response = self.services.list_all_db_files(dir_id=dir_id, all=all_files) - except Exception, e: + except Exception as e: response = {} logger.error("Exception: %s", e) try: @@ -483,12 +483,12 @@ class AirtimeApiClient(object): post_data = {"msg_post": msg} #encoded_msg is no longer used server_side!! - encoded_msg = urllib.quote('dummy') + encoded_msg = urllib.parse.quote('dummy') self.services.update_liquidsoap_status.req(post_data, msg=encoded_msg, stream_id=stream_id, boot_time=time).retry(5) - except Exception, e: + except Exception as e: #TODO logger.error("Exception: %s", e) @@ -497,7 +497,7 @@ class AirtimeApiClient(object): logger = self.logger return self.services.update_source_status.req(sourcename=sourcename, status=status).retry(5) - except Exception, e: + except Exception as e: #TODO logger.error("Exception: %s", e) @@ -514,7 +514,7 @@ class AirtimeApiClient(object): #http://localhost/api/get-files-without-replay-gain/dir_id/1 try: return self.services.get_files_without_replay_gain(dir_id=dir_id) - except Exception, e: + except Exception as e: self.logger.error(str(e)) return [] @@ -526,7 +526,7 @@ class AirtimeApiClient(object): """ try: return self.services.get_files_without_silan_value() - except Exception, e: + except Exception as e: self.logger.error(str(e)) return [] @@ -569,7 +569,7 @@ class AirtimeApiClient(object): try: response = self.services.update_stream_setting_table(_post_data={'data': json.dumps(data)}) return response - except Exception, e: + except Exception as e: #TODO self.logger.error(str(e)) diff --git a/python_apps/api_clients/setup.py b/python_apps/api_clients/setup.py index a615e6c0f..f9e69264d 100644 --- a/python_apps/api_clients/setup.py +++ b/python_apps/api_clients/setup.py @@ -24,11 +24,10 @@ setup(name='api_clients', # 'docopt', # 'kombu', # 'mutagen', -# 'poster', +# 'poster3', # 'PyDispatcher', # 'pyinotify', # 'pytz', -# 'wsgiref' ], zip_safe=False, data_files=[]) diff --git a/python_apps/api_clients/tests/test_apcurl.py b/python_apps/api_clients/tests/test_apcurl.py index 61880e45c..1cad874eb 100644 --- a/python_apps/api_clients/tests/test_apcurl.py +++ b/python_apps/api_clients/tests/test_apcurl.py @@ -5,16 +5,16 @@ class TestApcUrl(unittest.TestCase): def test_init(self): url = "/testing" u = ApcUrl(url) - self.assertEquals( u.base_url, url) + self.assertEqual( u.base_url, url) def test_params_1(self): u = ApcUrl("/testing/%%key%%") - self.assertEquals(u.params(key='val').url(), '/testing/val') + self.assertEqual(u.params(key='val').url(), '/testing/val') def test_params_2(self): u = ApcUrl('/testing/%%key%%/%%api%%/more_testing') full_url = u.params(key="AAA",api="BBB").url() - self.assertEquals(full_url, '/testing/AAA/BBB/more_testing') + self.assertEqual(full_url, '/testing/AAA/BBB/more_testing') def test_params_ex(self): u = ApcUrl("/testing/%%key%%") @@ -23,7 +23,7 @@ class TestApcUrl(unittest.TestCase): def test_url(self): u = "one/two/three" - self.assertEquals( ApcUrl(u).url(), u ) + self.assertEqual( ApcUrl(u).url(), u ) def test_url_ex(self): u = ApcUrl('/%%one%%/%%two%%/three').params(two='testing') diff --git a/python_apps/api_clients/tests/test_apirequest.py b/python_apps/api_clients/tests/test_apirequest.py index 09c1f6f3c..55735672e 100644 --- a/python_apps/api_clients/tests/test_apirequest.py +++ b/python_apps/api_clients/tests/test_apirequest.py @@ -6,16 +6,16 @@ from .. api_client import ApcUrl, ApiRequest class TestApiRequest(unittest.TestCase): def test_init(self): u = ApiRequest('request_name', ApcUrl('/test/ing')) - self.assertEquals(u.name, "request_name") + self.assertEqual(u.name, "request_name") def test_call(self): - ret = json.dumps( {u'ok':u'ok'} ) + ret = json.dumps( {'ok':'ok'} ) read = MagicMock() read.read = MagicMock(return_value=ret) u = '/testing' with patch('urllib2.urlopen') as mock_method: mock_method.return_value = read request = ApiRequest('mm', ApcUrl(u))() - self.assertEquals(request, json.loads(ret)) + self.assertEqual(request, json.loads(ret)) if __name__ == '__main__': unittest.main() diff --git a/python_apps/api_clients/tests/test_requestprovider.py b/python_apps/api_clients/tests/test_requestprovider.py index 98244e423..69f7294c6 100644 --- a/python_apps/api_clients/tests/test_requestprovider.py +++ b/python_apps/api_clients/tests/test_requestprovider.py @@ -19,7 +19,7 @@ class TestRequestProvider(unittest.TestCase): self.assertTrue( meth in rp ) def test_notify_webstream_data(self): - ret = json.dumps( {u'testing' : u'123' } ) + ret = json.dumps( {'testing' : '123' } ) rp = RequestProvider(self.cfg) read = MagicMock() read.read = MagicMock(return_value=ret) @@ -27,6 +27,6 @@ class TestRequestProvider(unittest.TestCase): mock_method.return_value = read response = rp.notify_webstream_data(media_id=123) mock_method.called_once_with(media_id=123) - self.assertEquals(json.loads(ret), response) + self.assertEqual(json.loads(ret), response) if __name__ == '__main__': unittest.main() diff --git a/python_apps/icecast2/install/icecast2-install.py b/python_apps/icecast2/install/icecast2-install.py index f1424f9bc..d12588ab7 100644 --- a/python_apps/icecast2/install/icecast2-install.py +++ b/python_apps/icecast2/install/icecast2-install.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from __future__ import print_function + import shutil import os import sys @@ -18,6 +18,6 @@ try: current_script_dir = get_current_script_dir() shutil.copy(current_script_dir+"/../airtime-icecast-status.xsl", "/usr/share/icecast2/web") -except Exception, e: +except Exception as e: print("exception: {}".format(e)) sys.exit(1) diff --git a/python_apps/pypo/bin/pyponotify b/python_apps/pypo/bin/pyponotify index b3ca44b4d..879f24c8c 100755 --- a/python_apps/pypo/bin/pyponotify +++ b/python_apps/pypo/bin/pyponotify @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import print_function + import traceback """ @@ -76,7 +76,7 @@ logger = rootLogger try: config = ConfigObj('/etc/airtime/airtime.conf') -except Exception, e: +except Exception as e: logger.error('Error loading config file: %s', e) sys.exit() diff --git a/python_apps/pypo/liquidsoap/__main__.py b/python_apps/pypo/liquidsoap/__main__.py index f696ef999..d7ac1bbb6 100644 --- a/python_apps/pypo/liquidsoap/__main__.py +++ b/python_apps/pypo/liquidsoap/__main__.py @@ -1,10 +1,10 @@ """ Runs Airtime liquidsoap """ -from __future__ import print_function + import argparse import os -import generate_liquidsoap_cfg +from . import generate_liquidsoap_cfg import logging import subprocess diff --git a/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py b/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py index 4043b83be..d35e2333f 100644 --- a/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py +++ b/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py @@ -1,4 +1,4 @@ -from __future__ import print_function + import logging import os import sys @@ -14,7 +14,7 @@ def generate_liquidsoap_config(ss): fh.write("################################################\n") fh.write("# The ignore() lines are to squash unused variable warnings\n") - for key, value in data.iteritems(): + for key, value in data.items(): try: if not "port" in key and not "bitrate" in key: # Stupid hack raise ValueError() @@ -49,7 +49,7 @@ def run(): ss = ac.get_stream_setting() generate_liquidsoap_config(ss) successful = True - except Exception, e: + except Exception as e: print("Unable to connect to the Airtime server.") logging.error(str(e)) logging.error("traceback: %s", traceback.format_exc()) diff --git a/python_apps/pypo/liquidsoap/liquidsoap_auth.py b/python_apps/pypo/liquidsoap/liquidsoap_auth.py index fb07905bd..604b2624d 100644 --- a/python_apps/pypo/liquidsoap/liquidsoap_auth.py +++ b/python_apps/pypo/liquidsoap/liquidsoap_auth.py @@ -1,4 +1,4 @@ -from __future__ import print_function + from api_clients import * import sys diff --git a/python_apps/pypo/liquidsoap/liquidsoap_prepare_terminate.py b/python_apps/pypo/liquidsoap/liquidsoap_prepare_terminate.py index 230b7c935..20766bebf 100644 --- a/python_apps/pypo/liquidsoap/liquidsoap_prepare_terminate.py +++ b/python_apps/pypo/liquidsoap/liquidsoap_prepare_terminate.py @@ -13,7 +13,7 @@ try: tn.write('exit\n') tn.read_all() -except Exception, e: - print('Error loading config file: %s', e) +except Exception as e: + print(('Error loading config file: %s', e)) sys.exit() diff --git a/python_apps/pypo/pypo/__main__.py b/python_apps/pypo/pypo/__main__.py index a51baffe3..94c8af0b5 100644 --- a/python_apps/pypo/pypo/__main__.py +++ b/python_apps/pypo/pypo/__main__.py @@ -1,7 +1,7 @@ """ Python part of radio playout (pypo) """ -from __future__ import absolute_import + import locale import logging @@ -16,10 +16,11 @@ from api_clients import api_client from configobj import ConfigObj from datetime import datetime from optparse import OptionParser +import importlib try: from queue import Queue except ImportError: # Python 2.7.5 (CentOS 7) - from Queue import Queue + from queue import Queue from threading import Lock from .listenerstat import ListenerStat @@ -119,7 +120,7 @@ try: consoleHandler.setFormatter(logFormatter) rootLogger.addHandler(consoleHandler) except Exception as e: - print("Couldn't configure logging", e) + print(("Couldn't configure logging", e)) sys.exit(1) @@ -160,7 +161,7 @@ def configure_locale(): "New locale set to: %s", locale.setlocale(locale.LC_ALL, new_locale) ) - reload(sys) + importlib.reload(sys) sys.setdefaultencoding("UTF-8") current_locale_encoding = locale.getlocale()[1].lower() logger.debug("sys default encoding %s", sys.getdefaultencoding()) diff --git a/python_apps/pypo/pypo/listenerstat.py b/python_apps/pypo/pypo/listenerstat.py index f22a537eb..91a9e9487 100644 --- a/python_apps/pypo/pypo/listenerstat.py +++ b/python_apps/pypo/pypo/listenerstat.py @@ -1,5 +1,5 @@ from threading import Thread -import urllib2 +import urllib.request, urllib.error, urllib.parse import defusedxml.minidom import base64 from datetime import datetime @@ -44,13 +44,13 @@ class ListenerStat(Thread): user_agent = "Mozilla/5.0 (Linux; rv:22.0) Gecko/20130405 Firefox/22.0" header["User-Agent"] = user_agent - req = urllib2.Request( + req = urllib.request.Request( #assuming that the icecast stats path is /admin/stats.xml #need to fix this url=url, headers=header) - f = urllib2.urlopen(req, timeout=ListenerStat.HTTP_REQUEST_TIMEOUT) + f = urllib.request.urlopen(req, timeout=ListenerStat.HTTP_REQUEST_TIMEOUT) document = f.read() return document @@ -109,7 +109,7 @@ class ListenerStat(Thread): #Note that there can be optimizations done, since if all three #streams are the same server, we will still initiate 3 separate #connections - for k, v in stream_parameters.items(): + for k, v in list(stream_parameters.items()): if v["enable"] == 'true': try: if v["output"] == "icecast": @@ -146,7 +146,7 @@ class ListenerStat(Thread): if stats: self.push_stream_stats(stats) - except Exception, e: + except Exception as e: self.logger.error('Exception: %s', e) time.sleep(120) diff --git a/python_apps/pypo/pypo/pypofetch.py b/python_apps/pypo/pypo/pypofetch.py index d39824aa7..827346888 100644 --- a/python_apps/pypo/pypo/pypofetch.py +++ b/python_apps/pypo/pypo/pypofetch.py @@ -11,14 +11,14 @@ import subprocess import signal from datetime import datetime import traceback -import pure +from . import pure import mimetypes -from Queue import Empty +from queue import Empty from threading import Thread, Timer from subprocess import Popen, PIPE from api_clients import api_client -from timeout import ls_timeout +from .timeout import ls_timeout def keyboardInterruptHandler(signum, frame): @@ -65,7 +65,7 @@ class PypoFetch(Thread): """ self.logger.debug("Cache dir does not exist. Creating...") os.makedirs(dir) - except Exception, e: + except Exception as e: pass self.schedule_data = [] @@ -120,7 +120,7 @@ class PypoFetch(Thread): if self.listener_timeout < 0: self.listener_timeout = 0 self.logger.info("New timeout: %s" % self.listener_timeout) - except Exception, e: + except Exception as e: top = traceback.format_exc() self.logger.error('Exception: %s', e) self.logger.error("traceback: %s", top) @@ -151,13 +151,13 @@ class PypoFetch(Thread): self.logger.debug('Getting information needed on bootstrap from Airtime') try: info = self.api_client.get_bootstrap_info() - except Exception, e: + except Exception as e: self.logger.error('Unable to get bootstrap info.. Exiting pypo...') self.logger.error(str(e)) self.logger.debug('info:%s', info) commands = [] - for k, v in info['switch_status'].iteritems(): + for k, v in info['switch_status'].items(): commands.append(self.switch_source_temp(k, v)) stream_format = info['stream_label'] @@ -194,11 +194,11 @@ class PypoFetch(Thread): tn.read_all() self.logger.info("Liquidsoap is up and running") break - except Exception, e: + except Exception as e: #sleep 0.5 seconds and try again time.sleep(0.5) - except Exception, e: + except Exception as e: self.logger.error(e) finally: if self.telnet_lock.locked(): @@ -237,7 +237,7 @@ class PypoFetch(Thread): tn.write('exit\n') output = tn.read_all() - except Exception, e: + except Exception as e: self.logger.error(str(e)) finally: self.telnet_lock.release() @@ -271,7 +271,7 @@ class PypoFetch(Thread): tn.write(command) tn.write('exit\n') tn.read_all() - except Exception, e: + except Exception as e: self.logger.error("Exception %s", e) finally: self.telnet_lock.release() @@ -288,7 +288,7 @@ class PypoFetch(Thread): tn.write(command) tn.write('exit\n') tn.read_all() - except Exception, e: + except Exception as e: self.logger.error("Exception %s", e) finally: self.telnet_lock.release() @@ -306,11 +306,11 @@ class PypoFetch(Thread): tn.write(command) tn.write('exit\n') tn.read_all() - except Exception, e: + except Exception as e: self.logger.error(str(e)) finally: self.telnet_lock.release() - except Exception, e: + except Exception as e: self.logger.error("Exception %s", e) """ @@ -336,7 +336,7 @@ class PypoFetch(Thread): download_dir = self.cache_dir try: os.makedirs(download_dir) - except Exception, e: + except Exception as e: pass media_copy = {} @@ -344,7 +344,7 @@ class PypoFetch(Thread): media_item = media[key] if (media_item['type'] == 'file'): fileExt = self.sanity_check_media_item(media_item) - dst = os.path.join(download_dir, unicode(media_item['id']) + unicode(fileExt)) + dst = os.path.join(download_dir, str(media_item['id']) + str(fileExt)) media_item['dst'] = dst media_item['file_ready'] = False media_filtered[key] = media_item @@ -357,7 +357,7 @@ class PypoFetch(Thread): self.media_prepare_queue.put(copy.copy(media_filtered)) - except Exception, e: self.logger.error("%s", e) + except Exception as e: self.logger.error("%s", e) # Send the data to pypo-push self.logger.debug("Pushing to pypo-push") @@ -366,7 +366,7 @@ class PypoFetch(Thread): # cleanup try: self.cache_cleanup(media) - except Exception, e: self.logger.error("%s", e) + except Exception as e: self.logger.error("%s", e) #do basic validation of file parameters. Useful for debugging #purposes @@ -408,7 +408,7 @@ class PypoFetch(Thread): for mkey in media: media_item = media[mkey] if media_item['type'] == 'file': - scheduled_file_set.add(unicode(media_item["id"]) + unicode(media_item["file_ext"])) + scheduled_file_set.add(str(media_item["id"]) + str(media_item["file_ext"])) expired_files = cached_file_set - scheduled_file_set @@ -426,7 +426,7 @@ class PypoFetch(Thread): self.logger.info("File '%s' removed" % path) else: self.logger.info("File '%s' not removed. Still busy!" % path) - except Exception, e: + except Exception as e: self.logger.error("Problem removing file '%s'" % f) self.logger.error(traceback.format_exc()) diff --git a/python_apps/pypo/pypo/pypofile.py b/python_apps/pypo/pypo/pypofile.py index 85b3d642c..3edf2c2d5 100644 --- a/python_apps/pypo/pypo/pypofile.py +++ b/python_apps/pypo/pypo/pypofile.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- from threading import Thread -from Queue import Empty -from ConfigParser import NoOptionError +from queue import Empty +from configparser import NoOptionError import logging import shutil @@ -12,7 +12,7 @@ import os import sys import stat import requests -import ConfigParser +import configparser import json import hashlib from requests.exceptions import ConnectionError, HTTPError, Timeout @@ -44,7 +44,7 @@ class PypoFile(Thread): dst_exists = True try: dst_size = os.path.getsize(dst) - except Exception, e: + except Exception as e: dst_exists = False do_copy = False @@ -69,11 +69,11 @@ class PypoFile(Thread): baseurl = self._config.get(CONFIG_SECTION, 'base_url') try: port = self._config.get(CONFIG_SECTION, 'base_port') - except NoOptionError, e: + except NoOptionError as e: port = 80 try: protocol = self._config.get(CONFIG_SECTION, 'protocol') - except NoOptionError, e: + except NoOptionError as e: protocol = str(("http", "https")[int(port) == 443]) try: @@ -103,7 +103,7 @@ class PypoFile(Thread): media_item["filesize"] = file_size media_item['file_ready'] = True - except Exception, e: + except Exception as e: self.logger.error("Could not copy from %s to %s" % (src, dst)) self.logger.error(e) @@ -172,7 +172,7 @@ class PypoFile(Thread): def read_config_file(self, config_path): """Parse the application's config file located at config_path.""" - config = ConfigParser.SafeConfigParser(allow_no_value=True) + config = configparser.SafeConfigParser(allow_no_value=True) try: config.readfp(open(config_path)) except IOError as e: @@ -202,14 +202,14 @@ class PypoFile(Thread): """ try: self.media = self.media_queue.get_nowait() - except Empty, e: + except Empty as e: pass media_item = self.get_highest_priority_media_item(self.media) if media_item is not None: self.copy_file(media_item) - except Exception, e: + except Exception as e: import traceback top = traceback.format_exc() self.logger.error(str(e)) @@ -221,7 +221,7 @@ class PypoFile(Thread): Entry point of the thread """ try: self.main() - except Exception, e: + except Exception as e: top = traceback.format_exc() self.logger.error('PypoFile Exception: %s', top) time.sleep(5) diff --git a/python_apps/pypo/pypo/pypoliqqueue.py b/python_apps/pypo/pypo/pypoliqqueue.py index 8f664c7d7..709b176a6 100644 --- a/python_apps/pypo/pypo/pypoliqqueue.py +++ b/python_apps/pypo/pypo/pypoliqqueue.py @@ -7,7 +7,7 @@ import sys import time -from Queue import Empty +from queue import Empty import signal def keyboardInterruptHandler(signum, frame): @@ -38,7 +38,7 @@ class PypoLiqQueue(Thread): time_until_next_play) media_schedule = self.queue.get(block=True, \ timeout=time_until_next_play) - except Empty, e: + except Empty as e: #Time to push a scheduled item. media_item = schedule_deque.popleft() self.pypo_liquidsoap.play(media_item) @@ -82,7 +82,7 @@ class PypoLiqQueue(Thread): def run(self): try: self.main() - except Exception, e: + except Exception as e: self.logger.error('PypoLiqQueue Exception: %s', traceback.format_exc()) diff --git a/python_apps/pypo/pypo/pypoliquidsoap.py b/python_apps/pypo/pypo/pypoliquidsoap.py index e1f341ed4..d9feca6d5 100644 --- a/python_apps/pypo/pypo/pypoliquidsoap.py +++ b/python_apps/pypo/pypo/pypoliquidsoap.py @@ -1,10 +1,10 @@ -from pypofetch import PypoFetch -from telnetliquidsoap import TelnetLiquidsoap +from .pypofetch import PypoFetch +from .telnetliquidsoap import TelnetLiquidsoap from datetime import datetime from datetime import timedelta -import eventtypes +from . import eventtypes import time class PypoLiquidsoap(): @@ -22,7 +22,7 @@ class PypoLiquidsoap(): logger,\ host,\ port,\ - self.liq_queue_tracker.keys()) + list(self.liq_queue_tracker.keys())) def get_telnet_dispatcher(self): return self.telnet_liquidsoap @@ -120,13 +120,12 @@ class PypoLiquidsoap(): try: scheduled_now_files = \ - filter(lambda x: x["type"] == eventtypes.FILE, scheduled_now) + [x for x in scheduled_now if x["type"] == eventtypes.FILE] scheduled_now_webstream = \ - filter(lambda x: x["type"] == eventtypes.STREAM_OUTPUT_START, \ - scheduled_now) + [x for x in scheduled_now if x["type"] == eventtypes.STREAM_OUTPUT_START] - schedule_ids = set(map(lambda x: x["row_id"], scheduled_now_files)) + schedule_ids = set([x["row_id"] for x in scheduled_now_files]) row_id_map = {} liq_queue_ids = set() @@ -200,7 +199,7 @@ class PypoLiquidsoap(): return media_item["type"] == eventtypes.FILE def clear_queue_tracker(self): - for i in self.liq_queue_tracker.keys(): + for i in list(self.liq_queue_tracker.keys()): self.liq_queue_tracker[i] = None def modify_cue_point(self, link): diff --git a/python_apps/pypo/pypo/pypomessagehandler.py b/python_apps/pypo/pypo/pypomessagehandler.py index 6e31d2f70..f5d1a9b51 100644 --- a/python_apps/pypo/pypo/pypomessagehandler.py +++ b/python_apps/pypo/pypo/pypomessagehandler.py @@ -53,7 +53,7 @@ class PypoMessageHandler(Thread): heartbeat = 5) as connection: rabbit = RabbitConsumer(connection, [schedule_queue], self) rabbit.run() - except Exception, e: + except Exception as e: self.logger.error(e) """ @@ -98,13 +98,13 @@ class PypoMessageHandler(Thread): self.recorder_queue.put(message) else: self.logger.info("Unknown command: %s" % command) - except Exception, e: + except Exception as e: self.logger.error("Exception in handling RabbitMQ message: %s", e) def main(self): try: self.init_rabbit_mq() - except Exception, e: + except Exception as e: self.logger.error('Exception: %s', e) self.logger.error("traceback: %s", traceback.format_exc()) self.logger.error("Error connecting to RabbitMQ Server. Trying again in few seconds") diff --git a/python_apps/pypo/pypo/pypopush.py b/python_apps/pypo/pypo/pypopush.py index e9b809473..c9f2c8076 100644 --- a/python_apps/pypo/pypo/pypopush.py +++ b/python_apps/pypo/pypo/pypopush.py @@ -13,15 +13,15 @@ import math import traceback import os -from pypofetch import PypoFetch -from pypoliqqueue import PypoLiqQueue +from .pypofetch import PypoFetch +from .pypoliqqueue import PypoLiqQueue -from Queue import Empty, Queue +from queue import Empty, Queue from threading import Thread from api_clients import api_client -from timeout import ls_timeout +from .timeout import ls_timeout logging.captureWarnings(True) @@ -67,7 +67,7 @@ class PypoPush(Thread): while True: try: media_schedule = self.queue.get(block=True) - except Exception, e: + except Exception as e: self.logger.error(str(e)) raise else: @@ -138,7 +138,7 @@ class PypoPush(Thread): tn.write("exit\n") self.logger.debug(tn.read_all()) - except Exception, e: + except Exception as e: self.logger.error(str(e)) finally: self.telnet_lock.release() @@ -146,7 +146,7 @@ class PypoPush(Thread): def run(self): while True: try: self.main() - except Exception, e: + except Exception as e: top = traceback.format_exc() self.logger.error('Pypo Push Exception: %s', top) time.sleep(5) diff --git a/python_apps/pypo/pypo/recorder.py b/python_apps/pypo/pypo/recorder.py index b98f5d4ef..24850e7cc 100644 --- a/python_apps/pypo/pypo/recorder.py +++ b/python_apps/pypo/pypo/recorder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from __future__ import print_function + import logging import json import time @@ -36,8 +36,8 @@ def api_client(logger): # loading config file try: config = ConfigObj('/etc/airtime/airtime.conf') -except Exception, e: - print('Error loading config file: %s', e) +except Exception as e: + print(('Error loading config file: %s', e)) sys.exit() # TODO : add docstrings everywhere in this module @@ -153,10 +153,10 @@ class ShowRecorder(Thread): recorded_file['title'] = "%s-%s-%s" % (self.show_name, full_date, full_time) #You cannot pass ints into the metadata of a file. Even tracknumber needs to be a string - recorded_file['tracknumber'] = unicode(self.show_instance) + recorded_file['tracknumber'] = str(self.show_instance) recorded_file.save() - except Exception, e: + except Exception as e: top = traceback.format_exc() self.logger.error('Exception: %s', e) self.logger.error("traceback: %s", top) @@ -173,7 +173,7 @@ class ShowRecorder(Thread): self.upload_file(filepath) os.remove(filepath) - except Exception, e: + except Exception as e: self.logger.error(e) else: self.logger.info("problem recording show") @@ -196,7 +196,7 @@ class Recorder(Thread): try: self.api_client.register_component('show-recorder') success = True - except Exception, e: + except Exception as e: self.logger.error(str(e)) time.sleep(10) @@ -221,12 +221,12 @@ class Recorder(Thread): temp_shows_to_record = {} shows = m['shows'] for show in shows: - show_starts = getDateTimeObj(show[u'starts']) - show_end = getDateTimeObj(show[u'ends']) + show_starts = getDateTimeObj(show['starts']) + show_end = getDateTimeObj(show['ends']) time_delta = show_end - show_starts - temp_shows_to_record[show[u'starts']] = [time_delta, - show[u'instance_id'], show[u'name'], m['server_timezone']] + temp_shows_to_record[show['starts']] = [time_delta, + show['instance_id'], show['name'], m['server_timezone']] self.shows_to_record = temp_shows_to_record def get_time_till_next_show(self): @@ -298,7 +298,7 @@ class Recorder(Thread): #remove show from shows to record. del self.shows_to_record[start_time] #self.time_till_next_show = self.get_time_till_next_show() - except Exception, e : + except Exception as e : top = traceback.format_exc() self.logger.error('Exception: %s', e) self.logger.error("traceback: %s", top) @@ -318,7 +318,7 @@ class Recorder(Thread): if temp is not None: self.process_recorder_schedule(temp) self.logger.info("Bootstrap recorder schedule received: %s", temp) - except Exception, e: + except Exception as e: self.logger.error( traceback.format_exc() ) self.logger.error(e) @@ -338,16 +338,16 @@ class Recorder(Thread): if temp is not None: self.process_recorder_schedule(temp) self.logger.info("updated recorder schedule received: %s", temp) - except Exception, e: + except Exception as e: self.logger.error( traceback.format_exc() ) self.logger.error(e) try: self.handle_message() - except Exception, e: + except Exception as e: self.logger.error( traceback.format_exc() ) self.logger.error('Pypo Recorder Exception: %s', e) time.sleep(PUSH_INTERVAL) self.loops += 1 - except Exception, e : + except Exception as e : top = traceback.format_exc() self.logger.error('Exception: %s', e) self.logger.error("traceback: %s", top) diff --git a/python_apps/pypo/pypo/telnetliquidsoap.py b/python_apps/pypo/pypo/telnetliquidsoap.py index d3a7d39d1..693957966 100644 --- a/python_apps/pypo/pypo/telnetliquidsoap.py +++ b/python_apps/pypo/pypo/telnetliquidsoap.py @@ -1,6 +1,6 @@ -from __future__ import print_function + import telnetlib -from timeout import ls_timeout +from .timeout import ls_timeout def create_liquidsoap_annotation(media): # We need liq_start_next value in the annotate. That is the value that controls overlap duration of crossfade. @@ -140,7 +140,7 @@ class TelnetLiquidsoap: tn.write("exit\n") self.logger.debug(tn.read_all()) - except Exception, e: + except Exception as e: self.logger.error(str(e)) finally: self.telnet_lock.release() @@ -159,7 +159,7 @@ class TelnetLiquidsoap: tn.write("exit\n") self.logger.debug(tn.read_all()) - except Exception, e: + except Exception as e: self.logger.error(str(e)) finally: self.telnet_lock.release() @@ -182,7 +182,7 @@ class TelnetLiquidsoap: self.logger.debug(tn.read_all()) self.current_prebuffering_stream_id = None - except Exception, e: + except Exception as e: self.logger.error(str(e)) finally: self.telnet_lock.release() @@ -205,7 +205,7 @@ class TelnetLiquidsoap: self.logger.debug(tn.read_all()) self.current_prebuffering_stream_id = media_item['row_id'] - except Exception, e: + except Exception as e: self.logger.error(str(e)) finally: self.telnet_lock.release() @@ -225,7 +225,7 @@ class TelnetLiquidsoap: self.logger.debug("stream_id: %s" % stream_id) return stream_id - except Exception, e: + except Exception as e: self.logger.error(str(e)) finally: self.telnet_lock.release() @@ -246,7 +246,7 @@ class TelnetLiquidsoap: tn.write(command) tn.write('exit\n') tn.read_all() - except Exception, e: + except Exception as e: self.logger.error(traceback.format_exc()) finally: self.telnet_lock.release() @@ -263,7 +263,7 @@ class TelnetLiquidsoap: tn.write('exit\n') tn.read_all() - except Exception, e: + except Exception as e: self.logger.error(str(e)) finally: self.telnet_lock.release() diff --git a/python_apps/pypo/pypo/testpypoliqqueue.py b/python_apps/pypo/pypo/testpypoliqqueue.py index d3fe2ead2..b3918a0bd 100644 --- a/python_apps/pypo/pypo/testpypoliqqueue.py +++ b/python_apps/pypo/pypo/testpypoliqqueue.py @@ -1,9 +1,9 @@ -from __future__ import print_function -from pypoliqqueue import PypoLiqQueue -from telnetliquidsoap import DummyTelnetLiquidsoap, TelnetLiquidsoap + +from .pypoliqqueue import PypoLiqQueue +from .telnetliquidsoap import DummyTelnetLiquidsoap, TelnetLiquidsoap -from Queue import Queue +from queue import Queue from threading import Lock import sys diff --git a/python_apps/pypo/pypo/timeout.py b/python_apps/pypo/pypo/timeout.py index 81ec927a0..0e409021a 100644 --- a/python_apps/pypo/pypo/timeout.py +++ b/python_apps/pypo/pypo/timeout.py @@ -1,5 +1,5 @@ import threading -import pypofetch +from . import pypofetch def __timeout(func, timeout_duration, default, args, kwargs): diff --git a/python_apps/pypo/setup.py b/python_apps/pypo/setup.py index 325b0a916..5eea754ca 100644 --- a/python_apps/pypo/setup.py +++ b/python_apps/pypo/setup.py @@ -17,7 +17,7 @@ else: for root, dirnames, filenames in os.walk('pypo'): for filename in filenames: pypo_files.append(os.path.join(root, filename)) - + data_files = [ ('/etc/init', ['install/upstart/airtime-playout.conf.template']), ('/etc/init', ['install/upstart/airtime-liquidsoap.conf.template']), @@ -55,12 +55,11 @@ setup(name='airtime-playout', 'future', 'kombu', 'mutagen', - 'poster', + 'poster3', 'PyDispatcher', 'pyinotify', 'pytz', 'requests', - 'wsgiref', 'defusedxml' ], zip_safe=False, diff --git a/python_apps/pypo/tests/test_modify_cue_in.py b/python_apps/pypo/tests/test_modify_cue_in.py index da17fd53f..11fd8ec73 100644 --- a/python_apps/pypo/tests/test_modify_cue_in.py +++ b/python_apps/pypo/tests/test_modify_cue_in.py @@ -1,6 +1,6 @@ from pypopush import PypoPush from threading import Lock -from Queue import Queue +from queue import Queue import datetime diff --git a/uninstall b/uninstall index 4432daff2..2c21d0307 100755 --- a/uninstall +++ b/uninstall @@ -65,7 +65,7 @@ echo -e "The following files, directories, and services will be removed:\n" for i in ${FILES[*]}; do echo $i done -echo "pip airtime-playout" +echo "pip3 airtime-playout" echo -e "\nIf your web root is not listed, you will need to manually remove it." @@ -103,6 +103,6 @@ if [[ "$IN" = "y" || "$IN" = "Y" ]]; then dropAirtimeDatabase fi -pip uninstall -y airtime-playout airtime-media-monitor airtime-analyzer +pip3 uninstall -y airtime-playout airtime-media-monitor airtime-analyzer service apache2 restart echo "...Done" From ea54493c214065077c3e72f5e6c3e65caee7750a Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Mon, 20 Jan 2020 14:44:17 +0200 Subject: [PATCH 06/38] add chaudum fixes --- .travis.yml | 1 + .../api_clients/api_clients/api_client.py | 23 +----------------- .../liquidsoap_prepare_terminate.py | 2 +- python_apps/pypo/pypo/__main__.py | 2 +- python_apps/pypo/pypo/listenerstat.py | 2 +- python_apps/pypo/pypo/pypofetch.py | 17 +++++++------ python_apps/pypo/pypo/pypoliquidsoap.py | 4 ++-- python_apps/pypo/pypo/recorder.py | 24 +++++++++---------- travis/python.sh | 7 +++--- 9 files changed, 33 insertions(+), 49 deletions(-) diff --git a/.travis.yml b/.travis.yml index 98b47313d..03fd55622 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,6 +58,7 @@ addons: - liquidsoap-plugin-faad - liquidsoap-plugin-vorbis - liquidsoap-plugin-opus + - python3 - python3-nose - python3-gst-1.0 - python3-magic diff --git a/python_apps/api_clients/api_clients/api_client.py b/python_apps/api_clients/api_clients/api_client.py index 1fd9300c2..95ad844e9 100644 --- a/python_apps/api_clients/api_clients/api_client.py +++ b/python_apps/api_clients/api_clients/api_client.py @@ -8,7 +8,6 @@ ############################################################################### import sys import time -import urllib.request, urllib.parse, urllib.error import urllib.request, urllib.error, urllib.parse import requests import socket @@ -21,26 +20,6 @@ from configobj import ConfigObj AIRTIME_API_VERSION = "1.1" -# TODO : Place these functions in some common module. Right now, media -# monitor uses the same functions and it would be better to reuse them -# instead of copy pasting them around - -def to_unicode(obj, encoding='utf-8'): - if isinstance(obj, str): - if not isinstance(obj, str): - obj = str(obj, encoding) - return obj - -def encode_to(obj, encoding='utf-8'): - if isinstance(obj, str): - obj = obj.encode(encoding) - return obj - -def convert_dict_value_to_utf8(md): - #list comprehension to convert all values of md to utf-8 - return dict([(item[0], encode_to(item[1], "utf-8")) for item in list(md.items())]) - - api_config = {} # URL to get the version number of the server API @@ -407,7 +386,7 @@ class AirtimeApiClient(object): # Note that we must prefix every key with: mdX where x is a number # Is there a way to format the next line a little better? The # parenthesis make the code almost unreadable - md_list = dict((("md%d" % i), json.dumps(convert_dict_value_to_utf8(md))) \ + md_list = dict((("md%d" % i), json.dumps(md)) \ for i,md in enumerate(valid_actions)) # For testing we add the following "dry" parameter to tell the # controller not to actually do any changes diff --git a/python_apps/pypo/liquidsoap/liquidsoap_prepare_terminate.py b/python_apps/pypo/liquidsoap/liquidsoap_prepare_terminate.py index 20766bebf..d40655fc9 100644 --- a/python_apps/pypo/liquidsoap/liquidsoap_prepare_terminate.py +++ b/python_apps/pypo/liquidsoap/liquidsoap_prepare_terminate.py @@ -14,6 +14,6 @@ try: tn.read_all() except Exception as e: - print(('Error loading config file: %s', e)) + print("Error loading config file: {}".format(e)) sys.exit() diff --git a/python_apps/pypo/pypo/__main__.py b/python_apps/pypo/pypo/__main__.py index 94c8af0b5..d9f3160a0 100644 --- a/python_apps/pypo/pypo/__main__.py +++ b/python_apps/pypo/pypo/__main__.py @@ -120,7 +120,7 @@ try: consoleHandler.setFormatter(logFormatter) rootLogger.addHandler(consoleHandler) except Exception as e: - print(("Couldn't configure logging", e)) + print("Couldn't configure logging: {}".format(e)) sys.exit(1) diff --git a/python_apps/pypo/pypo/listenerstat.py b/python_apps/pypo/pypo/listenerstat.py index 91a9e9487..e94c7f795 100644 --- a/python_apps/pypo/pypo/listenerstat.py +++ b/python_apps/pypo/pypo/listenerstat.py @@ -109,7 +109,7 @@ class ListenerStat(Thread): #Note that there can be optimizations done, since if all three #streams are the same server, we will still initiate 3 separate #connections - for k, v in list(stream_parameters.items()): + for k, v in stream_parameters.items(): if v["enable"] == 'true': try: if v["output"] == "icecast": diff --git a/python_apps/pypo/pypo/pypofetch.py b/python_apps/pypo/pypo/pypofetch.py index 827346888..b04aab7f3 100644 --- a/python_apps/pypo/pypo/pypofetch.py +++ b/python_apps/pypo/pypo/pypofetch.py @@ -344,20 +344,21 @@ class PypoFetch(Thread): media_item = media[key] if (media_item['type'] == 'file'): fileExt = self.sanity_check_media_item(media_item) - dst = os.path.join(download_dir, str(media_item['id']) + str(fileExt)) + dst = os.path.join(download_dir, media_item['id'] + fileExt) media_item['dst'] = dst media_item['file_ready'] = False media_filtered[key] = media_item - media_item['start'] = datetime.strptime(media_item['start'], + media_item['start'] = datetime.strptime(media_item['start'], "%Y-%m-%d-%H-%M-%S") - media_item['end'] = datetime.strptime(media_item['end'], + media_item['end'] = datetime.strptime(media_item['end'], "%Y-%m-%d-%H-%M-%S") media_copy[key] = media_item self.media_prepare_queue.put(copy.copy(media_filtered)) - except Exception as e: self.logger.error("%s", e) + except Exception as e: + self.logger.error(e) # Send the data to pypo-push self.logger.debug("Pushing to pypo-push") @@ -365,8 +366,10 @@ class PypoFetch(Thread): # cleanup - try: self.cache_cleanup(media) - except Exception as e: self.logger.error("%s", e) + try: + self.cache_cleanup(media) + except Exception as e: + self.logger.error(e) #do basic validation of file parameters. Useful for debugging #purposes @@ -408,7 +411,7 @@ class PypoFetch(Thread): for mkey in media: media_item = media[mkey] if media_item['type'] == 'file': - scheduled_file_set.add(str(media_item["id"]) + str(media_item["file_ext"])) + scheduled_file_set.add(media_item["id"] + media_item["file_ext"]) expired_files = cached_file_set - scheduled_file_set diff --git a/python_apps/pypo/pypo/pypoliquidsoap.py b/python_apps/pypo/pypo/pypoliquidsoap.py index d9feca6d5..773879020 100644 --- a/python_apps/pypo/pypo/pypoliquidsoap.py +++ b/python_apps/pypo/pypo/pypoliquidsoap.py @@ -125,7 +125,7 @@ class PypoLiquidsoap(): scheduled_now_webstream = \ [x for x in scheduled_now if x["type"] == eventtypes.STREAM_OUTPUT_START] - schedule_ids = set([x["row_id"] for x in scheduled_now_files]) + schedule_ids = {x["row_id"] for x in scheduled_now_files]} row_id_map = {} liq_queue_ids = set() @@ -199,7 +199,7 @@ class PypoLiquidsoap(): return media_item["type"] == eventtypes.FILE def clear_queue_tracker(self): - for i in list(self.liq_queue_tracker.keys()): + for i in self.liq_queue_tracker.keys(): self.liq_queue_tracker[i] = None def modify_cue_point(self, link): diff --git a/python_apps/pypo/pypo/recorder.py b/python_apps/pypo/pypo/recorder.py index 24850e7cc..6ed3f301d 100644 --- a/python_apps/pypo/pypo/recorder.py +++ b/python_apps/pypo/pypo/recorder.py @@ -37,7 +37,7 @@ def api_client(logger): try: config = ConfigObj('/etc/airtime/airtime.conf') except Exception as e: - print(('Error loading config file: %s', e)) + print("Error loading config file: {}".format(e)) sys.exit() # TODO : add docstrings everywhere in this module @@ -95,7 +95,7 @@ class ShowRecorder(Thread): self.logger.info("starting record") self.logger.info("command " + command) - + self.p = Popen(args,stdout=PIPE,stderr=PIPE) #blocks at the following line until the child process @@ -131,7 +131,7 @@ class ShowRecorder(Thread): register_openers() # files is what requests actually expects - files = {'file': open(filepath, "rb"), 'name': filename, 'show_instance': str(self.show_instance)} + files = {'file': open(filepath, "rb"), 'name': filename, 'show_instance': self.show_instance} self.api_client.upload_recorded_show(files, self.show_instance) @@ -153,7 +153,7 @@ class ShowRecorder(Thread): recorded_file['title'] = "%s-%s-%s" % (self.show_name, full_date, full_time) #You cannot pass ints into the metadata of a file. Even tracknumber needs to be a string - recorded_file['tracknumber'] = str(self.show_instance) + recorded_file['tracknumber'] = self.show_instance recorded_file.save() except Exception as e: @@ -246,11 +246,11 @@ class Recorder(Thread): self.logger.debug("Next show %s", next_show) self.logger.debug("Now %s", tnow) return out - + def cancel_recording(self): self.sr.cancel_recording() self.sr = None - + def currently_recording(self): if self.sr is not None and self.sr.is_recording(): return True @@ -278,23 +278,23 @@ class Recorder(Thread): start_time_formatted = '%(year)d-%(month)02d-%(day)02d %(hour)02d:%(min)02d:%(sec)02d' % \ {'year': start_time_on_server.year, 'month': start_time_on_server.month, 'day': start_time_on_server.day, \ 'hour': start_time_on_server.hour, 'min': start_time_on_server.minute, 'sec': start_time_on_server.second} - - + + seconds_waiting = 0 - + #avoiding CC-5299 while(True): if self.currently_recording(): self.logger.info("Previous record not finished, sleeping 100ms") seconds_waiting = seconds_waiting + 0.1 - time.sleep(0.1) + time.sleep(0.1) else: show_length_seconds = show_length.seconds - seconds_waiting - + self.sr = ShowRecorder(show_instance, show_name, show_length_seconds, start_time_formatted) self.sr.start() break - + #remove show from shows to record. del self.shows_to_record[start_time] #self.time_till_next_show = self.get_time_till_next_show() diff --git a/travis/python.sh b/travis/python.sh index 477d277a0..28ac33a6b 100755 --- a/travis/python.sh +++ b/travis/python.sh @@ -4,11 +4,12 @@ set -xe [[ "$PYTHON" == false ]] && exit 0 +python3 --version + pushd python_apps/airtime_analyzer -pyenv local 3.7 +pyenv local 3.4 pip3 install -e . -nosetests -a '!rgain' -echo "replaygain tests where skipped due to not having a reliable replaygain install on travis." +nosetests popd echo "Building docs..." From 6573e1d74cb267368b0aef649b12e04cf0b9c70f Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Mon, 20 Jan 2020 14:58:00 +0200 Subject: [PATCH 07/38] Set python versions in travis --- .travis.yml | 3 ++- travis/python.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 03fd55622..4824a8758 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,8 +70,9 @@ install: fi - > if [[ "$PYTHON" == true ]]; then + pyenv local 3.6 pip3 install --user mkdocs rgain3 - pushd python_apps/airtime_analyzer + pushd python_apps/airtime_analyzer python3 setup.py install --dry-run --no-init-script popd fi diff --git a/travis/python.sh b/travis/python.sh index 28ac33a6b..5e32cdeb7 100755 --- a/travis/python.sh +++ b/travis/python.sh @@ -7,7 +7,7 @@ set -xe python3 --version pushd python_apps/airtime_analyzer -pyenv local 3.4 +pyenv local 3.6 pip3 install -e . nosetests popd From d6c8d821075b8ebf9ff59bf3e1c947fe6eebc493 Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Mon, 20 Jan 2020 14:59:46 +0200 Subject: [PATCH 08/38] Update travis ubuntu version to 16.04 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4824a8758..b76af6469 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: trusty +dist: xenial version: ~> 1.0 language: php php: From 4f078da67965b4d3c3e3893968c0716a30fd1f9c Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Mon, 20 Jan 2020 15:04:35 +0200 Subject: [PATCH 09/38] bump travis python version --- .travis.yml | 2 +- travis/python.sh | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index b76af6469..abaf61498 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,7 +70,7 @@ install: fi - > if [[ "$PYTHON" == true ]]; then - pyenv local 3.6 + pyenv local 3.7 pip3 install --user mkdocs rgain3 pushd python_apps/airtime_analyzer python3 setup.py install --dry-run --no-init-script diff --git a/travis/python.sh b/travis/python.sh index 5e32cdeb7..5ad3b7a93 100755 --- a/travis/python.sh +++ b/travis/python.sh @@ -4,10 +4,8 @@ set -xe [[ "$PYTHON" == false ]] && exit 0 -python3 --version - pushd python_apps/airtime_analyzer -pyenv local 3.6 +pyenv local 3.7 pip3 install -e . nosetests popd From 4188baf851edc4a91c4bbdd73a2f92321ad7d662 Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Mon, 20 Jan 2020 15:47:05 +0200 Subject: [PATCH 10/38] update requirements to use python3 --- installer/lib/requirements-debian-stretch.apt | 102 +++++++--------- installer/lib/requirements-ubuntu-bionic.apt | 113 ++++++++---------- installer/lib/requirements-ubuntu-xenial.apt | 113 ++++++++---------- ...uirements-ubuntu-xenial_docker_minimal.apt | 105 +++++++--------- 4 files changed, 180 insertions(+), 253 deletions(-) diff --git a/installer/lib/requirements-debian-stretch.apt b/installer/lib/requirements-debian-stretch.apt index c71175f6a..9e7f461bc 100644 --- a/installer/lib/requirements-debian-stretch.apt +++ b/installer/lib/requirements-debian-stretch.apt @@ -1,71 +1,53 @@ apache2 +coreutils +curl +ecasound +flac git +gstreamer1.0-plugins-ugly +icecast2 +lame +libao-ocaml libapache2-mod-php7.0 -php7.0 -php7.0-dev -php7.0-bcmath -php7.0-mbstring -php-pear -php7.0-gd -php-amqplib - -lsb-release - -zip -unzip - -rabbitmq-server - -postgresql -postgresql-client -php7.0-pgsql - -python -python-virtualenv -python-pip - +libcamomile-ocaml-data +libfaad2 +libmad-ocaml +libopus0 +libportaudio2 +libpulse0 +libsamplerate0 libsoundtouch-ocaml libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 libvo-aacenc0 - -python-rgain -python-gst-1.0 -gstreamer1.0-plugins-ugly -python-pika - -patch - -icecast2 - -curl -php7.0-curl -mpg123 - -libcamomile-ocaml-data -libpulse0 -vorbis-tools +liquidsoap +lsb-release lsb-release lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 +mpg123 +patch +php7.0 +php7.0-bcmath +php7.0-curl +php7.0-dev +php7.0-gd +php7.0-mbstring +php7.0-pgsql +php-amqplib php-apcu - -lame - -coreutils - -liquidsoap - -libopus0 - +php-pear +postgresql +postgresql-client +pwgen +python3 +python3-gst-1.0 +python3-pika +python3-pip +python3-virtualenv +rabbitmq-server systemd-sysv - +unzip +vorbisgain +vorbis-tools +vorbis-tools xmlstarlet +zip diff --git a/installer/lib/requirements-ubuntu-bionic.apt b/installer/lib/requirements-ubuntu-bionic.apt index 58be7819f..5487b09af 100644 --- a/installer/lib/requirements-ubuntu-bionic.apt +++ b/installer/lib/requirements-ubuntu-bionic.apt @@ -1,62 +1,25 @@ apache2 -libapache2-mod-php7.2 -php7.2 -php-pear -php7.2-gd -php-bcmath -php-mbstring - -lsb-release - -zip -unzip - -rabbitmq-server - -postgresql -postgresql-client -php7.2-pgsql - -python -python-virtualenv -python-pip - -libsoundtouch-ocaml -libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 - -python-rgain -python-gst-1.0 -gstreamer1.0-plugins-ugly -python-pika - -patch - -php7.2-curl -mpg123 -curl - -icecast2 - -libcamomile-ocaml-data -libpulse0 -vorbis-tools -lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 -php-apcu - -lame - +build-essential coreutils - +curl +ecasound +flac +gstreamer1.0-plugins-ugly +icecast2 +lame +libao-ocaml +libapache2-mod-php7.2 +libcamomile-ocaml-data +libfaad2 +libffi-dev +libmad-ocaml +libopus0 +libportaudio2 +libpulse0 +libsamplerate0 +libsoundtouch-ocaml +libssl-dev +libtaglib-ocaml liquidsoap liquidsoap-plugin-alsa liquidsoap-plugin-ao @@ -71,15 +34,33 @@ liquidsoap-plugin-pulseaudio liquidsoap-plugin-taglib liquidsoap-plugin-voaacenc liquidsoap-plugin-vorbis - +lsb-release +lsof +mpg123 +patch +php7.2 +php7.2-curl +php7.2-gd +php7.2-pgsql +php-apcu +php-bcmath +php-mbstring +php-pear +postgresql +postgresql-client +pwgen +python3 +python3-dev +python3-gst-1.0 +python3-pika +python3-pip +python3-virtualenv +rabbitmq-server silan -libopus0 - sysvinit-utils - -build-essential -libssl-dev -libffi-dev -python-dev - +unzip +vorbisgain +vorbis-tools +vorbis-tools xmlstarlet +zip diff --git a/installer/lib/requirements-ubuntu-xenial.apt b/installer/lib/requirements-ubuntu-xenial.apt index 41381915f..a348b64a6 100644 --- a/installer/lib/requirements-ubuntu-xenial.apt +++ b/installer/lib/requirements-ubuntu-xenial.apt @@ -1,62 +1,25 @@ apache2 -libapache2-mod-php7.0 -php7.0 -php-pear -php7.0-gd -php-bcmath -php-mbstring - -lsb-release - -zip -unzip - -rabbitmq-server - -postgresql -postgresql-client -php7.0-pgsql - -python -python-virtualenv -python-pip - -libsoundtouch-ocaml -libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 - -python-rgain -python-gst-1.0 -gstreamer1.0-plugins-ugly -python-pika - -patch - -php7.0-curl -mpg123 -curl - -icecast2 - -libcamomile-ocaml-data -libpulse0 -vorbis-tools -lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 -php-apcu - -lame - +build-essential coreutils - +curl +ecasound +flac +gstreamer1.0-plugins-ugly +icecast2 +lame +libao-ocaml +libapache2-mod-php7.0 +libcamomile-ocaml-data +libfaad2 +libffi-dev +libmad-ocaml +libopus0 +libportaudio2 +libpulse0 +libsamplerate0 +libsoundtouch-ocaml +libssl-dev +libtaglib-ocaml liquidsoap liquidsoap-plugin-alsa liquidsoap-plugin-ao @@ -71,15 +34,33 @@ liquidsoap-plugin-pulseaudio liquidsoap-plugin-taglib liquidsoap-plugin-voaacenc liquidsoap-plugin-vorbis - +lsb-release +lsof +mpg123 +patch +php7.0 +php7.0-curl +php7.0-gd +php7.0-pgsql +php-apcu +php-bcmath +php-mbstring +php-pear +postgresql +postgresql-client +pwgen +python3 +python3-dev +python3-gst-1.0 +python3-pika +python3-pip +python3-virtualenv +rabbitmq-server silan -libopus0 - sysvinit-utils - -build-essential -libssl-dev -libffi-dev -python-dev - +unzip +vorbisgain +vorbis-tools +vorbis-tools xmlstarlet +zip diff --git a/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt b/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt index 09c94f817..8a05299e5 100644 --- a/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt +++ b/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt @@ -1,57 +1,24 @@ apache2 -libapache2-mod-php7.0 -php7.0 -php-pear -php7.0-gd -php-bcmath -php-mbstring - -lsb-release - -zip -unzip - -postgresql-client -php7.0-pgsql - -python -python-virtualenv -python-pip - -libsoundtouch-ocaml -libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 - -python-rgain -python-gst-1.0 -gstreamer1.0-plugins-ugly -python-pika - -patch - -php7.0-curl -mpg123 -curl - -libcamomile-ocaml-data -libpulse0 -vorbis-tools -lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 -php-apcu - -lame - +build-essential coreutils - +curl +ecasound +flac +gstreamer1.0-plugins-ugly +lame +libao-ocaml +libapache2-mod-php7.0 +libcamomile-ocaml-data +libfaad2 +libffi-dev +libmad-ocaml +libopus0 +libportaudio2 +libpulse0 +libsamplerate0 +libsoundtouch-ocaml +libssl-dev +libtaglib-ocaml liquidsoap liquidsoap-plugin-alsa liquidsoap-plugin-ao @@ -66,15 +33,31 @@ liquidsoap-plugin-pulseaudio liquidsoap-plugin-taglib liquidsoap-plugin-voaacenc liquidsoap-plugin-vorbis - +lsb-release +lsof +mpg123 +patch +php7.0 +php7.0-curl +php7.0-gd +php7.0-pgsql +php-apcu +php-bcmath +php-mbstring +php-pear +postgresql-client +pwgen +python3 +python3-dev +python3-gst-1.0 +python3-pika +python3-pip +python3-virtualenv silan -libopus0 - sysvinit-utils - -build-essential -libssl-dev -libffi-dev -python-dev - +unzip +vorbisgain +vorbis-tools +vorbis-tools xmlstarlet +zip From b8225b429e87d83dc4f0fc69bf63ce6a8bb1b2a5 Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Mon, 20 Jan 2020 15:47:35 +0200 Subject: [PATCH 11/38] remove lists for unsupported releases --- installer/lib/requirements-debian-jessie.apt | 67 ----------------- installer/lib/requirements-ubuntu-precise.apt | 74 ------------------- 2 files changed, 141 deletions(-) delete mode 100644 installer/lib/requirements-debian-jessie.apt delete mode 100644 installer/lib/requirements-ubuntu-precise.apt diff --git a/installer/lib/requirements-debian-jessie.apt b/installer/lib/requirements-debian-jessie.apt deleted file mode 100644 index 4e22102cc..000000000 --- a/installer/lib/requirements-debian-jessie.apt +++ /dev/null @@ -1,67 +0,0 @@ -apache2 -libapache2-mod-php5 -php5 -php-pear -php5-gd - -lsb-release - -rabbitmq-server - -zip -unzip - -postgresql -postgresql-client -php5-pgsql - -python -python-virtualenv -python-pip - -libsoundtouch-ocaml -libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 -libvo-aacenc0 - -python-rgain -python-gst-1.0 -gstreamer1.0-plugins-ugly -python-pika - -patch - -icecast2 - -curl -php5-curl -mpg123 - -libcamomile-ocaml-data -libpulse0 -vorbis-tools -lsb-release -lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 -php-apc - -lame - -coreutils - -liquidsoap - -libopus0 - -sysvinit -sysvinit-utils - -xmlstarlet diff --git a/installer/lib/requirements-ubuntu-precise.apt b/installer/lib/requirements-ubuntu-precise.apt deleted file mode 100644 index ed31b628f..000000000 --- a/installer/lib/requirements-ubuntu-precise.apt +++ /dev/null @@ -1,74 +0,0 @@ -apache2 -libapache2-mod-php5 -php5 -php-pear -php5-gd - -lsb-release - -zip -unzip - -rabbitmq-server - -postgresql -postgresql-client -php5-pgsql - -python -python-virtualenv -python-pip - -libsoundtouch-ocaml -libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 - -python-rgain -python-gst0.10 -gstreamer0.10-plugins-ugly -gir1.2-gstreamer-0.10 -patch - -curl -php5-curl -mpg123 - -icecast2 - -libcamomile-ocaml-data -libpulse0 -vorbis-tools -lsb-release -lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 -php-apc -dbus - -lame - -coreutils - -liquidsoap -liquidsoap-plugin-alsa -liquidsoap-plugin-ao -liquidsoap-plugin-faad -liquidsoap-plugin-flac -liquidsoap-plugin-icecast -liquidsoap-plugin-lame -liquidsoap-plugin-mad -liquidsoap-plugin-ogg -liquidsoap-plugin-portaudio -liquidsoap-plugin-pulseaudio -liquidsoap-plugin-taglib -liquidsoap-plugin-voaacenc -liquidsoap-plugin-vorbis - -xmlstarlet From 39e986a39ca23dd60c28a58ee9573b97d632a54e Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Mon, 20 Jan 2020 16:03:02 +0200 Subject: [PATCH 12/38] use packaged version of pycairo --- installer/lib/requirements-debian-buster.apt | 1 + installer/lib/requirements-debian-stretch.apt | 1 + installer/lib/requirements-ubuntu-bionic.apt | 1 + installer/lib/requirements-ubuntu-xenial.apt | 1 + installer/lib/requirements-ubuntu-xenial_docker_minimal.apt | 1 + 5 files changed, 5 insertions(+) diff --git a/installer/lib/requirements-debian-buster.apt b/installer/lib/requirements-debian-buster.apt index a12d84728..3dc1121c3 100644 --- a/installer/lib/requirements-debian-buster.apt +++ b/installer/lib/requirements-debian-buster.apt @@ -45,6 +45,7 @@ python3-gst-1.0 python3-pika python3-pip python3-virtualenv +python3-cairo rabbitmq-server silan systemd-sysv diff --git a/installer/lib/requirements-debian-stretch.apt b/installer/lib/requirements-debian-stretch.apt index 9e7f461bc..ace712897 100644 --- a/installer/lib/requirements-debian-stretch.apt +++ b/installer/lib/requirements-debian-stretch.apt @@ -43,6 +43,7 @@ python3-gst-1.0 python3-pika python3-pip python3-virtualenv +python3-cairo rabbitmq-server systemd-sysv unzip diff --git a/installer/lib/requirements-ubuntu-bionic.apt b/installer/lib/requirements-ubuntu-bionic.apt index 5487b09af..d45f05564 100644 --- a/installer/lib/requirements-ubuntu-bionic.apt +++ b/installer/lib/requirements-ubuntu-bionic.apt @@ -55,6 +55,7 @@ python3-gst-1.0 python3-pika python3-pip python3-virtualenv +python3-cairo rabbitmq-server silan sysvinit-utils diff --git a/installer/lib/requirements-ubuntu-xenial.apt b/installer/lib/requirements-ubuntu-xenial.apt index a348b64a6..275cc0200 100644 --- a/installer/lib/requirements-ubuntu-xenial.apt +++ b/installer/lib/requirements-ubuntu-xenial.apt @@ -55,6 +55,7 @@ python3-gst-1.0 python3-pika python3-pip python3-virtualenv +python3-cairo rabbitmq-server silan sysvinit-utils diff --git a/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt b/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt index 8a05299e5..41257a70e 100644 --- a/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt +++ b/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt @@ -53,6 +53,7 @@ python3-gst-1.0 python3-pika python3-pip python3-virtualenv +python3-cairo silan sysvinit-utils unzip From 6d27871c1cada97f92585c46f8546aab376d4c1b Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Mon, 20 Jan 2020 16:27:31 +0200 Subject: [PATCH 13/38] update pip in travis tests --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index abaf61498..db44c953f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -71,6 +71,7 @@ install: - > if [[ "$PYTHON" == true ]]; then pyenv local 3.7 + pip3 install -U pip wheel pip3 install --user mkdocs rgain3 pushd python_apps/airtime_analyzer python3 setup.py install --dry-run --no-init-script From 5d67172dd0afba3d4bd2a01471679369c8ec2b25 Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Mon, 20 Jan 2020 16:31:46 +0200 Subject: [PATCH 14/38] install libgi --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index db44c953f..d0c7222c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,6 +49,7 @@ addons: apt: packages: - silan + - libgirepository1.0-dev - liquidsoap - liquidsoap-plugin-mad - liquidsoap-plugin-taglib From 82042e8c69c4cffbc3303e0d2dd505ed598a58f2 Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Tue, 21 Jan 2020 09:13:42 +0200 Subject: [PATCH 15/38] fix test failures --- .travis.yml | 8 +- .../airtime_analyzer/analyzer_pipeline.py | 24 +++--- .../airtime_analyzer/cuepoint_analyzer.py | 2 +- .../airtime_analyzer/filemover_analyzer.py | 32 ++++---- .../airtime_analyzer/metadata_analyzer.py | 34 ++++---- .../airtime_analyzer/replaygain_analyzer.py | 14 ++-- .../tests/analyzer_pipeline_tests.py | 13 ++-- .../tests/filemover_analyzer_tests.py | 35 +++++---- .../tests/metadata_analyzer_tests.py | 78 +++++++++---------- .../tests/replaygain_analyzer_tests.py | 14 ---- 10 files changed, 125 insertions(+), 129 deletions(-) diff --git a/.travis.yml b/.travis.yml index d0c7222c9..7f863b7b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,6 @@ php: - 7.0 # folks who prefer running on 5.x should be using 5.6 in most cases, 5.4 is no in the matrix since noone should use it - 5.6 -# this is in for centos support, it's still the default on CentOS 7.3 and there were some lang changes after 5.4 -- 5.4 services: - postgresql - rabbitmq @@ -50,6 +48,12 @@ addons: packages: - silan - libgirepository1.0-dev + - gir1.2-gstreamer-1.0 + - gstreamer1.0-plugins-base + - gstreamer1.0-plugins-good + - gstreamer1.0-plugins-bad + - gstreamer1.0-plugins-ugly + - libcairo2-dev - liquidsoap - liquidsoap-plugin-mad - liquidsoap-plugin-taglib diff --git a/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py b/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py index 54892838c..c6b383127 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py @@ -1,9 +1,9 @@ -""" Analyzes and imports an audio file into the Airtime library. +""" Analyzes and imports an audio file into the Airtime library. """ import logging import threading import multiprocessing -import queue +from queue import Queue import configparser from .metadata_analyzer import MetadataAnalyzer from .filemover_analyzer import FileMoverAnalyzer @@ -12,8 +12,8 @@ from .replaygain_analyzer import ReplayGainAnalyzer from .playability_analyzer import * class AnalyzerPipeline: - """ Analyzes and imports an audio file into the Airtime library. - + """ Analyzes and imports an audio file into the Airtime library. + This currently performs metadata extraction (eg. gets the ID3 tags from an MP3), then moves the file to the Airtime music library (stor/imported), and returns the results back to the parent process. This class is used in an isolated process @@ -26,27 +26,27 @@ class AnalyzerPipeline: @staticmethod def run_analysis(queue, audio_file_path, import_directory, original_filename, storage_backend, file_prefix): """Analyze and import an audio file, and put all extracted metadata into queue. - + Keyword arguments: queue: A multiprocessing.queues.Queue which will be used to pass the extracted metadata back to the parent process. audio_file_path: Path on disk to the audio file to analyze. - import_directory: Path to the final Airtime "import" directory where + import_directory: Path to the final Airtime "import" directory where we will move the file. - original_filename: The original filename of the file, which we'll try to - preserve. The file at audio_file_path typically has a + original_filename: The original filename of the file, which we'll try to + preserve. The file at audio_file_path typically has a temporary randomly generated name, which is why we want - to know what the original name was. + to know what the original name was. storage_backend: String indicating the storage backend (amazon_s3 or file) file_prefix: """ - # It is super critical to initialize a separate log file here so that we + # It is super critical to initialize a separate log file here so that we # don't inherit logging/locks from the parent process. Supposedly # this can lead to Bad Things (deadlocks): http://bugs.python.org/issue6721 AnalyzerPipeline.python_logger_deadlock_workaround() try: - if not isinstance(queue, queue.Queue): + if not isinstance(queue, Queue): raise TypeError("queue must be a Queue.Queue()") if not isinstance(audio_file_path, str): raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__ + " instead.") @@ -72,7 +72,7 @@ class AnalyzerPipeline: metadata["import_status"] = 0 # Successfully imported - # Note that the queue we're putting the results into is our interprocess communication + # Note that the queue we're putting the results into is our interprocess communication # back to the main process. # Pass all the file metadata back to the main analyzer process, which then passes diff --git a/python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py index b10f573cb..49e90bce6 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py @@ -64,7 +64,7 @@ class CuePointAnalyzer(Analyzer): except OSError as e: # silan was not found logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have silan installed?")) except subprocess.CalledProcessError as e: # silan returned an error code - logging.warn("%s %s %s", e.cmd, e.message, e.returncode) + logging.warn("%s %s %s", e.cmd, e.output, e.returncode) except Exception as e: logging.warn(e) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py index f0e0e030e..8bc8bb94d 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py @@ -4,24 +4,24 @@ import time import shutil import os, errno import time -import uuid +import uuid from .analyzer import Analyzer class FileMoverAnalyzer(Analyzer): - """This analyzer copies a file over from a temporary directory (stor/organize) + """This analyzer copies a file over from a temporary directory (stor/organize) into the Airtime library (stor/imported). """ @staticmethod def analyze(audio_file_path, metadata): """Dummy method because we need more info than analyze gets passed to it""" raise Exception("Use FileMoverAnalyzer.move() instead.") - + @staticmethod def move(audio_file_path, import_directory, original_filename, metadata): """Move the file at audio_file_path over into the import_directory/import, renaming it to original_filename. - + Keyword arguments: audio_file_path: Path to the file to be imported. import_directory: Path to the "import" directory inside the Airtime stor directory. @@ -30,18 +30,20 @@ class FileMoverAnalyzer(Analyzer): metadata: A dictionary where the "full_path" of where the file is moved to will be added. """ if not isinstance(audio_file_path, str): - raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__) + raise TypeError("audio_file_path must be string. Was of type " + type(audio_file_path).__name__) if not isinstance(import_directory, str): - raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__) + raise TypeError("import_directory must be string. Was of type " + type(import_directory).__name__) if not isinstance(original_filename, str): - raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__) + raise TypeError("original_filename must be string. Was of type " + type(original_filename).__name__) if not isinstance(metadata, dict): raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__) - + if not os.path.exists(audio_file_path): + raise FileNotFoundError("audio file not found: {}".format(audio_file_path)) + #Import the file over to it's final location. # TODO: Also, handle the case where the move fails and write some code # to possibly move the file to problem_files. - + max_dir_len = 48 max_file_len = 48 final_file_path = import_directory @@ -58,11 +60,11 @@ class FileMoverAnalyzer(Analyzer): #If a file with the same name already exists in the "import" directory, then #we add a unique string to the end of this one. We never overwrite a file on import - #because if we did that, it would mean Airtime's database would have + #because if we did that, it would mean Airtime's database would have #the wrong information for the file we just overwrote (eg. the song length would be wrong!) #If the final file path is the same as the file we've been told to import (which #you often do when you're debugging), then don't move the file at all. - + if os.path.exists(final_file_path): if os.path.samefile(audio_file_path, final_file_path): metadata["full_path"] = final_file_path @@ -77,14 +79,14 @@ class FileMoverAnalyzer(Analyzer): #Ensure the full path to the file exists mkdir_p(os.path.dirname(final_file_path)) - - #Move the file into its final destination directory + + #Move the file into its final destination directory logging.debug("Moving %s to %s" % (audio_file_path, final_file_path)) shutil.move(audio_file_path, final_file_path) - + metadata["full_path"] = final_file_path return metadata - + def mkdir_p(path): """ Make all directories in a tree (like mkdir -p)""" if path == "": diff --git a/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py index d7a4210cb..d590d95a9 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py @@ -13,15 +13,17 @@ class MetadataAnalyzer(Analyzer): @staticmethod def analyze(filename, metadata): ''' Extract audio metadata from tags embedded in the file (eg. ID3 tags) - + Keyword arguments: filename: The path to the audio file to extract metadata from. - metadata: A dictionary that the extracted metadata will be added to. + metadata: A dictionary that the extracted metadata will be added to. ''' if not isinstance(filename, str): - raise TypeError("filename must be unicode. Was of type " + type(filename).__name__) + raise TypeError("filename must be string. Was of type " + type(filename).__name__) if not isinstance(metadata, dict): raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__) + if not os.path.exists(filename): + raise FileNotFoundError("audio file not found: {}".format(filename)) #Airtime <= 2.5.x nonsense: metadata["ftype"] = "audioclip" @@ -40,7 +42,7 @@ class MetadataAnalyzer(Analyzer): m.update(data) metadata["md5"] = m.hexdigest() - # Mutagen doesn't handle WAVE files so we use a different package + # Mutagen doesn't handle WAVE files so we use a different package ms = magic.open(magic.MIME_TYPE) ms.load() with open(filename, 'rb') as fh: @@ -57,15 +59,15 @@ class MetadataAnalyzer(Analyzer): if audio_file == None: # Don't use "if not" here. It is wrong due to mutagen's design. return metadata # Note that audio_file can equal {} if the file is valid but there's no metadata tags. - # We can still try to grab the info variables below. - + # We can still try to grab the info variables below. + #Grab other file information that isn't encoded in a tag, but instead usually #in the file header. Mutagen breaks that out into a separate "info" object: info = audio_file.info if hasattr(info, "sample_rate"): # Mutagen is annoying and inconsistent metadata["sample_rate"] = info.sample_rate if hasattr(info, "length"): - metadata["length_seconds"] = info.length + metadata["length_seconds"] = info.length #Converting the length in seconds (float) to a formatted time string track_length = datetime.timedelta(seconds=info.length) metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length) @@ -77,12 +79,12 @@ class MetadataAnalyzer(Analyzer): if hasattr(info, "bitrate"): metadata["bit_rate"] = info.bitrate - + # Use the mutagen to get the MIME type, if it has one. This is more reliable and # consistent for certain types of MP3s or MPEG files than the MIMEs returned by magic. if audio_file.mime: metadata["mime"] = audio_file.mime[0] - + #Try to get the number of channels if mutagen can... try: #Special handling for getting the # of channels from MP3s. It's in the "mode" field @@ -97,13 +99,13 @@ class MetadataAnalyzer(Analyzer): except (AttributeError, KeyError): #If mutagen can't figure out the number of channels, we'll just leave it out... pass - + #Try to extract the number of tracks on the album if we can (the "track total") try: track_number = audio_file["tracknumber"] - if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh + if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh track_number = track_number[0] - track_number_tokens = track_number + track_number_tokens = track_number if '/' in track_number: track_number_tokens = track_number.split('/') track_number = track_number_tokens[0] @@ -118,7 +120,7 @@ class MetadataAnalyzer(Analyzer): pass #We normalize the mutagen tags slightly here, so in case mutagen changes, - #we find the + #we find the mutagen_to_airtime_mapping = { 'title': 'track_title', 'artist': 'artist_name', @@ -153,13 +155,13 @@ class MetadataAnalyzer(Analyzer): # Some tags are returned as lists because there could be multiple values. # This is unusual so we're going to always just take the first item in the list. if isinstance(metadata[airtime_tag], list): - if metadata[airtime_tag]: + if metadata[airtime_tag]: metadata[airtime_tag] = metadata[airtime_tag][0] else: # Handle empty lists metadata[airtime_tag] = "" except KeyError: - continue + continue return metadata @@ -174,7 +176,7 @@ class MetadataAnalyzer(Analyzer): track_length = datetime.timedelta(seconds=length_seconds) metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length) metadata["length_seconds"] = length_seconds - metadata["cueout"] = metadata["length"] + metadata["cueout"] = metadata["length"] except wave.Error as ex: logging.error("Invalid WAVE file: {}".format(str(ex))) raise diff --git a/python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py index 4290ee059..309f73ce6 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py @@ -1,12 +1,13 @@ import subprocess import logging from .analyzer import Analyzer +import re class ReplayGainAnalyzer(Analyzer): ''' This class extracts the ReplayGain using a tool from the python-rgain package. ''' - REPLAYGAIN_EXECUTABLE = 'replaygain' # From the python-rgain package + REPLAYGAIN_EXECUTABLE = 'replaygain' # From the rgain3 python package @staticmethod def analyze(filename, metadata): @@ -19,17 +20,16 @@ class ReplayGainAnalyzer(Analyzer): ''' command = [ReplayGainAnalyzer.REPLAYGAIN_EXECUTABLE, '-d', filename] try: - results = subprocess.check_output(command, stderr=subprocess.STDOUT, close_fds=True) - filename_token = "%s: " % filename - rg_pos = results.find(filename_token, results.find("Calculating Replay Gain information")) + len(filename_token) - db_pos = results.find(" dB", rg_pos) - replaygain = results[rg_pos:db_pos] + results = subprocess.check_output(command, stderr=subprocess.STDOUT, + close_fds=True, text=True) + gain_match = r'Calculating Replay Gain information \.\.\.(?:\n|.)*?:([\d.-]*) dB' + replaygain = re.search(gain_match, results).group(1) metadata['replay_gain'] = float(replaygain) except OSError as e: # replaygain was not found logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have python-rgain installed?")) except subprocess.CalledProcessError as e: # replaygain returned an error code - logging.warn("%s %s %s", e.cmd, e.message, e.returncode) + logging.warn("%s %s %s", e.cmd, e.output, e.returncode) except Exception as e: logging.warn(e) diff --git a/python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py b/python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py index 8e986bc11..57ae8bcf1 100644 --- a/python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py +++ b/python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py @@ -1,9 +1,8 @@ from nose.tools import * -from ConfigParser import SafeConfigParser import os import shutil import multiprocessing -import Queue +from queue import Queue import datetime from airtime_analyzer.analyzer_pipeline import AnalyzerPipeline from airtime_analyzer import config_file @@ -21,7 +20,7 @@ def teardown(): def test_basic(): filename = os.path.basename(DEFAULT_AUDIO_FILE) - q = Queue.Queue() + q = Queue() file_prefix = u'' storage_backend = "file" #This actually imports the file into the "./Test Artist" directory. @@ -39,17 +38,17 @@ def test_basic(): @raises(TypeError) def test_wrong_type_queue_param(): - AnalyzerPipeline.run_analysis(Queue.Queue(), u'', u'', u'') + AnalyzerPipeline.run_analysis(Queue(), u'', u'', u'') @raises(TypeError) def test_wrong_type_string_param2(): - AnalyzerPipeline.run_analysis(Queue.Queue(), '', u'', u'') + AnalyzerPipeline.run_analysis(Queue(), '', u'', u'') @raises(TypeError) def test_wrong_type_string_param3(): - AnalyzerPipeline.run_analysis(Queue.Queue(), u'', '', u'') + AnalyzerPipeline.run_analysis(Queue(), u'', '', u'') @raises(TypeError) def test_wrong_type_string_param4(): - AnalyzerPipeline.run_analysis(Queue.Queue(), u'', u'', '') + AnalyzerPipeline.run_analysis(Queue(), u'', u'', '') diff --git a/python_apps/airtime_analyzer/tests/filemover_analyzer_tests.py b/python_apps/airtime_analyzer/tests/filemover_analyzer_tests.py index dbdbb2feb..4e7f9e304 100644 --- a/python_apps/airtime_analyzer/tests/filemover_analyzer_tests.py +++ b/python_apps/airtime_analyzer/tests/filemover_analyzer_tests.py @@ -2,7 +2,6 @@ from nose.tools import * import os import shutil import multiprocessing -import Queue import time import mock from pprint import pprint @@ -23,30 +22,34 @@ def test_dont_use_analyze(): @raises(TypeError) def test_move_wrong_string_param1(): - FileMoverAnalyzer.move('', u'', u'', dict()) + FileMoverAnalyzer.move(42, '', '', dict()) @raises(TypeError) def test_move_wrong_string_param2(): - FileMoverAnalyzer.move(u'', '', u'', dict()) + FileMoverAnalyzer.move(u'', 23, u'', dict()) @raises(TypeError) def test_move_wrong_string_param3(): - FileMoverAnalyzer.move(u'', u'', '', dict()) + FileMoverAnalyzer.move('', '', 5, dict()) @raises(TypeError) def test_move_wrong_dict_param(): - FileMoverAnalyzer.move(u'', u'', u'', 12345) + FileMoverAnalyzer.move('', '', '', 12345) + +@raises(FileNotFoundError) +def test_move_wrong_string_param3(): + FileMoverAnalyzer.move('', '', '', dict()) def test_basic(): filename = os.path.basename(DEFAULT_AUDIO_FILE) - FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict()) + FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict()) #Move the file back shutil.move("./" + filename, DEFAULT_AUDIO_FILE) assert os.path.exists(DEFAULT_AUDIO_FILE) def test_basic_samefile(): filename = os.path.basename(DEFAULT_AUDIO_FILE) - FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'tests/test_data', filename, dict()) + FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'tests/test_data', filename, dict()) assert os.path.exists(DEFAULT_AUDIO_FILE) def test_duplicate_file(): @@ -55,9 +58,9 @@ def test_duplicate_file(): FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict()) #Copy it back to the original location shutil.copy("./" + filename, DEFAULT_AUDIO_FILE) - #Import it again. It shouldn't overwrite the old file and instead create a new + #Import it again. It shouldn't overwrite the old file and instead create a new metadata = dict() - metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, metadata) + metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, metadata) #Cleanup: move the file (eg. 44100Hz-16bit-mono.mp3) back shutil.move("./" + filename, DEFAULT_AUDIO_FILE) #Remove the renamed duplicate, eg. 44100Hz-16bit-mono_03-26-2014-11-58.mp3 @@ -71,7 +74,7 @@ def test_duplicate_file(): it's imported within 1 second of the second file (ie. if the timestamp is the same). ''' def test_double_duplicate_files(): - # Here we use mock to patch out the time.localtime() function so that it + # Here we use mock to patch out the time.localtime() function so that it # always returns the same value. This allows us to consistently simulate this test cases # where the last two of the three files are imported at the same time as the timestamp. with mock.patch('airtime_analyzer.filemover_analyzer.time') as mock_time: @@ -83,17 +86,17 @@ def test_double_duplicate_files(): FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict()) #Copy it back to the original location shutil.copy("./" + filename, DEFAULT_AUDIO_FILE) - #Import it again. It shouldn't overwrite the old file and instead create a new + #Import it again. It shouldn't overwrite the old file and instead create a new first_dup_metadata = dict() - first_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, - first_dup_metadata) + first_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, + first_dup_metadata) #Copy it back again! shutil.copy("./" + filename, DEFAULT_AUDIO_FILE) #Reimport for the third time, which should have the same timestamp as the second one #thanks to us mocking out time.localtime() second_dup_metadata = dict() - second_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, - second_dup_metadata) + second_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, + second_dup_metadata) #Cleanup: move the file (eg. 44100Hz-16bit-mono.mp3) back shutil.move("./" + filename, DEFAULT_AUDIO_FILE) #Remove the renamed duplicate, eg. 44100Hz-16bit-mono_03-26-2014-11-58.mp3 @@ -105,7 +108,7 @@ def test_double_duplicate_files(): def test_bad_permissions_destination_dir(): filename = os.path.basename(DEFAULT_AUDIO_FILE) dest_dir = u'/sys/foobar' # /sys is using sysfs on Linux, which is unwritable - FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, dest_dir, filename, dict()) + FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, dest_dir, filename, dict()) #Move the file back shutil.move(os.path.join(dest_dir, filename), DEFAULT_AUDIO_FILE) assert os.path.exists(DEFAULT_AUDIO_FILE) diff --git a/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py b/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py index 3c60244bc..c92b82db6 100644 --- a/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py +++ b/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py @@ -4,7 +4,7 @@ import datetime import mutagen import mock from nose.tools import * -from airtime_analyzer.metadata_analyzer import MetadataAnalyzer +from airtime_analyzer.metadata_analyzer import MetadataAnalyzer def setup(): pass @@ -13,73 +13,73 @@ def teardown(): pass def check_default_metadata(metadata): - assert metadata['track_title'] == u'Test Title' - assert metadata['artist_name'] == u'Test Artist' - assert metadata['album_title'] == u'Test Album' - assert metadata['year'] == u'1999' - assert metadata['genre'] == u'Test Genre' - assert metadata['track_number'] == u'1' + assert metadata['track_title'] == 'Test Title' + assert metadata['artist_name'] == 'Test Artist' + assert metadata['album_title'] == 'Test Album' + assert metadata['year'] == '1999' + assert metadata['genre'] == 'Test Genre' + assert metadata['track_number'] == '1' assert metadata["length"] == str(datetime.timedelta(seconds=metadata["length_seconds"])) def test_mp3_mono(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.mp3', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-mono.mp3', dict()) check_default_metadata(metadata) assert metadata['channels'] == 1 assert metadata['bit_rate'] == 63998 assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mp3' # Not unicode because MIMEs aren't. - assert metadata['track_total'] == u'10' # MP3s can have a track_total + assert metadata['track_total'] == '10' # MP3s can have a track_total #Mutagen doesn't extract comments from mp3s it seems def test_mp3_jointstereo(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-jointstereo.mp3', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-jointstereo.mp3', dict()) check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 127998 assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mp3' - assert metadata['track_total'] == u'10' # MP3s can have a track_total + assert metadata['track_total'] == '10' # MP3s can have a track_total def test_mp3_simplestereo(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-simplestereo.mp3', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-simplestereo.mp3', dict()) check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 127998 assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mp3' - assert metadata['track_total'] == u'10' # MP3s can have a track_total + assert metadata['track_total'] == '10' # MP3s can have a track_total def test_mp3_dualmono(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-dualmono.mp3', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-dualmono.mp3', dict()) check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 127998 assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mp3' - assert metadata['track_total'] == u'10' # MP3s can have a track_total + assert metadata['track_total'] == '10' # MP3s can have a track_total def test_ogg_mono(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.ogg', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-mono.ogg', dict()) check_default_metadata(metadata) assert metadata['channels'] == 1 assert metadata['bit_rate'] == 80000 assert abs(metadata['length_seconds'] - 3.8) < 0.1 assert metadata['mime'] == 'audio/vorbis' - assert metadata['comment'] == u'Test Comment' + assert metadata['comment'] == 'Test Comment' def test_ogg_stereo(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.ogg', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo.ogg', dict()) check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 112000 assert abs(metadata['length_seconds'] - 3.8) < 0.1 assert metadata['mime'] == 'audio/vorbis' - assert metadata['comment'] == u'Test Comment' + assert metadata['comment'] == 'Test Comment' ''' faac and avconv can't seem to create a proper mono AAC file... ugh def test_aac_mono(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.m4a') + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-mono.m4a') print("Mono AAC metadata:") print(metadata) check_default_metadata(metadata) @@ -87,41 +87,41 @@ def test_aac_mono(): assert metadata['bit_rate'] == 80000 assert abs(metadata['length_seconds'] - 3.8) < 0.1 assert metadata['mime'] == 'audio/mp4' - assert metadata['comment'] == u'Test Comment' + assert metadata['comment'] == 'Test Comment' ''' def test_aac_stereo(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.m4a', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo.m4a', dict()) check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 102619 assert abs(metadata['length_seconds'] - 3.8) < 0.1 assert metadata['mime'] == 'audio/mp4' - assert metadata['comment'] == u'Test Comment' + assert metadata['comment'] == 'Test Comment' def test_mp3_utf8(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict()) # Using a bunch of different UTF-8 codepages here. Test data is from: # http://winrus.com/utf8-jap.htm - assert metadata['track_title'] == u'ｱｲｳｴｵｶｷｸｹｺｻｼｽｾｿﾀﾁﾂﾃ' - assert metadata['artist_name'] == u'てすと' - assert metadata['album_title'] == u'Ä ä Ü ü ß' - assert metadata['year'] == u'1999' - assert metadata['genre'] == u'Я Б Г Д Ж Й' - assert metadata['track_number'] == u'1' + assert metadata['track_title'] == 'ｱｲｳｴｵｶｷｸｹｺｻｼｽｾｿﾀﾁﾂﾃ' + assert metadata['artist_name'] == 'てすと' + assert metadata['album_title'] == 'Ä ä Ü ü ß' + assert metadata['year'] == '1999' + assert metadata['genre'] == 'Я Б Г Д Ж Й' + assert metadata['track_number'] == '1' assert metadata['channels'] == 2 assert metadata['bit_rate'] < 130000 assert metadata['bit_rate'] > 127000 assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mp3' - assert metadata['track_total'] == u'10' # MP3s can have a track_total + assert metadata['track_total'] == '10' # MP3s can have a track_total def test_invalid_wma(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-invalid.wma', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo-invalid.wma', dict()) assert metadata['mime'] == 'audio/x-ms-wma' def test_wav_stereo(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.wav', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo.wav', dict()) assert metadata['mime'] == 'audio/x-wav' assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['channels'] == 2 @@ -129,7 +129,7 @@ def test_wav_stereo(): # Make sure the parameter checking works -@raises(TypeError) +@raises(FileNotFoundError) def test_move_wrong_string_param1(): not_unicode = 'asdfasdf' MetadataAnalyzer.analyze(not_unicode, dict()) @@ -137,12 +137,12 @@ def test_move_wrong_string_param1(): @raises(TypeError) def test_move_wrong_metadata_dict(): not_a_dict = list() - MetadataAnalyzer.analyze(u'asdfasdf', not_a_dict) + MetadataAnalyzer.analyze('asdfasdf', not_a_dict) # Test an mp3 file where the number of channels is invalid or missing: def test_mp3_bad_channels(): - filename = u'tests/test_data/44100Hz-16bit-mono.mp3' - ''' + filename = 'tests/test_data/44100Hz-16bit-mono.mp3' + ''' It'd be a pain in the ass to construct a real MP3 with an invalid number of channels by hand because that value is stored in every MP3 frame in the file ''' @@ -158,8 +158,8 @@ def test_mp3_bad_channels(): assert metadata['bit_rate'] == 63998 assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mp3' # Not unicode because MIMEs aren't. - assert metadata['track_total'] == u'10' # MP3s can have a track_total + assert metadata['track_total'] == '10' # MP3s can have a track_total #Mutagen doesn't extract comments from mp3s it seems def test_unparsable_file(): - MetadataAnalyzer.analyze(u'README.rst', dict()) + MetadataAnalyzer.analyze('README.rst', dict()) diff --git a/python_apps/airtime_analyzer/tests/replaygain_analyzer_tests.py b/python_apps/airtime_analyzer/tests/replaygain_analyzer_tests.py index 0739e3126..cc988d652 100644 --- a/python_apps/airtime_analyzer/tests/replaygain_analyzer_tests.py +++ b/python_apps/airtime_analyzer/tests/replaygain_analyzer_tests.py @@ -2,20 +2,6 @@ from __future__ import print_function from nose.tools import * from airtime_analyzer.replaygain_analyzer import ReplayGainAnalyzer -''' -The tests in here were all tagged with the 'rgain' tag so the can be exluded from being run -with nosetest -a '!rgain'. This was needed due to the fact that it is not readily possible -to install replaygain on a containerized travis instance. - -We can either give running replaygain test on travis another shot after ubuntu getsan updated -gi instrospection allowing us to install gi and gobject into the virtualenv, or we can switch -to a full machine and stop using 'sudo: false' on travis. - -Deactivating these tests is a bad fix for now and I plan on looking into it again after -most everything else is up and running. For those interesed the tests seem to work locally -albeit my results not being up to the given tolerance of 0.30 (which I'm assuming is my rig's -problem and would work on travis if replaygain was available). -''' def check_default_metadata(metadata): ''' Check that the values extract by Silan/CuePointAnalyzer on our test audio files match what we expect. From 6213604341707764f7eaf5bd9e503bd51bf683ab Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Wed, 22 Jan 2020 12:10:37 +0200 Subject: [PATCH 16/38] test api_clients and pypo --- travis/python.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/travis/python.sh b/travis/python.sh index 5ad3b7a93..4f8c59674 100755 --- a/travis/python.sh +++ b/travis/python.sh @@ -4,8 +4,18 @@ set -xe [[ "$PYTHON" == false ]] && exit 0 -pushd python_apps/airtime_analyzer pyenv local 3.7 +pushd python_apps/airtime_analyzer +pip3 install -e . +nosetests +popd + +pushd python_apps/api_clients +pip3 install -e . +nosetests +popd + +pushd python_apps/pypo pip3 install -e . nosetests popd From e8a07831397224c38f0bdeaf0da7db94448efe8e Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Wed, 22 Jan 2020 12:14:40 +0200 Subject: [PATCH 17/38] Fix tests --- python_apps/api_clients/setup.py | 10 ------- python_apps/api_clients/tests/test_apcurl.py | 2 +- .../api_clients/tests/test_apirequest.py | 11 +++++--- .../api_clients/tests/test_requestprovider.py | 25 +++++++----------- python_apps/pypo/pypo/pypoliquidsoap.py | 2 +- python_apps/pypo/tests/run_tests.sh | 18 ------------- python_apps/pypo/tests/test_modify_cue_in.py | 26 ------------------- travis/python.sh | 5 ---- 8 files changed, 20 insertions(+), 79 deletions(-) delete mode 100755 python_apps/pypo/tests/run_tests.sh delete mode 100644 python_apps/pypo/tests/test_modify_cue_in.py diff --git a/python_apps/api_clients/setup.py b/python_apps/api_clients/setup.py index f9e69264d..ea9bad1f2 100644 --- a/python_apps/api_clients/setup.py +++ b/python_apps/api_clients/setup.py @@ -17,17 +17,7 @@ setup(name='api_clients', packages=['api_clients'], scripts=[], install_requires=[ -# 'amqplib', -# 'anyjson', -# 'argparse', 'configobj' -# 'docopt', -# 'kombu', -# 'mutagen', -# 'poster3', -# 'PyDispatcher', -# 'pyinotify', -# 'pytz', ], zip_safe=False, data_files=[]) diff --git a/python_apps/api_clients/tests/test_apcurl.py b/python_apps/api_clients/tests/test_apcurl.py index 1cad874eb..e3c8b29d5 100644 --- a/python_apps/api_clients/tests/test_apcurl.py +++ b/python_apps/api_clients/tests/test_apcurl.py @@ -1,5 +1,5 @@ import unittest -from .. api_client import ApcUrl, UrlBadParam, IncompleteUrl +from api_clients.api_client import ApcUrl, UrlBadParam, IncompleteUrl class TestApcUrl(unittest.TestCase): def test_init(self): diff --git a/python_apps/api_clients/tests/test_apirequest.py b/python_apps/api_clients/tests/test_apirequest.py index 55735672e..0fe13129a 100644 --- a/python_apps/api_clients/tests/test_apirequest.py +++ b/python_apps/api_clients/tests/test_apirequest.py @@ -1,7 +1,11 @@ import unittest import json from mock import MagicMock, patch -from .. api_client import ApcUrl, ApiRequest +from api_clients.api_client import ApcUrl, ApiRequest + +class ResponseInfo: + def getheader(self, name): + return 'application/json' class TestApiRequest(unittest.TestCase): def test_init(self): @@ -12,8 +16,9 @@ class TestApiRequest(unittest.TestCase): ret = json.dumps( {'ok':'ok'} ) read = MagicMock() read.read = MagicMock(return_value=ret) - u = '/testing' - with patch('urllib2.urlopen') as mock_method: + read.info = MagicMock(return_value=ResponseInfo()) + u = 'http://localhost/testing' + with patch('urllib.request.urlopen') as mock_method: mock_method.return_value = read request = ApiRequest('mm', ApcUrl(u))() self.assertEqual(request, json.loads(ret)) diff --git a/python_apps/api_clients/tests/test_requestprovider.py b/python_apps/api_clients/tests/test_requestprovider.py index 69f7294c6..c210aad85 100644 --- a/python_apps/api_clients/tests/test_requestprovider.py +++ b/python_apps/api_clients/tests/test_requestprovider.py @@ -2,13 +2,19 @@ import unittest import json from mock import patch, MagicMock from configobj import ConfigObj -from .. api_client import RequestProvider +from api_clients.api_client import RequestProvider, api_config class TestRequestProvider(unittest.TestCase): def setUp(self): - self.cfg = ConfigObj('api_client.cfg') + self.cfg = api_config + self.cfg['general'] = {} + self.cfg['general']['base_dir'] = '/test' + self.cfg['general']['base_port'] = 80 + self.cfg['general']['base_url'] = 'localhost' + self.cfg['general']['api_key'] = 'TEST_KEY' + self.cfg['api_base'] = 'api' def test_test(self): - self.assertTrue('api_key' in self.cfg) + self.assertTrue('general' in self.cfg) def test_init(self): rp = RequestProvider(self.cfg) self.assertTrue( len( rp.available_requests() ) > 0 ) @@ -16,17 +22,6 @@ class TestRequestProvider(unittest.TestCase): rp = RequestProvider(self.cfg) methods = ['upload_recorded', 'update_media_url', 'list_all_db_files'] for meth in methods: - self.assertTrue( meth in rp ) - - def test_notify_webstream_data(self): - ret = json.dumps( {'testing' : '123' } ) - rp = RequestProvider(self.cfg) - read = MagicMock() - read.read = MagicMock(return_value=ret) - with patch('urllib2.urlopen') as mock_method: - mock_method.return_value = read - response = rp.notify_webstream_data(media_id=123) - mock_method.called_once_with(media_id=123) - self.assertEqual(json.loads(ret), response) + self.assertTrue( meth in rp.requests ) if __name__ == '__main__': unittest.main() diff --git a/python_apps/pypo/pypo/pypoliquidsoap.py b/python_apps/pypo/pypo/pypoliquidsoap.py index 773879020..0c4399539 100644 --- a/python_apps/pypo/pypo/pypoliquidsoap.py +++ b/python_apps/pypo/pypo/pypoliquidsoap.py @@ -125,7 +125,7 @@ class PypoLiquidsoap(): scheduled_now_webstream = \ [x for x in scheduled_now if x["type"] == eventtypes.STREAM_OUTPUT_START] - schedule_ids = {x["row_id"] for x in scheduled_now_files]} + schedule_ids = [x["row_id"] for x in scheduled_now_files] row_id_map = {} liq_queue_ids = set() diff --git a/python_apps/pypo/tests/run_tests.sh b/python_apps/pypo/tests/run_tests.sh deleted file mode 100755 index 830a9bb85..000000000 --- a/python_apps/pypo/tests/run_tests.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -which py.test -pytest_exist=$? - -if [ "$pytest_exist" != "0" ]; then - echo "Need to have py.test installed. Exiting..." - exit 1 -fi - -SCRIPT=`readlink -f $0` -# Absolute directory this script is in -SCRIPTPATH=`dirname $SCRIPT` - -export PYTHONPATH=$PYTHONPATH:$SCRIPTPATH/..:$SCRIPTPATH/../.. - -py.test - diff --git a/python_apps/pypo/tests/test_modify_cue_in.py b/python_apps/pypo/tests/test_modify_cue_in.py deleted file mode 100644 index 11fd8ec73..000000000 --- a/python_apps/pypo/tests/test_modify_cue_in.py +++ /dev/null @@ -1,26 +0,0 @@ -from pypopush import PypoPush -from threading import Lock -from queue import Queue - -import datetime - -pypoPush_q = Queue() -telnet_lock = Lock() - -pp = PypoPush(pypoPush_q, telnet_lock) - -def test_modify_cue_in(): - link = pp.modify_first_link_cue_point([]) - assert len(link) == 0 - - min_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes = 1) - link = [{"start":min_ago.strftime("%Y-%m-%d-%H-%M-%S"), - "cue_in":"0", "cue_out":"30"}] - link = pp.modify_first_link_cue_point(link) - assert len(link) == 0 - - link = [{"start":min_ago.strftime("%Y-%m-%d-%H-%M-%S"), - "cue_in":"0", "cue_out":"70"}] - link = pp.modify_first_link_cue_point(link) - assert len(link) == 1 - diff --git a/travis/python.sh b/travis/python.sh index 4f8c59674..02ab72783 100755 --- a/travis/python.sh +++ b/travis/python.sh @@ -15,11 +15,6 @@ pip3 install -e . nosetests popd -pushd python_apps/pypo -pip3 install -e . -nosetests -popd - echo "Building docs..." mkdocs build --clean -q > /dev/null echo -n "done" From 5923dee83938fe0b0a8f7e959f1068a6a6eedf71 Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Wed, 22 Jan 2020 13:54:04 +0200 Subject: [PATCH 18/38] Fix running apps --- python_apps/airtime-celery/setup.py | 2 +- .../api_clients/api_clients/api_client.py | 2 +- .../api_clients/tests/test_apirequest.py | 2 +- python_apps/pypo/pypo/__main__.py | 53 ------------------- python_apps/pypo/pypo/recorder.py | 6 --- python_apps/pypo/setup.py | 1 - 6 files changed, 3 insertions(+), 63 deletions(-) diff --git a/python_apps/airtime-celery/setup.py b/python_apps/airtime-celery/setup.py index cd873dec0..79d382da7 100644 --- a/python_apps/airtime-celery/setup.py +++ b/python_apps/airtime-celery/setup.py @@ -45,7 +45,7 @@ setup( author_email="duncan.sommerville@sourcefabric.org", license="MIT", packages=["airtime-celery"], - install_requires=["soundcloud", "celery < 4", "kombu < 3.1", "configobj"], + install_requires=["soundcloud", "celery", "kombu", "configobj"], zip_safe=False, data_files=data_files, ) diff --git a/python_apps/api_clients/api_clients/api_client.py b/python_apps/api_clients/api_clients/api_client.py index 95ad844e9..911fa6179 100644 --- a/python_apps/api_clients/api_clients/api_client.py +++ b/python_apps/api_clients/api_clients/api_client.py @@ -122,7 +122,7 @@ class ApiRequest(object): try: req = urllib.request.Request(final_url, _post_data) f = urllib.request.urlopen(req, timeout=ApiRequest.API_HTTP_REQUEST_TIMEOUT) - content_type = f.info().getheader('Content-Type') + content_type = f.info().get_content_type() response = f.read() #Everything that calls an ApiRequest should be catching URLError explicitly #(according to the other comments in this file and a cursory grep through the code) diff --git a/python_apps/api_clients/tests/test_apirequest.py b/python_apps/api_clients/tests/test_apirequest.py index 0fe13129a..3ca7772e8 100644 --- a/python_apps/api_clients/tests/test_apirequest.py +++ b/python_apps/api_clients/tests/test_apirequest.py @@ -4,7 +4,7 @@ from mock import MagicMock, patch from api_clients.api_client import ApcUrl, ApiRequest class ResponseInfo: - def getheader(self, name): + def get_content_type(self): return 'application/json' class TestApiRequest(unittest.TestCase): diff --git a/python_apps/pypo/pypo/__main__.py b/python_apps/pypo/pypo/__main__.py index d9f3160a0..94ec277ef 100644 --- a/python_apps/pypo/pypo/__main__.py +++ b/python_apps/pypo/pypo/__main__.py @@ -123,59 +123,6 @@ except Exception as e: print("Couldn't configure logging: {}".format(e)) sys.exit(1) - -def configure_locale(): - """ - Silly hacks to force Python 2.x to run in UTF-8 mode. Not portable at all, - however serves our purpose at the moment. - - More information available here: - http://stackoverflow.com/questions/3828723/why-we-need-sys-setdefaultencodingutf-8-in-a-py-script - """ - logger.debug("Before %s", locale.nl_langinfo(locale.CODESET)) - current_locale = locale.getlocale() - - if current_locale[1] is None: - logger.debug("No locale currently set. Attempting to get default locale.") - default_locale = locale.getdefaultlocale() - - if default_locale[1] is None: - logger.debug( - "No default locale exists. Let's try loading from \ - /etc/default/locale" - ) - if os.path.exists("/etc/default/locale"): - locale_config = ConfigObj("/etc/default/locale") - lang = locale_config.get("LANG") - new_locale = lang - else: - logger.error( - "/etc/default/locale could not be found! Please \ - run 'sudo update-locale' from command-line." - ) - sys.exit(1) - else: - new_locale = default_locale - - logger.info( - "New locale set to: %s", locale.setlocale(locale.LC_ALL, new_locale) - ) - - importlib.reload(sys) - sys.setdefaultencoding("UTF-8") - current_locale_encoding = locale.getlocale()[1].lower() - logger.debug("sys default encoding %s", sys.getdefaultencoding()) - logger.debug("After %s", locale.nl_langinfo(locale.CODESET)) - - if current_locale_encoding not in ["utf-8", "utf8"]: - logger.error( - "Need a UTF-8 locale. Currently '%s'. Exiting..." % current_locale_encoding - ) - sys.exit(1) - - -configure_locale() - # loading config file try: config = ConfigObj("/etc/airtime/airtime.conf") diff --git a/python_apps/pypo/pypo/recorder.py b/python_apps/pypo/pypo/recorder.py index 6ed3f301d..ce2d3a4e0 100644 --- a/python_apps/pypo/pypo/recorder.py +++ b/python_apps/pypo/pypo/recorder.py @@ -15,9 +15,6 @@ import re from configobj import ConfigObj -from poster.encode import multipart_encode -from poster.streaminghttp import register_openers - from subprocess import Popen from subprocess import PIPE from threading import Thread @@ -127,9 +124,6 @@ class ShowRecorder(Thread): filename = os.path.split(filepath)[1] - # Register the streaming http handlers with urllib2 - register_openers() - # files is what requests actually expects files = {'file': open(filepath, "rb"), 'name': filename, 'show_instance': self.show_instance} diff --git a/python_apps/pypo/setup.py b/python_apps/pypo/setup.py index 5eea754ca..649ef331c 100644 --- a/python_apps/pypo/setup.py +++ b/python_apps/pypo/setup.py @@ -55,7 +55,6 @@ setup(name='airtime-playout', 'future', 'kombu', 'mutagen', - 'poster3', 'PyDispatcher', 'pyinotify', 'pytz', From 3a8b1e207f101cda03bf1dd33078edc5d625e2c5 Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Wed, 22 Jan 2020 15:15:22 +0200 Subject: [PATCH 19/38] deprecate media_monitor completely --- .../airtime_analyzer/bin/airtime_analyzer | 28 ++----------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/python_apps/airtime_analyzer/bin/airtime_analyzer b/python_apps/airtime_analyzer/bin/airtime_analyzer index 8dfb99341..7b8ca69bb 100755 --- a/python_apps/airtime_analyzer/bin/airtime_analyzer +++ b/python_apps/airtime_analyzer/bin/airtime_analyzer @@ -23,8 +23,6 @@ def run(): parser.add_argument("--http-retry-queue-file", help="specify where incompleted HTTP requests will be serialized (default is %s)" % DEFAULT_HTTP_RETRY_PATH) args = parser.parse_args() - check_if_media_monitor_is_running() - #Default config file path rmq_config_path = DEFAULT_RMQ_CONFIG_PATH http_retry_queue_path = DEFAULT_HTTP_RETRY_PATH @@ -36,32 +34,12 @@ def run(): if args.daemon: with daemon.DaemonContext(): aa.AirtimeAnalyzerServer(rmq_config_path=rmq_config_path, - http_retry_queue_path=http_retry_queue_path, + http_retry_queue_path=http_retry_queue_path, debug=args.debug) - else: + else: # Run without daemonizing aa.AirtimeAnalyzerServer(rmq_config_path=rmq_config_path, - http_retry_queue_path=http_retry_queue_path, + http_retry_queue_path=http_retry_queue_path, debug=args.debug) - -def check_if_media_monitor_is_running(): - """Ensure media_monitor isn't running before we start. - - We do this because media_monitor will move newly uploaded - files into the library on us and screw up the operation of airtime_analyzer. - media_monitor is deprecated. - """ - pids = [pid for pid in os.listdir('/proc') if pid.isdigit()] - - for pid in pids: - try: - process_name = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read() - if 'media_monitor.py' in process_name: - print("Error: This process conflicts with media_monitor, and media_monitor is running.") - print(" Please terminate the running media_monitor.py process and try again.") - exit(1) - except IOError: # proc has already terminated - continue - run() From 44a0cb50e1a2af355c1912e5cf6ddfd3a7bc16ba Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Wed, 22 Jan 2020 15:18:13 +0200 Subject: [PATCH 20/38] Follow pip best practices Let apt manage system pip3 version --- docs/scripts/install.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/scripts/install.sh b/docs/scripts/install.sh index 34bbb55fb..20adae162 100755 --- a/docs/scripts/install.sh +++ b/docs/scripts/install.sh @@ -4,7 +4,5 @@ echo "Updating Apt." apt-get update > /dev/null echo "Ensuring Pip is installed." DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3-pip > /dev/null -echo "Updating Pip." -pip3 install pip -q -q --upgrade > /dev/null echo "Ensuring Mkdocs is installed." pip3 install mkdocs From 6ebb1fd5558eb6388cee243ed1ef312754f8f1fb Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Thu, 23 Jan 2020 12:37:49 +0200 Subject: [PATCH 21/38] more pypo fixes --- installer/lib/requirements-debian-buster.apt | 2 + installer/lib/requirements-debian-stretch.apt | 2 + installer/lib/requirements-ubuntu-bionic.apt | 2 + installer/lib/requirements-ubuntu-xenial.apt | 2 + ...uirements-ubuntu-xenial_docker_minimal.apt | 2 + .../api_clients/api_clients/api_client.py | 43 +++++----- python_apps/pypo/liquidsoap/__main__.py | 11 ++- .../liquidsoap/generate_liquidsoap_cfg.py | 8 +- python_apps/pypo/pypo/__main__.py | 8 +- python_apps/pypo/pypo/pure.py | 12 ++- python_apps/pypo/pypo/pypofetch.py | 45 ++++------ python_apps/pypo/pypo/pypoliquidsoap.py | 41 +++++----- python_apps/pypo/pypo/telnetliquidsoap.py | 82 +++++++++---------- python_apps/pypo/setup.py | 3 +- 14 files changed, 132 insertions(+), 131 deletions(-) diff --git a/installer/lib/requirements-debian-buster.apt b/installer/lib/requirements-debian-buster.apt index 3dc1121c3..7438d1e0f 100644 --- a/installer/lib/requirements-debian-buster.apt +++ b/installer/lib/requirements-debian-buster.apt @@ -4,6 +4,8 @@ curl ecasound flac git +gstreamer1.0-plugins-bad +gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly icecast2 lame diff --git a/installer/lib/requirements-debian-stretch.apt b/installer/lib/requirements-debian-stretch.apt index ace712897..5dcf1e199 100644 --- a/installer/lib/requirements-debian-stretch.apt +++ b/installer/lib/requirements-debian-stretch.apt @@ -4,6 +4,8 @@ curl ecasound flac git +gstreamer1.0-plugins-bad +gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly icecast2 lame diff --git a/installer/lib/requirements-ubuntu-bionic.apt b/installer/lib/requirements-ubuntu-bionic.apt index d45f05564..0254b3d3c 100644 --- a/installer/lib/requirements-ubuntu-bionic.apt +++ b/installer/lib/requirements-ubuntu-bionic.apt @@ -4,6 +4,8 @@ coreutils curl ecasound flac +gstreamer1.0-plugins-bad +gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly icecast2 lame diff --git a/installer/lib/requirements-ubuntu-xenial.apt b/installer/lib/requirements-ubuntu-xenial.apt index 275cc0200..df64d2fd2 100644 --- a/installer/lib/requirements-ubuntu-xenial.apt +++ b/installer/lib/requirements-ubuntu-xenial.apt @@ -4,6 +4,8 @@ coreutils curl ecasound flac +gstreamer1.0-plugins-bad +gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly icecast2 lame diff --git a/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt b/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt index 41257a70e..549db5c9d 100644 --- a/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt +++ b/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt @@ -4,6 +4,8 @@ coreutils curl ecasound flac +gstreamer1.0-plugins-bad +gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly lame libao-ocaml diff --git a/python_apps/api_clients/api_clients/api_client.py b/python_apps/api_clients/api_clients/api_client.py index 911fa6179..fb9cad33e 100644 --- a/python_apps/api_clients/api_clients/api_client.py +++ b/python_apps/api_clients/api_clients/api_client.py @@ -117,7 +117,8 @@ class ApiRequest(object): def __call__(self,_post_data=None, **kwargs): final_url = self.url.params(**kwargs).url() - if _post_data is not None: _post_data = urllib.parse.urlencode(_post_data) + if _post_data is not None: + _post_data = urllib.parse.urlencode(_post_data).encode('utf-8') self.logger.debug(final_url) try: req = urllib.request.Request(final_url, _post_data) @@ -131,8 +132,7 @@ class ApiRequest(object): self.logger.error('HTTP request to %s timed out', final_url) raise except Exception as e: - #self.logger.error('Exception: %s', e) - #self.logger.error("traceback: %s", traceback.format_exc()) + #self.logger.exception(e) raise try: @@ -142,8 +142,7 @@ class ApiRequest(object): else: raise InvalidContentType() except Exception: - #self.logger.error(response) - #self.logger.error("traceback: %s", traceback.format_exc()) + #self.logger.exception(e) raise def req(self, *args, **kwargs): @@ -182,8 +181,10 @@ class RequestProvider(object): def __contains__(self, request) : return request in self.requests def __getattr__(self, attr): - if attr in self: return self.requests[attr] - else: return super(RequestProvider, self).__getattribute__(attr) + if attr in self: + return self.requests[attr] + else: + return super(RequestProvider, self).__getattribute__(attr) class AirtimeApiClient(object): @@ -197,8 +198,7 @@ class AirtimeApiClient(object): self.config.update(api_config) self.services = RequestProvider(self.config) except Exception as e: - self.logger.error('Error loading config file: %s', config_path) - self.logger.error("traceback: %s", traceback.format_exc()) + self.logger.exception('Error loading config file: %s', config_path) sys.exit(1) def __get_airtime_version(self): @@ -239,7 +239,7 @@ class AirtimeApiClient(object): try: self.services.notify_liquidsoap_started() except Exception as e: - self.logger.error(str(e)) + self.logger.exception(e) def notify_media_item_start_playing(self, media_id): """ This is a callback from liquidsoap, we use this to notify @@ -248,14 +248,14 @@ class AirtimeApiClient(object): try: return self.services.update_start_playing_url(media_id=media_id) except Exception as e: - self.logger.error(str(e)) + self.logger.exception(e) return None def get_shows_to_record(self): try: return self.services.show_schedule_url() except Exception as e: - self.logger.error(str(e)) + self.logger.exception(e) return None def upload_recorded_show(self, files, show_id): @@ -307,8 +307,7 @@ class AirtimeApiClient(object): logger.error("Server is down: %s", e.args) logger.error("traceback: %s", traceback.format_exc()) except Exception as e: - logger.error("Exception: %s", e) - logger.error("traceback: %s", traceback.format_exc()) + self.logger.exception(e) #wait some time before next retry time.sleep(retries_wait) @@ -320,7 +319,7 @@ class AirtimeApiClient(object): return self.services.check_live_stream_auth( username=username, password=password, djtype=dj_type) except Exception as e: - self.logger.error(str(e)) + self.logger.exception(e) return {} def construct_url(self,config_action_key): @@ -468,17 +467,14 @@ class AirtimeApiClient(object): stream_id=stream_id, boot_time=time).retry(5) except Exception as e: - #TODO - logger.error("Exception: %s", e) + self.logger.exception(e) def notify_source_status(self, sourcename, status): try: - logger = self.logger return self.services.update_source_status.req(sourcename=sourcename, status=status).retry(5) except Exception as e: - #TODO - logger.error("Exception: %s", e) + self.logger.exception(e) def get_bootstrap_info(self): """ Retrieve infomations needed on bootstrap time """ @@ -494,7 +490,7 @@ class AirtimeApiClient(object): try: return self.services.get_files_without_replay_gain(dir_id=dir_id) except Exception as e: - self.logger.error(str(e)) + self.logger.exception(e) return [] def get_files_without_silan_value(self): @@ -506,7 +502,7 @@ class AirtimeApiClient(object): try: return self.services.get_files_without_silan_value() except Exception as e: - self.logger.error(str(e)) + self.logger.exception(e) return [] def update_replay_gain_values(self, pairs): @@ -549,8 +545,7 @@ class AirtimeApiClient(object): response = self.services.update_stream_setting_table(_post_data={'data': json.dumps(data)}) return response except Exception as e: - #TODO - self.logger.error(str(e)) + self.logger.exception(e) def update_metadata_on_tunein(self): self.services.update_metadata_on_tunein() diff --git a/python_apps/pypo/liquidsoap/__main__.py b/python_apps/pypo/liquidsoap/__main__.py index d7ac1bbb6..4be598d1c 100644 --- a/python_apps/pypo/liquidsoap/__main__.py +++ b/python_apps/pypo/liquidsoap/__main__.py @@ -1,12 +1,11 @@ """ Runs Airtime liquidsoap """ - - import argparse import os from . import generate_liquidsoap_cfg import logging import subprocess +from pypo import pure PYPO_HOME = '/var/tmp/airtime/pypo/' @@ -16,16 +15,16 @@ def run(): parser = argparse.ArgumentParser() parser.add_argument("-d", "--debug", help="run in debug mode", action="store_true") args = parser.parse_args() - + os.environ["HOME"] = PYPO_HOME if args.debug: logging.basicConfig(level=getattr(logging, 'DEBUG', None)) - + generate_liquidsoap_cfg.run() ''' check liquidsoap version if less than 1.3 use legacy liquidsoap script ''' - liquidsoap_version=subprocess.check_output("liquidsoap --version", shell=True) - if "1.1.1" not in liquidsoap_version: + liquidsoap_version = subprocess.check_output("liquidsoap --version", shell=True, text=True) + if pure.version_cmp(liquidsoap_version, "1.3") < 0: script_path = os.path.join(os.path.dirname(__file__), 'ls_script.liq') else: script_path = os.path.join(os.path.dirname(__file__), 'ls_script_legacy.liq') diff --git a/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py b/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py index d35e2333f..27bc927f2 100644 --- a/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py +++ b/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py @@ -28,21 +28,21 @@ def generate_liquidsoap_config(ss): except: #Everything else is a string str_buffer = "%s = \"%s\"\n" % (key, value) - fh.write(str_buffer.encode('utf-8')) + fh.write(str_buffer) # ignore squashes unused variable errors from Liquidsoap - fh.write(("ignore(%s)\n" % key).encode('utf-8')) + fh.write("ignore(%s)\n" % key) auth_path = os.path.dirname(os.path.realpath(__file__)) fh.write('log_file = "/var/log/airtime/pypo-liquidsoap/