diff --git a/.travis.yml b/.travis.yml index e9cd45305..7f863b7b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: trusty +dist: xenial version: ~> 1.0 language: php php: @@ -10,8 +10,6 @@ php: - 7.0 # folks who prefer running on 5.x should be using 5.6 in most cases, 5.4 is no in the matrix since noone should use it - 5.6 -# this is in for centos support, it's still the default on CentOS 7.3 and there were some lang changes after 5.4 -- 5.4 services: - postgresql - rabbitmq @@ -49,6 +47,13 @@ addons: apt: packages: - silan + - libgirepository1.0-dev + - gir1.2-gstreamer-1.0 + - gstreamer1.0-plugins-base + - gstreamer1.0-plugins-good + - gstreamer1.0-plugins-bad + - gstreamer1.0-plugins-ugly + - libcairo2-dev - liquidsoap - liquidsoap-plugin-mad - liquidsoap-plugin-taglib @@ -58,10 +63,10 @@ addons: - liquidsoap-plugin-faad - liquidsoap-plugin-vorbis - liquidsoap-plugin-opus - - python-nose - - python-rgain - - python-gst-1.0 - - python-magic + - python3 + - python3-nose + - python3-gst-1.0 + - python3-magic - dos2unix install: - > @@ -70,9 +75,11 @@ install: fi - > if [[ "$PYTHON" == true ]]; then - pip install --user mkdocs - pushd python_apps/airtime_analyzer - python setup.py install --dry-run --no-init-script + pyenv local 3.7 + pip3 install -U pip wheel + pip3 install --user mkdocs rgain3 + pushd python_apps/airtime_analyzer + python3 setup.py install --dry-run --no-init-script popd fi before_script: diff --git a/docs/scripts/install.sh b/docs/scripts/install.sh index d7fb756f3..20adae162 100755 --- a/docs/scripts/install.sh +++ b/docs/scripts/install.sh @@ -3,8 +3,6 @@ echo "Updating Apt." apt-get update > /dev/null echo "Ensuring Pip is installed." -DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python-pip > /dev/null -echo "Updating Pip." -pip install pip -q -q --upgrade > /dev/null +DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3-pip > /dev/null echo "Ensuring Mkdocs is installed." -pip install -q mkdocs > /dev/null +pip3 install mkdocs diff --git a/install b/install index 4d4a629d5..bbbe94860 100755 --- a/install +++ b/install @@ -465,7 +465,7 @@ while :; do ;; --no-rabbitmq) skip_rabbitmq=1 - ;; + ;; --) shift break @@ -923,22 +923,12 @@ loud "\n-----------------------------------------------------" loud " * Installing Airtime Services * " loud "-----------------------------------------------------" -verbose "\n * Installing necessary python services..." -loudCmd "pip install setuptools --upgrade" -loudCmd "pip install zipp==1.0.0" -verbose "...Done" - -# Ubuntu Trusty and Debian Wheezy needs a workaround for python version SSL downloads -# This affects all python installs where python < 2.7.9 -python_version=$(python --version 2>&1 | awk '{ print $2 }') +python_version=$(python3 --version 2>&1 | awk '{ print $2 }') verbose "Detected Python version: $python_version" -# Convert version so each segment is zero padded for easy comparison -python_version_formatted=$(awk 'BEGIN {FS = "."} {printf "%03d.%03d.%03d\n", $1,$2,$3}' <<< $python_version) -if [[ "$python_version_formatted" < "002.007.009" ]]; then - verbose "\n * Installing pyOpenSSL and ca db for SNI support..." - loudCmd "pip install pyOpenSSL cryptography idna certifi --upgrade" - verbose "...Done" -fi + +verbose "\n * Installing necessary python services..." +loudCmd "pip3 install setuptools --upgrade" +verbose "...Done" verbose "\n * Creating /run/airtime..." mkdir -p /run/airtime @@ -960,11 +950,11 @@ if [ ! -d /var/log/airtime ]; then fi verbose "\n * Installing API client..." -loudCmd "python ${AIRTIMEROOT}/python_apps/api_clients/setup.py install --install-scripts=/usr/bin" +loudCmd "python3 ${AIRTIMEROOT}/python_apps/api_clients/setup.py install --install-scripts=/usr/bin" verbose "...Done" verbose "\n * Installing pypo and liquidsoap..." -loudCmd "python ${AIRTIMEROOT}/python_apps/pypo/setup.py install --install-scripts=/usr/bin --no-init-script" +loudCmd "python3 ${AIRTIMEROOT}/python_apps/pypo/setup.py install --install-scripts=/usr/bin --no-init-script" loudCmd "mkdir -p /var/log/airtime/{pypo,pypo-liquidsoap} /var/tmp/airtime/pypo/{cache,files,tmp} /var/tmp/airtime/show-recorder/" loudCmd "chown -R ${web_user}:${web_user} /var/log/airtime/{pypo,pypo-liquidsoap} /var/tmp/airtime/pypo/{cache,files,tmp} /var/tmp/airtime/show-recorder/" systemInitInstall airtime-liquidsoap $web_user @@ -972,7 +962,7 @@ systemInitInstall airtime-playout $web_user verbose "...Done" verbose "\n * Installing airtime-celery..." -loudCmd "python ${AIRTIMEROOT}/python_apps/airtime-celery/setup.py install --no-init-script" +loudCmd "python3 ${AIRTIMEROOT}/python_apps/airtime-celery/setup.py install --no-init-script" # Create the Celery user if $is_centos_dist; then loudCmd "id celery 2>/dev/null || adduser --no-create-home -c 'LibreTime Celery' -r celery || true" @@ -988,7 +978,7 @@ systemInitInstall airtime-celery verbose "...Done" verbose "\n * Installing airtime_analyzer..." -loudCmd "python ${AIRTIMEROOT}/python_apps/airtime_analyzer/setup.py install --install-scripts=/usr/bin --no-init-script" +loudCmd "python3 ${AIRTIMEROOT}/python_apps/airtime_analyzer/setup.py install --install-scripts=/usr/bin --no-init-script" systemInitInstall airtime_analyzer $web_user verbose "...Done" diff --git a/installer/lib/requirements-debian-buster.apt b/installer/lib/requirements-debian-buster.apt index aceab8a25..7438d1e0f 100644 --- a/installer/lib/requirements-debian-buster.apt +++ b/installer/lib/requirements-debian-buster.apt @@ -1,71 +1,59 @@ apache2 +coreutils +curl +ecasound +flac git +gstreamer1.0-plugins-bad +gstreamer1.0-plugins-good +gstreamer1.0-plugins-ugly +icecast2 +lame +libao-ocaml libapache2-mod-php7.3 -php7.3 -php7.3-dev -php7.3-bcmath -php7.3-mbstring -php-pear -php7.3-gd -php-amqplib - -lsb-release - -zip -unzip - -rabbitmq-server - -postgresql -postgresql-client -php7.3-pgsql - -python -python-virtualenv -python-pip - +libcairo2-dev +libcamomile-ocaml-data +libfaad2 +libmad-ocaml +libopus0 +libportaudio2 +libpulse0 +libsamplerate0 libsoundtouch-ocaml libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 libvo-aacenc0 - -python-rgain -python-gst-1.0 -gstreamer1.0-plugins-ugly -python-pika - -patch - -icecast2 - -curl -php7.3-curl -mpg123 - -libcamomile-ocaml-data -libpulse0 -vorbis-tools +liquidsoap +lsb-release lsb-release lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 +mpg123 +patch +php7.3 +php7.3-bcmath +php7.3-curl +php7.3-dev +php7.3-gd +php7.3-mbstring +php7.3-pgsql +php-amqplib php-apcu - -lame +php-pear +pkg-config +postgresql +postgresql-client +pwgen +python3 +python3-gst-1.0 +python3-pika +python3-pip +python3-virtualenv +python3-cairo +rabbitmq-server silan -coreutils - -liquidsoap - -libopus0 - systemd-sysv - +unzip +vorbisgain +vorbis-tools +vorbis-tools xmlstarlet +zip diff --git a/installer/lib/requirements-debian-jessie.apt b/installer/lib/requirements-debian-jessie.apt deleted file mode 100644 index 4e22102cc..000000000 --- a/installer/lib/requirements-debian-jessie.apt +++ /dev/null @@ -1,67 +0,0 @@ -apache2 -libapache2-mod-php5 -php5 -php-pear -php5-gd - -lsb-release - -rabbitmq-server - -zip -unzip - -postgresql -postgresql-client -php5-pgsql - -python -python-virtualenv -python-pip - -libsoundtouch-ocaml -libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 -libvo-aacenc0 - -python-rgain -python-gst-1.0 -gstreamer1.0-plugins-ugly -python-pika - -patch - -icecast2 - -curl -php5-curl -mpg123 - -libcamomile-ocaml-data -libpulse0 -vorbis-tools -lsb-release -lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 -php-apc - -lame - -coreutils - -liquidsoap - -libopus0 - -sysvinit -sysvinit-utils - -xmlstarlet diff --git a/installer/lib/requirements-debian-stretch.apt b/installer/lib/requirements-debian-stretch.apt index c71175f6a..5dcf1e199 100644 --- a/installer/lib/requirements-debian-stretch.apt +++ b/installer/lib/requirements-debian-stretch.apt @@ -1,71 +1,56 @@ apache2 +coreutils +curl +ecasound +flac git +gstreamer1.0-plugins-bad +gstreamer1.0-plugins-good +gstreamer1.0-plugins-ugly +icecast2 +lame +libao-ocaml libapache2-mod-php7.0 -php7.0 -php7.0-dev -php7.0-bcmath -php7.0-mbstring -php-pear -php7.0-gd -php-amqplib - -lsb-release - -zip -unzip - -rabbitmq-server - -postgresql -postgresql-client -php7.0-pgsql - -python -python-virtualenv -python-pip - +libcamomile-ocaml-data +libfaad2 +libmad-ocaml +libopus0 +libportaudio2 +libpulse0 +libsamplerate0 libsoundtouch-ocaml libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 libvo-aacenc0 - -python-rgain -python-gst-1.0 -gstreamer1.0-plugins-ugly -python-pika - -patch - -icecast2 - -curl -php7.0-curl -mpg123 - -libcamomile-ocaml-data -libpulse0 -vorbis-tools +liquidsoap +lsb-release lsb-release lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 +mpg123 +patch +php7.0 +php7.0-bcmath +php7.0-curl +php7.0-dev +php7.0-gd +php7.0-mbstring +php7.0-pgsql +php-amqplib php-apcu - -lame - -coreutils - -liquidsoap - -libopus0 - +php-pear +postgresql +postgresql-client +pwgen +python3 +python3-gst-1.0 +python3-pika +python3-pip +python3-virtualenv +python3-cairo +rabbitmq-server systemd-sysv - +unzip +vorbisgain +vorbis-tools +vorbis-tools xmlstarlet +zip diff --git a/installer/lib/requirements-ubuntu-bionic.apt b/installer/lib/requirements-ubuntu-bionic.apt index 58be7819f..0254b3d3c 100644 --- a/installer/lib/requirements-ubuntu-bionic.apt +++ b/installer/lib/requirements-ubuntu-bionic.apt @@ -1,62 +1,27 @@ apache2 -libapache2-mod-php7.2 -php7.2 -php-pear -php7.2-gd -php-bcmath -php-mbstring - -lsb-release - -zip -unzip - -rabbitmq-server - -postgresql -postgresql-client -php7.2-pgsql - -python -python-virtualenv -python-pip - -libsoundtouch-ocaml -libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 - -python-rgain -python-gst-1.0 -gstreamer1.0-plugins-ugly -python-pika - -patch - -php7.2-curl -mpg123 -curl - -icecast2 - -libcamomile-ocaml-data -libpulse0 -vorbis-tools -lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 -php-apcu - -lame - +build-essential coreutils - +curl +ecasound +flac +gstreamer1.0-plugins-bad +gstreamer1.0-plugins-good +gstreamer1.0-plugins-ugly +icecast2 +lame +libao-ocaml +libapache2-mod-php7.2 +libcamomile-ocaml-data +libfaad2 +libffi-dev +libmad-ocaml +libopus0 +libportaudio2 +libpulse0 +libsamplerate0 +libsoundtouch-ocaml +libssl-dev +libtaglib-ocaml liquidsoap liquidsoap-plugin-alsa liquidsoap-plugin-ao @@ -71,15 +36,34 @@ liquidsoap-plugin-pulseaudio liquidsoap-plugin-taglib liquidsoap-plugin-voaacenc liquidsoap-plugin-vorbis - +lsb-release +lsof +mpg123 +patch +php7.2 +php7.2-curl +php7.2-gd +php7.2-pgsql +php-apcu +php-bcmath +php-mbstring +php-pear +postgresql +postgresql-client +pwgen +python3 +python3-dev +python3-gst-1.0 +python3-pika +python3-pip +python3-virtualenv +python3-cairo +rabbitmq-server silan -libopus0 - sysvinit-utils - -build-essential -libssl-dev -libffi-dev -python-dev - +unzip +vorbisgain +vorbis-tools +vorbis-tools xmlstarlet +zip diff --git a/installer/lib/requirements-ubuntu-precise.apt b/installer/lib/requirements-ubuntu-precise.apt deleted file mode 100644 index ed31b628f..000000000 --- a/installer/lib/requirements-ubuntu-precise.apt +++ /dev/null @@ -1,74 +0,0 @@ -apache2 -libapache2-mod-php5 -php5 -php-pear -php5-gd - -lsb-release - -zip -unzip - -rabbitmq-server - -postgresql -postgresql-client -php5-pgsql - -python -python-virtualenv -python-pip - -libsoundtouch-ocaml -libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 - -python-rgain -python-gst0.10 -gstreamer0.10-plugins-ugly -gir1.2-gstreamer-0.10 -patch - -curl -php5-curl -mpg123 - -icecast2 - -libcamomile-ocaml-data -libpulse0 -vorbis-tools -lsb-release -lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 -php-apc -dbus - -lame - -coreutils - -liquidsoap -liquidsoap-plugin-alsa -liquidsoap-plugin-ao -liquidsoap-plugin-faad -liquidsoap-plugin-flac -liquidsoap-plugin-icecast -liquidsoap-plugin-lame -liquidsoap-plugin-mad -liquidsoap-plugin-ogg -liquidsoap-plugin-portaudio -liquidsoap-plugin-pulseaudio -liquidsoap-plugin-taglib -liquidsoap-plugin-voaacenc -liquidsoap-plugin-vorbis - -xmlstarlet diff --git a/installer/lib/requirements-ubuntu-xenial.apt b/installer/lib/requirements-ubuntu-xenial.apt index 41381915f..df64d2fd2 100644 --- a/installer/lib/requirements-ubuntu-xenial.apt +++ b/installer/lib/requirements-ubuntu-xenial.apt @@ -1,62 +1,27 @@ apache2 -libapache2-mod-php7.0 -php7.0 -php-pear -php7.0-gd -php-bcmath -php-mbstring - -lsb-release - -zip -unzip - -rabbitmq-server - -postgresql -postgresql-client -php7.0-pgsql - -python -python-virtualenv -python-pip - -libsoundtouch-ocaml -libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 - -python-rgain -python-gst-1.0 -gstreamer1.0-plugins-ugly -python-pika - -patch - -php7.0-curl -mpg123 -curl - -icecast2 - -libcamomile-ocaml-data -libpulse0 -vorbis-tools -lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 -php-apcu - -lame - +build-essential coreutils - +curl +ecasound +flac +gstreamer1.0-plugins-bad +gstreamer1.0-plugins-good +gstreamer1.0-plugins-ugly +icecast2 +lame +libao-ocaml +libapache2-mod-php7.0 +libcamomile-ocaml-data +libfaad2 +libffi-dev +libmad-ocaml +libopus0 +libportaudio2 +libpulse0 +libsamplerate0 +libsoundtouch-ocaml +libssl-dev +libtaglib-ocaml liquidsoap liquidsoap-plugin-alsa liquidsoap-plugin-ao @@ -71,15 +36,34 @@ liquidsoap-plugin-pulseaudio liquidsoap-plugin-taglib liquidsoap-plugin-voaacenc liquidsoap-plugin-vorbis - +lsb-release +lsof +mpg123 +patch +php7.0 +php7.0-curl +php7.0-gd +php7.0-pgsql +php-apcu +php-bcmath +php-mbstring +php-pear +postgresql +postgresql-client +pwgen +python3 +python3-dev +python3-gst-1.0 +python3-pika +python3-pip +python3-virtualenv +python3-cairo +rabbitmq-server silan -libopus0 - sysvinit-utils - -build-essential -libssl-dev -libffi-dev -python-dev - +unzip +vorbisgain +vorbis-tools +vorbis-tools xmlstarlet +zip diff --git a/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt b/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt index 09c94f817..549db5c9d 100644 --- a/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt +++ b/installer/lib/requirements-ubuntu-xenial_docker_minimal.apt @@ -1,57 +1,26 @@ apache2 -libapache2-mod-php7.0 -php7.0 -php-pear -php7.0-gd -php-bcmath -php-mbstring - -lsb-release - -zip -unzip - -postgresql-client -php7.0-pgsql - -python -python-virtualenv -python-pip - -libsoundtouch-ocaml -libtaglib-ocaml -libao-ocaml -libmad-ocaml -ecasound -libportaudio2 -libsamplerate0 - -python-rgain -python-gst-1.0 -gstreamer1.0-plugins-ugly -python-pika - -patch - -php7.0-curl -mpg123 -curl - -libcamomile-ocaml-data -libpulse0 -vorbis-tools -lsof -vorbisgain -flac -vorbis-tools -pwgen -libfaad2 -php-apcu - -lame - +build-essential coreutils - +curl +ecasound +flac +gstreamer1.0-plugins-bad +gstreamer1.0-plugins-good +gstreamer1.0-plugins-ugly +lame +libao-ocaml +libapache2-mod-php7.0 +libcamomile-ocaml-data +libfaad2 +libffi-dev +libmad-ocaml +libopus0 +libportaudio2 +libpulse0 +libsamplerate0 +libsoundtouch-ocaml +libssl-dev +libtaglib-ocaml liquidsoap liquidsoap-plugin-alsa liquidsoap-plugin-ao @@ -66,15 +35,32 @@ liquidsoap-plugin-pulseaudio liquidsoap-plugin-taglib liquidsoap-plugin-voaacenc liquidsoap-plugin-vorbis - +lsb-release +lsof +mpg123 +patch +php7.0 +php7.0-curl +php7.0-gd +php7.0-pgsql +php-apcu +php-bcmath +php-mbstring +php-pear +postgresql-client +pwgen +python3 +python3-dev +python3-gst-1.0 +python3-pika +python3-pip +python3-virtualenv +python3-cairo silan -libopus0 - sysvinit-utils - -build-essential -libssl-dev -libffi-dev -python-dev - +unzip +vorbisgain +vorbis-tools +vorbis-tools xmlstarlet +zip diff --git a/python_apps/airtime-celery/airtime-celery/tasks.py b/python_apps/airtime-celery/airtime-celery/tasks.py index 2ef843b89..097d072ec 100644 --- a/python_apps/airtime-celery/airtime-celery/tasks.py +++ b/python_apps/airtime-celery/airtime-celery/tasks.py @@ -86,8 +86,12 @@ def soundcloud_download(token, callback_url, api_key, track_id): auth=requests.auth.HTTPBasicAuth(api_key, ""), ) re.raise_for_status() + try: + response = re.content.decode() + except (UnicodeDecodeError, AttributeError): + response = re.content f = json.loads( - re.content + response ) # Read the response from the media API to get the file id obj["fileid"] = f["id"] else: @@ -203,8 +207,12 @@ def podcast_download( auth=requests.auth.HTTPBasicAuth(api_key, ""), ) re.raise_for_status() + try: + response = re.content.decode() + except (UnicodeDecodeError, AttributeError): + response = re.content f = json.loads( - re.content + response ) # Read the response from the media API to get the file id obj["fileid"] = f["id"] obj["status"] = 1 diff --git a/python_apps/airtime-celery/setup.py b/python_apps/airtime-celery/setup.py index cd873dec0..79d382da7 100644 --- a/python_apps/airtime-celery/setup.py +++ b/python_apps/airtime-celery/setup.py @@ -45,7 +45,7 @@ setup( author_email="duncan.sommerville@sourcefabric.org", license="MIT", packages=["airtime-celery"], - install_requires=["soundcloud", "celery < 4", "kombu < 3.1", "configobj"], + install_requires=["soundcloud", "celery", "kombu", "configobj"], zip_safe=False, data_files=data_files, ) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/airtime_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/airtime_analyzer.py index eafa8f743..325b2df3d 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/airtime_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/airtime_analyzer.py @@ -5,12 +5,12 @@ import logging.handlers import sys import signal import traceback -import config_file +from . import config_file from functools import partial -from metadata_analyzer import MetadataAnalyzer -from replaygain_analyzer import ReplayGainAnalyzer -from status_reporter import StatusReporter -from message_listener import MessageListener +from .metadata_analyzer import MetadataAnalyzer +from .replaygain_analyzer import ReplayGainAnalyzer +from .status_reporter import StatusReporter +from .message_listener import MessageListener class AirtimeAnalyzerServer: @@ -76,7 +76,7 @@ class AirtimeAnalyzerServer: def dump_stacktrace(stack): ''' Dump a stacktrace for all threads ''' code = [] - for threadId, stack in sys._current_frames().items(): + for threadId, stack in list(sys._current_frames().items()): code.append("\n# ThreadID: %s" % threadId) for filename, lineno, name, line in traceback.extract_stack(stack): code.append('File: "%s", line %d, in %s' % (filename, lineno, name)) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py b/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py index 20973fdb4..c6b383127 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py @@ -1,19 +1,19 @@ -""" Analyzes and imports an audio file into the Airtime library. +""" Analyzes and imports an audio file into the Airtime library. """ import logging import threading import multiprocessing -import Queue -import ConfigParser -from metadata_analyzer import MetadataAnalyzer -from filemover_analyzer import FileMoverAnalyzer -from cuepoint_analyzer import CuePointAnalyzer -from replaygain_analyzer import ReplayGainAnalyzer -from playability_analyzer import * +from queue import Queue +import configparser +from .metadata_analyzer import MetadataAnalyzer +from .filemover_analyzer import FileMoverAnalyzer +from .cuepoint_analyzer import CuePointAnalyzer +from .replaygain_analyzer import ReplayGainAnalyzer +from .playability_analyzer import * class AnalyzerPipeline: - """ Analyzes and imports an audio file into the Airtime library. - + """ Analyzes and imports an audio file into the Airtime library. + This currently performs metadata extraction (eg. gets the ID3 tags from an MP3), then moves the file to the Airtime music library (stor/imported), and returns the results back to the parent process. This class is used in an isolated process @@ -26,35 +26,35 @@ class AnalyzerPipeline: @staticmethod def run_analysis(queue, audio_file_path, import_directory, original_filename, storage_backend, file_prefix): """Analyze and import an audio file, and put all extracted metadata into queue. - + Keyword arguments: queue: A multiprocessing.queues.Queue which will be used to pass the extracted metadata back to the parent process. audio_file_path: Path on disk to the audio file to analyze. - import_directory: Path to the final Airtime "import" directory where + import_directory: Path to the final Airtime "import" directory where we will move the file. - original_filename: The original filename of the file, which we'll try to - preserve. The file at audio_file_path typically has a + original_filename: The original filename of the file, which we'll try to + preserve. The file at audio_file_path typically has a temporary randomly generated name, which is why we want - to know what the original name was. + to know what the original name was. storage_backend: String indicating the storage backend (amazon_s3 or file) file_prefix: """ - # It is super critical to initialize a separate log file here so that we + # It is super critical to initialize a separate log file here so that we # don't inherit logging/locks from the parent process. Supposedly # this can lead to Bad Things (deadlocks): http://bugs.python.org/issue6721 AnalyzerPipeline.python_logger_deadlock_workaround() try: - if not isinstance(queue, Queue.Queue): + if not isinstance(queue, Queue): raise TypeError("queue must be a Queue.Queue()") - if not isinstance(audio_file_path, unicode): + if not isinstance(audio_file_path, str): raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__ + " instead.") - if not isinstance(import_directory, unicode): + if not isinstance(import_directory, str): raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__ + " instead.") - if not isinstance(original_filename, unicode): + if not isinstance(original_filename, str): raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__ + " instead.") - if not isinstance(file_prefix, unicode): + if not isinstance(file_prefix, str): raise TypeError("file_prefix must be unicode. Was of type " + type(file_prefix).__name__ + " instead.") @@ -72,7 +72,7 @@ class AnalyzerPipeline: metadata["import_status"] = 0 # Successfully imported - # Note that the queue we're putting the results into is our interprocess communication + # Note that the queue we're putting the results into is our interprocess communication # back to the main process. # Pass all the file metadata back to the main analyzer process, which then passes @@ -91,7 +91,7 @@ class AnalyzerPipeline: @staticmethod def python_logger_deadlock_workaround(): # Workaround for: http://bugs.python.org/issue6721#msg140215 - logger_names = logging.Logger.manager.loggerDict.keys() + logger_names = list(logging.Logger.manager.loggerDict.keys()) logger_names.append(None) # Root logger for name in logger_names: for handler in logging.getLogger(name).handlers: diff --git a/python_apps/airtime_analyzer/airtime_analyzer/config_file.py b/python_apps/airtime_analyzer/airtime_analyzer/config_file.py index 7bd5a0b59..e98bd529d 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/config_file.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/config_file.py @@ -1,15 +1,16 @@ -import ConfigParser + +import configparser def read_config_file(config_path): """Parse the application's config file located at config_path.""" - config = ConfigParser.SafeConfigParser() + config = configparser.SafeConfigParser() try: config.readfp(open(config_path)) except IOError as e: - print "Failed to open config file at " + config_path + ": " + e.strerror + print("Failed to open config file at {}: {}".format(config_path, e.strerror)) exit(-1) except Exception as e: - print e.strerror + print(e.strerror) exit(-1) - return config \ No newline at end of file + return config diff --git a/python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py index 47a517b59..739df2478 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py @@ -3,7 +3,7 @@ import logging import traceback import json import datetime -from analyzer import Analyzer +from .analyzer import Analyzer class CuePointAnalyzer(Analyzer): @@ -27,6 +27,10 @@ class CuePointAnalyzer(Analyzer): command = [CuePointAnalyzer.SILAN_EXECUTABLE, '-b', '-F', '0.99', '-f', 'JSON', '-t', '1.0', filename] try: results_json = subprocess.check_output(command, stderr=subprocess.STDOUT, close_fds=True) + try: + results_json = results_json.decode() + except (UnicodeDecodeError, AttributeError): + pass silan_results = json.loads(results_json) # Defensive coding against Silan wildly miscalculating the cue in and out times: @@ -64,7 +68,7 @@ class CuePointAnalyzer(Analyzer): except OSError as e: # silan was not found logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have silan installed?")) except subprocess.CalledProcessError as e: # silan returned an error code - logging.warn("%s %s %s", e.cmd, e.message, e.returncode) + logging.warn("%s %s %s", e.cmd, e.output, e.returncode) except Exception as e: logging.warn(e) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py index 65badaf50..8bc8bb94d 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py @@ -4,24 +4,24 @@ import time import shutil import os, errno import time -import uuid +import uuid -from analyzer import Analyzer +from .analyzer import Analyzer class FileMoverAnalyzer(Analyzer): - """This analyzer copies a file over from a temporary directory (stor/organize) + """This analyzer copies a file over from a temporary directory (stor/organize) into the Airtime library (stor/imported). """ @staticmethod def analyze(audio_file_path, metadata): """Dummy method because we need more info than analyze gets passed to it""" raise Exception("Use FileMoverAnalyzer.move() instead.") - + @staticmethod def move(audio_file_path, import_directory, original_filename, metadata): """Move the file at audio_file_path over into the import_directory/import, renaming it to original_filename. - + Keyword arguments: audio_file_path: Path to the file to be imported. import_directory: Path to the "import" directory inside the Airtime stor directory. @@ -29,26 +29,28 @@ class FileMoverAnalyzer(Analyzer): original_filename: The filename of the file when it was uploaded to Airtime. metadata: A dictionary where the "full_path" of where the file is moved to will be added. """ - if not isinstance(audio_file_path, unicode): - raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__) - if not isinstance(import_directory, unicode): - raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__) - if not isinstance(original_filename, unicode): - raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__) + if not isinstance(audio_file_path, str): + raise TypeError("audio_file_path must be string. Was of type " + type(audio_file_path).__name__) + if not isinstance(import_directory, str): + raise TypeError("import_directory must be string. Was of type " + type(import_directory).__name__) + if not isinstance(original_filename, str): + raise TypeError("original_filename must be string. Was of type " + type(original_filename).__name__) if not isinstance(metadata, dict): raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__) - + if not os.path.exists(audio_file_path): + raise FileNotFoundError("audio file not found: {}".format(audio_file_path)) + #Import the file over to it's final location. # TODO: Also, handle the case where the move fails and write some code # to possibly move the file to problem_files. - + max_dir_len = 48 max_file_len = 48 final_file_path = import_directory orig_file_basename, orig_file_extension = os.path.splitext(original_filename) - if metadata.has_key("artist_name"): + if "artist_name" in metadata: final_file_path += "/" + metadata["artist_name"][0:max_dir_len] # truncating with array slicing - if metadata.has_key("album_title"): + if "album_title" in metadata: final_file_path += "/" + metadata["album_title"][0:max_dir_len] # Note that orig_file_extension includes the "." already final_file_path += "/" + orig_file_basename[0:max_file_len] + orig_file_extension @@ -58,11 +60,11 @@ class FileMoverAnalyzer(Analyzer): #If a file with the same name already exists in the "import" directory, then #we add a unique string to the end of this one. We never overwrite a file on import - #because if we did that, it would mean Airtime's database would have + #because if we did that, it would mean Airtime's database would have #the wrong information for the file we just overwrote (eg. the song length would be wrong!) #If the final file path is the same as the file we've been told to import (which #you often do when you're debugging), then don't move the file at all. - + if os.path.exists(final_file_path): if os.path.samefile(audio_file_path, final_file_path): metadata["full_path"] = final_file_path @@ -77,14 +79,14 @@ class FileMoverAnalyzer(Analyzer): #Ensure the full path to the file exists mkdir_p(os.path.dirname(final_file_path)) - - #Move the file into its final destination directory + + #Move the file into its final destination directory logging.debug("Moving %s to %s" % (audio_file_path, final_file_path)) shutil.move(audio_file_path, final_file_path) - + metadata["full_path"] = final_file_path return metadata - + def mkdir_p(path): """ Make all directories in a tree (like mkdir -p)""" if path == "": diff --git a/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py b/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py index c261f737b..5d28cc3f2 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py @@ -6,9 +6,9 @@ import select import signal import logging import multiprocessing -import Queue -from analyzer_pipeline import AnalyzerPipeline -from status_reporter import StatusReporter +import queue +from .analyzer_pipeline import AnalyzerPipeline +from .status_reporter import StatusReporter EXCHANGE = "airtime-uploads" EXCHANGE_TYPE = "topic" @@ -112,8 +112,7 @@ class MessageListener: self._channel.queue_bind(exchange=EXCHANGE, queue=QUEUE, routing_key=ROUTING_KEY) logging.info(" Listening for messages...") - self._channel.basic_consume(self.msg_received_callback, - queue=QUEUE, no_ack=False) + self._channel.basic_consume(QUEUE, self.msg_received_callback, auto_ack=False) def wait_for_messages(self): '''Wait until we've received a RabbitMQ message.''' @@ -158,6 +157,10 @@ class MessageListener: We avoid cascading failure this way. ''' try: + try: + body = body.decode() + except (UnicodeDecodeError, AttributeError): + pass msg_dict = json.loads(body) api_key = msg_dict["api_key"] callback_url = msg_dict["callback_url"] @@ -198,7 +201,7 @@ class MessageListener: if callback_url: # If we got an invalid message, there might be no callback_url in the JSON # Report this as a failed upload to the File Upload REST API. StatusReporter.report_failure_to_callback_url(callback_url, api_key, import_status=2, - reason=u'An error occurred while importing this file') + reason='An error occurred while importing this file') else: @@ -224,7 +227,7 @@ class MessageListener: ''' metadata = {} - q = Queue.Queue() + q = queue.Queue() try: AnalyzerPipeline.run_analysis(q, audio_file_path, import_directory, original_filename, storage_backend, file_prefix) metadata = q.get() diff --git a/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py index 489e7ef86..d590d95a9 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py @@ -6,22 +6,24 @@ import wave import logging import os import hashlib -from analyzer import Analyzer +from .analyzer import Analyzer class MetadataAnalyzer(Analyzer): @staticmethod def analyze(filename, metadata): ''' Extract audio metadata from tags embedded in the file (eg. ID3 tags) - + Keyword arguments: filename: The path to the audio file to extract metadata from. - metadata: A dictionary that the extracted metadata will be added to. + metadata: A dictionary that the extracted metadata will be added to. ''' - if not isinstance(filename, unicode): - raise TypeError("filename must be unicode. Was of type " + type(filename).__name__) + if not isinstance(filename, str): + raise TypeError("filename must be string. Was of type " + type(filename).__name__) if not isinstance(metadata, dict): raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__) + if not os.path.exists(filename): + raise FileNotFoundError("audio file not found: {}".format(filename)) #Airtime <= 2.5.x nonsense: metadata["ftype"] = "audioclip" @@ -40,7 +42,7 @@ class MetadataAnalyzer(Analyzer): m.update(data) metadata["md5"] = m.hexdigest() - # Mutagen doesn't handle WAVE files so we use a different package + # Mutagen doesn't handle WAVE files so we use a different package ms = magic.open(magic.MIME_TYPE) ms.load() with open(filename, 'rb') as fh: @@ -57,15 +59,15 @@ class MetadataAnalyzer(Analyzer): if audio_file == None: # Don't use "if not" here. It is wrong due to mutagen's design. return metadata # Note that audio_file can equal {} if the file is valid but there's no metadata tags. - # We can still try to grab the info variables below. - + # We can still try to grab the info variables below. + #Grab other file information that isn't encoded in a tag, but instead usually #in the file header. Mutagen breaks that out into a separate "info" object: info = audio_file.info if hasattr(info, "sample_rate"): # Mutagen is annoying and inconsistent metadata["sample_rate"] = info.sample_rate if hasattr(info, "length"): - metadata["length_seconds"] = info.length + metadata["length_seconds"] = info.length #Converting the length in seconds (float) to a formatted time string track_length = datetime.timedelta(seconds=info.length) metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length) @@ -77,12 +79,12 @@ class MetadataAnalyzer(Analyzer): if hasattr(info, "bitrate"): metadata["bit_rate"] = info.bitrate - + # Use the mutagen to get the MIME type, if it has one. This is more reliable and # consistent for certain types of MP3s or MPEG files than the MIMEs returned by magic. if audio_file.mime: metadata["mime"] = audio_file.mime[0] - + #Try to get the number of channels if mutagen can... try: #Special handling for getting the # of channels from MP3s. It's in the "mode" field @@ -97,18 +99,18 @@ class MetadataAnalyzer(Analyzer): except (AttributeError, KeyError): #If mutagen can't figure out the number of channels, we'll just leave it out... pass - + #Try to extract the number of tracks on the album if we can (the "track total") try: track_number = audio_file["tracknumber"] - if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh + if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh track_number = track_number[0] - track_number_tokens = track_number - if u'/' in track_number: - track_number_tokens = track_number.split(u'/') + track_number_tokens = track_number + if '/' in track_number: + track_number_tokens = track_number.split('/') track_number = track_number_tokens[0] - elif u'-' in track_number: - track_number_tokens = track_number.split(u'-') + elif '-' in track_number: + track_number_tokens = track_number.split('-') track_number = track_number_tokens[0] metadata["track_number"] = track_number track_total = track_number_tokens[1] @@ -118,7 +120,7 @@ class MetadataAnalyzer(Analyzer): pass #We normalize the mutagen tags slightly here, so in case mutagen changes, - #we find the + #we find the mutagen_to_airtime_mapping = { 'title': 'track_title', 'artist': 'artist_name', @@ -146,20 +148,20 @@ class MetadataAnalyzer(Analyzer): #'mime_type': 'mime', } - for mutagen_tag, airtime_tag in mutagen_to_airtime_mapping.iteritems(): + for mutagen_tag, airtime_tag in mutagen_to_airtime_mapping.items(): try: metadata[airtime_tag] = audio_file[mutagen_tag] # Some tags are returned as lists because there could be multiple values. # This is unusual so we're going to always just take the first item in the list. if isinstance(metadata[airtime_tag], list): - if metadata[airtime_tag]: + if metadata[airtime_tag]: metadata[airtime_tag] = metadata[airtime_tag][0] else: # Handle empty lists metadata[airtime_tag] = "" except KeyError: - continue + continue return metadata @@ -174,7 +176,7 @@ class MetadataAnalyzer(Analyzer): track_length = datetime.timedelta(seconds=length_seconds) metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length) metadata["length_seconds"] = length_seconds - metadata["cueout"] = metadata["length"] + metadata["cueout"] = metadata["length"] except wave.Error as ex: logging.error("Invalid WAVE file: {}".format(str(ex))) raise diff --git a/python_apps/airtime_analyzer/airtime_analyzer/playability_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/playability_analyzer.py index eb9062713..d7b2d546e 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/playability_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/playability_analyzer.py @@ -2,7 +2,7 @@ __author__ = 'asantoni' import subprocess import logging -from analyzer import Analyzer +from .analyzer import Analyzer class UnplayableFileError(Exception): pass diff --git a/python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py index 39ea2e439..309f73ce6 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py @@ -1,12 +1,13 @@ import subprocess import logging -from analyzer import Analyzer +from .analyzer import Analyzer +import re class ReplayGainAnalyzer(Analyzer): ''' This class extracts the ReplayGain using a tool from the python-rgain package. ''' - REPLAYGAIN_EXECUTABLE = 'replaygain' # From the python-rgain package + REPLAYGAIN_EXECUTABLE = 'replaygain' # From the rgain3 python package @staticmethod def analyze(filename, metadata): @@ -19,17 +20,16 @@ class ReplayGainAnalyzer(Analyzer): ''' command = [ReplayGainAnalyzer.REPLAYGAIN_EXECUTABLE, '-d', filename] try: - results = subprocess.check_output(command, stderr=subprocess.STDOUT, close_fds=True) - filename_token = "%s: " % filename - rg_pos = results.find(filename_token, results.find("Calculating Replay Gain information")) + len(filename_token) - db_pos = results.find(" dB", rg_pos) - replaygain = results[rg_pos:db_pos] + results = subprocess.check_output(command, stderr=subprocess.STDOUT, + close_fds=True, text=True) + gain_match = r'Calculating Replay Gain information \.\.\.(?:\n|.)*?:([\d.-]*) dB' + replaygain = re.search(gain_match, results).group(1) metadata['replay_gain'] = float(replaygain) except OSError as e: # replaygain was not found logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have python-rgain installed?")) except subprocess.CalledProcessError as e: # replaygain returned an error code - logging.warn("%s %s %s", e.cmd, e.message, e.returncode) + logging.warn("%s %s %s", e.cmd, e.output, e.returncode) except Exception as e: logging.warn(e) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py b/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py index bb1d03d34..ee3c78fac 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py @@ -2,12 +2,12 @@ import requests import json import logging import collections -import Queue +import queue import time import traceback import pickle import threading -from urlparse import urlparse +from urllib.parse import urlparse # Disable urllib3 warnings because these can cause a rare deadlock due to Python 2's crappy internal non-reentrant locking # around POSIX stuff. See SAAS-714. The hasattr() is for compatibility with older versions of requests. @@ -68,7 +68,7 @@ def process_http_requests(ipc_queue, http_retry_queue_path): break if not isinstance(request, PicklableHttpRequest): raise TypeError("request must be a PicklableHttpRequest. Was of type " + type(request).__name__) - except Queue.Empty: + except queue.Empty: request = None # If there's no new HTTP request we need to execute, let's check our "retry @@ -159,7 +159,7 @@ class StatusReporter(): ''' We use multiprocessing.Process again here because we need a thread for this stuff anyways, and Python gives us process isolation for free (crash safety). ''' - _ipc_queue = Queue.Queue() + _ipc_queue = queue.Queue() #_http_thread = multiprocessing.Process(target=process_http_requests, # args=(_ipc_queue,)) _http_thread = None @@ -222,7 +222,7 @@ class StatusReporter(): @classmethod def report_failure_to_callback_url(self, callback_url, api_key, import_status, reason): - if not isinstance(import_status, (int, long) ): + if not isinstance(import_status, int ): raise TypeError("import_status must be an integer. Was of type " + type(import_status).__name__) logging.debug("Reporting import failure to Airtime REST API...") diff --git a/python_apps/airtime_analyzer/bin/airtime_analyzer b/python_apps/airtime_analyzer/bin/airtime_analyzer index b3e4ab0aa..7b8ca69bb 100755 --- a/python_apps/airtime_analyzer/bin/airtime_analyzer +++ b/python_apps/airtime_analyzer/bin/airtime_analyzer @@ -2,6 +2,7 @@ """Runs the airtime_analyzer application. """ + import daemon import argparse import os @@ -14,7 +15,7 @@ DEFAULT_HTTP_RETRY_PATH = '/tmp/airtime_analyzer_http_retries' def run(): '''Entry-point for this application''' - print "Airtime Analyzer " + VERSION + print("Airtime Analyzer {}".format(VERSION)) parser = argparse.ArgumentParser() parser.add_argument("-d", "--daemon", help="run as a daemon", action="store_true") parser.add_argument("--debug", help="log full debugging output", action="store_true") @@ -22,8 +23,6 @@ def run(): parser.add_argument("--http-retry-queue-file", help="specify where incompleted HTTP requests will be serialized (default is %s)" % DEFAULT_HTTP_RETRY_PATH) args = parser.parse_args() - check_if_media_monitor_is_running() - #Default config file path rmq_config_path = DEFAULT_RMQ_CONFIG_PATH http_retry_queue_path = DEFAULT_HTTP_RETRY_PATH @@ -35,32 +34,12 @@ def run(): if args.daemon: with daemon.DaemonContext(): aa.AirtimeAnalyzerServer(rmq_config_path=rmq_config_path, - http_retry_queue_path=http_retry_queue_path, + http_retry_queue_path=http_retry_queue_path, debug=args.debug) - else: + else: # Run without daemonizing aa.AirtimeAnalyzerServer(rmq_config_path=rmq_config_path, - http_retry_queue_path=http_retry_queue_path, + http_retry_queue_path=http_retry_queue_path, debug=args.debug) - -def check_if_media_monitor_is_running(): - """Ensure media_monitor isn't running before we start. - - We do this because media_monitor will move newly uploaded - files into the library on us and screw up the operation of airtime_analyzer. - media_monitor is deprecated. - """ - pids = [pid for pid in os.listdir('/proc') if pid.isdigit()] - - for pid in pids: - try: - process_name = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read() - if 'media_monitor.py' in process_name: - print "Error: This process conflicts with media_monitor, and media_monitor is running." - print " Please terminate the running media_monitor.py process and try again." - exit(1) - except IOError: # proc has already terminated - continue - run() diff --git a/python_apps/airtime_analyzer/setup.py b/python_apps/airtime_analyzer/setup.py index eb697d941..186c10fc1 100644 --- a/python_apps/airtime_analyzer/setup.py +++ b/python_apps/airtime_analyzer/setup.py @@ -1,3 +1,4 @@ +from __future__ import print_function from setuptools import setup from subprocess import call import sys @@ -5,7 +6,7 @@ import os # Change directory since setuptools uses relative paths script_path = os.path.dirname(os.path.realpath(__file__)) -print script_path +print(script_path) os.chdir(script_path) # Allows us to avoid installing the upstart init script when deploying airtime_analyzer @@ -16,7 +17,7 @@ if '--no-init-script' in sys.argv: else: data_files = [('/etc/init', ['install/upstart/airtime_analyzer.conf']), ('/etc/init.d', ['install/sysvinit/airtime_analyzer'])] - print data_files + print(data_files) setup(name='airtime_analyzer', version='0.1', @@ -28,16 +29,15 @@ setup(name='airtime_analyzer', packages=['airtime_analyzer'], scripts=['bin/airtime_analyzer'], install_requires=[ - 'mutagen~=1.43.0', # got rid of specific version requirement - 'pika', - 'daemon', + 'mutagen~=1.43', + 'pika~=1.1.0', 'file-magic', 'nose', 'coverage', 'mock', - 'python-daemon==1.6', + 'python-daemon', 'requests>=2.7.0', - 'rgain', + 'rgain3', # These next 3 are required for requests to support SSL with SNI. Learned this the hard way... # What sucks is that GCC is required to pip install these. #'ndg-httpsclient', @@ -49,8 +49,8 @@ setup(name='airtime_analyzer', # Remind users to reload the initctl config so that "service start airtime_analyzer" works if data_files: - print "Remember to reload the initctl configuration" - print "Run \"sudo initctl reload-configuration; sudo service airtime_analyzer restart\" now." - print "Or on Ubuntu Xenial (16.04)" - print "Remember to reload the systemd configuration" - print "Run \"sudo systemctl daemon-reload; sudo service airtime_analyzer restart\" now." + print("Remember to reload the initctl configuration") + print("Run \"sudo initctl reload-configuration; sudo service airtime_analyzer restart\" now.") + print("Or on Ubuntu Xenial (16.04)") + print("Remember to reload the systemd configuration") + print("Run \"sudo systemctl daemon-reload; sudo service airtime_analyzer restart\" now.") diff --git a/python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py b/python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py index 8e986bc11..57ae8bcf1 100644 --- a/python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py +++ b/python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py @@ -1,9 +1,8 @@ from nose.tools import * -from ConfigParser import SafeConfigParser import os import shutil import multiprocessing -import Queue +from queue import Queue import datetime from airtime_analyzer.analyzer_pipeline import AnalyzerPipeline from airtime_analyzer import config_file @@ -21,7 +20,7 @@ def teardown(): def test_basic(): filename = os.path.basename(DEFAULT_AUDIO_FILE) - q = Queue.Queue() + q = Queue() file_prefix = u'' storage_backend = "file" #This actually imports the file into the "./Test Artist" directory. @@ -39,17 +38,17 @@ def test_basic(): @raises(TypeError) def test_wrong_type_queue_param(): - AnalyzerPipeline.run_analysis(Queue.Queue(), u'', u'', u'') + AnalyzerPipeline.run_analysis(Queue(), u'', u'', u'') @raises(TypeError) def test_wrong_type_string_param2(): - AnalyzerPipeline.run_analysis(Queue.Queue(), '', u'', u'') + AnalyzerPipeline.run_analysis(Queue(), '', u'', u'') @raises(TypeError) def test_wrong_type_string_param3(): - AnalyzerPipeline.run_analysis(Queue.Queue(), u'', '', u'') + AnalyzerPipeline.run_analysis(Queue(), u'', '', u'') @raises(TypeError) def test_wrong_type_string_param4(): - AnalyzerPipeline.run_analysis(Queue.Queue(), u'', u'', '') + AnalyzerPipeline.run_analysis(Queue(), u'', u'', '') diff --git a/python_apps/airtime_analyzer/tests/filemover_analyzer_tests.py b/python_apps/airtime_analyzer/tests/filemover_analyzer_tests.py index dbdbb2feb..4e7f9e304 100644 --- a/python_apps/airtime_analyzer/tests/filemover_analyzer_tests.py +++ b/python_apps/airtime_analyzer/tests/filemover_analyzer_tests.py @@ -2,7 +2,6 @@ from nose.tools import * import os import shutil import multiprocessing -import Queue import time import mock from pprint import pprint @@ -23,30 +22,34 @@ def test_dont_use_analyze(): @raises(TypeError) def test_move_wrong_string_param1(): - FileMoverAnalyzer.move('', u'', u'', dict()) + FileMoverAnalyzer.move(42, '', '', dict()) @raises(TypeError) def test_move_wrong_string_param2(): - FileMoverAnalyzer.move(u'', '', u'', dict()) + FileMoverAnalyzer.move(u'', 23, u'', dict()) @raises(TypeError) def test_move_wrong_string_param3(): - FileMoverAnalyzer.move(u'', u'', '', dict()) + FileMoverAnalyzer.move('', '', 5, dict()) @raises(TypeError) def test_move_wrong_dict_param(): - FileMoverAnalyzer.move(u'', u'', u'', 12345) + FileMoverAnalyzer.move('', '', '', 12345) + +@raises(FileNotFoundError) +def test_move_wrong_string_param3(): + FileMoverAnalyzer.move('', '', '', dict()) def test_basic(): filename = os.path.basename(DEFAULT_AUDIO_FILE) - FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict()) + FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict()) #Move the file back shutil.move("./" + filename, DEFAULT_AUDIO_FILE) assert os.path.exists(DEFAULT_AUDIO_FILE) def test_basic_samefile(): filename = os.path.basename(DEFAULT_AUDIO_FILE) - FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'tests/test_data', filename, dict()) + FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'tests/test_data', filename, dict()) assert os.path.exists(DEFAULT_AUDIO_FILE) def test_duplicate_file(): @@ -55,9 +58,9 @@ def test_duplicate_file(): FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict()) #Copy it back to the original location shutil.copy("./" + filename, DEFAULT_AUDIO_FILE) - #Import it again. It shouldn't overwrite the old file and instead create a new + #Import it again. It shouldn't overwrite the old file and instead create a new metadata = dict() - metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, metadata) + metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, metadata) #Cleanup: move the file (eg. 44100Hz-16bit-mono.mp3) back shutil.move("./" + filename, DEFAULT_AUDIO_FILE) #Remove the renamed duplicate, eg. 44100Hz-16bit-mono_03-26-2014-11-58.mp3 @@ -71,7 +74,7 @@ def test_duplicate_file(): it's imported within 1 second of the second file (ie. if the timestamp is the same). ''' def test_double_duplicate_files(): - # Here we use mock to patch out the time.localtime() function so that it + # Here we use mock to patch out the time.localtime() function so that it # always returns the same value. This allows us to consistently simulate this test cases # where the last two of the three files are imported at the same time as the timestamp. with mock.patch('airtime_analyzer.filemover_analyzer.time') as mock_time: @@ -83,17 +86,17 @@ def test_double_duplicate_files(): FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict()) #Copy it back to the original location shutil.copy("./" + filename, DEFAULT_AUDIO_FILE) - #Import it again. It shouldn't overwrite the old file and instead create a new + #Import it again. It shouldn't overwrite the old file and instead create a new first_dup_metadata = dict() - first_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, - first_dup_metadata) + first_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, + first_dup_metadata) #Copy it back again! shutil.copy("./" + filename, DEFAULT_AUDIO_FILE) #Reimport for the third time, which should have the same timestamp as the second one #thanks to us mocking out time.localtime() second_dup_metadata = dict() - second_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, - second_dup_metadata) + second_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, + second_dup_metadata) #Cleanup: move the file (eg. 44100Hz-16bit-mono.mp3) back shutil.move("./" + filename, DEFAULT_AUDIO_FILE) #Remove the renamed duplicate, eg. 44100Hz-16bit-mono_03-26-2014-11-58.mp3 @@ -105,7 +108,7 @@ def test_double_duplicate_files(): def test_bad_permissions_destination_dir(): filename = os.path.basename(DEFAULT_AUDIO_FILE) dest_dir = u'/sys/foobar' # /sys is using sysfs on Linux, which is unwritable - FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, dest_dir, filename, dict()) + FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, dest_dir, filename, dict()) #Move the file back shutil.move(os.path.join(dest_dir, filename), DEFAULT_AUDIO_FILE) assert os.path.exists(DEFAULT_AUDIO_FILE) diff --git a/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py b/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py index 2af30326b..c92b82db6 100644 --- a/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py +++ b/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- +from __future__ import print_function import datetime import mutagen import mock from nose.tools import * -from airtime_analyzer.metadata_analyzer import MetadataAnalyzer +from airtime_analyzer.metadata_analyzer import MetadataAnalyzer def setup(): pass @@ -12,115 +13,115 @@ def teardown(): pass def check_default_metadata(metadata): - assert metadata['track_title'] == u'Test Title' - assert metadata['artist_name'] == u'Test Artist' - assert metadata['album_title'] == u'Test Album' - assert metadata['year'] == u'1999' - assert metadata['genre'] == u'Test Genre' - assert metadata['track_number'] == u'1' + assert metadata['track_title'] == 'Test Title' + assert metadata['artist_name'] == 'Test Artist' + assert metadata['album_title'] == 'Test Album' + assert metadata['year'] == '1999' + assert metadata['genre'] == 'Test Genre' + assert metadata['track_number'] == '1' assert metadata["length"] == str(datetime.timedelta(seconds=metadata["length_seconds"])) def test_mp3_mono(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.mp3', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-mono.mp3', dict()) check_default_metadata(metadata) assert metadata['channels'] == 1 assert metadata['bit_rate'] == 63998 assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mp3' # Not unicode because MIMEs aren't. - assert metadata['track_total'] == u'10' # MP3s can have a track_total + assert metadata['track_total'] == '10' # MP3s can have a track_total #Mutagen doesn't extract comments from mp3s it seems def test_mp3_jointstereo(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-jointstereo.mp3', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-jointstereo.mp3', dict()) check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 127998 assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mp3' - assert metadata['track_total'] == u'10' # MP3s can have a track_total + assert metadata['track_total'] == '10' # MP3s can have a track_total def test_mp3_simplestereo(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-simplestereo.mp3', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-simplestereo.mp3', dict()) check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 127998 assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mp3' - assert metadata['track_total'] == u'10' # MP3s can have a track_total + assert metadata['track_total'] == '10' # MP3s can have a track_total def test_mp3_dualmono(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-dualmono.mp3', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-dualmono.mp3', dict()) check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 127998 assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mp3' - assert metadata['track_total'] == u'10' # MP3s can have a track_total + assert metadata['track_total'] == '10' # MP3s can have a track_total def test_ogg_mono(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.ogg', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-mono.ogg', dict()) check_default_metadata(metadata) assert metadata['channels'] == 1 assert metadata['bit_rate'] == 80000 assert abs(metadata['length_seconds'] - 3.8) < 0.1 assert metadata['mime'] == 'audio/vorbis' - assert metadata['comment'] == u'Test Comment' + assert metadata['comment'] == 'Test Comment' def test_ogg_stereo(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.ogg', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo.ogg', dict()) check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 112000 assert abs(metadata['length_seconds'] - 3.8) < 0.1 assert metadata['mime'] == 'audio/vorbis' - assert metadata['comment'] == u'Test Comment' + assert metadata['comment'] == 'Test Comment' ''' faac and avconv can't seem to create a proper mono AAC file... ugh def test_aac_mono(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.m4a') - print "Mono AAC metadata:" - print metadata + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-mono.m4a') + print("Mono AAC metadata:") + print(metadata) check_default_metadata(metadata) assert metadata['channels'] == 1 assert metadata['bit_rate'] == 80000 assert abs(metadata['length_seconds'] - 3.8) < 0.1 assert metadata['mime'] == 'audio/mp4' - assert metadata['comment'] == u'Test Comment' + assert metadata['comment'] == 'Test Comment' ''' def test_aac_stereo(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.m4a', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo.m4a', dict()) check_default_metadata(metadata) assert metadata['channels'] == 2 assert metadata['bit_rate'] == 102619 assert abs(metadata['length_seconds'] - 3.8) < 0.1 assert metadata['mime'] == 'audio/mp4' - assert metadata['comment'] == u'Test Comment' + assert metadata['comment'] == 'Test Comment' def test_mp3_utf8(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict()) # Using a bunch of different UTF-8 codepages here. Test data is from: # http://winrus.com/utf8-jap.htm - assert metadata['track_title'] == u'ｱｲｳｴｵｶｷｸｹｺｻｼｽｾｿﾀﾁﾂﾃ' - assert metadata['artist_name'] == u'てすと' - assert metadata['album_title'] == u'Ä ä Ü ü ß' - assert metadata['year'] == u'1999' - assert metadata['genre'] == u'Я Б Г Д Ж Й' - assert metadata['track_number'] == u'1' + assert metadata['track_title'] == 'ｱｲｳｴｵｶｷｸｹｺｻｼｽｾｿﾀﾁﾂﾃ' + assert metadata['artist_name'] == 'てすと' + assert metadata['album_title'] == 'Ä ä Ü ü ß' + assert metadata['year'] == '1999' + assert metadata['genre'] == 'Я Б Г Д Ж Й' + assert metadata['track_number'] == '1' assert metadata['channels'] == 2 assert metadata['bit_rate'] < 130000 assert metadata['bit_rate'] > 127000 assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mp3' - assert metadata['track_total'] == u'10' # MP3s can have a track_total + assert metadata['track_total'] == '10' # MP3s can have a track_total def test_invalid_wma(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-invalid.wma', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo-invalid.wma', dict()) assert metadata['mime'] == 'audio/x-ms-wma' def test_wav_stereo(): - metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.wav', dict()) + metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo.wav', dict()) assert metadata['mime'] == 'audio/x-wav' assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['channels'] == 2 @@ -128,7 +129,7 @@ def test_wav_stereo(): # Make sure the parameter checking works -@raises(TypeError) +@raises(FileNotFoundError) def test_move_wrong_string_param1(): not_unicode = 'asdfasdf' MetadataAnalyzer.analyze(not_unicode, dict()) @@ -136,12 +137,12 @@ def test_move_wrong_string_param1(): @raises(TypeError) def test_move_wrong_metadata_dict(): not_a_dict = list() - MetadataAnalyzer.analyze(u'asdfasdf', not_a_dict) + MetadataAnalyzer.analyze('asdfasdf', not_a_dict) # Test an mp3 file where the number of channels is invalid or missing: def test_mp3_bad_channels(): - filename = u'tests/test_data/44100Hz-16bit-mono.mp3' - ''' + filename = 'tests/test_data/44100Hz-16bit-mono.mp3' + ''' It'd be a pain in the ass to construct a real MP3 with an invalid number of channels by hand because that value is stored in every MP3 frame in the file ''' @@ -157,8 +158,8 @@ def test_mp3_bad_channels(): assert metadata['bit_rate'] == 63998 assert abs(metadata['length_seconds'] - 3.9) < 0.1 assert metadata['mime'] == 'audio/mp3' # Not unicode because MIMEs aren't. - assert metadata['track_total'] == u'10' # MP3s can have a track_total + assert metadata['track_total'] == '10' # MP3s can have a track_total #Mutagen doesn't extract comments from mp3s it seems def test_unparsable_file(): - MetadataAnalyzer.analyze(u'README.rst', dict()) + MetadataAnalyzer.analyze('README.rst', dict()) diff --git a/python_apps/airtime_analyzer/tests/replaygain_analyzer_tests.py b/python_apps/airtime_analyzer/tests/replaygain_analyzer_tests.py index 4a4e8ca58..cc988d652 100644 --- a/python_apps/airtime_analyzer/tests/replaygain_analyzer_tests.py +++ b/python_apps/airtime_analyzer/tests/replaygain_analyzer_tests.py @@ -1,20 +1,7 @@ +from __future__ import print_function from nose.tools import * from airtime_analyzer.replaygain_analyzer import ReplayGainAnalyzer -''' -The tests in here were all tagged with the 'rgain' tag so the can be exluded from being run -with nosetest -a '!rgain'. This was needed due to the fact that it is not readily possible -to install replaygain on a containerized travis instance. - -We can either give running replaygain test on travis another shot after ubuntu getsan updated -gi instrospection allowing us to install gi and gobject into the virtualenv, or we can switch -to a full machine and stop using 'sudo: false' on travis. - -Deactivating these tests is a bad fix for now and I plan on looking into it again after -most everything else is up and running. For those interesed the tests seem to work locally -albeit my results not being up to the given tolerance of 0.30 (which I'm assuming is my rig's -problem and would work on travis if replaygain was available). -''' def check_default_metadata(metadata): ''' Check that the values extract by Silan/CuePointAnalyzer on our test audio files match what we expect. @@ -28,7 +15,7 @@ def check_default_metadata(metadata): ''' tolerance = 0.30 expected_replaygain = 5.0 - print metadata['replay_gain'] + print(metadata['replay_gain']) assert abs(metadata['replay_gain'] - expected_replaygain) < tolerance def test_missing_replaygain(): diff --git a/python_apps/api_clients/api_clients/api_client.py b/python_apps/api_clients/api_clients/api_client.py index 49bac65e1..d983c5c6c 100644 --- a/python_apps/api_clients/api_clients/api_client.py +++ b/python_apps/api_clients/api_clients/api_client.py @@ -8,8 +8,7 @@ ############################################################################### import sys import time -import urllib -import urllib2 +import urllib.request, urllib.error, urllib.parse import requests import socket import logging @@ -21,26 +20,6 @@ from configobj import ConfigObj AIRTIME_API_VERSION = "1.1" -# TODO : Place these functions in some common module. Right now, media -# monitor uses the same functions and it would be better to reuse them -# instead of copy pasting them around - -def to_unicode(obj, encoding='utf-8'): - if isinstance(obj, basestring): - if not isinstance(obj, unicode): - obj = unicode(obj, encoding) - return obj - -def encode_to(obj, encoding='utf-8'): - if isinstance(obj, unicode): - obj = obj.encode(encoding) - return obj - -def convert_dict_value_to_utf8(md): - #list comprehension to convert all values of md to utf-8 - return dict([(item[0], encode_to(item[1], "utf-8")) for item in md.items()]) - - api_config = {} # URL to get the version number of the server API @@ -114,7 +93,7 @@ class ApcUrl(object): def params(self, **params): temp_url = self.base_url - for k, v in params.iteritems(): + for k, v in params.items(): wrapped_param = "%%" + k + "%%" if wrapped_param in temp_url: temp_url = temp_url.replace(wrapped_param, str(v)) @@ -138,12 +117,13 @@ class ApiRequest(object): def __call__(self,_post_data=None, **kwargs): final_url = self.url.params(**kwargs).url() - if _post_data is not None: _post_data = urllib.urlencode(_post_data) + if _post_data is not None: + _post_data = urllib.parse.urlencode(_post_data).encode('utf-8') self.logger.debug(final_url) try: - req = urllib2.Request(final_url, _post_data) - f = urllib2.urlopen(req, timeout=ApiRequest.API_HTTP_REQUEST_TIMEOUT) - content_type = f.info().getheader('Content-Type') + req = urllib.request.Request(final_url, _post_data) + f = urllib.request.urlopen(req, timeout=ApiRequest.API_HTTP_REQUEST_TIMEOUT) + content_type = f.info().get_content_type() response = f.read() #Everything that calls an ApiRequest should be catching URLError explicitly #(according to the other comments in this file and a cursory grep through the code) @@ -151,20 +131,22 @@ class ApiRequest(object): except socket.timeout: self.logger.error('HTTP request to %s timed out', final_url) raise - except Exception, e: - #self.logger.error('Exception: %s', e) - #self.logger.error("traceback: %s", traceback.format_exc()) + except Exception as e: + #self.logger.exception(e) raise try: if content_type == 'application/json': + try: + response = response.decode() + except (UnicodeDecodeError, AttributeError): + pass data = json.loads(response) return data else: raise InvalidContentType() except Exception: - #self.logger.error(response) - #self.logger.error("traceback: %s", traceback.format_exc()) + #self.logger.exception(e) raise def req(self, *args, **kwargs): @@ -193,18 +175,20 @@ class RequestProvider(object): self.config["general"]["base_dir"], self.config["api_base"], '%%action%%')) # Now we must discover the possible actions - actions = dict( (k,v) for k,v in cfg.iteritems() if '%%api_key%%' in v) - for action_name, action_value in actions.iteritems(): + actions = dict( (k,v) for k,v in cfg.items() if '%%api_key%%' in v) + for action_name, action_value in actions.items(): new_url = self.url.params(action=action_value).params( api_key=self.config["general"]['api_key']) self.requests[action_name] = ApiRequest(action_name, new_url) - def available_requests(self) : return self.requests.keys() + def available_requests(self) : return list(self.requests.keys()) def __contains__(self, request) : return request in self.requests def __getattr__(self, attr): - if attr in self: return self.requests[attr] - else: return super(RequestProvider, self).__getattribute__(attr) + if attr in self: + return self.requests[attr] + else: + return super(RequestProvider, self).__getattribute__(attr) class AirtimeApiClient(object): @@ -217,17 +201,16 @@ class AirtimeApiClient(object): self.config = ConfigObj(config_path) self.config.update(api_config) self.services = RequestProvider(self.config) - except Exception, e: - self.logger.error('Error loading config file: %s', config_path) - self.logger.error("traceback: %s", traceback.format_exc()) + except Exception as e: + self.logger.exception('Error loading config file: %s', config_path) sys.exit(1) def __get_airtime_version(self): - try: return self.services.version_url()[u'airtime_version'] + try: return self.services.version_url()['airtime_version'] except Exception: return -1 def __get_api_version(self): - try: return self.services.version_url()[u'api_version'] + try: return self.services.version_url()['api_version'] except Exception: return -1 def is_server_compatible(self, verbose=True): @@ -259,8 +242,8 @@ class AirtimeApiClient(object): def notify_liquidsoap_started(self): try: self.services.notify_liquidsoap_started() - except Exception, e: - self.logger.error(str(e)) + except Exception as e: + self.logger.exception(e) def notify_media_item_start_playing(self, media_id): """ This is a callback from liquidsoap, we use this to notify @@ -268,15 +251,15 @@ class AirtimeApiClient(object): which we handed to liquidsoap in get_liquidsoap_data(). """ try: return self.services.update_start_playing_url(media_id=media_id) - except Exception, e: - self.logger.error(str(e)) + except Exception as e: + self.logger.exception(e) return None def get_shows_to_record(self): try: return self.services.show_schedule_url() - except Exception, e: - self.logger.error(str(e)) + except Exception as e: + self.logger.exception(e) return None def upload_recorded_show(self, files, show_id): @@ -321,15 +304,14 @@ class AirtimeApiClient(object): """ break - except requests.exceptions.HTTPError, e: + except requests.exceptions.HTTPError as e: logger.error("Http error code: %s", e.code) logger.error("traceback: %s", traceback.format_exc()) - except requests.exceptions.ConnectionError, e: + except requests.exceptions.ConnectionError as e: logger.error("Server is down: %s", e.args) logger.error("traceback: %s", traceback.format_exc()) - except Exception, e: - logger.error("Exception: %s", e) - logger.error("traceback: %s", traceback.format_exc()) + except Exception as e: + self.logger.exception(e) #wait some time before next retry time.sleep(retries_wait) @@ -340,8 +322,8 @@ class AirtimeApiClient(object): try: return self.services.check_live_stream_auth( username=username, password=password, djtype=dj_type) - except Exception, e: - self.logger.error(str(e)) + except Exception as e: + self.logger.exception(e) return {} def construct_url(self,config_action_key): @@ -407,7 +389,7 @@ class AirtimeApiClient(object): # Note that we must prefix every key with: mdX where x is a number # Is there a way to format the next line a little better? The # parenthesis make the code almost unreadable - md_list = dict((("md%d" % i), json.dumps(convert_dict_value_to_utf8(md))) \ + md_list = dict((("md%d" % i), json.dumps(md)) \ for i,md in enumerate(valid_actions)) # For testing we add the following "dry" parameter to tell the # controller not to actually do any changes @@ -422,10 +404,10 @@ class AirtimeApiClient(object): def list_all_db_files(self, dir_id, all_files=True): logger = self.logger try: - all_files = u"1" if all_files else u"0" + all_files = "1" if all_files else "0" response = self.services.list_all_db_files(dir_id=dir_id, all=all_files) - except Exception, e: + except Exception as e: response = {} logger.error("Exception: %s", e) try: @@ -483,23 +465,20 @@ class AirtimeApiClient(object): post_data = {"msg_post": msg} #encoded_msg is no longer used server_side!! - encoded_msg = urllib.quote('dummy') + encoded_msg = urllib.parse.quote('dummy') self.services.update_liquidsoap_status.req(post_data, msg=encoded_msg, stream_id=stream_id, boot_time=time).retry(5) - except Exception, e: - #TODO - logger.error("Exception: %s", e) + except Exception as e: + self.logger.exception(e) def notify_source_status(self, sourcename, status): try: - logger = self.logger return self.services.update_source_status.req(sourcename=sourcename, status=status).retry(5) - except Exception, e: - #TODO - logger.error("Exception: %s", e) + except Exception as e: + self.logger.exception(e) def get_bootstrap_info(self): """ Retrieve infomations needed on bootstrap time """ @@ -514,8 +493,8 @@ class AirtimeApiClient(object): #http://localhost/api/get-files-without-replay-gain/dir_id/1 try: return self.services.get_files_without_replay_gain(dir_id=dir_id) - except Exception, e: - self.logger.error(str(e)) + except Exception as e: + self.logger.exception(e) return [] def get_files_without_silan_value(self): @@ -526,8 +505,8 @@ class AirtimeApiClient(object): """ try: return self.services.get_files_without_silan_value() - except Exception, e: - self.logger.error(str(e)) + except Exception as e: + self.logger.exception(e) return [] def update_replay_gain_values(self, pairs): @@ -569,9 +548,8 @@ class AirtimeApiClient(object): try: response = self.services.update_stream_setting_table(_post_data={'data': json.dumps(data)}) return response - except Exception, e: - #TODO - self.logger.error(str(e)) + except Exception as e: + self.logger.exception(e) def update_metadata_on_tunein(self): self.services.update_metadata_on_tunein() diff --git a/python_apps/api_clients/setup.py b/python_apps/api_clients/setup.py index b71f509a2..ea9bad1f2 100644 --- a/python_apps/api_clients/setup.py +++ b/python_apps/api_clients/setup.py @@ -1,10 +1,11 @@ +from __future__ import print_function from setuptools import setup from subprocess import call import sys import os script_path = os.path.dirname(os.path.realpath(__file__)) -print script_path +print(script_path) os.chdir(script_path) setup(name='api_clients', @@ -16,18 +17,7 @@ setup(name='api_clients', packages=['api_clients'], scripts=[], install_requires=[ -# 'amqplib', -# 'anyjson', -# 'argparse', 'configobj' -# 'docopt', -# 'kombu', -# 'mutagen', -# 'poster', -# 'PyDispatcher', -# 'pyinotify', -# 'pytz', -# 'wsgiref' ], zip_safe=False, - data_files=[]) \ No newline at end of file + data_files=[]) diff --git a/python_apps/api_clients/tests/test_apcurl.py b/python_apps/api_clients/tests/test_apcurl.py index 61880e45c..e3c8b29d5 100644 --- a/python_apps/api_clients/tests/test_apcurl.py +++ b/python_apps/api_clients/tests/test_apcurl.py @@ -1,20 +1,20 @@ import unittest -from .. api_client import ApcUrl, UrlBadParam, IncompleteUrl +from api_clients.api_client import ApcUrl, UrlBadParam, IncompleteUrl class TestApcUrl(unittest.TestCase): def test_init(self): url = "/testing" u = ApcUrl(url) - self.assertEquals( u.base_url, url) + self.assertEqual( u.base_url, url) def test_params_1(self): u = ApcUrl("/testing/%%key%%") - self.assertEquals(u.params(key='val').url(), '/testing/val') + self.assertEqual(u.params(key='val').url(), '/testing/val') def test_params_2(self): u = ApcUrl('/testing/%%key%%/%%api%%/more_testing') full_url = u.params(key="AAA",api="BBB").url() - self.assertEquals(full_url, '/testing/AAA/BBB/more_testing') + self.assertEqual(full_url, '/testing/AAA/BBB/more_testing') def test_params_ex(self): u = ApcUrl("/testing/%%key%%") @@ -23,7 +23,7 @@ class TestApcUrl(unittest.TestCase): def test_url(self): u = "one/two/three" - self.assertEquals( ApcUrl(u).url(), u ) + self.assertEqual( ApcUrl(u).url(), u ) def test_url_ex(self): u = ApcUrl('/%%one%%/%%two%%/three').params(two='testing') diff --git a/python_apps/api_clients/tests/test_apirequest.py b/python_apps/api_clients/tests/test_apirequest.py index 09c1f6f3c..3ca7772e8 100644 --- a/python_apps/api_clients/tests/test_apirequest.py +++ b/python_apps/api_clients/tests/test_apirequest.py @@ -1,21 +1,26 @@ import unittest import json from mock import MagicMock, patch -from .. api_client import ApcUrl, ApiRequest +from api_clients.api_client import ApcUrl, ApiRequest + +class ResponseInfo: + def get_content_type(self): + return 'application/json' class TestApiRequest(unittest.TestCase): def test_init(self): u = ApiRequest('request_name', ApcUrl('/test/ing')) - self.assertEquals(u.name, "request_name") + self.assertEqual(u.name, "request_name") def test_call(self): - ret = json.dumps( {u'ok':u'ok'} ) + ret = json.dumps( {'ok':'ok'} ) read = MagicMock() read.read = MagicMock(return_value=ret) - u = '/testing' - with patch('urllib2.urlopen') as mock_method: + read.info = MagicMock(return_value=ResponseInfo()) + u = 'http://localhost/testing' + with patch('urllib.request.urlopen') as mock_method: mock_method.return_value = read request = ApiRequest('mm', ApcUrl(u))() - self.assertEquals(request, json.loads(ret)) + self.assertEqual(request, json.loads(ret)) if __name__ == '__main__': unittest.main() diff --git a/python_apps/api_clients/tests/test_requestprovider.py b/python_apps/api_clients/tests/test_requestprovider.py index 98244e423..c210aad85 100644 --- a/python_apps/api_clients/tests/test_requestprovider.py +++ b/python_apps/api_clients/tests/test_requestprovider.py @@ -2,13 +2,19 @@ import unittest import json from mock import patch, MagicMock from configobj import ConfigObj -from .. api_client import RequestProvider +from api_clients.api_client import RequestProvider, api_config class TestRequestProvider(unittest.TestCase): def setUp(self): - self.cfg = ConfigObj('api_client.cfg') + self.cfg = api_config + self.cfg['general'] = {} + self.cfg['general']['base_dir'] = '/test' + self.cfg['general']['base_port'] = 80 + self.cfg['general']['base_url'] = 'localhost' + self.cfg['general']['api_key'] = 'TEST_KEY' + self.cfg['api_base'] = 'api' def test_test(self): - self.assertTrue('api_key' in self.cfg) + self.assertTrue('general' in self.cfg) def test_init(self): rp = RequestProvider(self.cfg) self.assertTrue( len( rp.available_requests() ) > 0 ) @@ -16,17 +22,6 @@ class TestRequestProvider(unittest.TestCase): rp = RequestProvider(self.cfg) methods = ['upload_recorded', 'update_media_url', 'list_all_db_files'] for meth in methods: - self.assertTrue( meth in rp ) - - def test_notify_webstream_data(self): - ret = json.dumps( {u'testing' : u'123' } ) - rp = RequestProvider(self.cfg) - read = MagicMock() - read.read = MagicMock(return_value=ret) - with patch('urllib2.urlopen') as mock_method: - mock_method.return_value = read - response = rp.notify_webstream_data(media_id=123) - mock_method.called_once_with(media_id=123) - self.assertEquals(json.loads(ret), response) + self.assertTrue( meth in rp.requests ) if __name__ == '__main__': unittest.main() diff --git a/python_apps/icecast2/install/icecast2-install.py b/python_apps/icecast2/install/icecast2-install.py index 0b411d8da..d12588ab7 100644 --- a/python_apps/icecast2/install/icecast2-install.py +++ b/python_apps/icecast2/install/icecast2-install.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- + import shutil import os import sys if os.geteuid() != 0: - print "Please run this as root." + print("Please run this as root.") sys.exit(1) def get_current_script_dir(): @@ -17,6 +18,6 @@ try: current_script_dir = get_current_script_dir() shutil.copy(current_script_dir+"/../airtime-icecast-status.xsl", "/usr/share/icecast2/web") -except Exception, e: - print "exception: %s" % e +except Exception as e: + print("exception: {}".format(e)) sys.exit(1) diff --git a/python_apps/pypo/bin/pyponotify b/python_apps/pypo/bin/pyponotify index 9d8fe4ab8..879f24c8c 100755 --- a/python_apps/pypo/bin/pyponotify +++ b/python_apps/pypo/bin/pyponotify @@ -1,5 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- + import traceback """ @@ -75,7 +76,7 @@ logger = rootLogger try: config = ConfigObj('/etc/airtime/airtime.conf') -except Exception, e: +except Exception as e: logger.error('Error loading config file: %s', e) sys.exit() @@ -133,21 +134,20 @@ class Notify: elif options.liquidsoap_started: self.notify_liquidsoap_started() else: - logger.debug("Unrecognized option in options(%s). Doing nothing" \ - % str(options)) + logger.debug("Unrecognized option in options({}). Doing nothing".format(options)) if __name__ == '__main__': - print - print '#########################################' - print '# *** pypo *** #' - print '# pypo notification gateway #' - print '#########################################' + print() + print('#########################################') + print('# *** pypo *** #') + print('# pypo notification gateway #') + print('#########################################') # initialize try: n = Notify() n.run_with_options(options) except Exception as e: - print( traceback.format_exc() ) + print(traceback.format_exc()) diff --git a/python_apps/pypo/liquidsoap/__main__.py b/python_apps/pypo/liquidsoap/__main__.py index 09e4abe5f..1377af109 100644 --- a/python_apps/pypo/liquidsoap/__main__.py +++ b/python_apps/pypo/liquidsoap/__main__.py @@ -1,29 +1,29 @@ """ Runs Airtime liquidsoap """ - import argparse import os -import generate_liquidsoap_cfg +from . import generate_liquidsoap_cfg import logging import subprocess +from pypo import pure PYPO_HOME = '/var/tmp/airtime/pypo/' def run(): '''Entry-point for this application''' - print "Airtime Liquidsoap" + print("Airtime Liquidsoap") parser = argparse.ArgumentParser() parser.add_argument("-d", "--debug", help="run in debug mode", action="store_true") args = parser.parse_args() - + os.environ["HOME"] = PYPO_HOME if args.debug: logging.basicConfig(level=getattr(logging, 'DEBUG', None)) - + generate_liquidsoap_cfg.run() ''' check liquidsoap version if less than 1.3 use legacy liquidsoap script ''' - liquidsoap_version=subprocess.check_output("liquidsoap --version", shell=True) + liquidsoap_version = subprocess.check_output("liquidsoap --version", shell=True, universal_newlines=True) if "1.1.1" not in liquidsoap_version: script_path = os.path.join(os.path.dirname(__file__), 'ls_script.liq') else: diff --git a/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py b/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py index 4f104c62c..27bc927f2 100644 --- a/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py +++ b/python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py @@ -1,3 +1,4 @@ + import logging import os import sys @@ -13,7 +14,7 @@ def generate_liquidsoap_config(ss): fh.write("################################################\n") fh.write("# The ignore() lines are to squash unused variable warnings\n") - for key, value in data.iteritems(): + for key, value in data.items(): try: if not "port" in key and not "bitrate" in key: # Stupid hack raise ValueError() @@ -27,29 +28,29 @@ def generate_liquidsoap_config(ss): except: #Everything else is a string str_buffer = "%s = \"%s\"\n" % (key, value) - fh.write(str_buffer.encode('utf-8')) + fh.write(str_buffer) # ignore squashes unused variable errors from Liquidsoap - fh.write(("ignore(%s)\n" % key).encode('utf-8')) + fh.write("ignore(%s)\n" % key) auth_path = os.path.dirname(os.path.realpath(__file__)) fh.write('log_file = "/var/log/airtime/pypo-liquidsoap/