Merge pull request #951 from paddatrapper/python3

Port to Python 3
This commit is contained in:
Robb 2020-05-04 20:43:25 -04:00 committed by GitHub
commit b73adda637
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
59 changed files with 834 additions and 1169 deletions

View File

@ -1,4 +1,4 @@
dist: trusty
dist: xenial
version: ~> 1.0
language: php
php:
@ -10,8 +10,6 @@ php:
- 7.0
# folks who prefer running on 5.x should be using 5.6 in most cases, 5.4 is no in the matrix since noone should use it
- 5.6
# this is in for centos support, it's still the default on CentOS 7.3 and there were some lang changes after 5.4
- 5.4
services:
- postgresql
- rabbitmq
@ -49,6 +47,13 @@ addons:
apt:
packages:
- silan
- libgirepository1.0-dev
- gir1.2-gstreamer-1.0
- gstreamer1.0-plugins-base
- gstreamer1.0-plugins-good
- gstreamer1.0-plugins-bad
- gstreamer1.0-plugins-ugly
- libcairo2-dev
- liquidsoap
- liquidsoap-plugin-mad
- liquidsoap-plugin-taglib
@ -58,10 +63,10 @@ addons:
- liquidsoap-plugin-faad
- liquidsoap-plugin-vorbis
- liquidsoap-plugin-opus
- python-nose
- python-rgain
- python-gst-1.0
- python-magic
- python3
- python3-nose
- python3-gst-1.0
- python3-magic
- dos2unix
install:
- >
@ -70,9 +75,11 @@ install:
fi
- >
if [[ "$PYTHON" == true ]]; then
pip install --user mkdocs
pushd python_apps/airtime_analyzer
python setup.py install --dry-run --no-init-script
pyenv local 3.7
pip3 install -U pip wheel
pip3 install --user mkdocs rgain3
pushd python_apps/airtime_analyzer
python3 setup.py install --dry-run --no-init-script
popd
fi
before_script:

View File

@ -3,8 +3,6 @@
echo "Updating Apt."
apt-get update > /dev/null
echo "Ensuring Pip is installed."
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python-pip > /dev/null
echo "Updating Pip."
pip install pip -q -q --upgrade > /dev/null
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3-pip > /dev/null
echo "Ensuring Mkdocs is installed."
pip install -q mkdocs > /dev/null
pip3 install mkdocs

30
install
View File

@ -465,7 +465,7 @@ while :; do
;;
--no-rabbitmq)
skip_rabbitmq=1
;;
;;
--)
shift
break
@ -923,22 +923,12 @@ loud "\n-----------------------------------------------------"
loud " * Installing Airtime Services * "
loud "-----------------------------------------------------"
verbose "\n * Installing necessary python services..."
loudCmd "pip install setuptools --upgrade"
loudCmd "pip install zipp==1.0.0"
verbose "...Done"
# Ubuntu Trusty and Debian Wheezy needs a workaround for python version SSL downloads
# This affects all python installs where python < 2.7.9
python_version=$(python --version 2>&1 | awk '{ print $2 }')
python_version=$(python3 --version 2>&1 | awk '{ print $2 }')
verbose "Detected Python version: $python_version"
# Convert version so each segment is zero padded for easy comparison
python_version_formatted=$(awk 'BEGIN {FS = "."} {printf "%03d.%03d.%03d\n", $1,$2,$3}' <<< $python_version)
if [[ "$python_version_formatted" < "002.007.009" ]]; then
verbose "\n * Installing pyOpenSSL and ca db for SNI support..."
loudCmd "pip install pyOpenSSL cryptography idna certifi --upgrade"
verbose "...Done"
fi
verbose "\n * Installing necessary python services..."
loudCmd "pip3 install setuptools --upgrade"
verbose "...Done"
verbose "\n * Creating /run/airtime..."
mkdir -p /run/airtime
@ -960,11 +950,11 @@ if [ ! -d /var/log/airtime ]; then
fi
verbose "\n * Installing API client..."
loudCmd "python ${AIRTIMEROOT}/python_apps/api_clients/setup.py install --install-scripts=/usr/bin"
loudCmd "python3 ${AIRTIMEROOT}/python_apps/api_clients/setup.py install --install-scripts=/usr/bin"
verbose "...Done"
verbose "\n * Installing pypo and liquidsoap..."
loudCmd "python ${AIRTIMEROOT}/python_apps/pypo/setup.py install --install-scripts=/usr/bin --no-init-script"
loudCmd "python3 ${AIRTIMEROOT}/python_apps/pypo/setup.py install --install-scripts=/usr/bin --no-init-script"
loudCmd "mkdir -p /var/log/airtime/{pypo,pypo-liquidsoap} /var/tmp/airtime/pypo/{cache,files,tmp} /var/tmp/airtime/show-recorder/"
loudCmd "chown -R ${web_user}:${web_user} /var/log/airtime/{pypo,pypo-liquidsoap} /var/tmp/airtime/pypo/{cache,files,tmp} /var/tmp/airtime/show-recorder/"
systemInitInstall airtime-liquidsoap $web_user
@ -972,7 +962,7 @@ systemInitInstall airtime-playout $web_user
verbose "...Done"
verbose "\n * Installing airtime-celery..."
loudCmd "python ${AIRTIMEROOT}/python_apps/airtime-celery/setup.py install --no-init-script"
loudCmd "python3 ${AIRTIMEROOT}/python_apps/airtime-celery/setup.py install --no-init-script"
# Create the Celery user
if $is_centos_dist; then
loudCmd "id celery 2>/dev/null || adduser --no-create-home -c 'LibreTime Celery' -r celery || true"
@ -988,7 +978,7 @@ systemInitInstall airtime-celery
verbose "...Done"
verbose "\n * Installing airtime_analyzer..."
loudCmd "python ${AIRTIMEROOT}/python_apps/airtime_analyzer/setup.py install --install-scripts=/usr/bin --no-init-script"
loudCmd "python3 ${AIRTIMEROOT}/python_apps/airtime_analyzer/setup.py install --install-scripts=/usr/bin --no-init-script"
systemInitInstall airtime_analyzer $web_user
verbose "...Done"

View File

@ -1,71 +1,59 @@
apache2
coreutils
curl
ecasound
flac
git
gstreamer1.0-plugins-bad
gstreamer1.0-plugins-good
gstreamer1.0-plugins-ugly
icecast2
lame
libao-ocaml
libapache2-mod-php7.3
php7.3
php7.3-dev
php7.3-bcmath
php7.3-mbstring
php-pear
php7.3-gd
php-amqplib
lsb-release
zip
unzip
rabbitmq-server
postgresql
postgresql-client
php7.3-pgsql
python
python-virtualenv
python-pip
libcairo2-dev
libcamomile-ocaml-data
libfaad2
libmad-ocaml
libopus0
libportaudio2
libpulse0
libsamplerate0
libsoundtouch-ocaml
libtaglib-ocaml
libao-ocaml
libmad-ocaml
ecasound
libportaudio2
libsamplerate0
libvo-aacenc0
python-rgain
python-gst-1.0
gstreamer1.0-plugins-ugly
python-pika
patch
icecast2
curl
php7.3-curl
mpg123
libcamomile-ocaml-data
libpulse0
vorbis-tools
liquidsoap
lsb-release
lsb-release
lsof
vorbisgain
flac
vorbis-tools
pwgen
libfaad2
mpg123
patch
php7.3
php7.3-bcmath
php7.3-curl
php7.3-dev
php7.3-gd
php7.3-mbstring
php7.3-pgsql
php-amqplib
php-apcu
lame
php-pear
pkg-config
postgresql
postgresql-client
pwgen
python3
python3-gst-1.0
python3-pika
python3-pip
python3-virtualenv
python3-cairo
rabbitmq-server
silan
coreutils
liquidsoap
libopus0
systemd-sysv
unzip
vorbisgain
vorbis-tools
vorbis-tools
xmlstarlet
zip

View File

@ -1,67 +0,0 @@
apache2
libapache2-mod-php5
php5
php-pear
php5-gd
lsb-release
rabbitmq-server
zip
unzip
postgresql
postgresql-client
php5-pgsql
python
python-virtualenv
python-pip
libsoundtouch-ocaml
libtaglib-ocaml
libao-ocaml
libmad-ocaml
ecasound
libportaudio2
libsamplerate0
libvo-aacenc0
python-rgain
python-gst-1.0
gstreamer1.0-plugins-ugly
python-pika
patch
icecast2
curl
php5-curl
mpg123
libcamomile-ocaml-data
libpulse0
vorbis-tools
lsb-release
lsof
vorbisgain
flac
vorbis-tools
pwgen
libfaad2
php-apc
lame
coreutils
liquidsoap
libopus0
sysvinit
sysvinit-utils
xmlstarlet

View File

@ -1,71 +1,56 @@
apache2
coreutils
curl
ecasound
flac
git
gstreamer1.0-plugins-bad
gstreamer1.0-plugins-good
gstreamer1.0-plugins-ugly
icecast2
lame
libao-ocaml
libapache2-mod-php7.0
php7.0
php7.0-dev
php7.0-bcmath
php7.0-mbstring
php-pear
php7.0-gd
php-amqplib
lsb-release
zip
unzip
rabbitmq-server
postgresql
postgresql-client
php7.0-pgsql
python
python-virtualenv
python-pip
libcamomile-ocaml-data
libfaad2
libmad-ocaml
libopus0
libportaudio2
libpulse0
libsamplerate0
libsoundtouch-ocaml
libtaglib-ocaml
libao-ocaml
libmad-ocaml
ecasound
libportaudio2
libsamplerate0
libvo-aacenc0
python-rgain
python-gst-1.0
gstreamer1.0-plugins-ugly
python-pika
patch
icecast2
curl
php7.0-curl
mpg123
libcamomile-ocaml-data
libpulse0
vorbis-tools
liquidsoap
lsb-release
lsb-release
lsof
vorbisgain
flac
vorbis-tools
pwgen
libfaad2
mpg123
patch
php7.0
php7.0-bcmath
php7.0-curl
php7.0-dev
php7.0-gd
php7.0-mbstring
php7.0-pgsql
php-amqplib
php-apcu
lame
coreutils
liquidsoap
libopus0
php-pear
postgresql
postgresql-client
pwgen
python3
python3-gst-1.0
python3-pika
python3-pip
python3-virtualenv
python3-cairo
rabbitmq-server
systemd-sysv
unzip
vorbisgain
vorbis-tools
vorbis-tools
xmlstarlet
zip

View File

@ -1,62 +1,27 @@
apache2
libapache2-mod-php7.2
php7.2
php-pear
php7.2-gd
php-bcmath
php-mbstring
lsb-release
zip
unzip
rabbitmq-server
postgresql
postgresql-client
php7.2-pgsql
python
python-virtualenv
python-pip
libsoundtouch-ocaml
libtaglib-ocaml
libao-ocaml
libmad-ocaml
ecasound
libportaudio2
libsamplerate0
python-rgain
python-gst-1.0
gstreamer1.0-plugins-ugly
python-pika
patch
php7.2-curl
mpg123
curl
icecast2
libcamomile-ocaml-data
libpulse0
vorbis-tools
lsof
vorbisgain
flac
vorbis-tools
pwgen
libfaad2
php-apcu
lame
build-essential
coreutils
curl
ecasound
flac
gstreamer1.0-plugins-bad
gstreamer1.0-plugins-good
gstreamer1.0-plugins-ugly
icecast2
lame
libao-ocaml
libapache2-mod-php7.2
libcamomile-ocaml-data
libfaad2
libffi-dev
libmad-ocaml
libopus0
libportaudio2
libpulse0
libsamplerate0
libsoundtouch-ocaml
libssl-dev
libtaglib-ocaml
liquidsoap
liquidsoap-plugin-alsa
liquidsoap-plugin-ao
@ -71,15 +36,34 @@ liquidsoap-plugin-pulseaudio
liquidsoap-plugin-taglib
liquidsoap-plugin-voaacenc
liquidsoap-plugin-vorbis
lsb-release
lsof
mpg123
patch
php7.2
php7.2-curl
php7.2-gd
php7.2-pgsql
php-apcu
php-bcmath
php-mbstring
php-pear
postgresql
postgresql-client
pwgen
python3
python3-dev
python3-gst-1.0
python3-pika
python3-pip
python3-virtualenv
python3-cairo
rabbitmq-server
silan
libopus0
sysvinit-utils
build-essential
libssl-dev
libffi-dev
python-dev
unzip
vorbisgain
vorbis-tools
vorbis-tools
xmlstarlet
zip

View File

@ -1,74 +0,0 @@
apache2
libapache2-mod-php5
php5
php-pear
php5-gd
lsb-release
zip
unzip
rabbitmq-server
postgresql
postgresql-client
php5-pgsql
python
python-virtualenv
python-pip
libsoundtouch-ocaml
libtaglib-ocaml
libao-ocaml
libmad-ocaml
ecasound
libportaudio2
libsamplerate0
python-rgain
python-gst0.10
gstreamer0.10-plugins-ugly
gir1.2-gstreamer-0.10
patch
curl
php5-curl
mpg123
icecast2
libcamomile-ocaml-data
libpulse0
vorbis-tools
lsb-release
lsof
vorbisgain
flac
vorbis-tools
pwgen
libfaad2
php-apc
dbus
lame
coreutils
liquidsoap
liquidsoap-plugin-alsa
liquidsoap-plugin-ao
liquidsoap-plugin-faad
liquidsoap-plugin-flac
liquidsoap-plugin-icecast
liquidsoap-plugin-lame
liquidsoap-plugin-mad
liquidsoap-plugin-ogg
liquidsoap-plugin-portaudio
liquidsoap-plugin-pulseaudio
liquidsoap-plugin-taglib
liquidsoap-plugin-voaacenc
liquidsoap-plugin-vorbis
xmlstarlet

View File

@ -1,62 +1,27 @@
apache2
libapache2-mod-php7.0
php7.0
php-pear
php7.0-gd
php-bcmath
php-mbstring
lsb-release
zip
unzip
rabbitmq-server
postgresql
postgresql-client
php7.0-pgsql
python
python-virtualenv
python-pip
libsoundtouch-ocaml
libtaglib-ocaml
libao-ocaml
libmad-ocaml
ecasound
libportaudio2
libsamplerate0
python-rgain
python-gst-1.0
gstreamer1.0-plugins-ugly
python-pika
patch
php7.0-curl
mpg123
curl
icecast2
libcamomile-ocaml-data
libpulse0
vorbis-tools
lsof
vorbisgain
flac
vorbis-tools
pwgen
libfaad2
php-apcu
lame
build-essential
coreutils
curl
ecasound
flac
gstreamer1.0-plugins-bad
gstreamer1.0-plugins-good
gstreamer1.0-plugins-ugly
icecast2
lame
libao-ocaml
libapache2-mod-php7.0
libcamomile-ocaml-data
libfaad2
libffi-dev
libmad-ocaml
libopus0
libportaudio2
libpulse0
libsamplerate0
libsoundtouch-ocaml
libssl-dev
libtaglib-ocaml
liquidsoap
liquidsoap-plugin-alsa
liquidsoap-plugin-ao
@ -71,15 +36,34 @@ liquidsoap-plugin-pulseaudio
liquidsoap-plugin-taglib
liquidsoap-plugin-voaacenc
liquidsoap-plugin-vorbis
lsb-release
lsof
mpg123
patch
php7.0
php7.0-curl
php7.0-gd
php7.0-pgsql
php-apcu
php-bcmath
php-mbstring
php-pear
postgresql
postgresql-client
pwgen
python3
python3-dev
python3-gst-1.0
python3-pika
python3-pip
python3-virtualenv
python3-cairo
rabbitmq-server
silan
libopus0
sysvinit-utils
build-essential
libssl-dev
libffi-dev
python-dev
unzip
vorbisgain
vorbis-tools
vorbis-tools
xmlstarlet
zip

View File

@ -1,57 +1,26 @@
apache2
libapache2-mod-php7.0
php7.0
php-pear
php7.0-gd
php-bcmath
php-mbstring
lsb-release
zip
unzip
postgresql-client
php7.0-pgsql
python
python-virtualenv
python-pip
libsoundtouch-ocaml
libtaglib-ocaml
libao-ocaml
libmad-ocaml
ecasound
libportaudio2
libsamplerate0
python-rgain
python-gst-1.0
gstreamer1.0-plugins-ugly
python-pika
patch
php7.0-curl
mpg123
curl
libcamomile-ocaml-data
libpulse0
vorbis-tools
lsof
vorbisgain
flac
vorbis-tools
pwgen
libfaad2
php-apcu
lame
build-essential
coreutils
curl
ecasound
flac
gstreamer1.0-plugins-bad
gstreamer1.0-plugins-good
gstreamer1.0-plugins-ugly
lame
libao-ocaml
libapache2-mod-php7.0
libcamomile-ocaml-data
libfaad2
libffi-dev
libmad-ocaml
libopus0
libportaudio2
libpulse0
libsamplerate0
libsoundtouch-ocaml
libssl-dev
libtaglib-ocaml
liquidsoap
liquidsoap-plugin-alsa
liquidsoap-plugin-ao
@ -66,15 +35,32 @@ liquidsoap-plugin-pulseaudio
liquidsoap-plugin-taglib
liquidsoap-plugin-voaacenc
liquidsoap-plugin-vorbis
lsb-release
lsof
mpg123
patch
php7.0
php7.0-curl
php7.0-gd
php7.0-pgsql
php-apcu
php-bcmath
php-mbstring
php-pear
postgresql-client
pwgen
python3
python3-dev
python3-gst-1.0
python3-pika
python3-pip
python3-virtualenv
python3-cairo
silan
libopus0
sysvinit-utils
build-essential
libssl-dev
libffi-dev
python-dev
unzip
vorbisgain
vorbis-tools
vorbis-tools
xmlstarlet
zip

View File

@ -86,8 +86,12 @@ def soundcloud_download(token, callback_url, api_key, track_id):
auth=requests.auth.HTTPBasicAuth(api_key, ""),
)
re.raise_for_status()
try:
response = re.content.decode()
except (UnicodeDecodeError, AttributeError):
response = re.content
f = json.loads(
re.content
response
) # Read the response from the media API to get the file id
obj["fileid"] = f["id"]
else:
@ -203,8 +207,12 @@ def podcast_download(
auth=requests.auth.HTTPBasicAuth(api_key, ""),
)
re.raise_for_status()
try:
response = re.content.decode()
except (UnicodeDecodeError, AttributeError):
response = re.content
f = json.loads(
re.content
response
) # Read the response from the media API to get the file id
obj["fileid"] = f["id"]
obj["status"] = 1

View File

@ -45,7 +45,7 @@ setup(
author_email="duncan.sommerville@sourcefabric.org",
license="MIT",
packages=["airtime-celery"],
install_requires=["soundcloud", "celery < 4", "kombu < 3.1", "configobj"],
install_requires=["soundcloud", "celery", "kombu", "configobj"],
zip_safe=False,
data_files=data_files,
)

View File

@ -5,12 +5,12 @@ import logging.handlers
import sys
import signal
import traceback
import config_file
from . import config_file
from functools import partial
from metadata_analyzer import MetadataAnalyzer
from replaygain_analyzer import ReplayGainAnalyzer
from status_reporter import StatusReporter
from message_listener import MessageListener
from .metadata_analyzer import MetadataAnalyzer
from .replaygain_analyzer import ReplayGainAnalyzer
from .status_reporter import StatusReporter
from .message_listener import MessageListener
class AirtimeAnalyzerServer:
@ -76,7 +76,7 @@ class AirtimeAnalyzerServer:
def dump_stacktrace(stack):
''' Dump a stacktrace for all threads '''
code = []
for threadId, stack in sys._current_frames().items():
for threadId, stack in list(sys._current_frames().items()):
code.append("\n# ThreadID: %s" % threadId)
for filename, lineno, name, line in traceback.extract_stack(stack):
code.append('File: "%s", line %d, in %s' % (filename, lineno, name))

View File

@ -1,19 +1,19 @@
""" Analyzes and imports an audio file into the Airtime library.
""" Analyzes and imports an audio file into the Airtime library.
"""
import logging
import threading
import multiprocessing
import Queue
import ConfigParser
from metadata_analyzer import MetadataAnalyzer
from filemover_analyzer import FileMoverAnalyzer
from cuepoint_analyzer import CuePointAnalyzer
from replaygain_analyzer import ReplayGainAnalyzer
from playability_analyzer import *
from queue import Queue
import configparser
from .metadata_analyzer import MetadataAnalyzer
from .filemover_analyzer import FileMoverAnalyzer
from .cuepoint_analyzer import CuePointAnalyzer
from .replaygain_analyzer import ReplayGainAnalyzer
from .playability_analyzer import *
class AnalyzerPipeline:
""" Analyzes and imports an audio file into the Airtime library.
""" Analyzes and imports an audio file into the Airtime library.
This currently performs metadata extraction (eg. gets the ID3 tags from an MP3),
then moves the file to the Airtime music library (stor/imported), and returns
the results back to the parent process. This class is used in an isolated process
@ -26,35 +26,35 @@ class AnalyzerPipeline:
@staticmethod
def run_analysis(queue, audio_file_path, import_directory, original_filename, storage_backend, file_prefix):
"""Analyze and import an audio file, and put all extracted metadata into queue.
Keyword arguments:
queue: A multiprocessing.queues.Queue which will be used to pass the
extracted metadata back to the parent process.
audio_file_path: Path on disk to the audio file to analyze.
import_directory: Path to the final Airtime "import" directory where
import_directory: Path to the final Airtime "import" directory where
we will move the file.
original_filename: The original filename of the file, which we'll try to
preserve. The file at audio_file_path typically has a
original_filename: The original filename of the file, which we'll try to
preserve. The file at audio_file_path typically has a
temporary randomly generated name, which is why we want
to know what the original name was.
to know what the original name was.
storage_backend: String indicating the storage backend (amazon_s3 or file)
file_prefix:
"""
# It is super critical to initialize a separate log file here so that we
# It is super critical to initialize a separate log file here so that we
# don't inherit logging/locks from the parent process. Supposedly
# this can lead to Bad Things (deadlocks): http://bugs.python.org/issue6721
AnalyzerPipeline.python_logger_deadlock_workaround()
try:
if not isinstance(queue, Queue.Queue):
if not isinstance(queue, Queue):
raise TypeError("queue must be a Queue.Queue()")
if not isinstance(audio_file_path, unicode):
if not isinstance(audio_file_path, str):
raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__ + " instead.")
if not isinstance(import_directory, unicode):
if not isinstance(import_directory, str):
raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__ + " instead.")
if not isinstance(original_filename, unicode):
if not isinstance(original_filename, str):
raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__ + " instead.")
if not isinstance(file_prefix, unicode):
if not isinstance(file_prefix, str):
raise TypeError("file_prefix must be unicode. Was of type " + type(file_prefix).__name__ + " instead.")
@ -72,7 +72,7 @@ class AnalyzerPipeline:
metadata["import_status"] = 0 # Successfully imported
# Note that the queue we're putting the results into is our interprocess communication
# Note that the queue we're putting the results into is our interprocess communication
# back to the main process.
# Pass all the file metadata back to the main analyzer process, which then passes
@ -91,7 +91,7 @@ class AnalyzerPipeline:
@staticmethod
def python_logger_deadlock_workaround():
# Workaround for: http://bugs.python.org/issue6721#msg140215
logger_names = logging.Logger.manager.loggerDict.keys()
logger_names = list(logging.Logger.manager.loggerDict.keys())
logger_names.append(None) # Root logger
for name in logger_names:
for handler in logging.getLogger(name).handlers:

View File

@ -1,15 +1,16 @@
import ConfigParser
import configparser
def read_config_file(config_path):
"""Parse the application's config file located at config_path."""
config = ConfigParser.SafeConfigParser()
config = configparser.SafeConfigParser()
try:
config.readfp(open(config_path))
except IOError as e:
print "Failed to open config file at " + config_path + ": " + e.strerror
print("Failed to open config file at {}: {}".format(config_path, e.strerror))
exit(-1)
except Exception as e:
print e.strerror
print(e.strerror)
exit(-1)
return config
return config

View File

@ -3,7 +3,7 @@ import logging
import traceback
import json
import datetime
from analyzer import Analyzer
from .analyzer import Analyzer
class CuePointAnalyzer(Analyzer):
@ -27,6 +27,10 @@ class CuePointAnalyzer(Analyzer):
command = [CuePointAnalyzer.SILAN_EXECUTABLE, '-b', '-F', '0.99', '-f', 'JSON', '-t', '1.0', filename]
try:
results_json = subprocess.check_output(command, stderr=subprocess.STDOUT, close_fds=True)
try:
results_json = results_json.decode()
except (UnicodeDecodeError, AttributeError):
pass
silan_results = json.loads(results_json)
# Defensive coding against Silan wildly miscalculating the cue in and out times:
@ -64,7 +68,7 @@ class CuePointAnalyzer(Analyzer):
except OSError as e: # silan was not found
logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have silan installed?"))
except subprocess.CalledProcessError as e: # silan returned an error code
logging.warn("%s %s %s", e.cmd, e.message, e.returncode)
logging.warn("%s %s %s", e.cmd, e.output, e.returncode)
except Exception as e:
logging.warn(e)

View File

@ -4,24 +4,24 @@ import time
import shutil
import os, errno
import time
import uuid
import uuid
from analyzer import Analyzer
from .analyzer import Analyzer
class FileMoverAnalyzer(Analyzer):
"""This analyzer copies a file over from a temporary directory (stor/organize)
"""This analyzer copies a file over from a temporary directory (stor/organize)
into the Airtime library (stor/imported).
"""
@staticmethod
def analyze(audio_file_path, metadata):
"""Dummy method because we need more info than analyze gets passed to it"""
raise Exception("Use FileMoverAnalyzer.move() instead.")
@staticmethod
def move(audio_file_path, import_directory, original_filename, metadata):
"""Move the file at audio_file_path over into the import_directory/import,
renaming it to original_filename.
Keyword arguments:
audio_file_path: Path to the file to be imported.
import_directory: Path to the "import" directory inside the Airtime stor directory.
@ -29,26 +29,28 @@ class FileMoverAnalyzer(Analyzer):
original_filename: The filename of the file when it was uploaded to Airtime.
metadata: A dictionary where the "full_path" of where the file is moved to will be added.
"""
if not isinstance(audio_file_path, unicode):
raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__)
if not isinstance(import_directory, unicode):
raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__)
if not isinstance(original_filename, unicode):
raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__)
if not isinstance(audio_file_path, str):
raise TypeError("audio_file_path must be string. Was of type " + type(audio_file_path).__name__)
if not isinstance(import_directory, str):
raise TypeError("import_directory must be string. Was of type " + type(import_directory).__name__)
if not isinstance(original_filename, str):
raise TypeError("original_filename must be string. Was of type " + type(original_filename).__name__)
if not isinstance(metadata, dict):
raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__)
if not os.path.exists(audio_file_path):
raise FileNotFoundError("audio file not found: {}".format(audio_file_path))
#Import the file over to it's final location.
# TODO: Also, handle the case where the move fails and write some code
# to possibly move the file to problem_files.
max_dir_len = 48
max_file_len = 48
final_file_path = import_directory
orig_file_basename, orig_file_extension = os.path.splitext(original_filename)
if metadata.has_key("artist_name"):
if "artist_name" in metadata:
final_file_path += "/" + metadata["artist_name"][0:max_dir_len] # truncating with array slicing
if metadata.has_key("album_title"):
if "album_title" in metadata:
final_file_path += "/" + metadata["album_title"][0:max_dir_len]
# Note that orig_file_extension includes the "." already
final_file_path += "/" + orig_file_basename[0:max_file_len] + orig_file_extension
@ -58,11 +60,11 @@ class FileMoverAnalyzer(Analyzer):
#If a file with the same name already exists in the "import" directory, then
#we add a unique string to the end of this one. We never overwrite a file on import
#because if we did that, it would mean Airtime's database would have
#because if we did that, it would mean Airtime's database would have
#the wrong information for the file we just overwrote (eg. the song length would be wrong!)
#If the final file path is the same as the file we've been told to import (which
#you often do when you're debugging), then don't move the file at all.
if os.path.exists(final_file_path):
if os.path.samefile(audio_file_path, final_file_path):
metadata["full_path"] = final_file_path
@ -77,14 +79,14 @@ class FileMoverAnalyzer(Analyzer):
#Ensure the full path to the file exists
mkdir_p(os.path.dirname(final_file_path))
#Move the file into its final destination directory
#Move the file into its final destination directory
logging.debug("Moving %s to %s" % (audio_file_path, final_file_path))
shutil.move(audio_file_path, final_file_path)
metadata["full_path"] = final_file_path
return metadata
def mkdir_p(path):
""" Make all directories in a tree (like mkdir -p)"""
if path == "":

View File

@ -6,9 +6,9 @@ import select
import signal
import logging
import multiprocessing
import Queue
from analyzer_pipeline import AnalyzerPipeline
from status_reporter import StatusReporter
import queue
from .analyzer_pipeline import AnalyzerPipeline
from .status_reporter import StatusReporter
EXCHANGE = "airtime-uploads"
EXCHANGE_TYPE = "topic"
@ -112,8 +112,7 @@ class MessageListener:
self._channel.queue_bind(exchange=EXCHANGE, queue=QUEUE, routing_key=ROUTING_KEY)
logging.info(" Listening for messages...")
self._channel.basic_consume(self.msg_received_callback,
queue=QUEUE, no_ack=False)
self._channel.basic_consume(QUEUE, self.msg_received_callback, auto_ack=False)
def wait_for_messages(self):
'''Wait until we've received a RabbitMQ message.'''
@ -158,6 +157,10 @@ class MessageListener:
We avoid cascading failure this way.
'''
try:
try:
body = body.decode()
except (UnicodeDecodeError, AttributeError):
pass
msg_dict = json.loads(body)
api_key = msg_dict["api_key"]
callback_url = msg_dict["callback_url"]
@ -198,7 +201,7 @@ class MessageListener:
if callback_url: # If we got an invalid message, there might be no callback_url in the JSON
# Report this as a failed upload to the File Upload REST API.
StatusReporter.report_failure_to_callback_url(callback_url, api_key, import_status=2,
reason=u'An error occurred while importing this file')
reason='An error occurred while importing this file')
else:
@ -224,7 +227,7 @@ class MessageListener:
'''
metadata = {}
q = Queue.Queue()
q = queue.Queue()
try:
AnalyzerPipeline.run_analysis(q, audio_file_path, import_directory, original_filename, storage_backend, file_prefix)
metadata = q.get()

View File

@ -6,22 +6,24 @@ import wave
import logging
import os
import hashlib
from analyzer import Analyzer
from .analyzer import Analyzer
class MetadataAnalyzer(Analyzer):
@staticmethod
def analyze(filename, metadata):
''' Extract audio metadata from tags embedded in the file (eg. ID3 tags)
Keyword arguments:
filename: The path to the audio file to extract metadata from.
metadata: A dictionary that the extracted metadata will be added to.
metadata: A dictionary that the extracted metadata will be added to.
'''
if not isinstance(filename, unicode):
raise TypeError("filename must be unicode. Was of type " + type(filename).__name__)
if not isinstance(filename, str):
raise TypeError("filename must be string. Was of type " + type(filename).__name__)
if not isinstance(metadata, dict):
raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__)
if not os.path.exists(filename):
raise FileNotFoundError("audio file not found: {}".format(filename))
#Airtime <= 2.5.x nonsense:
metadata["ftype"] = "audioclip"
@ -40,7 +42,7 @@ class MetadataAnalyzer(Analyzer):
m.update(data)
metadata["md5"] = m.hexdigest()
# Mutagen doesn't handle WAVE files so we use a different package
# Mutagen doesn't handle WAVE files so we use a different package
ms = magic.open(magic.MIME_TYPE)
ms.load()
with open(filename, 'rb') as fh:
@ -57,15 +59,15 @@ class MetadataAnalyzer(Analyzer):
if audio_file == None: # Don't use "if not" here. It is wrong due to mutagen's design.
return metadata
# Note that audio_file can equal {} if the file is valid but there's no metadata tags.
# We can still try to grab the info variables below.
# We can still try to grab the info variables below.
#Grab other file information that isn't encoded in a tag, but instead usually
#in the file header. Mutagen breaks that out into a separate "info" object:
info = audio_file.info
if hasattr(info, "sample_rate"): # Mutagen is annoying and inconsistent
metadata["sample_rate"] = info.sample_rate
if hasattr(info, "length"):
metadata["length_seconds"] = info.length
metadata["length_seconds"] = info.length
#Converting the length in seconds (float) to a formatted time string
track_length = datetime.timedelta(seconds=info.length)
metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length)
@ -77,12 +79,12 @@ class MetadataAnalyzer(Analyzer):
if hasattr(info, "bitrate"):
metadata["bit_rate"] = info.bitrate
# Use the mutagen to get the MIME type, if it has one. This is more reliable and
# consistent for certain types of MP3s or MPEG files than the MIMEs returned by magic.
if audio_file.mime:
metadata["mime"] = audio_file.mime[0]
#Try to get the number of channels if mutagen can...
try:
#Special handling for getting the # of channels from MP3s. It's in the "mode" field
@ -97,18 +99,18 @@ class MetadataAnalyzer(Analyzer):
except (AttributeError, KeyError):
#If mutagen can't figure out the number of channels, we'll just leave it out...
pass
#Try to extract the number of tracks on the album if we can (the "track total")
try:
track_number = audio_file["tracknumber"]
if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh
if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh
track_number = track_number[0]
track_number_tokens = track_number
if u'/' in track_number:
track_number_tokens = track_number.split(u'/')
track_number_tokens = track_number
if '/' in track_number:
track_number_tokens = track_number.split('/')
track_number = track_number_tokens[0]
elif u'-' in track_number:
track_number_tokens = track_number.split(u'-')
elif '-' in track_number:
track_number_tokens = track_number.split('-')
track_number = track_number_tokens[0]
metadata["track_number"] = track_number
track_total = track_number_tokens[1]
@ -118,7 +120,7 @@ class MetadataAnalyzer(Analyzer):
pass
#We normalize the mutagen tags slightly here, so in case mutagen changes,
#we find the
#we find the
mutagen_to_airtime_mapping = {
'title': 'track_title',
'artist': 'artist_name',
@ -146,20 +148,20 @@ class MetadataAnalyzer(Analyzer):
#'mime_type': 'mime',
}
for mutagen_tag, airtime_tag in mutagen_to_airtime_mapping.iteritems():
for mutagen_tag, airtime_tag in mutagen_to_airtime_mapping.items():
try:
metadata[airtime_tag] = audio_file[mutagen_tag]
# Some tags are returned as lists because there could be multiple values.
# This is unusual so we're going to always just take the first item in the list.
if isinstance(metadata[airtime_tag], list):
if metadata[airtime_tag]:
if metadata[airtime_tag]:
metadata[airtime_tag] = metadata[airtime_tag][0]
else: # Handle empty lists
metadata[airtime_tag] = ""
except KeyError:
continue
continue
return metadata
@ -174,7 +176,7 @@ class MetadataAnalyzer(Analyzer):
track_length = datetime.timedelta(seconds=length_seconds)
metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length)
metadata["length_seconds"] = length_seconds
metadata["cueout"] = metadata["length"]
metadata["cueout"] = metadata["length"]
except wave.Error as ex:
logging.error("Invalid WAVE file: {}".format(str(ex)))
raise

View File

@ -2,7 +2,7 @@ __author__ = 'asantoni'
import subprocess
import logging
from analyzer import Analyzer
from .analyzer import Analyzer
class UnplayableFileError(Exception):
pass

View File

@ -1,12 +1,13 @@
import subprocess
import logging
from analyzer import Analyzer
from .analyzer import Analyzer
import re
class ReplayGainAnalyzer(Analyzer):
''' This class extracts the ReplayGain using a tool from the python-rgain package. '''
REPLAYGAIN_EXECUTABLE = 'replaygain' # From the python-rgain package
REPLAYGAIN_EXECUTABLE = 'replaygain' # From the rgain3 python package
@staticmethod
def analyze(filename, metadata):
@ -19,17 +20,16 @@ class ReplayGainAnalyzer(Analyzer):
'''
command = [ReplayGainAnalyzer.REPLAYGAIN_EXECUTABLE, '-d', filename]
try:
results = subprocess.check_output(command, stderr=subprocess.STDOUT, close_fds=True)
filename_token = "%s: " % filename
rg_pos = results.find(filename_token, results.find("Calculating Replay Gain information")) + len(filename_token)
db_pos = results.find(" dB", rg_pos)
replaygain = results[rg_pos:db_pos]
results = subprocess.check_output(command, stderr=subprocess.STDOUT,
close_fds=True, text=True)
gain_match = r'Calculating Replay Gain information \.\.\.(?:\n|.)*?:([\d.-]*) dB'
replaygain = re.search(gain_match, results).group(1)
metadata['replay_gain'] = float(replaygain)
except OSError as e: # replaygain was not found
logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have python-rgain installed?"))
except subprocess.CalledProcessError as e: # replaygain returned an error code
logging.warn("%s %s %s", e.cmd, e.message, e.returncode)
logging.warn("%s %s %s", e.cmd, e.output, e.returncode)
except Exception as e:
logging.warn(e)

View File

@ -2,12 +2,12 @@ import requests
import json
import logging
import collections
import Queue
import queue
import time
import traceback
import pickle
import threading
from urlparse import urlparse
from urllib.parse import urlparse
# Disable urllib3 warnings because these can cause a rare deadlock due to Python 2's crappy internal non-reentrant locking
# around POSIX stuff. See SAAS-714. The hasattr() is for compatibility with older versions of requests.
@ -68,7 +68,7 @@ def process_http_requests(ipc_queue, http_retry_queue_path):
break
if not isinstance(request, PicklableHttpRequest):
raise TypeError("request must be a PicklableHttpRequest. Was of type " + type(request).__name__)
except Queue.Empty:
except queue.Empty:
request = None
# If there's no new HTTP request we need to execute, let's check our "retry
@ -159,7 +159,7 @@ class StatusReporter():
''' We use multiprocessing.Process again here because we need a thread for this stuff
anyways, and Python gives us process isolation for free (crash safety).
'''
_ipc_queue = Queue.Queue()
_ipc_queue = queue.Queue()
#_http_thread = multiprocessing.Process(target=process_http_requests,
# args=(_ipc_queue,))
_http_thread = None
@ -222,7 +222,7 @@ class StatusReporter():
@classmethod
def report_failure_to_callback_url(self, callback_url, api_key, import_status, reason):
if not isinstance(import_status, (int, long) ):
if not isinstance(import_status, int ):
raise TypeError("import_status must be an integer. Was of type " + type(import_status).__name__)
logging.debug("Reporting import failure to Airtime REST API...")

View File

@ -2,6 +2,7 @@
"""Runs the airtime_analyzer application.
"""
import daemon
import argparse
import os
@ -14,7 +15,7 @@ DEFAULT_HTTP_RETRY_PATH = '/tmp/airtime_analyzer_http_retries'
def run():
'''Entry-point for this application'''
print "Airtime Analyzer " + VERSION
print("Airtime Analyzer {}".format(VERSION))
parser = argparse.ArgumentParser()
parser.add_argument("-d", "--daemon", help="run as a daemon", action="store_true")
parser.add_argument("--debug", help="log full debugging output", action="store_true")
@ -22,8 +23,6 @@ def run():
parser.add_argument("--http-retry-queue-file", help="specify where incompleted HTTP requests will be serialized (default is %s)" % DEFAULT_HTTP_RETRY_PATH)
args = parser.parse_args()
check_if_media_monitor_is_running()
#Default config file path
rmq_config_path = DEFAULT_RMQ_CONFIG_PATH
http_retry_queue_path = DEFAULT_HTTP_RETRY_PATH
@ -35,32 +34,12 @@ def run():
if args.daemon:
with daemon.DaemonContext():
aa.AirtimeAnalyzerServer(rmq_config_path=rmq_config_path,
http_retry_queue_path=http_retry_queue_path,
http_retry_queue_path=http_retry_queue_path,
debug=args.debug)
else:
else:
# Run without daemonizing
aa.AirtimeAnalyzerServer(rmq_config_path=rmq_config_path,
http_retry_queue_path=http_retry_queue_path,
http_retry_queue_path=http_retry_queue_path,
debug=args.debug)
def check_if_media_monitor_is_running():
"""Ensure media_monitor isn't running before we start.
We do this because media_monitor will move newly uploaded
files into the library on us and screw up the operation of airtime_analyzer.
media_monitor is deprecated.
"""
pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
for pid in pids:
try:
process_name = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read()
if 'media_monitor.py' in process_name:
print "Error: This process conflicts with media_monitor, and media_monitor is running."
print " Please terminate the running media_monitor.py process and try again."
exit(1)
except IOError: # proc has already terminated
continue
run()

View File

@ -1,3 +1,4 @@
from __future__ import print_function
from setuptools import setup
from subprocess import call
import sys
@ -5,7 +6,7 @@ import os
# Change directory since setuptools uses relative paths
script_path = os.path.dirname(os.path.realpath(__file__))
print script_path
print(script_path)
os.chdir(script_path)
# Allows us to avoid installing the upstart init script when deploying airtime_analyzer
@ -16,7 +17,7 @@ if '--no-init-script' in sys.argv:
else:
data_files = [('/etc/init', ['install/upstart/airtime_analyzer.conf']),
('/etc/init.d', ['install/sysvinit/airtime_analyzer'])]
print data_files
print(data_files)
setup(name='airtime_analyzer',
version='0.1',
@ -28,16 +29,15 @@ setup(name='airtime_analyzer',
packages=['airtime_analyzer'],
scripts=['bin/airtime_analyzer'],
install_requires=[
'mutagen~=1.43.0', # got rid of specific version requirement
'pika',
'daemon',
'mutagen~=1.43',
'pika~=1.1.0',
'file-magic',
'nose',
'coverage',
'mock',
'python-daemon==1.6',
'python-daemon',
'requests>=2.7.0',
'rgain',
'rgain3',
# These next 3 are required for requests to support SSL with SNI. Learned this the hard way...
# What sucks is that GCC is required to pip install these.
#'ndg-httpsclient',
@ -49,8 +49,8 @@ setup(name='airtime_analyzer',
# Remind users to reload the initctl config so that "service start airtime_analyzer" works
if data_files:
print "Remember to reload the initctl configuration"
print "Run \"sudo initctl reload-configuration; sudo service airtime_analyzer restart\" now."
print "Or on Ubuntu Xenial (16.04)"
print "Remember to reload the systemd configuration"
print "Run \"sudo systemctl daemon-reload; sudo service airtime_analyzer restart\" now."
print("Remember to reload the initctl configuration")
print("Run \"sudo initctl reload-configuration; sudo service airtime_analyzer restart\" now.")
print("Or on Ubuntu Xenial (16.04)")
print("Remember to reload the systemd configuration")
print("Run \"sudo systemctl daemon-reload; sudo service airtime_analyzer restart\" now.")

View File

@ -1,9 +1,8 @@
from nose.tools import *
from ConfigParser import SafeConfigParser
import os
import shutil
import multiprocessing
import Queue
from queue import Queue
import datetime
from airtime_analyzer.analyzer_pipeline import AnalyzerPipeline
from airtime_analyzer import config_file
@ -21,7 +20,7 @@ def teardown():
def test_basic():
filename = os.path.basename(DEFAULT_AUDIO_FILE)
q = Queue.Queue()
q = Queue()
file_prefix = u''
storage_backend = "file"
#This actually imports the file into the "./Test Artist" directory.
@ -39,17 +38,17 @@ def test_basic():
@raises(TypeError)
def test_wrong_type_queue_param():
AnalyzerPipeline.run_analysis(Queue.Queue(), u'', u'', u'')
AnalyzerPipeline.run_analysis(Queue(), u'', u'', u'')
@raises(TypeError)
def test_wrong_type_string_param2():
AnalyzerPipeline.run_analysis(Queue.Queue(), '', u'', u'')
AnalyzerPipeline.run_analysis(Queue(), '', u'', u'')
@raises(TypeError)
def test_wrong_type_string_param3():
AnalyzerPipeline.run_analysis(Queue.Queue(), u'', '', u'')
AnalyzerPipeline.run_analysis(Queue(), u'', '', u'')
@raises(TypeError)
def test_wrong_type_string_param4():
AnalyzerPipeline.run_analysis(Queue.Queue(), u'', u'', '')
AnalyzerPipeline.run_analysis(Queue(), u'', u'', '')

View File

@ -2,7 +2,6 @@ from nose.tools import *
import os
import shutil
import multiprocessing
import Queue
import time
import mock
from pprint import pprint
@ -23,30 +22,34 @@ def test_dont_use_analyze():
@raises(TypeError)
def test_move_wrong_string_param1():
FileMoverAnalyzer.move('', u'', u'', dict())
FileMoverAnalyzer.move(42, '', '', dict())
@raises(TypeError)
def test_move_wrong_string_param2():
FileMoverAnalyzer.move(u'', '', u'', dict())
FileMoverAnalyzer.move(u'', 23, u'', dict())
@raises(TypeError)
def test_move_wrong_string_param3():
FileMoverAnalyzer.move(u'', u'', '', dict())
FileMoverAnalyzer.move('', '', 5, dict())
@raises(TypeError)
def test_move_wrong_dict_param():
FileMoverAnalyzer.move(u'', u'', u'', 12345)
FileMoverAnalyzer.move('', '', '', 12345)
@raises(FileNotFoundError)
def test_move_wrong_string_param3():
FileMoverAnalyzer.move('', '', '', dict())
def test_basic():
filename = os.path.basename(DEFAULT_AUDIO_FILE)
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict())
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict())
#Move the file back
shutil.move("./" + filename, DEFAULT_AUDIO_FILE)
assert os.path.exists(DEFAULT_AUDIO_FILE)
def test_basic_samefile():
filename = os.path.basename(DEFAULT_AUDIO_FILE)
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'tests/test_data', filename, dict())
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'tests/test_data', filename, dict())
assert os.path.exists(DEFAULT_AUDIO_FILE)
def test_duplicate_file():
@ -55,9 +58,9 @@ def test_duplicate_file():
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict())
#Copy it back to the original location
shutil.copy("./" + filename, DEFAULT_AUDIO_FILE)
#Import it again. It shouldn't overwrite the old file and instead create a new
#Import it again. It shouldn't overwrite the old file and instead create a new
metadata = dict()
metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, metadata)
metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, metadata)
#Cleanup: move the file (eg. 44100Hz-16bit-mono.mp3) back
shutil.move("./" + filename, DEFAULT_AUDIO_FILE)
#Remove the renamed duplicate, eg. 44100Hz-16bit-mono_03-26-2014-11-58.mp3
@ -71,7 +74,7 @@ def test_duplicate_file():
it's imported within 1 second of the second file (ie. if the timestamp is the same).
'''
def test_double_duplicate_files():
# Here we use mock to patch out the time.localtime() function so that it
# Here we use mock to patch out the time.localtime() function so that it
# always returns the same value. This allows us to consistently simulate this test cases
# where the last two of the three files are imported at the same time as the timestamp.
with mock.patch('airtime_analyzer.filemover_analyzer.time') as mock_time:
@ -83,17 +86,17 @@ def test_double_duplicate_files():
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict())
#Copy it back to the original location
shutil.copy("./" + filename, DEFAULT_AUDIO_FILE)
#Import it again. It shouldn't overwrite the old file and instead create a new
#Import it again. It shouldn't overwrite the old file and instead create a new
first_dup_metadata = dict()
first_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename,
first_dup_metadata)
first_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename,
first_dup_metadata)
#Copy it back again!
shutil.copy("./" + filename, DEFAULT_AUDIO_FILE)
#Reimport for the third time, which should have the same timestamp as the second one
#thanks to us mocking out time.localtime()
second_dup_metadata = dict()
second_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename,
second_dup_metadata)
second_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename,
second_dup_metadata)
#Cleanup: move the file (eg. 44100Hz-16bit-mono.mp3) back
shutil.move("./" + filename, DEFAULT_AUDIO_FILE)
#Remove the renamed duplicate, eg. 44100Hz-16bit-mono_03-26-2014-11-58.mp3
@ -105,7 +108,7 @@ def test_double_duplicate_files():
def test_bad_permissions_destination_dir():
filename = os.path.basename(DEFAULT_AUDIO_FILE)
dest_dir = u'/sys/foobar' # /sys is using sysfs on Linux, which is unwritable
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, dest_dir, filename, dict())
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, dest_dir, filename, dict())
#Move the file back
shutil.move(os.path.join(dest_dir, filename), DEFAULT_AUDIO_FILE)
assert os.path.exists(DEFAULT_AUDIO_FILE)

View File

@ -1,9 +1,10 @@
# -*- coding: utf-8 -*-
from __future__ import print_function
import datetime
import mutagen
import mock
from nose.tools import *
from airtime_analyzer.metadata_analyzer import MetadataAnalyzer
from airtime_analyzer.metadata_analyzer import MetadataAnalyzer
def setup():
pass
@ -12,115 +13,115 @@ def teardown():
pass
def check_default_metadata(metadata):
assert metadata['track_title'] == u'Test Title'
assert metadata['artist_name'] == u'Test Artist'
assert metadata['album_title'] == u'Test Album'
assert metadata['year'] == u'1999'
assert metadata['genre'] == u'Test Genre'
assert metadata['track_number'] == u'1'
assert metadata['track_title'] == 'Test Title'
assert metadata['artist_name'] == 'Test Artist'
assert metadata['album_title'] == 'Test Album'
assert metadata['year'] == '1999'
assert metadata['genre'] == 'Test Genre'
assert metadata['track_number'] == '1'
assert metadata["length"] == str(datetime.timedelta(seconds=metadata["length_seconds"]))
def test_mp3_mono():
metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.mp3', dict())
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-mono.mp3', dict())
check_default_metadata(metadata)
assert metadata['channels'] == 1
assert metadata['bit_rate'] == 63998
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['mime'] == 'audio/mp3' # Not unicode because MIMEs aren't.
assert metadata['track_total'] == u'10' # MP3s can have a track_total
assert metadata['track_total'] == '10' # MP3s can have a track_total
#Mutagen doesn't extract comments from mp3s it seems
def test_mp3_jointstereo():
metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-jointstereo.mp3', dict())
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-jointstereo.mp3', dict())
check_default_metadata(metadata)
assert metadata['channels'] == 2
assert metadata['bit_rate'] == 127998
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['mime'] == 'audio/mp3'
assert metadata['track_total'] == u'10' # MP3s can have a track_total
assert metadata['track_total'] == '10' # MP3s can have a track_total
def test_mp3_simplestereo():
metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-simplestereo.mp3', dict())
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-simplestereo.mp3', dict())
check_default_metadata(metadata)
assert metadata['channels'] == 2
assert metadata['bit_rate'] == 127998
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['mime'] == 'audio/mp3'
assert metadata['track_total'] == u'10' # MP3s can have a track_total
assert metadata['track_total'] == '10' # MP3s can have a track_total
def test_mp3_dualmono():
metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-dualmono.mp3', dict())
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-dualmono.mp3', dict())
check_default_metadata(metadata)
assert metadata['channels'] == 2
assert metadata['bit_rate'] == 127998
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['mime'] == 'audio/mp3'
assert metadata['track_total'] == u'10' # MP3s can have a track_total
assert metadata['track_total'] == '10' # MP3s can have a track_total
def test_ogg_mono():
metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.ogg', dict())
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-mono.ogg', dict())
check_default_metadata(metadata)
assert metadata['channels'] == 1
assert metadata['bit_rate'] == 80000
assert abs(metadata['length_seconds'] - 3.8) < 0.1
assert metadata['mime'] == 'audio/vorbis'
assert metadata['comment'] == u'Test Comment'
assert metadata['comment'] == 'Test Comment'
def test_ogg_stereo():
metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.ogg', dict())
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo.ogg', dict())
check_default_metadata(metadata)
assert metadata['channels'] == 2
assert metadata['bit_rate'] == 112000
assert abs(metadata['length_seconds'] - 3.8) < 0.1
assert metadata['mime'] == 'audio/vorbis'
assert metadata['comment'] == u'Test Comment'
assert metadata['comment'] == 'Test Comment'
''' faac and avconv can't seem to create a proper mono AAC file... ugh
def test_aac_mono():
metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.m4a')
print "Mono AAC metadata:"
print metadata
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-mono.m4a')
print("Mono AAC metadata:")
print(metadata)
check_default_metadata(metadata)
assert metadata['channels'] == 1
assert metadata['bit_rate'] == 80000
assert abs(metadata['length_seconds'] - 3.8) < 0.1
assert metadata['mime'] == 'audio/mp4'
assert metadata['comment'] == u'Test Comment'
assert metadata['comment'] == 'Test Comment'
'''
def test_aac_stereo():
metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.m4a', dict())
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo.m4a', dict())
check_default_metadata(metadata)
assert metadata['channels'] == 2
assert metadata['bit_rate'] == 102619
assert abs(metadata['length_seconds'] - 3.8) < 0.1
assert metadata['mime'] == 'audio/mp4'
assert metadata['comment'] == u'Test Comment'
assert metadata['comment'] == 'Test Comment'
def test_mp3_utf8():
metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict())
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict())
# Using a bunch of different UTF-8 codepages here. Test data is from:
# http://winrus.com/utf8-jap.htm
assert metadata['track_title'] == u'アイウエオカキクケコサシスセソタチツテ'
assert metadata['artist_name'] == u'てすと'
assert metadata['album_title'] == u'Ä ä Ü ü ß'
assert metadata['year'] == u'1999'
assert metadata['genre'] == u'Я Б Г Д Ж Й'
assert metadata['track_number'] == u'1'
assert metadata['track_title'] == 'アイウエオカキクケコサシスセソタチツテ'
assert metadata['artist_name'] == 'てすと'
assert metadata['album_title'] == 'Ä ä Ü ü ß'
assert metadata['year'] == '1999'
assert metadata['genre'] == 'Я Б Г Д Ж Й'
assert metadata['track_number'] == '1'
assert metadata['channels'] == 2
assert metadata['bit_rate'] < 130000
assert metadata['bit_rate'] > 127000
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['mime'] == 'audio/mp3'
assert metadata['track_total'] == u'10' # MP3s can have a track_total
assert metadata['track_total'] == '10' # MP3s can have a track_total
def test_invalid_wma():
metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-invalid.wma', dict())
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo-invalid.wma', dict())
assert metadata['mime'] == 'audio/x-ms-wma'
def test_wav_stereo():
metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.wav', dict())
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo.wav', dict())
assert metadata['mime'] == 'audio/x-wav'
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['channels'] == 2
@ -128,7 +129,7 @@ def test_wav_stereo():
# Make sure the parameter checking works
@raises(TypeError)
@raises(FileNotFoundError)
def test_move_wrong_string_param1():
not_unicode = 'asdfasdf'
MetadataAnalyzer.analyze(not_unicode, dict())
@ -136,12 +137,12 @@ def test_move_wrong_string_param1():
@raises(TypeError)
def test_move_wrong_metadata_dict():
not_a_dict = list()
MetadataAnalyzer.analyze(u'asdfasdf', not_a_dict)
MetadataAnalyzer.analyze('asdfasdf', not_a_dict)
# Test an mp3 file where the number of channels is invalid or missing:
def test_mp3_bad_channels():
filename = u'tests/test_data/44100Hz-16bit-mono.mp3'
'''
filename = 'tests/test_data/44100Hz-16bit-mono.mp3'
'''
It'd be a pain in the ass to construct a real MP3 with an invalid number
of channels by hand because that value is stored in every MP3 frame in the file
'''
@ -157,8 +158,8 @@ def test_mp3_bad_channels():
assert metadata['bit_rate'] == 63998
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['mime'] == 'audio/mp3' # Not unicode because MIMEs aren't.
assert metadata['track_total'] == u'10' # MP3s can have a track_total
assert metadata['track_total'] == '10' # MP3s can have a track_total
#Mutagen doesn't extract comments from mp3s it seems
def test_unparsable_file():
MetadataAnalyzer.analyze(u'README.rst', dict())
MetadataAnalyzer.analyze('README.rst', dict())

View File

@ -1,20 +1,7 @@
from __future__ import print_function
from nose.tools import *
from airtime_analyzer.replaygain_analyzer import ReplayGainAnalyzer
'''
The tests in here were all tagged with the 'rgain' tag so the can be exluded from being run
with nosetest -a '!rgain'. This was needed due to the fact that it is not readily possible
to install replaygain on a containerized travis instance.
We can either give running replaygain test on travis another shot after ubuntu getsan updated
gi instrospection allowing us to install gi and gobject into the virtualenv, or we can switch
to a full machine and stop using 'sudo: false' on travis.
Deactivating these tests is a bad fix for now and I plan on looking into it again after
most everything else is up and running. For those interesed the tests seem to work locally
albeit my results not being up to the given tolerance of 0.30 (which I'm assuming is my rig's
problem and would work on travis if replaygain was available).
'''
def check_default_metadata(metadata):
''' Check that the values extract by Silan/CuePointAnalyzer on our test audio files match what we expect.
@ -28,7 +15,7 @@ def check_default_metadata(metadata):
'''
tolerance = 0.30
expected_replaygain = 5.0
print metadata['replay_gain']
print(metadata['replay_gain'])
assert abs(metadata['replay_gain'] - expected_replaygain) < tolerance
def test_missing_replaygain():

View File

@ -8,8 +8,7 @@
###############################################################################
import sys
import time
import urllib
import urllib2
import urllib.request, urllib.error, urllib.parse
import requests
import socket
import logging
@ -21,26 +20,6 @@ from configobj import ConfigObj
AIRTIME_API_VERSION = "1.1"
# TODO : Place these functions in some common module. Right now, media
# monitor uses the same functions and it would be better to reuse them
# instead of copy pasting them around
def to_unicode(obj, encoding='utf-8'):
if isinstance(obj, basestring):
if not isinstance(obj, unicode):
obj = unicode(obj, encoding)
return obj
def encode_to(obj, encoding='utf-8'):
if isinstance(obj, unicode):
obj = obj.encode(encoding)
return obj
def convert_dict_value_to_utf8(md):
#list comprehension to convert all values of md to utf-8
return dict([(item[0], encode_to(item[1], "utf-8")) for item in md.items()])
api_config = {}
# URL to get the version number of the server API
@ -114,7 +93,7 @@ class ApcUrl(object):
def params(self, **params):
temp_url = self.base_url
for k, v in params.iteritems():
for k, v in params.items():
wrapped_param = "%%" + k + "%%"
if wrapped_param in temp_url:
temp_url = temp_url.replace(wrapped_param, str(v))
@ -138,12 +117,13 @@ class ApiRequest(object):
def __call__(self,_post_data=None, **kwargs):
final_url = self.url.params(**kwargs).url()
if _post_data is not None: _post_data = urllib.urlencode(_post_data)
if _post_data is not None:
_post_data = urllib.parse.urlencode(_post_data).encode('utf-8')
self.logger.debug(final_url)
try:
req = urllib2.Request(final_url, _post_data)
f = urllib2.urlopen(req, timeout=ApiRequest.API_HTTP_REQUEST_TIMEOUT)
content_type = f.info().getheader('Content-Type')
req = urllib.request.Request(final_url, _post_data)
f = urllib.request.urlopen(req, timeout=ApiRequest.API_HTTP_REQUEST_TIMEOUT)
content_type = f.info().get_content_type()
response = f.read()
#Everything that calls an ApiRequest should be catching URLError explicitly
#(according to the other comments in this file and a cursory grep through the code)
@ -151,20 +131,22 @@ class ApiRequest(object):
except socket.timeout:
self.logger.error('HTTP request to %s timed out', final_url)
raise
except Exception, e:
#self.logger.error('Exception: %s', e)
#self.logger.error("traceback: %s", traceback.format_exc())
except Exception as e:
#self.logger.exception(e)
raise
try:
if content_type == 'application/json':
try:
response = response.decode()
except (UnicodeDecodeError, AttributeError):
pass
data = json.loads(response)
return data
else:
raise InvalidContentType()
except Exception:
#self.logger.error(response)
#self.logger.error("traceback: %s", traceback.format_exc())
#self.logger.exception(e)
raise
def req(self, *args, **kwargs):
@ -193,18 +175,20 @@ class RequestProvider(object):
self.config["general"]["base_dir"], self.config["api_base"],
'%%action%%'))
# Now we must discover the possible actions
actions = dict( (k,v) for k,v in cfg.iteritems() if '%%api_key%%' in v)
for action_name, action_value in actions.iteritems():
actions = dict( (k,v) for k,v in cfg.items() if '%%api_key%%' in v)
for action_name, action_value in actions.items():
new_url = self.url.params(action=action_value).params(
api_key=self.config["general"]['api_key'])
self.requests[action_name] = ApiRequest(action_name, new_url)
def available_requests(self) : return self.requests.keys()
def available_requests(self) : return list(self.requests.keys())
def __contains__(self, request) : return request in self.requests
def __getattr__(self, attr):
if attr in self: return self.requests[attr]
else: return super(RequestProvider, self).__getattribute__(attr)
if attr in self:
return self.requests[attr]
else:
return super(RequestProvider, self).__getattribute__(attr)
class AirtimeApiClient(object):
@ -217,17 +201,16 @@ class AirtimeApiClient(object):
self.config = ConfigObj(config_path)
self.config.update(api_config)
self.services = RequestProvider(self.config)
except Exception, e:
self.logger.error('Error loading config file: %s', config_path)
self.logger.error("traceback: %s", traceback.format_exc())
except Exception as e:
self.logger.exception('Error loading config file: %s', config_path)
sys.exit(1)
def __get_airtime_version(self):
try: return self.services.version_url()[u'airtime_version']
try: return self.services.version_url()['airtime_version']
except Exception: return -1
def __get_api_version(self):
try: return self.services.version_url()[u'api_version']
try: return self.services.version_url()['api_version']
except Exception: return -1
def is_server_compatible(self, verbose=True):
@ -259,8 +242,8 @@ class AirtimeApiClient(object):
def notify_liquidsoap_started(self):
try:
self.services.notify_liquidsoap_started()
except Exception, e:
self.logger.error(str(e))
except Exception as e:
self.logger.exception(e)
def notify_media_item_start_playing(self, media_id):
""" This is a callback from liquidsoap, we use this to notify
@ -268,15 +251,15 @@ class AirtimeApiClient(object):
which we handed to liquidsoap in get_liquidsoap_data(). """
try:
return self.services.update_start_playing_url(media_id=media_id)
except Exception, e:
self.logger.error(str(e))
except Exception as e:
self.logger.exception(e)
return None
def get_shows_to_record(self):
try:
return self.services.show_schedule_url()
except Exception, e:
self.logger.error(str(e))
except Exception as e:
self.logger.exception(e)
return None
def upload_recorded_show(self, files, show_id):
@ -321,15 +304,14 @@ class AirtimeApiClient(object):
"""
break
except requests.exceptions.HTTPError, e:
except requests.exceptions.HTTPError as e:
logger.error("Http error code: %s", e.code)
logger.error("traceback: %s", traceback.format_exc())
except requests.exceptions.ConnectionError, e:
except requests.exceptions.ConnectionError as e:
logger.error("Server is down: %s", e.args)
logger.error("traceback: %s", traceback.format_exc())
except Exception, e:
logger.error("Exception: %s", e)
logger.error("traceback: %s", traceback.format_exc())
except Exception as e:
self.logger.exception(e)
#wait some time before next retry
time.sleep(retries_wait)
@ -340,8 +322,8 @@ class AirtimeApiClient(object):
try:
return self.services.check_live_stream_auth(
username=username, password=password, djtype=dj_type)
except Exception, e:
self.logger.error(str(e))
except Exception as e:
self.logger.exception(e)
return {}
def construct_url(self,config_action_key):
@ -407,7 +389,7 @@ class AirtimeApiClient(object):
# Note that we must prefix every key with: mdX where x is a number
# Is there a way to format the next line a little better? The
# parenthesis make the code almost unreadable
md_list = dict((("md%d" % i), json.dumps(convert_dict_value_to_utf8(md))) \
md_list = dict((("md%d" % i), json.dumps(md)) \
for i,md in enumerate(valid_actions))
# For testing we add the following "dry" parameter to tell the
# controller not to actually do any changes
@ -422,10 +404,10 @@ class AirtimeApiClient(object):
def list_all_db_files(self, dir_id, all_files=True):
logger = self.logger
try:
all_files = u"1" if all_files else u"0"
all_files = "1" if all_files else "0"
response = self.services.list_all_db_files(dir_id=dir_id,
all=all_files)
except Exception, e:
except Exception as e:
response = {}
logger.error("Exception: %s", e)
try:
@ -483,23 +465,20 @@ class AirtimeApiClient(object):
post_data = {"msg_post": msg}
#encoded_msg is no longer used server_side!!
encoded_msg = urllib.quote('dummy')
encoded_msg = urllib.parse.quote('dummy')
self.services.update_liquidsoap_status.req(post_data,
msg=encoded_msg,
stream_id=stream_id,
boot_time=time).retry(5)
except Exception, e:
#TODO
logger.error("Exception: %s", e)
except Exception as e:
self.logger.exception(e)
def notify_source_status(self, sourcename, status):
try:
logger = self.logger
return self.services.update_source_status.req(sourcename=sourcename,
status=status).retry(5)
except Exception, e:
#TODO
logger.error("Exception: %s", e)
except Exception as e:
self.logger.exception(e)
def get_bootstrap_info(self):
""" Retrieve infomations needed on bootstrap time """
@ -514,8 +493,8 @@ class AirtimeApiClient(object):
#http://localhost/api/get-files-without-replay-gain/dir_id/1
try:
return self.services.get_files_without_replay_gain(dir_id=dir_id)
except Exception, e:
self.logger.error(str(e))
except Exception as e:
self.logger.exception(e)
return []
def get_files_without_silan_value(self):
@ -526,8 +505,8 @@ class AirtimeApiClient(object):
"""
try:
return self.services.get_files_without_silan_value()
except Exception, e:
self.logger.error(str(e))
except Exception as e:
self.logger.exception(e)
return []
def update_replay_gain_values(self, pairs):
@ -569,9 +548,8 @@ class AirtimeApiClient(object):
try:
response = self.services.update_stream_setting_table(_post_data={'data': json.dumps(data)})
return response
except Exception, e:
#TODO
self.logger.error(str(e))
except Exception as e:
self.logger.exception(e)
def update_metadata_on_tunein(self):
self.services.update_metadata_on_tunein()

View File

@ -1,10 +1,11 @@
from __future__ import print_function
from setuptools import setup
from subprocess import call
import sys
import os
script_path = os.path.dirname(os.path.realpath(__file__))
print script_path
print(script_path)
os.chdir(script_path)
setup(name='api_clients',
@ -16,18 +17,7 @@ setup(name='api_clients',
packages=['api_clients'],
scripts=[],
install_requires=[
# 'amqplib',
# 'anyjson',
# 'argparse',
'configobj'
# 'docopt',
# 'kombu',
# 'mutagen',
# 'poster',
# 'PyDispatcher',
# 'pyinotify',
# 'pytz',
# 'wsgiref'
],
zip_safe=False,
data_files=[])
data_files=[])

View File

@ -1,20 +1,20 @@
import unittest
from .. api_client import ApcUrl, UrlBadParam, IncompleteUrl
from api_clients.api_client import ApcUrl, UrlBadParam, IncompleteUrl
class TestApcUrl(unittest.TestCase):
def test_init(self):
url = "/testing"
u = ApcUrl(url)
self.assertEquals( u.base_url, url)
self.assertEqual( u.base_url, url)
def test_params_1(self):
u = ApcUrl("/testing/%%key%%")
self.assertEquals(u.params(key='val').url(), '/testing/val')
self.assertEqual(u.params(key='val').url(), '/testing/val')
def test_params_2(self):
u = ApcUrl('/testing/%%key%%/%%api%%/more_testing')
full_url = u.params(key="AAA",api="BBB").url()
self.assertEquals(full_url, '/testing/AAA/BBB/more_testing')
self.assertEqual(full_url, '/testing/AAA/BBB/more_testing')
def test_params_ex(self):
u = ApcUrl("/testing/%%key%%")
@ -23,7 +23,7 @@ class TestApcUrl(unittest.TestCase):
def test_url(self):
u = "one/two/three"
self.assertEquals( ApcUrl(u).url(), u )
self.assertEqual( ApcUrl(u).url(), u )
def test_url_ex(self):
u = ApcUrl('/%%one%%/%%two%%/three').params(two='testing')

View File

@ -1,21 +1,26 @@
import unittest
import json
from mock import MagicMock, patch
from .. api_client import ApcUrl, ApiRequest
from api_clients.api_client import ApcUrl, ApiRequest
class ResponseInfo:
def get_content_type(self):
return 'application/json'
class TestApiRequest(unittest.TestCase):
def test_init(self):
u = ApiRequest('request_name', ApcUrl('/test/ing'))
self.assertEquals(u.name, "request_name")
self.assertEqual(u.name, "request_name")
def test_call(self):
ret = json.dumps( {u'ok':u'ok'} )
ret = json.dumps( {'ok':'ok'} )
read = MagicMock()
read.read = MagicMock(return_value=ret)
u = '/testing'
with patch('urllib2.urlopen') as mock_method:
read.info = MagicMock(return_value=ResponseInfo())
u = 'http://localhost/testing'
with patch('urllib.request.urlopen') as mock_method:
mock_method.return_value = read
request = ApiRequest('mm', ApcUrl(u))()
self.assertEquals(request, json.loads(ret))
self.assertEqual(request, json.loads(ret))
if __name__ == '__main__': unittest.main()

View File

@ -2,13 +2,19 @@ import unittest
import json
from mock import patch, MagicMock
from configobj import ConfigObj
from .. api_client import RequestProvider
from api_clients.api_client import RequestProvider, api_config
class TestRequestProvider(unittest.TestCase):
def setUp(self):
self.cfg = ConfigObj('api_client.cfg')
self.cfg = api_config
self.cfg['general'] = {}
self.cfg['general']['base_dir'] = '/test'
self.cfg['general']['base_port'] = 80
self.cfg['general']['base_url'] = 'localhost'
self.cfg['general']['api_key'] = 'TEST_KEY'
self.cfg['api_base'] = 'api'
def test_test(self):
self.assertTrue('api_key' in self.cfg)
self.assertTrue('general' in self.cfg)
def test_init(self):
rp = RequestProvider(self.cfg)
self.assertTrue( len( rp.available_requests() ) > 0 )
@ -16,17 +22,6 @@ class TestRequestProvider(unittest.TestCase):
rp = RequestProvider(self.cfg)
methods = ['upload_recorded', 'update_media_url', 'list_all_db_files']
for meth in methods:
self.assertTrue( meth in rp )
def test_notify_webstream_data(self):
ret = json.dumps( {u'testing' : u'123' } )
rp = RequestProvider(self.cfg)
read = MagicMock()
read.read = MagicMock(return_value=ret)
with patch('urllib2.urlopen') as mock_method:
mock_method.return_value = read
response = rp.notify_webstream_data(media_id=123)
mock_method.called_once_with(media_id=123)
self.assertEquals(json.loads(ret), response)
self.assertTrue( meth in rp.requests )
if __name__ == '__main__': unittest.main()

View File

@ -1,11 +1,12 @@
# -*- coding: utf-8 -*-
import shutil
import os
import sys
if os.geteuid() != 0:
print "Please run this as root."
print("Please run this as root.")
sys.exit(1)
def get_current_script_dir():
@ -17,6 +18,6 @@ try:
current_script_dir = get_current_script_dir()
shutil.copy(current_script_dir+"/../airtime-icecast-status.xsl", "/usr/share/icecast2/web")
except Exception, e:
print "exception: %s" % e
except Exception as e:
print("exception: {}".format(e))
sys.exit(1)

View File

@ -1,5 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import traceback
"""
@ -75,7 +76,7 @@ logger = rootLogger
try:
config = ConfigObj('/etc/airtime/airtime.conf')
except Exception, e:
except Exception as e:
logger.error('Error loading config file: %s', e)
sys.exit()
@ -133,21 +134,20 @@ class Notify:
elif options.liquidsoap_started:
self.notify_liquidsoap_started()
else:
logger.debug("Unrecognized option in options(%s). Doing nothing" \
% str(options))
logger.debug("Unrecognized option in options({}). Doing nothing".format(options))
if __name__ == '__main__':
print
print '#########################################'
print '# *** pypo *** #'
print '# pypo notification gateway #'
print '#########################################'
print()
print('#########################################')
print('# *** pypo *** #')
print('# pypo notification gateway #')
print('#########################################')
# initialize
try:
n = Notify()
n.run_with_options(options)
except Exception as e:
print( traceback.format_exc() )
print(traceback.format_exc())

View File

@ -1,29 +1,29 @@
""" Runs Airtime liquidsoap
"""
import argparse
import os
import generate_liquidsoap_cfg
from . import generate_liquidsoap_cfg
import logging
import subprocess
from pypo import pure
PYPO_HOME = '/var/tmp/airtime/pypo/'
def run():
'''Entry-point for this application'''
print "Airtime Liquidsoap"
print("Airtime Liquidsoap")
parser = argparse.ArgumentParser()
parser.add_argument("-d", "--debug", help="run in debug mode", action="store_true")
args = parser.parse_args()
os.environ["HOME"] = PYPO_HOME
if args.debug:
logging.basicConfig(level=getattr(logging, 'DEBUG', None))
generate_liquidsoap_cfg.run()
''' check liquidsoap version if less than 1.3 use legacy liquidsoap script '''
liquidsoap_version=subprocess.check_output("liquidsoap --version", shell=True)
liquidsoap_version = subprocess.check_output("liquidsoap --version", shell=True, universal_newlines=True)
if "1.1.1" not in liquidsoap_version:
script_path = os.path.join(os.path.dirname(__file__), 'ls_script.liq')
else:

View File

@ -1,3 +1,4 @@
import logging
import os
import sys
@ -13,7 +14,7 @@ def generate_liquidsoap_config(ss):
fh.write("################################################\n")
fh.write("# The ignore() lines are to squash unused variable warnings\n")
for key, value in data.iteritems():
for key, value in data.items():
try:
if not "port" in key and not "bitrate" in key: # Stupid hack
raise ValueError()
@ -27,29 +28,29 @@ def generate_liquidsoap_config(ss):
except: #Everything else is a string
str_buffer = "%s = \"%s\"\n" % (key, value)
fh.write(str_buffer.encode('utf-8'))
fh.write(str_buffer)
# ignore squashes unused variable errors from Liquidsoap
fh.write(("ignore(%s)\n" % key).encode('utf-8'))
fh.write("ignore(%s)\n" % key)
auth_path = os.path.dirname(os.path.realpath(__file__))
fh.write('log_file = "/var/log/airtime/pypo-liquidsoap/<script>.log"\n')
fh.write('auth_path = "%s/liquidsoap_auth.py"\n' % auth_path)
fh.close()
def run():
logging.basicConfig(format='%(message)s')
attempts = 0
max_attempts = 10
successful = False
while not successful:
try:
ac = AirtimeApiClient(logging.getLogger())
ss = ac.get_stream_setting()
generate_liquidsoap_config(ss)
successful = True
except Exception, e:
print "Unable to connect to the Airtime server."
except Exception as e:
print("Unable to connect to the Airtime server.")
logging.error(str(e))
logging.error("traceback: %s", traceback.format_exc())
if attempts == max_attempts:

View File

@ -1,3 +1,4 @@
from api_clients import *
import sys
@ -16,8 +17,8 @@ elif dj_type == '--dj':
response = api_clients.check_live_stream_auth(username, password, source_type)
if 'msg' in response and response['msg'] == True:
print response['msg']
print(response['msg'])
sys.exit(0)
else:
print False
print(False)
sys.exit(1)

View File

@ -13,7 +13,7 @@ try:
tn.write('exit\n')
tn.read_all()
except Exception, e:
print('Error loading config file: %s', e)
except Exception as e:
print("Error loading config file: {}".format(e))
sys.exit()

View File

@ -244,7 +244,7 @@ def check_auth(user="", password="", ~type="master") =
log("#{type} user #{user} connected",label="#{type}_source")
# Check auth based on return value from auth script
ret = snd(snd(run_process("python #{auth_path} --#{type} #{user} #{password}"))) == "0"
ret = snd(snd(run_process("python3 #{auth_path} --#{type} #{user} #{password}"))) == "0"
if ret then
log("#{type} user #{user} authenticated",label="#{type}_source")

View File

@ -239,7 +239,7 @@ end
def check_master_dj_client(user,password) =
log("master connected")
#get the output of the php script
ret = get_process_lines("python #{auth_path} --master #{user} #{password}")
ret = get_process_lines("python3 #{auth_path} --master #{user} #{password}")
#ret has now the value of the live client (dj1,dj2, or djx), or "ERROR"/"unknown" ...
ret = list.hd(ret)
@ -250,7 +250,7 @@ end
def check_dj_client(user,password) =
log("live dj connected")
#get the output of the php script
ret = get_process_lines("python #{auth_path} --dj #{user} #{password}")
ret = get_process_lines("python3 #{auth_path} --dj #{user} #{password}")
#ret has now the value of the live client (dj1,dj2, or djx), or "ERROR"/"unknown" ...
hd = list.hd(ret)
log("Live DJ authenticated: #{hd}")

View File

@ -1,7 +1,7 @@
"""
Python part of radio playout (pypo)
"""
from __future__ import absolute_import
import locale
import logging
@ -16,10 +16,11 @@ from api_clients import api_client
from configobj import ConfigObj
from datetime import datetime
from optparse import OptionParser
import importlib
try:
from queue import Queue
except ImportError: # Python 2.7.5 (CentOS 7)
from Queue import Queue
from queue import Queue
from threading import Lock
from .listenerstat import ListenerStat
@ -119,62 +120,9 @@ try:
consoleHandler.setFormatter(logFormatter)
rootLogger.addHandler(consoleHandler)
except Exception as e:
print("Couldn't configure logging", e)
print("Couldn't configure logging: {}".format(e))
sys.exit(1)
def configure_locale():
"""
Silly hacks to force Python 2.x to run in UTF-8 mode. Not portable at all,
however serves our purpose at the moment.
More information available here:
http://stackoverflow.com/questions/3828723/why-we-need-sys-setdefaultencodingutf-8-in-a-py-script
"""
logger.debug("Before %s", locale.nl_langinfo(locale.CODESET))
current_locale = locale.getlocale()
if current_locale[1] is None:
logger.debug("No locale currently set. Attempting to get default locale.")
default_locale = locale.getdefaultlocale()
if default_locale[1] is None:
logger.debug(
"No default locale exists. Let's try loading from \
/etc/default/locale"
)
if os.path.exists("/etc/default/locale"):
locale_config = ConfigObj("/etc/default/locale")
lang = locale_config.get("LANG")
new_locale = lang
else:
logger.error(
"/etc/default/locale could not be found! Please \
run 'sudo update-locale' from command-line."
)
sys.exit(1)
else:
new_locale = default_locale
logger.info(
"New locale set to: %s", locale.setlocale(locale.LC_ALL, new_locale)
)
reload(sys)
sys.setdefaultencoding("UTF-8")
current_locale_encoding = locale.getlocale()[1].lower()
logger.debug("sys default encoding %s", sys.getdefaultencoding())
logger.debug("After %s", locale.nl_langinfo(locale.CODESET))
if current_locale_encoding not in ["utf-8", "utf8"]:
logger.error(
"Need a UTF-8 locale. Currently '%s'. Exiting..." % current_locale_encoding
)
sys.exit(1)
configure_locale()
# loading config file
try:
config = ConfigObj("/etc/airtime/airtime.conf")
@ -207,11 +155,11 @@ def liquidsoap_get_info(telnet_lock, host, port, logger):
telnet_lock.acquire()
tn = telnetlib.Telnet(host, port)
msg = "version\n"
tn.write(msg)
tn.write("exit\n")
response = tn.read_all()
tn.write(msg.encode("utf-8"))
tn.write("exit\n".encode("utf-8"))
response = tn.read_all().decode("utf-8")
except Exception as e:
logger.error(str(e))
logger.error(e)
return None
finally:
telnet_lock.release()

View File

@ -1,5 +1,5 @@
from threading import Thread
import urllib2
import urllib.request, urllib.error, urllib.parse
import defusedxml.minidom
import base64
from datetime import datetime
@ -44,13 +44,13 @@ class ListenerStat(Thread):
user_agent = "Mozilla/5.0 (Linux; rv:22.0) Gecko/20130405 Firefox/22.0"
header["User-Agent"] = user_agent
req = urllib2.Request(
req = urllib.request.Request(
#assuming that the icecast stats path is /admin/stats.xml
#need to fix this
url=url,
headers=header)
f = urllib2.urlopen(req, timeout=ListenerStat.HTTP_REQUEST_TIMEOUT)
f = urllib.request.urlopen(req, timeout=ListenerStat.HTTP_REQUEST_TIMEOUT)
document = f.read()
return document
@ -146,7 +146,7 @@ class ListenerStat(Thread):
if stats:
self.push_stream_stats(stats)
except Exception, e:
except Exception as e:
self.logger.error('Exception: %s', e)
time.sleep(120)

View File

@ -1,10 +1,14 @@
import re
from packaging.version import Version, parse
def version_cmp(version1, version2):
def normalize(v):
return [int(x) for x in re.sub(r'(\.0+)*$','', v).split(".")]
return cmp(normalize(version1), normalize(version2))
version1 = parse(version1)
version2 = parse(version2)
if version1 > version2:
return 1
if version1 == version2:
return 0
return -1
def date_interval_to_seconds(interval):
"""

View File

@ -10,15 +10,14 @@ import copy
import subprocess
import signal
from datetime import datetime
import traceback
import pure
import mimetypes
from Queue import Empty
from . import pure
from queue import Empty
from threading import Thread, Timer
from subprocess import Popen, PIPE
from api_clients import api_client
from timeout import ls_timeout
from .timeout import ls_timeout
def keyboardInterruptHandler(signum, frame):
@ -65,7 +64,7 @@ class PypoFetch(Thread):
"""
self.logger.debug("Cache dir does not exist. Creating...")
os.makedirs(dir)
except Exception, e:
except Exception as e:
pass
self.schedule_data = []
@ -79,6 +78,10 @@ class PypoFetch(Thread):
try:
self.logger.info("Received event from Pypo Message Handler: %s" % message)
try:
message = message.decode()
except (UnicodeDecodeError, AttributeError):
pass
m = json.loads(message)
command = m['event_type']
self.logger.info("Handling command: " + command)
@ -120,11 +123,8 @@ class PypoFetch(Thread):
if self.listener_timeout < 0:
self.listener_timeout = 0
self.logger.info("New timeout: %s" % self.listener_timeout)
except Exception, e:
top = traceback.format_exc()
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", top)
self.logger.error("Exception in handling Message Handler message: %s", e)
except Exception as e:
self.logger.exception("Exception in handling Message Handler message")
def switch_source_temp(self, sourcename, status):
@ -151,13 +151,12 @@ class PypoFetch(Thread):
self.logger.debug('Getting information needed on bootstrap from Airtime')
try:
info = self.api_client.get_bootstrap_info()
except Exception, e:
self.logger.error('Unable to get bootstrap info.. Exiting pypo...')
self.logger.error(str(e))
except Exception as e:
self.logger.exception('Unable to get bootstrap info.. Exiting pypo...')
self.logger.debug('info:%s', info)
commands = []
for k, v in info['switch_status'].iteritems():
for k, v in info['switch_status'].items():
commands.append(self.switch_source_temp(k, v))
stream_format = info['stream_label']
@ -190,16 +189,16 @@ class PypoFetch(Thread):
while True:
try:
tn = telnetlib.Telnet(self.config['ls_host'], self.config['ls_port'])
tn.write("exit\n")
tn.write('exit\n'.encode('utf-8'))
tn.read_all()
self.logger.info("Liquidsoap is up and running")
break
except Exception, e:
except Exception as e:
#sleep 0.5 seconds and try again
time.sleep(0.5)
except Exception, e:
self.logger.error(e)
except Exception as e:
self.logger.exception(e)
finally:
if self.telnet_lock.locked():
self.telnet_lock.release()
@ -226,19 +225,19 @@ class PypoFetch(Thread):
# we are manually adjusting the bootup time variable so the status msg will get
# updated.
current_time = time.time()
boot_up_time_command = "vars.bootup_time " + str(current_time) + "\n"
boot_up_time_command = ("vars.bootup_time " + str(current_time) + "\n").encode('utf-8')
self.logger.info(boot_up_time_command)
tn.write(boot_up_time_command)
connection_status = "streams.connection_status\n"
connection_status = ("streams.connection_status\n").encode('utf-8')
self.logger.info(connection_status)
tn.write(connection_status)
tn.write('exit\n')
tn.write('exit\n'.encode('utf-8'))
output = tn.read_all()
except Exception, e:
self.logger.error(str(e))
except Exception as e:
self.logger.exception(e)
finally:
self.telnet_lock.release()
@ -269,10 +268,10 @@ class PypoFetch(Thread):
command = ('vars.stream_metadata_type %s\n' % stream_format).encode('utf-8')
self.logger.info(command)
tn.write(command)
tn.write('exit\n')
tn.write('exit\n'.encode('utf-8'))
tn.read_all()
except Exception, e:
self.logger.error("Exception %s", e)
except Exception as e:
self.logger.exception(e)
finally:
self.telnet_lock.release()
@ -286,10 +285,10 @@ class PypoFetch(Thread):
command = ('vars.default_dj_fade %s\n' % fade).encode('utf-8')
self.logger.info(command)
tn.write(command)
tn.write('exit\n')
tn.write('exit\n'.encode('utf-8'))
tn.read_all()
except Exception, e:
self.logger.error("Exception %s", e)
except Exception as e:
self.logger.exception(e)
finally:
self.telnet_lock.release()
@ -304,14 +303,14 @@ class PypoFetch(Thread):
command = ('vars.station_name %s\n' % station_name).encode('utf-8')
self.logger.info(command)
tn.write(command)
tn.write('exit\n')
tn.write('exit\n'.encode('utf-8'))
tn.read_all()
except Exception, e:
self.logger.error(str(e))
except Exception as e:
self.logger.exception(e)
finally:
self.telnet_lock.release()
except Exception, e:
self.logger.error("Exception %s", e)
except Exception as e:
self.logger.exception(e)
"""
Process the schedule
@ -336,7 +335,7 @@ class PypoFetch(Thread):
download_dir = self.cache_dir
try:
os.makedirs(download_dir)
except Exception, e:
except Exception as e:
pass
media_copy = {}
@ -344,20 +343,21 @@ class PypoFetch(Thread):
media_item = media[key]
if (media_item['type'] == 'file'):
fileExt = self.sanity_check_media_item(media_item)
dst = os.path.join(download_dir, unicode(media_item['id']) + unicode(fileExt))
dst = os.path.join(download_dir, "{}{}".format(media_item['id'], fileExt))
media_item['dst'] = dst
media_item['file_ready'] = False
media_filtered[key] = media_item
media_item['start'] = datetime.strptime(media_item['start'],
media_item['start'] = datetime.strptime(media_item['start'],
"%Y-%m-%d-%H-%M-%S")
media_item['end'] = datetime.strptime(media_item['end'],
media_item['end'] = datetime.strptime(media_item['end'],
"%Y-%m-%d-%H-%M-%S")
media_copy[key] = media_item
self.media_prepare_queue.put(copy.copy(media_filtered))
except Exception, e: self.logger.error("%s", e)
except Exception as e:
self.logger.exception(e)
# Send the data to pypo-push
self.logger.debug("Pushing to pypo-push")
@ -365,8 +365,10 @@ class PypoFetch(Thread):
# cleanup
try: self.cache_cleanup(media)
except Exception, e: self.logger.error("%s", e)
try:
self.cache_cleanup(media)
except Exception as e:
self.logger.exception(e)
#do basic validation of file parameters. Useful for debugging
#purposes
@ -408,7 +410,9 @@ class PypoFetch(Thread):
for mkey in media:
media_item = media[mkey]
if media_item['type'] == 'file':
scheduled_file_set.add(unicode(media_item["id"]) + unicode(media_item["file_ext"]))
if "file_ext" not in media_item.keys():
media_item["file_ext"] = mimetypes.guess_extension(media_item['metadata']['mime'], strict=False)
scheduled_file_set.add("{}{}".format(media_item["id"], media_item["file_ext"]))
expired_files = cached_file_set - scheduled_file_set
@ -426,9 +430,8 @@ class PypoFetch(Thread):
self.logger.info("File '%s' removed" % path)
else:
self.logger.info("File '%s' not removed. Still busy!" % path)
except Exception, e:
self.logger.error("Problem removing file '%s'" % f)
self.logger.error(traceback.format_exc())
except Exception as e:
self.logger.exception("Problem removing file '%s'" % f)
def manual_schedule_fetch(self):
success, self.schedule_data = self.api_client.get_schedule()
@ -498,18 +501,13 @@ class PypoFetch(Thread):
self.logger.info("Queue timeout. Fetching schedule manually")
manual_fetch_needed = True
except Exception as e:
top = traceback.format_exc()
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", top)
self.logger.exception(e)
try:
if manual_fetch_needed:
self.persistent_manual_schedule_fetch(max_attempts=5)
except Exception as e:
top = traceback.format_exc()
self.logger.error('Failed to manually fetch the schedule.')
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", top)
self.logger.exception('Failed to manually fetch the schedule.')
loops += 1

View File

@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
from threading import Thread
from Queue import Empty
from ConfigParser import NoOptionError
from queue import Empty
from configparser import NoOptionError
import logging
import shutil
@ -12,7 +12,7 @@ import os
import sys
import stat
import requests
import ConfigParser
import configparser
import json
import hashlib
from requests.exceptions import ConnectionError, HTTPError, Timeout
@ -44,7 +44,7 @@ class PypoFile(Thread):
dst_exists = True
try:
dst_size = os.path.getsize(dst)
except Exception, e:
except Exception as e:
dst_exists = False
do_copy = False
@ -69,11 +69,11 @@ class PypoFile(Thread):
baseurl = self._config.get(CONFIG_SECTION, 'base_url')
try:
port = self._config.get(CONFIG_SECTION, 'base_port')
except NoOptionError, e:
except NoOptionError as e:
port = 80
try:
protocol = self._config.get(CONFIG_SECTION, 'protocol')
except NoOptionError, e:
except NoOptionError as e:
protocol = str(("http", "https")[int(port) == 443])
try:
@ -103,7 +103,7 @@ class PypoFile(Thread):
media_item["filesize"] = file_size
media_item['file_ready'] = True
except Exception, e:
except Exception as e:
self.logger.error("Could not copy from %s to %s" % (src, dst))
self.logger.error(e)
@ -172,7 +172,7 @@ class PypoFile(Thread):
def read_config_file(self, config_path):
"""Parse the application's config file located at config_path."""
config = ConfigParser.SafeConfigParser(allow_no_value=True)
config = configparser.SafeConfigParser(allow_no_value=True)
try:
config.readfp(open(config_path))
except IOError as e:
@ -202,14 +202,14 @@ class PypoFile(Thread):
"""
try:
self.media = self.media_queue.get_nowait()
except Empty, e:
except Empty as e:
pass
media_item = self.get_highest_priority_media_item(self.media)
if media_item is not None:
self.copy_file(media_item)
except Exception, e:
except Exception as e:
import traceback
top = traceback.format_exc()
self.logger.error(str(e))
@ -221,7 +221,7 @@ class PypoFile(Thread):
Entry point of the thread
"""
try: self.main()
except Exception, e:
except Exception as e:
top = traceback.format_exc()
self.logger.error('PypoFile Exception: %s', top)
time.sleep(5)

View File

@ -7,7 +7,7 @@ import sys
import time
from Queue import Empty
from queue import Empty
import signal
def keyboardInterruptHandler(signum, frame):
@ -38,7 +38,7 @@ class PypoLiqQueue(Thread):
time_until_next_play)
media_schedule = self.queue.get(block=True, \
timeout=time_until_next_play)
except Empty, e:
except Empty as e:
#Time to push a scheduled item.
media_item = schedule_deque.popleft()
self.pypo_liquidsoap.play(media_item)
@ -82,7 +82,7 @@ class PypoLiqQueue(Thread):
def run(self):
try: self.main()
except Exception, e:
except Exception as e:
self.logger.error('PypoLiqQueue Exception: %s', traceback.format_exc())

View File

@ -1,10 +1,10 @@
from pypofetch import PypoFetch
from telnetliquidsoap import TelnetLiquidsoap
from .pypofetch import PypoFetch
from .telnetliquidsoap import TelnetLiquidsoap
from datetime import datetime
from datetime import timedelta
import eventtypes
from . import eventtypes
import time
class PypoLiquidsoap():
@ -22,7 +22,7 @@ class PypoLiquidsoap():
logger,\
host,\
port,\
self.liq_queue_tracker.keys())
list(self.liq_queue_tracker.keys()))
def get_telnet_dispatcher(self):
return self.telnet_liquidsoap
@ -97,36 +97,37 @@ class PypoLiquidsoap():
def verify_correct_present_media(self, scheduled_now):
#verify whether Liquidsoap is currently playing the correct files.
#if we find an item that Liquidsoap is not playing, then push it
#into one of Liquidsoap's queues. If Liquidsoap is already playing
#it do nothing. If Liquidsoap is playing a track that isn't in
#currently_playing then stop it.
"""
verify whether Liquidsoap is currently playing the correct files.
if we find an item that Liquidsoap is not playing, then push it
into one of Liquidsoap's queues. If Liquidsoap is already playing
it do nothing. If Liquidsoap is playing a track that isn't in
currently_playing then stop it.
#Check for Liquidsoap media we should source.skip
#get liquidsoap items for each queue. Since each queue can only have one
#item, we should have a max of 8 items.
Check for Liquidsoap media we should source.skip
get liquidsoap items for each queue. Since each queue can only have one
item, we should have a max of 8 items.
#2013-03-21-22-56-00_0: {
#id: 1,
#type: "stream_output_start",
#row_id: 41,
#uri: "http://stream2.radioblackout.org:80/blackout.ogg",
#start: "2013-03-21-22-56-00",
#end: "2013-03-21-23-26-00",
#show_name: "Untitled Show",
#independent_event: true
#},
2013-03-21-22-56-00_0: {
id: 1,
type: "stream_output_start",
row_id: 41,
uri: "http://stream2.radioblackout.org:80/blackout.ogg",
start: "2013-03-21-22-56-00",
end: "2013-03-21-23-26-00",
show_name: "Untitled Show",
independent_event: true
},
"""
try:
scheduled_now_files = \
filter(lambda x: x["type"] == eventtypes.FILE, scheduled_now)
[x for x in scheduled_now if x["type"] == eventtypes.FILE]
scheduled_now_webstream = \
filter(lambda x: x["type"] == eventtypes.STREAM_OUTPUT_START, \
scheduled_now)
[x for x in scheduled_now if x["type"] == eventtypes.STREAM_OUTPUT_START]
schedule_ids = set(map(lambda x: x["row_id"], scheduled_now_files))
schedule_ids = set([x["row_id"] for x in scheduled_now_files])
row_id_map = {}
liq_queue_ids = set()
@ -156,7 +157,6 @@ class PypoLiquidsoap():
to_be_removed.add(i["row_id"])
to_be_added.add(i["row_id"])
to_be_removed.update(liq_queue_ids - schedule_ids)
to_be_added.update(schedule_ids - liq_queue_ids)

View File

@ -53,7 +53,7 @@ class PypoMessageHandler(Thread):
heartbeat = 5) as connection:
rabbit = RabbitConsumer(connection, [schedule_queue], self)
rabbit.run()
except Exception, e:
except Exception as e:
self.logger.error(e)
"""
@ -64,6 +64,10 @@ class PypoMessageHandler(Thread):
try:
self.logger.info("Received event from RabbitMQ: %s" % message)
try:
message = message.decode()
except (UnicodeDecodeError, AttributeError):
pass
m = json.loads(message)
command = m['event_type']
self.logger.info("Handling command: " + command)
@ -98,13 +102,13 @@ class PypoMessageHandler(Thread):
self.recorder_queue.put(message)
else:
self.logger.info("Unknown command: %s" % command)
except Exception, e:
except Exception as e:
self.logger.error("Exception in handling RabbitMQ message: %s", e)
def main(self):
try:
self.init_rabbit_mq()
except Exception, e:
except Exception as e:
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", traceback.format_exc())
self.logger.error("Error connecting to RabbitMQ Server. Trying again in few seconds")

View File

@ -13,15 +13,15 @@ import math
import traceback
import os
from pypofetch import PypoFetch
from pypoliqqueue import PypoLiqQueue
from .pypofetch import PypoFetch
from .pypoliqqueue import PypoLiqQueue
from Queue import Empty, Queue
from queue import Empty, Queue
from threading import Thread
from api_clients import api_client
from timeout import ls_timeout
from .timeout import ls_timeout
logging.captureWarnings(True)
@ -67,7 +67,7 @@ class PypoPush(Thread):
while True:
try:
media_schedule = self.queue.get(block=True)
except Exception, e:
except Exception as e:
self.logger.error(str(e))
raise
else:
@ -138,7 +138,7 @@ class PypoPush(Thread):
tn.write("exit\n")
self.logger.debug(tn.read_all())
except Exception, e:
except Exception as e:
self.logger.error(str(e))
finally:
self.telnet_lock.release()
@ -146,7 +146,7 @@ class PypoPush(Thread):
def run(self):
while True:
try: self.main()
except Exception, e:
except Exception as e:
top = traceback.format_exc()
self.logger.error('Pypo Push Exception: %s', top)
time.sleep(5)

View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
import logging
import json
import time
@ -14,9 +15,6 @@ import re
from configobj import ConfigObj
from poster.encode import multipart_encode
from poster.streaminghttp import register_openers
from subprocess import Popen
from subprocess import PIPE
from threading import Thread
@ -35,8 +33,8 @@ def api_client(logger):
# loading config file
try:
config = ConfigObj('/etc/airtime/airtime.conf')
except Exception, e:
print ('Error loading config file: %s', e)
except Exception as e:
print("Error loading config file: {}".format(e))
sys.exit()
# TODO : add docstrings everywhere in this module
@ -94,7 +92,7 @@ class ShowRecorder(Thread):
self.logger.info("starting record")
self.logger.info("command " + command)
self.p = Popen(args,stdout=PIPE,stderr=PIPE)
#blocks at the following line until the child process
@ -126,11 +124,8 @@ class ShowRecorder(Thread):
filename = os.path.split(filepath)[1]
# Register the streaming http handlers with urllib2
register_openers()
# files is what requests actually expects
files = {'file': open(filepath, "rb"), 'name': filename, 'show_instance': str(self.show_instance)}
files = {'file': open(filepath, "rb"), 'name': filename, 'show_instance': self.show_instance}
self.api_client.upload_recorded_show(files, self.show_instance)
@ -152,10 +147,10 @@ class ShowRecorder(Thread):
recorded_file['title'] = "%s-%s-%s" % (self.show_name,
full_date, full_time)
#You cannot pass ints into the metadata of a file. Even tracknumber needs to be a string
recorded_file['tracknumber'] = unicode(self.show_instance)
recorded_file['tracknumber'] = self.show_instance
recorded_file.save()
except Exception, e:
except Exception as e:
top = traceback.format_exc()
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", top)
@ -172,7 +167,7 @@ class ShowRecorder(Thread):
self.upload_file(filepath)
os.remove(filepath)
except Exception, e:
except Exception as e:
self.logger.error(e)
else:
self.logger.info("problem recording show")
@ -195,14 +190,18 @@ class Recorder(Thread):
try:
self.api_client.register_component('show-recorder')
success = True
except Exception, e:
except Exception as e:
self.logger.error(str(e))
time.sleep(10)
def handle_message(self):
if not self.queue.empty():
message = self.queue.get()
msg = json.loads(message)
try:
message = message.decode()
except (UnicodeDecodeError, AttributeError):
pass
msg = json.loads(message)
command = msg["event_type"]
self.logger.info("Received msg from Pypo Message Handler: %s", msg)
if command == 'cancel_recording':
@ -220,12 +219,12 @@ class Recorder(Thread):
temp_shows_to_record = {}
shows = m['shows']
for show in shows:
show_starts = getDateTimeObj(show[u'starts'])
show_end = getDateTimeObj(show[u'ends'])
show_starts = getDateTimeObj(show['starts'])
show_end = getDateTimeObj(show['ends'])
time_delta = show_end - show_starts
temp_shows_to_record[show[u'starts']] = [time_delta,
show[u'instance_id'], show[u'name'], m['server_timezone']]
temp_shows_to_record[show['starts']] = [time_delta,
show['instance_id'], show['name'], m['server_timezone']]
self.shows_to_record = temp_shows_to_record
def get_time_till_next_show(self):
@ -245,11 +244,11 @@ class Recorder(Thread):
self.logger.debug("Next show %s", next_show)
self.logger.debug("Now %s", tnow)
return out
def cancel_recording(self):
self.sr.cancel_recording()
self.sr = None
def currently_recording(self):
if self.sr is not None and self.sr.is_recording():
return True
@ -277,27 +276,27 @@ class Recorder(Thread):
start_time_formatted = '%(year)d-%(month)02d-%(day)02d %(hour)02d:%(min)02d:%(sec)02d' % \
{'year': start_time_on_server.year, 'month': start_time_on_server.month, 'day': start_time_on_server.day, \
'hour': start_time_on_server.hour, 'min': start_time_on_server.minute, 'sec': start_time_on_server.second}
seconds_waiting = 0
#avoiding CC-5299
while(True):
if self.currently_recording():
self.logger.info("Previous record not finished, sleeping 100ms")
seconds_waiting = seconds_waiting + 0.1
time.sleep(0.1)
time.sleep(0.1)
else:
show_length_seconds = show_length.seconds - seconds_waiting
self.sr = ShowRecorder(show_instance, show_name, show_length_seconds, start_time_formatted)
self.sr.start()
break
#remove show from shows to record.
del self.shows_to_record[start_time]
#self.time_till_next_show = self.get_time_till_next_show()
except Exception, e :
except Exception as e :
top = traceback.format_exc()
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", top)
@ -317,7 +316,7 @@ class Recorder(Thread):
if temp is not None:
self.process_recorder_schedule(temp)
self.logger.info("Bootstrap recorder schedule received: %s", temp)
except Exception, e:
except Exception as e:
self.logger.error( traceback.format_exc() )
self.logger.error(e)
@ -337,16 +336,16 @@ class Recorder(Thread):
if temp is not None:
self.process_recorder_schedule(temp)
self.logger.info("updated recorder schedule received: %s", temp)
except Exception, e:
except Exception as e:
self.logger.error( traceback.format_exc() )
self.logger.error(e)
try: self.handle_message()
except Exception, e:
except Exception as e:
self.logger.error( traceback.format_exc() )
self.logger.error('Pypo Recorder Exception: %s', e)
time.sleep(PUSH_INTERVAL)
self.loops += 1
except Exception, e :
except Exception as e :
top = traceback.format_exc()
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", top)

View File

@ -1,5 +1,7 @@
import telnetlib
from timeout import ls_timeout
from .timeout import ls_timeout
import traceback
def create_liquidsoap_annotation(media):
# We need liq_start_next value in the annotate. That is the value that controls overlap duration of crossfade.
@ -51,8 +53,8 @@ class TelnetLiquidsoap:
return True
tn = self.__connect()
msg = '%s.queue\nexit\n' % queue_id
tn.write(msg)
output = tn.read_all().splitlines()
tn.write(msg.encode('utf-8'))
output = tn.read_all().decode('utf-8').splitlines()
if len(output) == 3:
return len(output[0]) == 0
else:
@ -67,10 +69,10 @@ class TelnetLiquidsoap:
for i in self.queues:
msg = 'queues.%s_skip\n' % i
self.logger.debug(msg)
tn.write(msg)
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write(msg.encode('utf-8'))
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
except Exception:
raise
finally:
@ -84,10 +86,10 @@ class TelnetLiquidsoap:
msg = 'queues.%s_skip\n' % queue_id
self.logger.debug(msg)
tn.write(msg)
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write(msg.encode('utf-8'))
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
except Exception:
raise
finally:
@ -104,17 +106,17 @@ class TelnetLiquidsoap:
tn = self.__connect()
annotation = create_liquidsoap_annotation(media_item)
msg = '%s.push %s\n' % (queue_id, annotation.encode('utf-8'))
msg = '%s.push %s\n' % (queue_id, annotation)
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
show_name = media_item['show_name']
msg = 'vars.show_name %s\n' % show_name.encode('utf-8')
tn.write(msg)
msg = 'vars.show_name %s\n' % show_name
tn.write(msg.encode('utf-8'))
self.logger.debug(msg)
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
except Exception:
raise
finally:
@ -130,17 +132,18 @@ class TelnetLiquidsoap:
msg = 'http.stop\n'
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
msg = 'dynamic_source.id -1\n'
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
except Exception, e:
except Exception as e:
self.logger.error(str(e))
self.logger.error(traceback.format_exc())
finally:
self.telnet_lock.release()
@ -153,13 +156,14 @@ class TelnetLiquidsoap:
msg = 'dynamic_source.output_stop\n'
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
except Exception, e:
except Exception as e:
self.logger.error(str(e))
self.logger.error(traceback.format_exc())
finally:
self.telnet_lock.release()
@ -171,21 +175,22 @@ class TelnetLiquidsoap:
#TODO: DO we need this?
msg = 'streams.scheduled_play_start\n'
tn.write(msg)
tn.write(msg.encode('utf-8'))
msg = 'dynamic_source.output_start\n'
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
self.current_prebuffering_stream_id = None
except Exception, e:
except Exception as e:
self.logger.error(str(e))
self.logger.error(traceback.format_exc())
finally:
self.telnet_lock.release()
@ls_timeout
def start_web_stream_buffer(self, media_item):
try:
@ -194,18 +199,19 @@ class TelnetLiquidsoap:
msg = 'dynamic_source.id %s\n' % media_item['row_id']
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
msg = 'http.restart %s\n' % media_item['uri'].encode('latin-1')
msg = 'http.restart %s\n' % media_item['uri']
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
self.current_prebuffering_stream_id = media_item['row_id']
except Exception, e:
except Exception as e:
self.logger.error(str(e))
self.logger.error(traceback.format_exc())
finally:
self.telnet_lock.release()
@ -217,15 +223,16 @@ class TelnetLiquidsoap:
msg = 'dynamic_source.get_id\n'
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
tn.write("exit\n")
stream_id = tn.read_all().splitlines()[0]
tn.write("exit\n".encode('utf-8'))
stream_id = tn.read_all().decode('utf-8').splitlines()[0]
self.logger.debug("stream_id: %s" % stream_id)
return stream_id
except Exception, e:
except Exception as e:
self.logger.error(str(e))
self.logger.error(traceback.format_exc())
finally:
self.telnet_lock.release()
@ -242,10 +249,10 @@ class TelnetLiquidsoap:
self.telnet_lock.acquire()
tn = telnetlib.Telnet(self.ls_host, self.ls_port)
self.logger.info(command)
tn.write(command)
tn.write('exit\n')
tn.read_all()
except Exception, e:
tn.write(command.encode('utf-8'))
tn.write('exit\n'.encode('utf-8'))
tn.read_all().decode('utf-8')
except Exception as e:
self.logger.error(traceback.format_exc())
finally:
self.telnet_lock.release()
@ -258,12 +265,15 @@ class TelnetLiquidsoap:
tn = telnetlib.Telnet(self.ls_host, self.ls_port)
for i in commands:
self.logger.info(i)
if type(i) is str:
i = i.encode('utf-8')
tn.write(i)
tn.write('exit\n')
tn.read_all()
except Exception, e:
tn.write('exit\n'.encode('utf-8'))
tn.read_all().decode('utf-8')
except Exception as e:
self.logger.error(str(e))
self.logger.error(traceback.format_exc())
finally:
self.telnet_lock.release()
@ -302,7 +312,7 @@ class DummyTelnetLiquidsoap:
self.logger.info("Pushing %s to queue %s" % (media_item, queue_id))
from datetime import datetime
print "Time now: %s" % datetime.utcnow()
print("Time now: {:s}".format(datetime.utcnow()))
annotation = create_liquidsoap_annotation(media_item)
self.liquidsoap_mock_queues[queue_id].append(annotation)
@ -318,7 +328,7 @@ class DummyTelnetLiquidsoap:
self.logger.info("Purging queue %s" % queue_id)
from datetime import datetime
print "Time now: %s" % datetime.utcnow()
print("Time now: {:s}".format(datetime.utcnow()))
except Exception:
raise

View File

@ -1,8 +1,9 @@
from pypoliqqueue import PypoLiqQueue
from telnetliquidsoap import DummyTelnetLiquidsoap, TelnetLiquidsoap
from .pypoliqqueue import PypoLiqQueue
from .telnetliquidsoap import DummyTelnetLiquidsoap, TelnetLiquidsoap
from Queue import Queue
from queue import Queue
from threading import Lock
import sys
@ -45,7 +46,7 @@ plq.daemon = True
plq.start()
print "Time now: %s" % datetime.utcnow()
print("Time now: {:s}".format(datetime.utcnow()))
media_schedule = {}

View File

@ -1,5 +1,5 @@
import threading
import pypofetch
from . import pypofetch
def __timeout(func, timeout_duration, default, args, kwargs):

View File

@ -1,10 +1,11 @@
from __future__ import print_function
from setuptools import setup
from subprocess import call
import sys
import os
script_path = os.path.dirname(os.path.realpath(__file__))
print script_path
print(script_path)
os.chdir(script_path)
# Allows us to avoid installing the upstart init script when deploying on Airtime Pro:
@ -16,7 +17,7 @@ else:
for root, dirnames, filenames in os.walk('pypo'):
for filename in filenames:
pypo_files.append(os.path.join(root, filename))
data_files = [
('/etc/init', ['install/upstart/airtime-playout.conf.template']),
('/etc/init', ['install/upstart/airtime-liquidsoap.conf.template']),
@ -29,7 +30,7 @@ else:
('/var/tmp/airtime/pypo/files', []),
('/var/tmp/airtime/pypo/tmp', []),
]
print data_files
print(data_files)
setup(name='airtime-playout',
version='1.0',
@ -54,19 +55,18 @@ setup(name='airtime-playout',
'future',
'kombu',
'mutagen',
'poster',
'PyDispatcher',
'pyinotify',
'pytz',
'requests',
'wsgiref',
'defusedxml'
'defusedxml',
'packaging',
],
zip_safe=False,
data_files=data_files)
# Reload the initctl config so that playout services works
if data_files:
print "Reloading initctl configuration"
print("Reloading initctl configuration")
#call(['initctl', 'reload-configuration'])
print "Run \"sudo service airtime-playout start\" and \"sudo service airtime-liquidsoap start\""
print("Run \"sudo service airtime-playout start\" and \"sudo service airtime-liquidsoap start\"")

View File

@ -1,18 +0,0 @@
#!/bin/bash
which py.test
pytest_exist=$?
if [ "$pytest_exist" != "0" ]; then
echo "Need to have py.test installed. Exiting..."
exit 1
fi
SCRIPT=`readlink -f $0`
# Absolute directory this script is in
SCRIPTPATH=`dirname $SCRIPT`
export PYTHONPATH=$PYTHONPATH:$SCRIPTPATH/..:$SCRIPTPATH/../..
py.test

View File

@ -1,26 +0,0 @@
from pypopush import PypoPush
from threading import Lock
from Queue import Queue
import datetime
pypoPush_q = Queue()
telnet_lock = Lock()
pp = PypoPush(pypoPush_q, telnet_lock)
def test_modify_cue_in():
link = pp.modify_first_link_cue_point([])
assert len(link) == 0
min_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes = 1)
link = [{"start":min_ago.strftime("%Y-%m-%d-%H-%M-%S"),
"cue_in":"0", "cue_out":"30"}]
link = pp.modify_first_link_cue_point(link)
assert len(link) == 0
link = [{"start":min_ago.strftime("%Y-%m-%d-%H-%M-%S"),
"cue_in":"0", "cue_out":"70"}]
link = pp.modify_first_link_cue_point(link)
assert len(link) == 1

View File

@ -4,11 +4,15 @@ set -xe
[[ "$PYTHON" == false ]] && exit 0
pyenv local 3.7
pushd python_apps/airtime_analyzer
pyenv local 2.7
pip install -e .
nosetests -a '!rgain'
echo "replaygain tests where skipped due to not having a reliable replaygain install on travis."
pip3 install -e .
nosetests
popd
pushd python_apps/api_clients
pip3 install -e .
nosetests
popd
echo "Building docs..."

View File

@ -65,7 +65,7 @@ echo -e "The following files, directories, and services will be removed:\n"
for i in ${FILES[*]}; do
echo $i
done
echo "pip airtime-playout"
echo "pip3 airtime-playout"
echo -e "\nIf your web root is not listed, you will need to manually remove it."
@ -103,6 +103,6 @@ if [[ "$IN" = "y" || "$IN" = "Y" ]]; then
dropAirtimeDatabase
fi
pip uninstall -y airtime-playout airtime-media-monitor airtime-analyzer
pip3 uninstall -y airtime-playout airtime-media-monitor airtime-analyzer
service apache2 restart
echo "...Done"