Congegni/sintonia
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Format code using black

Browse source
This commit is contained in:
jo 2021-05-27 16:23:02 +02:00
parent efe4fa027e
commit c27f020d73
85 changed files with 3238 additions and 2243 deletions
Download patch file Download diff file Expand all files Collapse all files

37
python_apps/airtime_analyzer/airtime_analyzer/airtime_analyzer.py
View file

@ -10,24 +10,25 @@ from . import config_file
from functools import partial
from .metadata_analyzer import MetadataAnalyzer
from .replaygain_analyzer import ReplayGainAnalyzer
from .status_reporter import StatusReporter
from .status_reporter import StatusReporter
from .message_listener import MessageListener
class AirtimeAnalyzerServer:
"""A server for importing uploads to Airtime as background jobs.
"""
"""A server for importing uploads to Airtime as background jobs."""
# Constants
# Constants
_LOG_PATH = "/var/log/airtime/airtime_analyzer.log"
# Variables
_log_level = logging.INFO
def __init__(self, rmq_config_path, http_retry_queue_path, debug=False):
# Dump a stacktrace with 'kill -SIGUSR2 <PID>'
signal.signal(signal.SIGUSR2, lambda sig, frame: AirtimeAnalyzerServer.dump_stacktrace())
signal.signal(
signal.SIGUSR2, lambda sig, frame: AirtimeAnalyzerServer.dump_stacktrace()
)
# Configure logging
self.setup_logging(debug)
@ -43,11 +44,10 @@ class AirtimeAnalyzerServer:
self._msg_listener = MessageListener(rmq_config)
StatusReporter.stop_thread()
def setup_logging(self, debug):
"""Set up nicely formatted logging and log rotation.
Keyword arguments:
debug -- a boolean indicating whether to enable super verbose logging
to the screen and disk.
@ -55,27 +55,30 @@ class AirtimeAnalyzerServer:
if debug:
self._log_level = logging.DEBUG
else:
#Disable most pika/rabbitmq logging:
pika_logger = logging.getLogger('pika')
# Disable most pika/rabbitmq logging:
pika_logger = logging.getLogger("pika")
pika_logger.setLevel(logging.CRITICAL)
# Set up logging
logFormatter = logging.Formatter("%(asctime)s [%(module)s] [%(levelname)-5.5s] %(message)s")
logFormatter = logging.Formatter(
"%(asctime)s [%(module)s] [%(levelname)-5.5s] %(message)s"
)
rootLogger = logging.getLogger()
rootLogger.setLevel(self._log_level)
fileHandler = logging.handlers.RotatingFileHandler(filename=self._LOG_PATH, maxBytes=1024*1024*30,
backupCount=8)
fileHandler = logging.handlers.RotatingFileHandler(
filename=self._LOG_PATH, maxBytes=1024 * 1024 * 30, backupCount=8
)
fileHandler.setFormatter(logFormatter)
rootLogger.addHandler(fileHandler)
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
rootLogger.addHandler(consoleHandler)
@classmethod
def dump_stacktrace(stack):
''' Dump a stacktrace for all threads '''
"""Dump a stacktrace for all threads"""
code = []
for threadId, stack in list(sys._current_frames().items()):
code.append("\n# ThreadID: %s" % threadId)
@ -83,4 +86,4 @@ class AirtimeAnalyzerServer:
code.append('File: "%s", line %d, in %s' % (filename, lineno, name))
if line:
code.append(" %s" % (line.strip()))
logging.info('\n'.join(code))
logging.info("\n".join(code))

3
python_apps/airtime_analyzer/airtime_analyzer/analyzer.py
View file

@ -3,8 +3,7 @@
class Analyzer:
""" Abstract base class for all "analyzers".
"""
"""Abstract base class for all "analyzers"."""
@staticmethod
def analyze(filename, metadata):

58
python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py
View file

@ -12,20 +12,28 @@ from .cuepoint_analyzer import CuePointAnalyzer
from .replaygain_analyzer import ReplayGainAnalyzer
from .playability_analyzer import *
class AnalyzerPipeline:
""" Analyzes and imports an audio file into the Airtime library.
This currently performs metadata extraction (eg. gets the ID3 tags from an MP3),
then moves the file to the Airtime music library (stor/imported), and returns
the results back to the parent process. This class is used in an isolated process
so that if it crashes, it does not kill the entire airtime_analyzer daemon and
the failure to import can be reported back to the web application.
class AnalyzerPipeline:
"""Analyzes and imports an audio file into the Airtime library.
This currently performs metadata extraction (eg. gets the ID3 tags from an MP3),
then moves the file to the Airtime music library (stor/imported), and returns
the results back to the parent process. This class is used in an isolated process
so that if it crashes, it does not kill the entire airtime_analyzer daemon and
the failure to import can be reported back to the web application.
"""
IMPORT_STATUS_FAILED = 2
@staticmethod
def run_analysis(queue, audio_file_path, import_directory, original_filename, storage_backend, file_prefix):
def run_analysis(
queue,
audio_file_path,
import_directory,
original_filename,
storage_backend,
file_prefix,
):
"""Analyze and import an audio file, and put all extracted metadata into queue.
Keyword arguments:
@ -50,14 +58,29 @@ class AnalyzerPipeline:
if not isinstance(queue, Queue):
raise TypeError("queue must be a Queue.Queue()")
if not isinstance(audio_file_path, str):
raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__ + " instead.")
raise TypeError(
"audio_file_path must be unicode. Was of type "
+ type(audio_file_path).__name__
+ " instead."
)
if not isinstance(import_directory, str):
raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__ + " instead.")
raise TypeError(
"import_directory must be unicode. Was of type "
+ type(import_directory).__name__
+ " instead."
)
if not isinstance(original_filename, str):
raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__ + " instead.")
raise TypeError(
"original_filename must be unicode. Was of type "
+ type(original_filename).__name__
+ " instead."
)
if not isinstance(file_prefix, str):
raise TypeError("file_prefix must be unicode. Was of type " + type(file_prefix).__name__ + " instead.")
raise TypeError(
"file_prefix must be unicode. Was of type "
+ type(file_prefix).__name__
+ " instead."
)
# Analyze the audio file we were told to analyze:
# First, we extract the ID3 tags and other metadata:
@ -69,9 +92,11 @@ class AnalyzerPipeline:
metadata = ReplayGainAnalyzer.analyze(audio_file_path, metadata)
metadata = PlayabilityAnalyzer.analyze(audio_file_path, metadata)
metadata = FileMoverAnalyzer.move(audio_file_path, import_directory, original_filename, metadata)
metadata = FileMoverAnalyzer.move(
audio_file_path, import_directory, original_filename, metadata
)
metadata["import_status"] = 0 # Successfully imported
metadata["import_status"] = 0 # Successfully imported
# Note that the queue we're putting the results into is our interprocess communication
# back to the main process.
@ -93,9 +118,8 @@ class AnalyzerPipeline:
def python_logger_deadlock_workaround():
# Workaround for: http://bugs.python.org/issue6721#msg140215
logger_names = list(logging.Logger.manager.loggerDict.keys())
logger_names.append(None) # Root logger
logger_names.append(None) # Root logger
for name in logger_names:
for handler in logging.getLogger(name).handlers:
handler.createLock()
logging._lock = threading.RLock()

44
python_apps/airtime_analyzer/airtime_analyzer/cli.py
View file

@ -9,21 +9,32 @@ import os
import airtime_analyzer.airtime_analyzer as aa
VERSION = "1.0"
LIBRETIME_CONF_DIR = os.getenv('LIBRETIME_CONF_DIR', '/etc/airtime')
DEFAULT_RMQ_CONFIG_PATH = os.path.join(LIBRETIME_CONF_DIR, 'airtime.conf')
DEFAULT_HTTP_RETRY_PATH = '/tmp/airtime_analyzer_http_retries'
LIBRETIME_CONF_DIR = os.getenv("LIBRETIME_CONF_DIR", "/etc/airtime")
DEFAULT_RMQ_CONFIG_PATH = os.path.join(LIBRETIME_CONF_DIR, "airtime.conf")
DEFAULT_HTTP_RETRY_PATH = "/tmp/airtime_analyzer_http_retries"
def main():
'''Entry-point for this application'''
"""Entry-point for this application"""
print("LibreTime Analyzer {}".format(VERSION))
parser = argparse.ArgumentParser()
parser.add_argument("-d", "--daemon", help="run as a daemon", action="store_true")
parser.add_argument("--debug", help="log full debugging output", action="store_true")
parser.add_argument("--rmq-config-file", help="specify a configuration file with RabbitMQ settings (default is %s)" % DEFAULT_RMQ_CONFIG_PATH)
parser.add_argument("--http-retry-queue-file", help="specify where incompleted HTTP requests will be serialized (default is %s)" % DEFAULT_HTTP_RETRY_PATH)
parser.add_argument(
"--debug", help="log full debugging output", action="store_true"
)
parser.add_argument(
"--rmq-config-file",
help="specify a configuration file with RabbitMQ settings (default is %s)"
% DEFAULT_RMQ_CONFIG_PATH,
)
parser.add_argument(
"--http-retry-queue-file",
help="specify where incompleted HTTP requests will be serialized (default is %s)"
% DEFAULT_HTTP_RETRY_PATH,
)
args = parser.parse_args()
#Default config file path
# Default config file path
rmq_config_path = DEFAULT_RMQ_CONFIG_PATH
http_retry_queue_path = DEFAULT_HTTP_RETRY_PATH
if args.rmq_config_file:
@ -33,14 +44,19 @@ def main():
if args.daemon:
with daemon.DaemonContext():
aa.AirtimeAnalyzerServer(rmq_config_path=rmq_config_path,
http_retry_queue_path=http_retry_queue_path,
debug=args.debug)
aa.AirtimeAnalyzerServer(
rmq_config_path=rmq_config_path,
http_retry_queue_path=http_retry_queue_path,
debug=args.debug,
)
else:
# Run without daemonizing
aa.AirtimeAnalyzerServer(rmq_config_path=rmq_config_path,
http_retry_queue_path=http_retry_queue_path,
debug=args.debug)
aa.AirtimeAnalyzerServer(
rmq_config_path=rmq_config_path,
http_retry_queue_path=http_retry_queue_path,
debug=args.debug,
)
if __name__ == "__main__":
main()

1
python_apps/airtime_analyzer/airtime_analyzer/config_file.py
View file

@ -2,6 +2,7 @@
import configparser
def read_config_file(config_path):
"""Parse the application's config file located at config_path."""
config = configparser.SafeConfigParser()

77
python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py
View file

@ -8,26 +8,38 @@ from .analyzer import Analyzer
class CuePointAnalyzer(Analyzer):
''' This class extracts the cue-in time, cue-out time, and length of a track using silan. '''
"""This class extracts the cue-in time, cue-out time, and length of a track using silan."""
SILAN_EXECUTABLE = 'silan'
SILAN_EXECUTABLE = "silan"
@staticmethod
def analyze(filename, metadata):
''' Extracts the cue-in and cue-out times along and sets the file duration based on that.
"""Extracts the cue-in and cue-out times along and sets the file duration based on that.
The cue points are there to skip the silence at the start and end of a track, and are determined
using "silan", which analyzes the loudness in a track.
:param filename: The full path to the file to analyzer
:param metadata: A metadata dictionary where the results will be put
:return: The metadata dictionary
'''
''' The silan -F 0.99 parameter tweaks the highpass filter. The default is 0.98, but at that setting,
"""
""" The silan -F 0.99 parameter tweaks the highpass filter. The default is 0.98, but at that setting,
the unit test on the short m4a file fails. With the new setting, it gets the correct cue-in time and
all the unit tests pass.
'''
command = [CuePointAnalyzer.SILAN_EXECUTABLE, '-b', '-F', '0.99', '-f', 'JSON', '-t', '1.0', filename]
"""
command = [
CuePointAnalyzer.SILAN_EXECUTABLE,
"-b",
"-F",
"0.99",
"-f",
"JSON",
"-t",
"1.0",
filename,
]
try:
results_json = subprocess.check_output(command, stderr=subprocess.STDOUT, close_fds=True)
results_json = subprocess.check_output(
command, stderr=subprocess.STDOUT, close_fds=True
)
try:
results_json = results_json.decode()
except (UnicodeDecodeError, AttributeError):
@ -35,40 +47,51 @@ class CuePointAnalyzer(Analyzer):
silan_results = json.loads(results_json)
# Defensive coding against Silan wildly miscalculating the cue in and out times:
silan_length_seconds = float(silan_results['file duration'])
silan_cuein = format(silan_results['sound'][0][0], 'f')
silan_cueout = format(silan_results['sound'][0][1], 'f')
silan_length_seconds = float(silan_results["file duration"])
silan_cuein = format(silan_results["sound"][0][0], "f")
silan_cueout = format(silan_results["sound"][0][1], "f")
# Sanity check the results against any existing metadata passed to us (presumably extracted by Mutagen):
if 'length_seconds' in metadata:
if "length_seconds" in metadata:
# Silan has a rare bug where it can massively overestimate the length or cue out time sometimes.
if (silan_length_seconds - metadata['length_seconds'] > 3) or (float(silan_cueout) - metadata['length_seconds'] > 2):
if (silan_length_seconds - metadata["length_seconds"] > 3) or (
float(silan_cueout) - metadata["length_seconds"] > 2
):
# Don't trust anything silan says then...
raise Exception("Silan cue out {0} or length {1} differs too much from the Mutagen length {2}. Ignoring Silan values."
.format(silan_cueout, silan_length_seconds, metadata['length_seconds']))
raise Exception(
"Silan cue out {0} or length {1} differs too much from the Mutagen length {2}. Ignoring Silan values.".format(
silan_cueout,
silan_length_seconds,
metadata["length_seconds"],
)
)
# Don't allow silan to trim more than the greater of 3 seconds or 5% off the start of a track
if float(silan_cuein) > max(silan_length_seconds*0.05, 3):
raise Exception("Silan cue in time {0} too big, ignoring.".format(silan_cuein))
if float(silan_cuein) > max(silan_length_seconds * 0.05, 3):
raise Exception(
"Silan cue in time {0} too big, ignoring.".format(silan_cuein)
)
else:
# Only use the Silan track length in the worst case, where Mutagen didn't give us one for some reason.
# (This is mostly to make the unit tests still pass.)
# Convert the length into a formatted time string.
metadata['length_seconds'] = silan_length_seconds #
track_length = datetime.timedelta(seconds=metadata['length_seconds'])
metadata["length_seconds"] = silan_length_seconds #
track_length = datetime.timedelta(seconds=metadata["length_seconds"])
metadata["length"] = str(track_length)
''' XXX: I've commented out the track_length stuff below because Mutagen seems more accurate than silan
""" XXX: I've commented out the track_length stuff below because Mutagen seems more accurate than silan
as of Mutagen version 1.31. We are always going to use Mutagen's length now because Silan's
length can be off by a few seconds reasonably often.
'''
"""
metadata['cuein'] = silan_cuein
metadata['cueout'] = silan_cueout
metadata["cuein"] = silan_cuein
metadata["cueout"] = silan_cueout
except OSError as e: # silan was not found
logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have silan installed?"))
except subprocess.CalledProcessError as e: # silan returned an error code
except OSError as e: # silan was not found
logging.warn(
"Failed to run: %s - %s. %s"
% (command[0], e.strerror, "Do you have silan installed?")
)
except subprocess.CalledProcessError as e: # silan returned an error code
logging.warn("%s %s %s", e.cmd, e.output, e.returncode)
except Exception as e:
logging.warn(e)

88
python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py
View file

@ -9,10 +9,12 @@ import uuid
from .analyzer import Analyzer
class FileMoverAnalyzer(Analyzer):
"""This analyzer copies a file over from a temporary directory (stor/organize)
into the Airtime library (stor/imported).
into the Airtime library (stor/imported).
"""
@staticmethod
def analyze(audio_file_path, metadata):
"""Dummy method because we need more info than analyze gets passed to it"""
@ -21,27 +23,38 @@ class FileMoverAnalyzer(Analyzer):
@staticmethod
def move(audio_file_path, import_directory, original_filename, metadata):
"""Move the file at audio_file_path over into the import_directory/import,
renaming it to original_filename.
renaming it to original_filename.
Keyword arguments:
audio_file_path: Path to the file to be imported.
import_directory: Path to the "import" directory inside the Airtime stor directory.
(eg. /srv/airtime/stor/import)
original_filename: The filename of the file when it was uploaded to Airtime.
metadata: A dictionary where the "full_path" of where the file is moved to will be added.
Keyword arguments:
audio_file_path: Path to the file to be imported.
import_directory: Path to the "import" directory inside the Airtime stor directory.
(eg. /srv/airtime/stor/import)
original_filename: The filename of the file when it was uploaded to Airtime.
metadata: A dictionary where the "full_path" of where the file is moved to will be added.
"""
if not isinstance(audio_file_path, str):
raise TypeError("audio_file_path must be string. Was of type " + type(audio_file_path).__name__)
raise TypeError(
"audio_file_path must be string. Was of type "
+ type(audio_file_path).__name__
)
if not isinstance(import_directory, str):
raise TypeError("import_directory must be string. Was of type " + type(import_directory).__name__)
raise TypeError(
"import_directory must be string. Was of type "
+ type(import_directory).__name__
)
if not isinstance(original_filename, str):
raise TypeError("original_filename must be string. Was of type " + type(original_filename).__name__)
raise TypeError(
"original_filename must be string. Was of type "
+ type(original_filename).__name__
)
if not isinstance(metadata, dict):
raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__)
raise TypeError(
"metadata must be a dict. Was of type " + type(metadata).__name__
)
if not os.path.exists(audio_file_path):
raise FileNotFoundError("audio file not found: {}".format(audio_file_path))
#Import the file over to it's final location.
# Import the file over to it's final location.
# TODO: Also, handle the case where the move fails and write some code
# to possibly move the file to problem_files.
@ -50,52 +63,65 @@ class FileMoverAnalyzer(Analyzer):
final_file_path = import_directory
orig_file_basename, orig_file_extension = os.path.splitext(original_filename)
if "artist_name" in metadata:
final_file_path += "/" + metadata["artist_name"][0:max_dir_len] # truncating with array slicing
final_file_path += (
"/" + metadata["artist_name"][0:max_dir_len]
) # truncating with array slicing
if "album_title" in metadata:
final_file_path += "/" + metadata["album_title"][0:max_dir_len]
# Note that orig_file_extension includes the "." already
final_file_path += "/" + orig_file_basename[0:max_file_len] + orig_file_extension
final_file_path += (
"/" + orig_file_basename[0:max_file_len] + orig_file_extension
)
#Ensure any redundant slashes are stripped
# Ensure any redundant slashes are stripped
final_file_path = os.path.normpath(final_file_path)
#If a file with the same name already exists in the "import" directory, then
#we add a unique string to the end of this one. We never overwrite a file on import
#because if we did that, it would mean Airtime's database would have
#the wrong information for the file we just overwrote (eg. the song length would be wrong!)
#If the final file path is the same as the file we've been told to import (which
#you often do when you're debugging), then don't move the file at all.
# If a file with the same name already exists in the "import" directory, then
# we add a unique string to the end of this one. We never overwrite a file on import
# because if we did that, it would mean Airtime's database would have
# the wrong information for the file we just overwrote (eg. the song length would be wrong!)
# If the final file path is the same as the file we've been told to import (which
# you often do when you're debugging), then don't move the file at all.
if os.path.exists(final_file_path):
if os.path.samefile(audio_file_path, final_file_path):
metadata["full_path"] = final_file_path
return metadata
base_file_path, file_extension = os.path.splitext(final_file_path)
final_file_path = "%s_%s%s" % (base_file_path, time.strftime("%m-%d-%Y-%H-%M-%S", time.localtime()), file_extension)
final_file_path = "%s_%s%s" % (
base_file_path,
time.strftime("%m-%d-%Y-%H-%M-%S", time.localtime()),
file_extension,
)
#If THAT path exists, append a UUID instead:
# If THAT path exists, append a UUID instead:
while os.path.exists(final_file_path):
base_file_path, file_extension = os.path.splitext(final_file_path)
final_file_path = "%s_%s%s" % (base_file_path, str(uuid.uuid4()), file_extension)
final_file_path = "%s_%s%s" % (
base_file_path,
str(uuid.uuid4()),
file_extension,
)
#Ensure the full path to the file exists
# Ensure the full path to the file exists
mkdir_p(os.path.dirname(final_file_path))
#Move the file into its final destination directory
# Move the file into its final destination directory
logging.debug("Moving %s to %s" % (audio_file_path, final_file_path))
shutil.move(audio_file_path, final_file_path)
metadata["full_path"] = final_file_path
return metadata
def mkdir_p(path):
""" Make all directories in a tree (like mkdir -p)"""
"""Make all directories in a tree (like mkdir -p)"""
if path == "":
return
try:
os.makedirs(path)
except OSError as exc: # Python >2.5
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else: raise
else:
raise

180
python_apps/airtime_analyzer/airtime_analyzer/message_listener.py
View file

@ -5,8 +5,8 @@ import json
import time
import select
import signal
import logging
import multiprocessing
import logging
import multiprocessing
import queue
from .analyzer_pipeline import AnalyzerPipeline
from .status_reporter import StatusReporter
@ -54,29 +54,30 @@ QUEUE = "airtime-uploads"
So that is a quick overview of the design constraints for this application, and
why airtime_analyzer is written this way.
"""
class MessageListener:
class MessageListener:
def __init__(self, rmq_config):
''' Start listening for file upload notification messages
from RabbitMQ
Keyword arguments:
rmq_config: A ConfigParser object containing the [rabbitmq] configuration.
'''
"""Start listening for file upload notification messages
from RabbitMQ
Keyword arguments:
rmq_config: A ConfigParser object containing the [rabbitmq] configuration.
"""
self._shutdown = False
# Read the RabbitMQ connection settings from the rmq_config file
# The exceptions throw here by default give good error messages.
# The exceptions throw here by default give good error messages.
RMQ_CONFIG_SECTION = "rabbitmq"
self._host = rmq_config.get(RMQ_CONFIG_SECTION, 'host')
self._port = rmq_config.getint(RMQ_CONFIG_SECTION, 'port')
self._username = rmq_config.get(RMQ_CONFIG_SECTION, 'user')
self._password = rmq_config.get(RMQ_CONFIG_SECTION, 'password')
self._vhost = rmq_config.get(RMQ_CONFIG_SECTION, 'vhost')
self._host = rmq_config.get(RMQ_CONFIG_SECTION, "host")
self._port = rmq_config.getint(RMQ_CONFIG_SECTION, "port")
self._username = rmq_config.get(RMQ_CONFIG_SECTION, "user")
self._password = rmq_config.get(RMQ_CONFIG_SECTION, "password")
self._vhost = rmq_config.get(RMQ_CONFIG_SECTION, "vhost")
# Set up a signal handler so we can shutdown gracefully
# For some reason, this signal handler must be set up here. I'd rather
# For some reason, this signal handler must be set up here. I'd rather
# put it in AirtimeAnalyzerServer, but it doesn't work there (something to do
# with pika's SIGTERM handler interfering with it, I think...)
signal.signal(signal.SIGTERM, self.graceful_shutdown)
@ -86,9 +87,9 @@ class MessageListener:
self.connect_to_messaging_server()
self.wait_for_messages()
except (KeyboardInterrupt, SystemExit):
break # Break out of the while loop and exit the application
break # Break out of the while loop and exit the application
except select.error:
pass
pass
except pika.exceptions.AMQPError as e:
if self._shutdown:
break
@ -100,27 +101,37 @@ class MessageListener:
self.disconnect_from_messaging_server()
logging.info("Exiting cleanly.")
def connect_to_messaging_server(self):
'''Connect to the RabbitMQ server and start listening for messages.'''
self._connection = pika.BlockingConnection(pika.ConnectionParameters(host=self._host,
port=self._port, virtual_host=self._vhost,
credentials=pika.credentials.PlainCredentials(self._username, self._password)))
"""Connect to the RabbitMQ server and start listening for messages."""
self._connection = pika.BlockingConnection(
pika.ConnectionParameters(
host=self._host,
port=self._port,
virtual_host=self._vhost,
credentials=pika.credentials.PlainCredentials(
self._username, self._password
),
)
)
self._channel = self._connection.channel()
self._channel.exchange_declare(exchange=EXCHANGE, exchange_type=EXCHANGE_TYPE, durable=True)
self._channel.exchange_declare(
exchange=EXCHANGE, exchange_type=EXCHANGE_TYPE, durable=True
)
result = self._channel.queue_declare(queue=QUEUE, durable=True)
self._channel.queue_bind(exchange=EXCHANGE, queue=QUEUE, routing_key=ROUTING_KEY)
self._channel.queue_bind(
exchange=EXCHANGE, queue=QUEUE, routing_key=ROUTING_KEY
)
logging.info(" Listening for messages...")
self._channel.basic_consume(QUEUE, self.msg_received_callback, auto_ack=False)
def wait_for_messages(self):
'''Wait until we've received a RabbitMQ message.'''
"""Wait until we've received a RabbitMQ message."""
self._channel.start_consuming()
def disconnect_from_messaging_server(self):
'''Stop consuming RabbitMQ messages and disconnect'''
"""Stop consuming RabbitMQ messages and disconnect"""
# If you try to close a connection that's already closed, you're going to have a bad time.
# We're breaking EAFP because this can be called multiple times depending on exception
# handling flow here.
@ -128,43 +139,45 @@ class MessageListener:
self._channel.stop_consuming()
if not self._connection.is_closed and not self._connection.is_closing:
self._connection.close()
def graceful_shutdown(self, signum, frame):
'''Disconnect and break out of the message listening loop'''
"""Disconnect and break out of the message listening loop"""
self._shutdown = True
self.disconnect_from_messaging_server()
def msg_received_callback(self, channel, method_frame, header_frame, body):
''' A callback method that runs when a RabbitMQ message is received.
Here we parse the message, spin up an analyzer process, and report the
metadata back to the Airtime web application (or report an error).
'''
logging.info(" - Received '%s' on routing_key '%s'" % (body, method_frame.routing_key))
#Declare all variables here so they exist in the exception handlers below, no matter what.
"""A callback method that runs when a RabbitMQ message is received.
Here we parse the message, spin up an analyzer process, and report the
metadata back to the Airtime web application (or report an error).
"""
logging.info(
" - Received '%s' on routing_key '%s'" % (body, method_frame.routing_key)
)
# Declare all variables here so they exist in the exception handlers below, no matter what.
audio_file_path = ""
#final_file_path = ""
# final_file_path = ""
import_directory = ""
original_filename = ""
callback_url = ""
api_key = ""
callback_url = ""
api_key = ""
file_prefix = ""
''' Spin up a worker process. We use the multiprocessing module and multiprocessing.Queue
""" Spin up a worker process. We use the multiprocessing module and multiprocessing.Queue
to pass objects between the processes so that if the analyzer process crashes, it does not
take down the rest of the daemon and we NACK that message so that it doesn't get
propagated to other airtime_analyzer daemons (eg. running on other servers).
We avoid cascading failure this way.
'''
"""
try:
try:
body = body.decode()
except (UnicodeDecodeError, AttributeError):
pass
msg_dict = json.loads(body)
api_key = msg_dict["api_key"]
callback_url = msg_dict["callback_url"]
api_key = msg_dict["api_key"]
callback_url = msg_dict["callback_url"]
audio_file_path = msg_dict["tmp_file_path"]
import_directory = msg_dict["import_directory"]
@ -172,48 +185,71 @@ class MessageListener:
file_prefix = msg_dict["file_prefix"]
storage_backend = msg_dict["storage_backend"]
audio_metadata = MessageListener.spawn_analyzer_process(audio_file_path, import_directory, original_filename, storage_backend, file_prefix)
StatusReporter.report_success_to_callback_url(callback_url, api_key, audio_metadata)
audio_metadata = MessageListener.spawn_analyzer_process(
audio_file_path,
import_directory,
original_filename,
storage_backend,
file_prefix,
)
StatusReporter.report_success_to_callback_url(
callback_url, api_key, audio_metadata
)
except KeyError as e:
# A field in msg_dict that we needed was missing (eg. audio_file_path)
logging.exception("A mandatory airtime_analyzer message field was missing from the message.")
logging.exception(
"A mandatory airtime_analyzer message field was missing from the message."
)
# See the huge comment about NACK below.
channel.basic_nack(delivery_tag=method_frame.delivery_tag, multiple=False,
requeue=False) #Important that it doesn't requeue the message
channel.basic_nack(
delivery_tag=method_frame.delivery_tag, multiple=False, requeue=False
) # Important that it doesn't requeue the message
except Exception as e:
logging.exception(e)
''' If ANY exception happens while processing a file, we're going to NACK to the
""" If ANY exception happens while processing a file, we're going to NACK to the
messaging server and tell it to remove the message from the queue.
(NACK is a negative acknowledgement. We could use ACK instead, but this might come
in handy in the future.)
Exceptions in this context are unexpected, unhandled errors. We try to recover
from as many errors as possible in AnalyzerPipeline, but we're safeguarding ourselves
here from any catastrophic or genuinely unexpected errors:
'''
channel.basic_nack(delivery_tag=method_frame.delivery_tag, multiple=False,
requeue=False) #Important that it doesn't requeue the message
"""
channel.basic_nack(
delivery_tag=method_frame.delivery_tag, multiple=False, requeue=False
) # Important that it doesn't requeue the message
#
# TODO: If the JSON was invalid or the web server is down,
# TODO: If the JSON was invalid or the web server is down,
# then don't report that failure to the REST API
#TODO: Catch exceptions from this HTTP request too:
if callback_url: # If we got an invalid message, there might be no callback_url in the JSON
# TODO: Catch exceptions from this HTTP request too:
if (
callback_url
): # If we got an invalid message, there might be no callback_url in the JSON
# Report this as a failed upload to the File Upload REST API.
StatusReporter.report_failure_to_callback_url(callback_url, api_key, import_status=2,
reason='An error occurred while importing this file')
StatusReporter.report_failure_to_callback_url(
callback_url,
api_key,
import_status=2,
reason="An error occurred while importing this file",
)
else:
# ACK at the very end, after the message has been successfully processed.
# If we don't ack, then RabbitMQ will redeliver the message in the future.
channel.basic_ack(delivery_tag=method_frame.delivery_tag)
@staticmethod
def spawn_analyzer_process(audio_file_path, import_directory, original_filename, storage_backend, file_prefix):
''' Spawn a child process to analyze and import a new audio file. '''
'''
def spawn_analyzer_process(
audio_file_path,
import_directory,
original_filename,
storage_backend,
file_prefix,
):
"""Spawn a child process to analyze and import a new audio file."""
"""
q = multiprocessing.Queue()
p = multiprocessing.Process(target=AnalyzerPipeline.run_analysis,
args=(q, audio_file_path, import_directory, original_filename, storage_backend, file_prefix))
@ -225,12 +261,19 @@ class MessageListener:
logging.info(results)
else:
raise Exception("Analyzer process terminated unexpectedly.")
'''
"""
metadata = {}
q = queue.Queue()
try:
AnalyzerPipeline.run_analysis(q, audio_file_path, import_directory, original_filename, storage_backend, file_prefix)
AnalyzerPipeline.run_analysis(
q,
audio_file_path,
import_directory,
original_filename,
storage_backend,
file_prefix,
)
metadata = q.get()
except Exception as e:
logging.error("Analyzer pipeline exception: %s" % str(e))
@ -241,4 +284,3 @@ class MessageListener:
q.get()
return metadata

132
python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py
View file

@ -9,32 +9,36 @@ import os
import hashlib
from .analyzer import Analyzer
class MetadataAnalyzer(Analyzer):
class MetadataAnalyzer(Analyzer):
@staticmethod
def analyze(filename, metadata):
''' Extract audio metadata from tags embedded in the file (eg. ID3 tags)
"""Extract audio metadata from tags embedded in the file (eg. ID3 tags)
Keyword arguments:
filename: The path to the audio file to extract metadata from.
metadata: A dictionary that the extracted metadata will be added to.
'''
Keyword arguments:
filename: The path to the audio file to extract metadata from.
metadata: A dictionary that the extracted metadata will be added to.
"""
if not isinstance(filename, str):
raise TypeError("filename must be string. Was of type " + type(filename).__name__)
raise TypeError(
"filename must be string. Was of type " + type(filename).__name__
)
if not isinstance(metadata, dict):
raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__)
raise TypeError(
"metadata must be a dict. Was of type " + type(metadata).__name__
)
if not os.path.exists(filename):
raise FileNotFoundError("audio file not found: {}".format(filename))
#Airtime <= 2.5.x nonsense:
# Airtime <= 2.5.x nonsense:
metadata["ftype"] = "audioclip"
#Other fields we'll want to set for Airtime:
# Other fields we'll want to set for Airtime:
metadata["hidden"] = False
# Get file size and md5 hash of the file
metadata["filesize"] = os.path.getsize(filename)
with open(filename, 'rb') as fh:
with open(filename, "rb") as fh:
m = hashlib.md5()
while True:
data = fh.read(8192)
@ -46,37 +50,41 @@ class MetadataAnalyzer(Analyzer):
# Mutagen doesn't handle WAVE files so we use a different package
ms = magic.open(magic.MIME_TYPE)
ms.load()
with open(filename, 'rb') as fh:
with open(filename, "rb") as fh:
mime_check = ms.buffer(fh.read(2014))
metadata["mime"] = mime_check
if mime_check == 'audio/x-wav':
if mime_check == "audio/x-wav":
return MetadataAnalyzer._analyze_wave(filename, metadata)
#Extract metadata from an audio file using mutagen
# Extract metadata from an audio file using mutagen
audio_file = mutagen.File(filename, easy=True)
#Bail if the file couldn't be parsed. The title should stay as the filename
#inside Airtime.
if audio_file == None: # Don't use "if not" here. It is wrong due to mutagen's design.
# Bail if the file couldn't be parsed. The title should stay as the filename
# inside Airtime.
if (
audio_file == None
): # Don't use "if not" here. It is wrong due to mutagen's design.
return metadata
# Note that audio_file can equal {} if the file is valid but there's no metadata tags.
# We can still try to grab the info variables below.
#Grab other file information that isn't encoded in a tag, but instead usually
#in the file header. Mutagen breaks that out into a separate "info" object:
# Grab other file information that isn't encoded in a tag, but instead usually
# in the file header. Mutagen breaks that out into a separate "info" object:
info = audio_file.info
if hasattr(info, "sample_rate"): # Mutagen is annoying and inconsistent
if hasattr(info, "sample_rate"): # Mutagen is annoying and inconsistent
metadata["sample_rate"] = info.sample_rate
if hasattr(info, "length"):
metadata["length_seconds"] = info.length
#Converting the length in seconds (float) to a formatted time string
# Converting the length in seconds (float) to a formatted time string
track_length = datetime.timedelta(seconds=info.length)
metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length)
metadata["length"] = str(
track_length
) # time.strftime("%H:%M:%S.%f", track_length)
# Other fields for Airtime
metadata["cueout"] = metadata["length"]
# Set a default cue in time in seconds
metadata["cuein"] = 0.0;
metadata["cuein"] = 0.0
if hasattr(info, "bitrate"):
metadata["bit_rate"] = info.bitrate
@ -86,11 +94,11 @@ class MetadataAnalyzer(Analyzer):
if audio_file.mime:
metadata["mime"] = audio_file.mime[0]
#Try to get the number of channels if mutagen can...
# Try to get the number of channels if mutagen can...
try:
#Special handling for getting the # of channels from MP3s. It's in the "mode" field
#which is 0=Stereo, 1=Joint Stereo, 2=Dual Channel, 3=Mono. Part of the ID3 spec...
if metadata["mime"] in ["audio/mpeg", 'audio/mp3']:
# Special handling for getting the # of channels from MP3s. It's in the "mode" field
# which is 0=Stereo, 1=Joint Stereo, 2=Dual Channel, 3=Mono. Part of the ID3 spec...
if metadata["mime"] in ["audio/mpeg", "audio/mp3"]:
if info.mode == 3:
metadata["channels"] = 1
else:
@ -98,54 +106,54 @@ class MetadataAnalyzer(Analyzer):
else:
metadata["channels"] = info.channels
except (AttributeError, KeyError):
#If mutagen can't figure out the number of channels, we'll just leave it out...
# If mutagen can't figure out the number of channels, we'll just leave it out...
pass
#Try to extract the number of tracks on the album if we can (the "track total")
# Try to extract the number of tracks on the album if we can (the "track total")
try:
track_number = audio_file["tracknumber"]
if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh
if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh
track_number = track_number[0]
track_number_tokens = track_number
if '/' in track_number:
track_number_tokens = track_number.split('/')
if "/" in track_number:
track_number_tokens = track_number.split("/")
track_number = track_number_tokens[0]
elif '-' in track_number:
track_number_tokens = track_number.split('-')
elif "-" in track_number:
track_number_tokens = track_number.split("-")
track_number = track_number_tokens[0]
metadata["track_number"] = track_number
track_total = track_number_tokens[1]
metadata["track_total"] = track_total
except (AttributeError, KeyError, IndexError):
#If we couldn't figure out the track_number or track_total, just ignore it...
# If we couldn't figure out the track_number or track_total, just ignore it...
pass
#We normalize the mutagen tags slightly here, so in case mutagen changes,
#we find the
# We normalize the mutagen tags slightly here, so in case mutagen changes,
# we find the
mutagen_to_airtime_mapping = {
'title': 'track_title',
'artist': 'artist_name',
'album': 'album_title',
'bpm': 'bpm',
'composer': 'composer',
'conductor': 'conductor',
'copyright': 'copyright',
'comment': 'comment',
'encoded_by': 'encoder',
'genre': 'genre',
'isrc': 'isrc',
'label': 'label',
'organization': 'label',
"title": "track_title",
"artist": "artist_name",
"album": "album_title",
"bpm": "bpm",
"composer": "composer",
"conductor": "conductor",
"copyright": "copyright",
"comment": "comment",
"encoded_by": "encoder",
"genre": "genre",
"isrc": "isrc",
"label": "label",
"organization": "label",
#'length': 'length',
'language': 'language',
'last_modified':'last_modified',
'mood': 'mood',
'bit_rate': 'bit_rate',
'replay_gain': 'replaygain',
"language": "language",
"last_modified": "last_modified",
"mood": "mood",
"bit_rate": "bit_rate",
"replay_gain": "replaygain",
#'tracknumber': 'track_number',
#'track_total': 'track_total',
'website': 'website',
'date': 'year',
"website": "website",
"date": "year",
#'mime_type': 'mime',
}
@ -158,7 +166,7 @@ class MetadataAnalyzer(Analyzer):
if isinstance(metadata[airtime_tag], list):
if metadata[airtime_tag]:
metadata[airtime_tag] = metadata[airtime_tag][0]
else: # Handle empty lists
else: # Handle empty lists
metadata[airtime_tag] = ""
except KeyError:
@ -169,13 +177,15 @@ class MetadataAnalyzer(Analyzer):
@staticmethod
def _analyze_wave(filename, metadata):
try:
reader = wave.open(filename, 'rb')
reader = wave.open(filename, "rb")
metadata["channels"] = reader.getnchannels()
metadata["sample_rate"] = reader.getframerate()
length_seconds = float(reader.getnframes()) / float(metadata["sample_rate"])
#Converting the length in seconds (float) to a formatted time string
# Converting the length in seconds (float) to a formatted time string
track_length = datetime.timedelta(seconds=length_seconds)
metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length)
metadata["length"] = str(
track_length
) # time.strftime("%H:%M:%S.%f", track_length)
metadata["length_seconds"] = length_seconds
metadata["cueout"] = metadata["length"]
except wave.Error as ex:

35
python_apps/airtime_analyzer/airtime_analyzer/playability_analyzer.py
View file

@ -1,32 +1,47 @@
# -*- coding: utf-8 -*-
__author__ = 'asantoni'
__author__ = "asantoni"
import subprocess
import logging
from .analyzer import Analyzer
class UnplayableFileError(Exception):
pass
class PlayabilityAnalyzer(Analyzer):
''' This class checks if a file can actually be played with Liquidsoap. '''
LIQUIDSOAP_EXECUTABLE = 'liquidsoap'
class PlayabilityAnalyzer(Analyzer):
"""This class checks if a file can actually be played with Liquidsoap."""
LIQUIDSOAP_EXECUTABLE = "liquidsoap"
@staticmethod
def analyze(filename, metadata):
''' Checks if a file can be played by Liquidsoap.
"""Checks if a file can be played by Liquidsoap.
:param filename: The full path to the file to analyzer
:param metadata: A metadata dictionary where the results will be put
:return: The metadata dictionary
'''
command = [PlayabilityAnalyzer.LIQUIDSOAP_EXECUTABLE, '-v', '-c', "output.dummy(audio_to_stereo(single(argv(1))))", '--', filename]
"""
command = [
PlayabilityAnalyzer.LIQUIDSOAP_EXECUTABLE,
"-v",
"-c",
"output.dummy(audio_to_stereo(single(argv(1))))",
"--",
filename,
]
try:
subprocess.check_output(command, stderr=subprocess.STDOUT, close_fds=True)
except OSError as e: # liquidsoap was not found
logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have liquidsoap installed?"))
except (subprocess.CalledProcessError, Exception) as e: # liquidsoap returned an error code
except OSError as e: # liquidsoap was not found
logging.warn(
"Failed to run: %s - %s. %s"
% (command[0], e.strerror, "Do you have liquidsoap installed?")
)
except (
subprocess.CalledProcessError,
Exception,
) as e: # liquidsoap returned an error code
logging.warn(e)
raise UnplayableFileError()

37
python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py
View file

@ -6,30 +6,39 @@ import re
class ReplayGainAnalyzer(Analyzer):
''' This class extracts the ReplayGain using a tool from the python-rgain package. '''
"""This class extracts the ReplayGain using a tool from the python-rgain package."""
REPLAYGAIN_EXECUTABLE = 'replaygain' # From the rgain3 python package
REPLAYGAIN_EXECUTABLE = "replaygain" # From the rgain3 python package
@staticmethod
def analyze(filename, metadata):
''' Extracts the Replaygain loudness normalization factor of a track.
"""Extracts the Replaygain loudness normalization factor of a track.
:param filename: The full path to the file to analyzer
:param metadata: A metadata dictionary where the results will be put
:return: The metadata dictionary
'''
''' The -d flag means do a dry-run, ie. don't modify the file directly.
'''
command = [ReplayGainAnalyzer.REPLAYGAIN_EXECUTABLE, '-d', filename]
"""
""" The -d flag means do a dry-run, ie. don't modify the file directly.
"""
command = [ReplayGainAnalyzer.REPLAYGAIN_EXECUTABLE, "-d", filename]
try:
results = subprocess.check_output(command, stderr=subprocess.STDOUT,
close_fds=True, universal_newlines=True)
gain_match = r'Calculating Replay Gain information \.\.\.(?:\n|.)*?:([\d.-]*) dB'
results = subprocess.check_output(
command,
stderr=subprocess.STDOUT,
close_fds=True,
universal_newlines=True,
)
gain_match = (
r"Calculating Replay Gain information \.\.\.(?:\n|.)*?:([\d.-]*) dB"
)
replaygain = re.search(gain_match, results).group(1)
metadata['replay_gain'] = float(replaygain)
metadata["replay_gain"] = float(replaygain)
except OSError as e: # replaygain was not found
logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have python-rgain installed?"))
except subprocess.CalledProcessError as e: # replaygain returned an error code
except OSError as e: # replaygain was not found
logging.warn(
"Failed to run: %s - %s. %s"
% (command[0], e.strerror, "Do you have python-rgain installed?")
)
except subprocess.CalledProcessError as e: # replaygain returned an error code
logging.warn("%s %s %s", e.cmd, e.output, e.returncode)
except Exception as e:
logging.warn(e)

163
python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py
View file

@ -7,14 +7,15 @@ import queue
import time
import traceback
import pickle
import threading
import threading
from urllib.parse import urlparse
# Disable urllib3 warnings because these can cause a rare deadlock due to Python 2's crappy internal non-reentrant locking
# around POSIX stuff. See SAAS-714. The hasattr() is for compatibility with older versions of requests.
if hasattr(requests, 'packages'):
if hasattr(requests, "packages"):
requests.packages.urllib3.disable_warnings()
class PicklableHttpRequest:
def __init__(self, method, url, data, api_key):
self.method = method
@ -23,18 +24,23 @@ class PicklableHttpRequest:
self.api_key = api_key
def create_request(self):
return requests.Request(method=self.method, url=self.url, data=self.data,
auth=requests.auth.HTTPBasicAuth(self.api_key, ''))
return requests.Request(
method=self.method,
url=self.url,
data=self.data,
auth=requests.auth.HTTPBasicAuth(self.api_key, ""),
)
def process_http_requests(ipc_queue, http_retry_queue_path):
''' Runs in a separate thread and performs all the HTTP requests where we're
reporting extracted audio file metadata or errors back to the Airtime web application.
"""Runs in a separate thread and performs all the HTTP requests where we're
reporting extracted audio file metadata or errors back to the Airtime web application.
This process also checks every 5 seconds if there's failed HTTP requests that we
need to retry. We retry failed HTTP requests so that we don't lose uploads if the
web server is temporarily down.
This process also checks every 5 seconds if there's failed HTTP requests that we
need to retry. We retry failed HTTP requests so that we don't lose uploads if the
web server is temporarily down.
'''
"""
# Store any failed requests (eg. due to web server errors or downtime) to be
# retried later:
@ -45,7 +51,7 @@ def process_http_requests(ipc_queue, http_retry_queue_path):
# if airtime_analyzer is shut down while the web server is down or unreachable,
# and there were failed HTTP requests pending, waiting to be retried.
try:
with open(http_retry_queue_path, 'rb') as pickle_file:
with open(http_retry_queue_path, "rb") as pickle_file:
retry_queue = pickle.load(pickle_file)
except IOError as e:
if e.errno == 2:
@ -64,11 +70,16 @@ def process_http_requests(ipc_queue, http_retry_queue_path):
while not shutdown:
try:
request = ipc_queue.get(block=True, timeout=5)
if isinstance(request, str) and request == "shutdown": # Bit of a cheat
if (
isinstance(request, str) and request == "shutdown"
): # Bit of a cheat
shutdown = True
break
if not isinstance(request, PicklableHttpRequest):
raise TypeError("request must be a PicklableHttpRequest. Was of type " + type(request).__name__)
raise TypeError(
"request must be a PicklableHttpRequest. Was of type "
+ type(request).__name__
)
except queue.Empty:
request = None
@ -85,32 +96,40 @@ def process_http_requests(ipc_queue, http_retry_queue_path):
logging.info("Shutting down status_reporter")
# Pickle retry_queue to disk so that we don't lose uploads if we're shut down while
# while the web server is down or unreachable.
with open(http_retry_queue_path, 'wb') as pickle_file:
with open(http_retry_queue_path, "wb") as pickle_file:
pickle.dump(retry_queue, pickle_file)
return
except Exception as e: # Terrible top-level exception handler to prevent the thread from dying, just in case.
except Exception as e: # Terrible top-level exception handler to prevent the thread from dying, just in case.
if shutdown:
return
logging.exception("Unhandled exception in StatusReporter")
logging.exception(e)
logging.info("Restarting StatusReporter thread")
time.sleep(2) # Throttle it
time.sleep(2) # Throttle it
def send_http_request(picklable_request, retry_queue):
if not isinstance(picklable_request, PicklableHttpRequest):
raise TypeError("picklable_request must be a PicklableHttpRequest. Was of type " + type(picklable_request).__name__)
try:
raise TypeError(
"picklable_request must be a PicklableHttpRequest. Was of type "
+ type(picklable_request).__name__
)
try:
bare_request = picklable_request.create_request()
s = requests.Session()
prepared_request = s.prepare_request(bare_request)
r = s.send(prepared_request, timeout=StatusReporter._HTTP_REQUEST_TIMEOUT, verify=False) # SNI is a pain in the ass
r.raise_for_status() # Raise an exception if there was an http error code returned
r = s.send(
prepared_request, timeout=StatusReporter._HTTP_REQUEST_TIMEOUT, verify=False
) # SNI is a pain in the ass
r.raise_for_status() # Raise an exception if there was an http error code returned
logging.info("HTTP request sent successfully.")
except requests.exceptions.HTTPError as e:
if e.response.status_code == 422:
# Do no retry the request if there was a metadata validation error
logging.error("HTTP request failed due to an HTTP exception. Exception was: %s" % str(e))
logging.error(
"HTTP request failed due to an HTTP exception. Exception was: %s"
% str(e)
)
else:
# The request failed with an error 500 probably, so let's check if Airtime and/or
# the web server are broken. If not, then our request was probably causing an
@ -124,8 +143,10 @@ def send_http_request(picklable_request, retry_queue):
# You will have to find these bad requests in logs or you'll be
# notified by sentry.
except requests.exceptions.ConnectionError as e:
logging.error("HTTP request failed due to a connection error. Retrying later. %s" % str(e))
retry_queue.append(picklable_request) # Retry it later
logging.error(
"HTTP request failed due to a connection error. Retrying later. %s" % str(e)
)
retry_queue.append(picklable_request) # Retry it later
except Exception as e:
logging.error("HTTP request failed with unhandled exception. %s" % str(e))
logging.error(traceback.format_exc())
@ -134,12 +155,13 @@ def send_http_request(picklable_request, retry_queue):
# that breaks our code. I don't want us pickling data that potentially
# breaks airtime_analyzer.
def is_web_server_broken(url):
''' Do a naive test to check if the web server we're trying to access is down.
We use this to try to differentiate between error 500s that are coming
from (for example) a bug in the Airtime Media REST API and error 500s
caused by Airtime or the webserver itself being broken temporarily.
'''
"""Do a naive test to check if the web server we're trying to access is down.
We use this to try to differentiate between error 500s that are coming
from (for example) a bug in the Airtime Media REST API and error 500s
caused by Airtime or the webserver itself being broken temporarily.
"""
try:
test_req = requests.get(url, verify=False)
test_req.raise_for_status()
@ -147,35 +169,38 @@ def is_web_server_broken(url):
return True
else:
# The request worked fine, so the web server and Airtime are still up.
return False
return False
return False
class StatusReporter():
''' Reports the extracted audio file metadata and job status back to the
Airtime web application.
'''
class StatusReporter:
"""Reports the extracted audio file metadata and job status back to the
Airtime web application.
"""
_HTTP_REQUEST_TIMEOUT = 30
''' We use multiprocessing.Process again here because we need a thread for this stuff
""" We use multiprocessing.Process again here because we need a thread for this stuff
anyways, and Python gives us process isolation for free (crash safety).
'''
"""
_ipc_queue = queue.Queue()
#_http_thread = multiprocessing.Process(target=process_http_requests,
# _http_thread = multiprocessing.Process(target=process_http_requests,
# args=(_ipc_queue,))
_http_thread = None
@classmethod
def start_thread(self, http_retry_queue_path):
StatusReporter._http_thread = threading.Thread(target=process_http_requests,
args=(StatusReporter._ipc_queue,http_retry_queue_path))
StatusReporter._http_thread = threading.Thread(
target=process_http_requests,
args=(StatusReporter._ipc_queue, http_retry_queue_path),
)
StatusReporter._http_thread.start()
@classmethod
def stop_thread(self):
logging.info("Terminating status_reporter process")
#StatusReporter._http_thread.terminate() # Triggers SIGTERM on the child process
StatusReporter._ipc_queue.put("shutdown") # Special trigger
# StatusReporter._http_thread.terminate() # Triggers SIGTERM on the child process
StatusReporter._ipc_queue.put("shutdown") # Special trigger
StatusReporter._http_thread.join()
@classmethod
@ -184,30 +209,33 @@ class StatusReporter():
@classmethod
def report_success_to_callback_url(self, callback_url, api_key, audio_metadata):
''' Report the extracted metadata and status of the successfully imported file
to the callback URL (which should be the Airtime File Upload API)
'''
"""Report the extracted metadata and status of the successfully imported file
to the callback URL (which should be the Airtime File Upload API)
"""
put_payload = json.dumps(audio_metadata)
#r = requests.Request(method='PUT', url=callback_url, data=put_payload,
# r = requests.Request(method='PUT', url=callback_url, data=put_payload,
# auth=requests.auth.HTTPBasicAuth(api_key, ''))
'''
"""
r = requests.Request(method='PUT', url=callback_url, data=put_payload,
auth=requests.auth.HTTPBasicAuth(api_key, ''))
StatusReporter._send_http_request(r)
'''
"""
StatusReporter._send_http_request(PicklableHttpRequest(method='PUT', url=callback_url,
data=put_payload, api_key=api_key))
StatusReporter._send_http_request(
PicklableHttpRequest(
method="PUT", url=callback_url, data=put_payload, api_key=api_key
)
)
'''
"""
try:
r.raise_for_status() # Raise an exception if there was an http error code returned
except requests.exceptions.RequestException:
StatusReporter._ipc_queue.put(r.prepare())
'''
"""
'''
"""
# Encode the audio metadata as json and post it back to the callback_url
put_payload = json.dumps(audio_metadata)
logging.debug("sending http put with payload: " + put_payload)
@ -219,31 +247,38 @@ class StatusReporter():
#TODO: queue up failed requests and try them again later.
r.raise_for_status() # Raise an exception if there was an http error code returned
'''
"""
@classmethod
def report_failure_to_callback_url(self, callback_url, api_key, import_status, reason):
if not isinstance(import_status, int ):
raise TypeError("import_status must be an integer. Was of type " + type(import_status).__name__)
def report_failure_to_callback_url(
self, callback_url, api_key, import_status, reason
):
if not isinstance(import_status, int):
raise TypeError(
"import_status must be an integer. Was of type "
+ type(import_status).__name__
)
logging.debug("Reporting import failure to Airtime REST API...")
audio_metadata = dict()
audio_metadata["import_status"] = import_status
audio_metadata["comment"] = reason # hack attack
put_payload = json.dumps(audio_metadata)
#logging.debug("sending http put with payload: " + put_payload)
'''
# logging.debug("sending http put with payload: " + put_payload)
"""
r = requests.put(callback_url, data=put_payload,
auth=requests.auth.HTTPBasicAuth(api_key, ''),
timeout=StatusReporter._HTTP_REQUEST_TIMEOUT)
'''
StatusReporter._send_http_request(PicklableHttpRequest(method='PUT', url=callback_url,
data=put_payload, api_key=api_key))
'''
"""
StatusReporter._send_http_request(
PicklableHttpRequest(
method="PUT", url=callback_url, data=put_payload, api_key=api_key
)
)
"""
logging.debug("HTTP request returned status: " + str(r.status_code))
logging.debug(r.text) # log the response body
#TODO: queue up failed requests and try them again later.
r.raise_for_status() # raise an exception if there was an http error code returned
'''
"""

4
python_apps/airtime_analyzer/tests/airtime_analyzer_tests.py
View file

@ -2,12 +2,14 @@
from nose.tools import *
import airtime_analyzer
def setup():
pass
def teardown():
pass
def test_basic():
pass

48
python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py
View file

@ -8,48 +8,58 @@ import datetime
from airtime_analyzer.analyzer_pipeline import AnalyzerPipeline
from airtime_analyzer import config_file
DEFAULT_AUDIO_FILE = u'tests/test_data/44100Hz-16bit-mono.mp3'
DEFAULT_IMPORT_DEST = u'Test Artist/Test Album/44100Hz-16bit-mono.mp3'
DEFAULT_AUDIO_FILE = u"tests/test_data/44100Hz-16bit-mono.mp3"
DEFAULT_IMPORT_DEST = u"Test Artist/Test Album/44100Hz-16bit-mono.mp3"
def setup():
pass
def teardown():
#Move the file back
# Move the file back
shutil.move(DEFAULT_IMPORT_DEST, DEFAULT_AUDIO_FILE)
assert os.path.exists(DEFAULT_AUDIO_FILE)
def test_basic():
filename = os.path.basename(DEFAULT_AUDIO_FILE)
q = Queue()
file_prefix = u''
file_prefix = u""
storage_backend = "file"
#This actually imports the file into the "./Test Artist" directory.
AnalyzerPipeline.run_analysis(q, DEFAULT_AUDIO_FILE, u'.', filename, storage_backend, file_prefix)
# This actually imports the file into the "./Test Artist" directory.
AnalyzerPipeline.run_analysis(
q, DEFAULT_AUDIO_FILE, u".", filename, storage_backend, file_prefix
)
metadata = q.get()
assert metadata['track_title'] == u'Test Title'
assert metadata['artist_name'] == u'Test Artist'
assert metadata['album_title'] == u'Test Album'
assert metadata['year'] == u'1999'
assert metadata['genre'] == u'Test Genre'
assert metadata['mime'] == 'audio/mp3' # Not unicode because MIMEs aren't.
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata["length"] == str(datetime.timedelta(seconds=metadata["length_seconds"]))
assert metadata["track_title"] == u"Test Title"
assert metadata["artist_name"] == u"Test Artist"
assert metadata["album_title"] == u"Test Album"
assert metadata["year"] == u"1999"
assert metadata["genre"] == u"Test Genre"
assert metadata["mime"] == "audio/mp3" # Not unicode because MIMEs aren't.
assert abs(metadata["length_seconds"] - 3.9) < 0.1
assert metadata["length"] == str(
datetime.timedelta(seconds=metadata["length_seconds"])
)
assert os.path.exists(DEFAULT_IMPORT_DEST)
@raises(TypeError)
def test_wrong_type_queue_param():
AnalyzerPipeline.run_analysis(Queue(), u'', u'', u'')
AnalyzerPipeline.run_analysis(Queue(), u"", u"", u"")
@raises(TypeError)
def test_wrong_type_string_param2():
AnalyzerPipeline.run_analysis(Queue(), '', u'', u'')
AnalyzerPipeline.run_analysis(Queue(), "", u"", u"")
@raises(TypeError)
def test_wrong_type_string_param3():
AnalyzerPipeline.run_analysis(Queue(), u'', '', u'')
AnalyzerPipeline.run_analysis(Queue(), u"", "", u"")
@raises(TypeError)
def test_wrong_type_string_param4():
AnalyzerPipeline.run_analysis(Queue(), u'', u'', '')
AnalyzerPipeline.run_analysis(Queue(), u"", u"", "")

5
python_apps/airtime_analyzer/tests/analyzer_tests.py
View file

@ -2,13 +2,16 @@
from nose.tools import *
from airtime_analyzer.analyzer import Analyzer
def setup():
pass
def teardown():
pass
@raises(NotImplementedError)
def test_analyze():
abstract_analyzer = Analyzer()
abstract_analyzer.analyze(u'foo', dict())
abstract_analyzer.analyze(u"foo", dict())

72
python_apps/airtime_analyzer/tests/cuepoint_analyzer_tests.py
View file

@ -2,63 +2,97 @@
from nose.tools import *
from airtime_analyzer.cuepoint_analyzer import CuePointAnalyzer
def check_default_metadata(metadata):
''' Check that the values extract by Silan/CuePointAnalyzer on our test audio files match what we expect.
"""Check that the values extract by Silan/CuePointAnalyzer on our test audio files match what we expect.
:param metadata: a metadata dictionary
:return: Nothing
'''
"""
# We give silan some leeway here by specifying a tolerance
tolerance_seconds = 0.1
length_seconds = 3.9
assert abs(metadata['length_seconds'] - length_seconds) < tolerance_seconds
assert abs(float(metadata['cuein'])) < tolerance_seconds
assert abs(float(metadata['cueout']) - length_seconds) < tolerance_seconds
assert abs(metadata["length_seconds"] - length_seconds) < tolerance_seconds
assert abs(float(metadata["cuein"])) < tolerance_seconds
assert abs(float(metadata["cueout"]) - length_seconds) < tolerance_seconds
def test_missing_silan():
old_silan = CuePointAnalyzer.SILAN_EXECUTABLE
CuePointAnalyzer.SILAN_EXECUTABLE = 'foosdaf'
metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict())
CuePointAnalyzer.SILAN_EXECUTABLE = old_silan # Need to put this back
CuePointAnalyzer.SILAN_EXECUTABLE = "foosdaf"
metadata = CuePointAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo-utf8.mp3", dict()
)
CuePointAnalyzer.SILAN_EXECUTABLE = old_silan # Need to put this back
def test_invalid_filepath():
metadata = CuePointAnalyzer.analyze(u'non-existent-file', dict())
metadata = CuePointAnalyzer.analyze(u"non-existent-file", dict())
def test_mp3_utf8():
metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict())
metadata = CuePointAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo-utf8.mp3", dict()
)
check_default_metadata(metadata)
def test_mp3_dualmono():
metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-dualmono.mp3', dict())
metadata = CuePointAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-dualmono.mp3", dict()
)
check_default_metadata(metadata)
def test_mp3_jointstereo():
metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-jointstereo.mp3', dict())
metadata = CuePointAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-jointstereo.mp3", dict()
)
check_default_metadata(metadata)
def test_mp3_simplestereo():
metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-simplestereo.mp3', dict())
metadata = CuePointAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-simplestereo.mp3", dict()
)
check_default_metadata(metadata)
def test_mp3_stereo():
metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.mp3', dict())
metadata = CuePointAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo.mp3", dict()
)
check_default_metadata(metadata)
def test_mp3_mono():
metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.mp3', dict())
metadata = CuePointAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-mono.mp3", dict()
)
check_default_metadata(metadata)
def test_ogg_stereo():
metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.ogg', dict())
metadata = CuePointAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo.ogg", dict()
)
check_default_metadata(metadata)
def test_invalid_wma():
metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-invalid.wma', dict())
metadata = CuePointAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo-invalid.wma", dict()
)
def test_m4a_stereo():
metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.m4a', dict())
metadata = CuePointAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo.m4a", dict()
)
check_default_metadata(metadata)
def test_wav_stereo():
metadata = CuePointAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.wav', dict())
metadata = CuePointAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo.wav", dict()
)
check_default_metadata(metadata)

92
python_apps/airtime_analyzer/tests/filemover_analyzer_tests.py
View file

@ -8,109 +8,125 @@ import mock
from pprint import pprint
from airtime_analyzer.filemover_analyzer import FileMoverAnalyzer
DEFAULT_AUDIO_FILE = u'tests/test_data/44100Hz-16bit-mono.mp3'
DEFAULT_IMPORT_DEST = u'Test Artist/Test Album/44100Hz-16bit-mono.mp3'
DEFAULT_AUDIO_FILE = u"tests/test_data/44100Hz-16bit-mono.mp3"
DEFAULT_IMPORT_DEST = u"Test Artist/Test Album/44100Hz-16bit-mono.mp3"
def setup():
pass
def teardown():
pass
@raises(Exception)
def test_dont_use_analyze():
FileMoverAnalyzer.analyze(u'foo', dict())
FileMoverAnalyzer.analyze(u"foo", dict())
@raises(TypeError)
def test_move_wrong_string_param1():
FileMoverAnalyzer.move(42, '', '', dict())
FileMoverAnalyzer.move(42, "", "", dict())
@raises(TypeError)
def test_move_wrong_string_param2():
FileMoverAnalyzer.move(u'', 23, u'', dict())
FileMoverAnalyzer.move(u"", 23, u"", dict())
@raises(TypeError)
def test_move_wrong_string_param3():
FileMoverAnalyzer.move('', '', 5, dict())
FileMoverAnalyzer.move("", "", 5, dict())
@raises(TypeError)
def test_move_wrong_dict_param():
FileMoverAnalyzer.move('', '', '', 12345)
FileMoverAnalyzer.move("", "", "", 12345)
@raises(FileNotFoundError)
def test_move_wrong_string_param3():
FileMoverAnalyzer.move('', '', '', dict())
FileMoverAnalyzer.move("", "", "", dict())
def test_basic():
filename = os.path.basename(DEFAULT_AUDIO_FILE)
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict())
#Move the file back
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u".", filename, dict())
# Move the file back
shutil.move("./" + filename, DEFAULT_AUDIO_FILE)
assert os.path.exists(DEFAULT_AUDIO_FILE)
def test_basic_samefile():
filename = os.path.basename(DEFAULT_AUDIO_FILE)
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'tests/test_data', filename, dict())
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u"tests/test_data", filename, dict())
assert os.path.exists(DEFAULT_AUDIO_FILE)
def test_duplicate_file():
filename = os.path.basename(DEFAULT_AUDIO_FILE)
#Import the file once
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict())
#Copy it back to the original location
# Import the file once
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u".", filename, dict())
# Copy it back to the original location
shutil.copy("./" + filename, DEFAULT_AUDIO_FILE)
#Import it again. It shouldn't overwrite the old file and instead create a new
# Import it again. It shouldn't overwrite the old file and instead create a new
metadata = dict()
metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, metadata)
#Cleanup: move the file (eg. 44100Hz-16bit-mono.mp3) back
metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u".", filename, metadata)
# Cleanup: move the file (eg. 44100Hz-16bit-mono.mp3) back
shutil.move("./" + filename, DEFAULT_AUDIO_FILE)
#Remove the renamed duplicate, eg. 44100Hz-16bit-mono_03-26-2014-11-58.mp3
# Remove the renamed duplicate, eg. 44100Hz-16bit-mono_03-26-2014-11-58.mp3
os.remove(metadata["full_path"])
assert os.path.exists(DEFAULT_AUDIO_FILE)
''' If you import three copies of the same file, the behaviour is:
""" If you import three copies of the same file, the behaviour is:
- The filename is of the first file preserved.
- The filename of the second file has the timestamp attached to it.
- The filename of the third file has a UUID placed after the timestamp, but ONLY IF
it's imported within 1 second of the second file (ie. if the timestamp is the same).
'''
"""
def test_double_duplicate_files():
# Here we use mock to patch out the time.localtime() function so that it
# always returns the same value. This allows us to consistently simulate this test cases
# where the last two of the three files are imported at the same time as the timestamp.
with mock.patch('airtime_analyzer.filemover_analyzer.time') as mock_time:
mock_time.localtime.return_value = time.localtime()#date(2010, 10, 8)
with mock.patch("airtime_analyzer.filemover_analyzer.time") as mock_time:
mock_time.localtime.return_value = time.localtime() # date(2010, 10, 8)
mock_time.side_effect = lambda *args, **kw: time(*args, **kw)
filename = os.path.basename(DEFAULT_AUDIO_FILE)
#Import the file once
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename, dict())
#Copy it back to the original location
# Import the file once
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u".", filename, dict())
# Copy it back to the original location
shutil.copy("./" + filename, DEFAULT_AUDIO_FILE)
#Import it again. It shouldn't overwrite the old file and instead create a new
# Import it again. It shouldn't overwrite the old file and instead create a new
first_dup_metadata = dict()
first_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename,
first_dup_metadata)
#Copy it back again!
first_dup_metadata = FileMoverAnalyzer.move(
DEFAULT_AUDIO_FILE, u".", filename, first_dup_metadata
)
# Copy it back again!
shutil.copy("./" + filename, DEFAULT_AUDIO_FILE)
#Reimport for the third time, which should have the same timestamp as the second one
#thanks to us mocking out time.localtime()
# Reimport for the third time, which should have the same timestamp as the second one
# thanks to us mocking out time.localtime()
second_dup_metadata = dict()
second_dup_metadata = FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, u'.', filename,
second_dup_metadata)
#Cleanup: move the file (eg. 44100Hz-16bit-mono.mp3) back
second_dup_metadata = FileMoverAnalyzer.move(
DEFAULT_AUDIO_FILE, u".", filename, second_dup_metadata
)
# Cleanup: move the file (eg. 44100Hz-16bit-mono.mp3) back
shutil.move("./" + filename, DEFAULT_AUDIO_FILE)
#Remove the renamed duplicate, eg. 44100Hz-16bit-mono_03-26-2014-11-58.mp3
# Remove the renamed duplicate, eg. 44100Hz-16bit-mono_03-26-2014-11-58.mp3
os.remove(first_dup_metadata["full_path"])
os.remove(second_dup_metadata["full_path"])
assert os.path.exists(DEFAULT_AUDIO_FILE)
@raises(OSError)
def test_bad_permissions_destination_dir():
filename = os.path.basename(DEFAULT_AUDIO_FILE)
dest_dir = u'/sys/foobar' # /sys is using sysfs on Linux, which is unwritable
dest_dir = u"/sys/foobar" # /sys is using sysfs on Linux, which is unwritable
FileMoverAnalyzer.move(DEFAULT_AUDIO_FILE, dest_dir, filename, dict())
#Move the file back
# Move the file back
shutil.move(os.path.join(dest_dir, filename), DEFAULT_AUDIO_FILE)
assert os.path.exists(DEFAULT_AUDIO_FILE)

210
python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py
View file

@ -6,78 +6,101 @@ import mock
from nose.tools import *
from airtime_analyzer.metadata_analyzer import MetadataAnalyzer
def setup():
pass
def teardown():
pass
def check_default_metadata(metadata):
assert metadata['track_title'] == 'Test Title'
assert metadata['artist_name'] == 'Test Artist'
assert metadata['album_title'] == 'Test Album'
assert metadata['year'] == '1999'
assert metadata['genre'] == 'Test Genre'
assert metadata['track_number'] == '1'
assert metadata["length"] == str(datetime.timedelta(seconds=metadata["length_seconds"]))
assert metadata["track_title"] == "Test Title"
assert metadata["artist_name"] == "Test Artist"
assert metadata["album_title"] == "Test Album"
assert metadata["year"] == "1999"
assert metadata["genre"] == "Test Genre"
assert metadata["track_number"] == "1"
assert metadata["length"] == str(
datetime.timedelta(seconds=metadata["length_seconds"])
)
def test_mp3_mono():
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-mono.mp3', dict())
metadata = MetadataAnalyzer.analyze(
"tests/test_data/44100Hz-16bit-mono.mp3", dict()
)
check_default_metadata(metadata)
assert metadata['channels'] == 1
assert metadata['bit_rate'] == 63998
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['mime'] == 'audio/mp3' # Not unicode because MIMEs aren't.
assert metadata['track_total'] == '10' # MP3s can have a track_total
#Mutagen doesn't extract comments from mp3s it seems
assert metadata["channels"] == 1
assert metadata["bit_rate"] == 63998
assert abs(metadata["length_seconds"] - 3.9) < 0.1
assert metadata["mime"] == "audio/mp3" # Not unicode because MIMEs aren't.
assert metadata["track_total"] == "10" # MP3s can have a track_total
# Mutagen doesn't extract comments from mp3s it seems
def test_mp3_jointstereo():
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-jointstereo.mp3', dict())
metadata = MetadataAnalyzer.analyze(
"tests/test_data/44100Hz-16bit-jointstereo.mp3", dict()
)
check_default_metadata(metadata)
assert metadata['channels'] == 2
assert metadata['bit_rate'] == 127998
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['mime'] == 'audio/mp3'
assert metadata['track_total'] == '10' # MP3s can have a track_total
assert metadata["channels"] == 2
assert metadata["bit_rate"] == 127998
assert abs(metadata["length_seconds"] - 3.9) < 0.1
assert metadata["mime"] == "audio/mp3"
assert metadata["track_total"] == "10" # MP3s can have a track_total
def test_mp3_simplestereo():
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-simplestereo.mp3', dict())
metadata = MetadataAnalyzer.analyze(
"tests/test_data/44100Hz-16bit-simplestereo.mp3", dict()
)
check_default_metadata(metadata)
assert metadata['channels'] == 2
assert metadata['bit_rate'] == 127998
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['mime'] == 'audio/mp3'
assert metadata['track_total'] == '10' # MP3s can have a track_total
assert metadata["channels"] == 2
assert metadata["bit_rate"] == 127998
assert abs(metadata["length_seconds"] - 3.9) < 0.1
assert metadata["mime"] == "audio/mp3"
assert metadata["track_total"] == "10" # MP3s can have a track_total
def test_mp3_dualmono():
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-dualmono.mp3', dict())
metadata = MetadataAnalyzer.analyze(
"tests/test_data/44100Hz-16bit-dualmono.mp3", dict()
)
check_default_metadata(metadata)
assert metadata['channels'] == 2
assert metadata['bit_rate'] == 127998
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['mime'] == 'audio/mp3'
assert metadata['track_total'] == '10' # MP3s can have a track_total
assert metadata["channels"] == 2
assert metadata["bit_rate"] == 127998
assert abs(metadata["length_seconds"] - 3.9) < 0.1
assert metadata["mime"] == "audio/mp3"
assert metadata["track_total"] == "10" # MP3s can have a track_total
def test_ogg_mono():
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-mono.ogg', dict())
metadata = MetadataAnalyzer.analyze(
"tests/test_data/44100Hz-16bit-mono.ogg", dict()
)
check_default_metadata(metadata)
assert metadata['channels'] == 1
assert metadata['bit_rate'] == 80000
assert abs(metadata['length_seconds'] - 3.8) < 0.1
assert metadata['mime'] == 'audio/vorbis'
assert metadata['comment'] == 'Test Comment'
assert metadata["channels"] == 1
assert metadata["bit_rate"] == 80000
assert abs(metadata["length_seconds"] - 3.8) < 0.1
assert metadata["mime"] == "audio/vorbis"
assert metadata["comment"] == "Test Comment"
def test_ogg_stereo():
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo.ogg', dict())
metadata = MetadataAnalyzer.analyze(
"tests/test_data/44100Hz-16bit-stereo.ogg", dict()
)
check_default_metadata(metadata)
assert metadata['channels'] == 2
assert metadata['bit_rate'] == 112000
assert abs(metadata['length_seconds'] - 3.8) < 0.1
assert metadata['mime'] == 'audio/vorbis'
assert metadata['comment'] == 'Test Comment'
assert metadata["channels"] == 2
assert metadata["bit_rate"] == 112000
assert abs(metadata["length_seconds"] - 3.8) < 0.1
assert metadata["mime"] == "audio/vorbis"
assert metadata["comment"] == "Test Comment"
''' faac and avconv can't seem to create a proper mono AAC file... ugh
""" faac and avconv can't seem to create a proper mono AAC file... ugh
def test_aac_mono():
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-mono.m4a')
print("Mono AAC metadata:")
@ -88,78 +111,93 @@ def test_aac_mono():
assert abs(metadata['length_seconds'] - 3.8) < 0.1
assert metadata['mime'] == 'audio/mp4'
assert metadata['comment'] == 'Test Comment'
'''
"""
def test_aac_stereo():
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo.m4a', dict())
metadata = MetadataAnalyzer.analyze(
"tests/test_data/44100Hz-16bit-stereo.m4a", dict()
)
check_default_metadata(metadata)
assert metadata['channels'] == 2
assert metadata['bit_rate'] == 102619
assert abs(metadata['length_seconds'] - 3.8) < 0.1
assert metadata['mime'] == 'audio/mp4'
assert metadata['comment'] == 'Test Comment'
assert metadata["channels"] == 2
assert metadata["bit_rate"] == 102619
assert abs(metadata["length_seconds"] - 3.8) < 0.1
assert metadata["mime"] == "audio/mp4"
assert metadata["comment"] == "Test Comment"
def test_mp3_utf8():
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict())
metadata = MetadataAnalyzer.analyze(
"tests/test_data/44100Hz-16bit-stereo-utf8.mp3", dict()
)
# Using a bunch of different UTF-8 codepages here. Test data is from:
# http://winrus.com/utf8-jap.htm
assert metadata['track_title'] == 'ｱｲｳｴｵｶｷｸｹｺｻｼｽｾｿﾀﾁﾂﾃ'
assert metadata['artist_name'] == 'てすと'
assert metadata['album_title'] == 'Ä ä Ü ü ß'
assert metadata['year'] == '1999'
assert metadata['genre'] == 'Я Б Г Д Ж Й'
assert metadata['track_number'] == '1'
assert metadata['channels'] == 2
assert metadata['bit_rate'] < 130000
assert metadata['bit_rate'] > 127000
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['mime'] == 'audio/mp3'
assert metadata['track_total'] == '10' # MP3s can have a track_total
assert metadata["track_title"] == "ｱｲｳｴｵｶｷｸｹｺｻｼｽｾｿﾀﾁﾂﾃ"
assert metadata["artist_name"] == "てすと"
assert metadata["album_title"] == "Ä ä Ü ü ß"
assert metadata["year"] == "1999"
assert metadata["genre"] == "Я Б Г Д Ж Й"
assert metadata["track_number"] == "1"
assert metadata["channels"] == 2
assert metadata["bit_rate"] < 130000
assert metadata["bit_rate"] > 127000
assert abs(metadata["length_seconds"] - 3.9) < 0.1
assert metadata["mime"] == "audio/mp3"
assert metadata["track_total"] == "10" # MP3s can have a track_total
def test_invalid_wma():
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo-invalid.wma', dict())
assert metadata['mime'] == 'audio/x-ms-wma'
metadata = MetadataAnalyzer.analyze(
"tests/test_data/44100Hz-16bit-stereo-invalid.wma", dict()
)
assert metadata["mime"] == "audio/x-ms-wma"
def test_wav_stereo():
metadata = MetadataAnalyzer.analyze('tests/test_data/44100Hz-16bit-stereo.wav', dict())
assert metadata['mime'] == 'audio/x-wav'
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['channels'] == 2
assert metadata['sample_rate'] == 44100
metadata = MetadataAnalyzer.analyze(
"tests/test_data/44100Hz-16bit-stereo.wav", dict()
)
assert metadata["mime"] == "audio/x-wav"
assert abs(metadata["length_seconds"] - 3.9) < 0.1
assert metadata["channels"] == 2
assert metadata["sample_rate"] == 44100
# Make sure the parameter checking works
@raises(FileNotFoundError)
def test_move_wrong_string_param1():
not_unicode = 'asdfasdf'
not_unicode = "asdfasdf"
MetadataAnalyzer.analyze(not_unicode, dict())
@raises(TypeError)
def test_move_wrong_metadata_dict():
not_a_dict = list()
MetadataAnalyzer.analyze('asdfasdf', not_a_dict)
MetadataAnalyzer.analyze("asdfasdf", not_a_dict)
# Test an mp3 file where the number of channels is invalid or missing:
def test_mp3_bad_channels():
filename = 'tests/test_data/44100Hz-16bit-mono.mp3'
'''
filename = "tests/test_data/44100Hz-16bit-mono.mp3"
"""
It'd be a pain in the ass to construct a real MP3 with an invalid number
of channels by hand because that value is stored in every MP3 frame in the file
'''
"""
audio_file = mutagen.File(filename, easy=True)
audio_file.info.mode = 1777
with mock.patch('airtime_analyzer.metadata_analyzer.mutagen') as mock_mutagen:
with mock.patch("airtime_analyzer.metadata_analyzer.mutagen") as mock_mutagen:
mock_mutagen.File.return_value = audio_file
#mock_mutagen.side_effect = lambda *args, **kw: audio_file #File(*args, **kw)
# mock_mutagen.side_effect = lambda *args, **kw: audio_file #File(*args, **kw)
metadata = MetadataAnalyzer.analyze(filename, dict())
check_default_metadata(metadata)
assert metadata['channels'] == 1
assert metadata['bit_rate'] == 63998
assert abs(metadata['length_seconds'] - 3.9) < 0.1
assert metadata['mime'] == 'audio/mp3' # Not unicode because MIMEs aren't.
assert metadata['track_total'] == '10' # MP3s can have a track_total
#Mutagen doesn't extract comments from mp3s it seems
assert metadata["channels"] == 1
assert metadata["bit_rate"] == 63998
assert abs(metadata["length_seconds"] - 3.9) < 0.1
assert metadata["mime"] == "audio/mp3" # Not unicode because MIMEs aren't.
assert metadata["track_total"] == "10" # MP3s can have a track_total
# Mutagen doesn't extract comments from mp3s it seems
def test_unparsable_file():
MetadataAnalyzer.analyze('tests/test_data/unparsable.txt', dict())
MetadataAnalyzer.analyze("tests/test_data/unparsable.txt", dict())

70
python_apps/airtime_analyzer/tests/playability_analyzer_tests.py
View file

@ -2,61 +2,97 @@
from nose.tools import *
from airtime_analyzer.playability_analyzer import *
def check_default_metadata(metadata):
''' Stub function for now in case we need it later.'''
"""Stub function for now in case we need it later."""
pass
def test_missing_liquidsoap():
old_ls = PlayabilityAnalyzer.LIQUIDSOAP_EXECUTABLE
PlayabilityAnalyzer.LIQUIDSOAP_EXECUTABLE = 'foosdaf'
metadata = PlayabilityAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict())
PlayabilityAnalyzer.LIQUIDSOAP_EXECUTABLE = old_ls # Need to put this back
PlayabilityAnalyzer.LIQUIDSOAP_EXECUTABLE = "foosdaf"
metadata = PlayabilityAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo-utf8.mp3", dict()
)
PlayabilityAnalyzer.LIQUIDSOAP_EXECUTABLE = old_ls # Need to put this back
@raises(UnplayableFileError)
def test_invalid_filepath():
metadata = PlayabilityAnalyzer.analyze(u'non-existent-file', dict())
metadata = PlayabilityAnalyzer.analyze(u"non-existent-file", dict())
def test_mp3_utf8():
metadata = PlayabilityAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict())
metadata = PlayabilityAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo-utf8.mp3", dict()
)
check_default_metadata(metadata)
def test_mp3_dualmono():
metadata = PlayabilityAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-dualmono.mp3', dict())
metadata = PlayabilityAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-dualmono.mp3", dict()
)
check_default_metadata(metadata)
def test_mp3_jointstereo():
metadata = PlayabilityAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-jointstereo.mp3', dict())
metadata = PlayabilityAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-jointstereo.mp3", dict()
)
check_default_metadata(metadata)
def test_mp3_simplestereo():
metadata = PlayabilityAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-simplestereo.mp3', dict())
metadata = PlayabilityAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-simplestereo.mp3", dict()
)
check_default_metadata(metadata)
def test_mp3_stereo():
metadata = PlayabilityAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.mp3', dict())
metadata = PlayabilityAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo.mp3", dict()
)
check_default_metadata(metadata)
def test_mp3_mono():
metadata = PlayabilityAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.mp3', dict())
metadata = PlayabilityAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-mono.mp3", dict()
)
check_default_metadata(metadata)
def test_ogg_stereo():
metadata = PlayabilityAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.ogg', dict())
metadata = PlayabilityAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo.ogg", dict()
)
check_default_metadata(metadata)
@raises(UnplayableFileError)
def test_invalid_wma():
metadata = PlayabilityAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-invalid.wma', dict())
metadata = PlayabilityAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo-invalid.wma", dict()
)
def test_m4a_stereo():
metadata = PlayabilityAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.m4a', dict())
metadata = PlayabilityAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo.m4a", dict()
)
check_default_metadata(metadata)
def test_wav_stereo():
metadata = PlayabilityAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.wav', dict())
metadata = PlayabilityAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo.wav", dict()
)
check_default_metadata(metadata)
@raises(UnplayableFileError)
def test_unknown():
metadata = PlayabilityAnalyzer.analyze(u'http://www.google.com', dict())
check_default_metadata(metadata)
metadata = PlayabilityAnalyzer.analyze(u"http://www.google.com", dict())
check_default_metadata(metadata)

98
python_apps/airtime_analyzer/tests/replaygain_analyzer_tests.py
View file

@ -5,80 +5,134 @@ from airtime_analyzer.replaygain_analyzer import ReplayGainAnalyzer
def check_default_metadata(metadata):
''' Check that the values extract by Silan/CuePointAnalyzer on our test audio files match what we expect.
"""Check that the values extract by Silan/CuePointAnalyzer on our test audio files match what we expect.
:param metadata: a metadata dictionary
:return: Nothing
'''
'''
"""
"""
# We give python-rgain some leeway here by specifying a tolerance. It's not perfectly consistent across codecs...
assert abs(metadata['cuein']) < tolerance_seconds
assert abs(metadata['cueout'] - length_seconds) < tolerance_seconds
'''
"""
tolerance = 0.60
expected_replaygain = 5.2
print(metadata['replay_gain'])
assert abs(metadata['replay_gain'] - expected_replaygain) < tolerance
print(metadata["replay_gain"])
assert abs(metadata["replay_gain"] - expected_replaygain) < tolerance
def test_missing_replaygain():
old_rg = ReplayGainAnalyzer.REPLAYGAIN_EXECUTABLE
ReplayGainAnalyzer.REPLAYGAIN_EXECUTABLE = 'foosdaf'
metadata = ReplayGainAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict())
ReplayGainAnalyzer.REPLAYGAIN_EXECUTABLE = old_rg # Need to put this back
ReplayGainAnalyzer.REPLAYGAIN_EXECUTABLE = "foosdaf"
metadata = ReplayGainAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo-utf8.mp3", dict()
)
ReplayGainAnalyzer.REPLAYGAIN_EXECUTABLE = old_rg # Need to put this back
def test_invalid_filepath():
metadata = ReplayGainAnalyzer.analyze(u'non-existent-file', dict())
metadata = ReplayGainAnalyzer.analyze(u"non-existent-file", dict())
def test_mp3_utf8():
metadata = ReplayGainAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-utf8.mp3', dict())
metadata = ReplayGainAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo-utf8.mp3", dict()
)
check_default_metadata(metadata)
test_mp3_utf8.rgain = True
def test_mp3_dualmono():
metadata = ReplayGainAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-dualmono.mp3', dict())
metadata = ReplayGainAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-dualmono.mp3", dict()
)
check_default_metadata(metadata)
test_mp3_dualmono.rgain = True
def test_mp3_jointstereo():
metadata = ReplayGainAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-jointstereo.mp3', dict())
metadata = ReplayGainAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-jointstereo.mp3", dict()
)
check_default_metadata(metadata)
test_mp3_jointstereo.rgain = True
def test_mp3_simplestereo():
metadata = ReplayGainAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-simplestereo.mp3', dict())
metadata = ReplayGainAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-simplestereo.mp3", dict()
)
check_default_metadata(metadata)
test_mp3_simplestereo.rgain = True
def test_mp3_stereo():
metadata = ReplayGainAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.mp3', dict())
metadata = ReplayGainAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo.mp3", dict()
)
check_default_metadata(metadata)
test_mp3_stereo.rgain = True
def test_mp3_mono():
metadata = ReplayGainAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.mp3', dict())
metadata = ReplayGainAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-mono.mp3", dict()
)
check_default_metadata(metadata)
test_mp3_mono.rgain = True
def test_ogg_stereo():
metadata = ReplayGainAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.ogg', dict())
metadata = ReplayGainAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo.ogg", dict()
)
check_default_metadata(metadata)
test_ogg_stereo = True
def test_invalid_wma():
metadata = ReplayGainAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo-invalid.wma', dict())
metadata = ReplayGainAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo-invalid.wma", dict()
)
test_invalid_wma.rgain = True
def test_mp3_missing_id3_header():
metadata = ReplayGainAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mp3-missingid3header.mp3', dict())
metadata = ReplayGainAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-mp3-missingid3header.mp3", dict()
)
test_mp3_missing_id3_header.rgain = True
def test_m4a_stereo():
metadata = ReplayGainAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.m4a', dict())
metadata = ReplayGainAnalyzer.analyze(
u"tests/test_data/44100Hz-16bit-stereo.m4a", dict()
)
check_default_metadata(metadata)
test_m4a_stereo.rgain = True
''' WAVE is not supported by python-rgain yet
""" WAVE is not supported by python-rgain yet
def test_wav_stereo():
metadata = ReplayGainAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-stereo.wav', dict())
check_default_metadata(metadata)
test_wav_stereo.rgain = True
'''
"""