Congegni/sintonia
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix test failures

Browse source
This commit is contained in:
Kyle Robbertze 2020-01-21 09:13:42 +02:00
parent 5d67172dd0
commit 82042e8c69
10 changed files with 125 additions and 129 deletions
Download patch file Download diff file Expand all files Collapse all files

24
python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py
View file

@ -1,9 +1,9 @@
""" Analyzes and imports an audio file into the Airtime library.
""" Analyzes and imports an audio file into the Airtime library.
"""
import logging
import threading
import multiprocessing
import queue
from queue import Queue
import configparser
from .metadata_analyzer import MetadataAnalyzer
from .filemover_analyzer import FileMoverAnalyzer
@ -12,8 +12,8 @@ from .replaygain_analyzer import ReplayGainAnalyzer
from .playability_analyzer import *
class AnalyzerPipeline:
""" Analyzes and imports an audio file into the Airtime library.
""" Analyzes and imports an audio file into the Airtime library.
This currently performs metadata extraction (eg. gets the ID3 tags from an MP3),
then moves the file to the Airtime music library (stor/imported), and returns
the results back to the parent process. This class is used in an isolated process
@ -26,27 +26,27 @@ class AnalyzerPipeline:
@staticmethod
def run_analysis(queue, audio_file_path, import_directory, original_filename, storage_backend, file_prefix):
"""Analyze and import an audio file, and put all extracted metadata into queue.
Keyword arguments:
queue: A multiprocessing.queues.Queue which will be used to pass the
extracted metadata back to the parent process.
audio_file_path: Path on disk to the audio file to analyze.
import_directory: Path to the final Airtime "import" directory where
import_directory: Path to the final Airtime "import" directory where
we will move the file.
original_filename: The original filename of the file, which we'll try to
preserve. The file at audio_file_path typically has a
original_filename: The original filename of the file, which we'll try to
preserve. The file at audio_file_path typically has a
temporary randomly generated name, which is why we want
to know what the original name was.
to know what the original name was.
storage_backend: String indicating the storage backend (amazon_s3 or file)
file_prefix:
"""
# It is super critical to initialize a separate log file here so that we
# It is super critical to initialize a separate log file here so that we
# don't inherit logging/locks from the parent process. Supposedly
# this can lead to Bad Things (deadlocks): http://bugs.python.org/issue6721
AnalyzerPipeline.python_logger_deadlock_workaround()
try:
if not isinstance(queue, queue.Queue):
if not isinstance(queue, Queue):
raise TypeError("queue must be a Queue.Queue()")
if not isinstance(audio_file_path, str):
raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__ + " instead.")
@ -72,7 +72,7 @@ class AnalyzerPipeline:
metadata["import_status"] = 0 # Successfully imported
# Note that the queue we're putting the results into is our interprocess communication
# Note that the queue we're putting the results into is our interprocess communication
# back to the main process.
# Pass all the file metadata back to the main analyzer process, which then passes

2
python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py
View file

@ -64,7 +64,7 @@ class CuePointAnalyzer(Analyzer):
except OSError as e: # silan was not found
logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have silan installed?"))
except subprocess.CalledProcessError as e: # silan returned an error code
logging.warn("%s %s %s", e.cmd, e.message, e.returncode)
logging.warn("%s %s %s", e.cmd, e.output, e.returncode)
except Exception as e:
logging.warn(e)

32
python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py
View file

@ -4,24 +4,24 @@ import time
import shutil
import os, errno
import time
import uuid
import uuid
from .analyzer import Analyzer
class FileMoverAnalyzer(Analyzer):
"""This analyzer copies a file over from a temporary directory (stor/organize)
"""This analyzer copies a file over from a temporary directory (stor/organize)
into the Airtime library (stor/imported).
"""
@staticmethod
def analyze(audio_file_path, metadata):
"""Dummy method because we need more info than analyze gets passed to it"""
raise Exception("Use FileMoverAnalyzer.move() instead.")
@staticmethod
def move(audio_file_path, import_directory, original_filename, metadata):
"""Move the file at audio_file_path over into the import_directory/import,
renaming it to original_filename.
Keyword arguments:
audio_file_path: Path to the file to be imported.
import_directory: Path to the "import" directory inside the Airtime stor directory.
@ -30,18 +30,20 @@ class FileMoverAnalyzer(Analyzer):
metadata: A dictionary where the "full_path" of where the file is moved to will be added.
"""
if not isinstance(audio_file_path, str):
raise TypeError("audio_file_path must be unicode. Was of type " + type(audio_file_path).__name__)
raise TypeError("audio_file_path must be string. Was of type " + type(audio_file_path).__name__)
if not isinstance(import_directory, str):
raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__)
raise TypeError("import_directory must be string. Was of type " + type(import_directory).__name__)
if not isinstance(original_filename, str):
raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__)
raise TypeError("original_filename must be string. Was of type " + type(original_filename).__name__)
if not isinstance(metadata, dict):
raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__)
if not os.path.exists(audio_file_path):
raise FileNotFoundError("audio file not found: {}".format(audio_file_path))
#Import the file over to it's final location.
# TODO: Also, handle the case where the move fails and write some code
# to possibly move the file to problem_files.
max_dir_len = 48
max_file_len = 48
final_file_path = import_directory
@ -58,11 +60,11 @@ class FileMoverAnalyzer(Analyzer):
#If a file with the same name already exists in the "import" directory, then
#we add a unique string to the end of this one. We never overwrite a file on import
#because if we did that, it would mean Airtime's database would have
#because if we did that, it would mean Airtime's database would have
#the wrong information for the file we just overwrote (eg. the song length would be wrong!)
#If the final file path is the same as the file we've been told to import (which
#you often do when you're debugging), then don't move the file at all.
if os.path.exists(final_file_path):
if os.path.samefile(audio_file_path, final_file_path):
metadata["full_path"] = final_file_path
@ -77,14 +79,14 @@ class FileMoverAnalyzer(Analyzer):
#Ensure the full path to the file exists
mkdir_p(os.path.dirname(final_file_path))
#Move the file into its final destination directory
#Move the file into its final destination directory
logging.debug("Moving %s to %s" % (audio_file_path, final_file_path))
shutil.move(audio_file_path, final_file_path)
metadata["full_path"] = final_file_path
return metadata
def mkdir_p(path):
""" Make all directories in a tree (like mkdir -p)"""
if path == "":

34
python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py
View file

@ -13,15 +13,17 @@ class MetadataAnalyzer(Analyzer):
@staticmethod
def analyze(filename, metadata):
''' Extract audio metadata from tags embedded in the file (eg. ID3 tags)
Keyword arguments:
filename: The path to the audio file to extract metadata from.
metadata: A dictionary that the extracted metadata will be added to.
metadata: A dictionary that the extracted metadata will be added to.
'''
if not isinstance(filename, str):
raise TypeError("filename must be unicode. Was of type " + type(filename).__name__)
raise TypeError("filename must be string. Was of type " + type(filename).__name__)
if not isinstance(metadata, dict):
raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__)
if not os.path.exists(filename):
raise FileNotFoundError("audio file not found: {}".format(filename))
#Airtime <= 2.5.x nonsense:
metadata["ftype"] = "audioclip"
@ -40,7 +42,7 @@ class MetadataAnalyzer(Analyzer):
m.update(data)
metadata["md5"] = m.hexdigest()
# Mutagen doesn't handle WAVE files so we use a different package
# Mutagen doesn't handle WAVE files so we use a different package
ms = magic.open(magic.MIME_TYPE)
ms.load()
with open(filename, 'rb') as fh:
@ -57,15 +59,15 @@ class MetadataAnalyzer(Analyzer):
if audio_file == None: # Don't use "if not" here. It is wrong due to mutagen's design.
return metadata
# Note that audio_file can equal {} if the file is valid but there's no metadata tags.
# We can still try to grab the info variables below.
# We can still try to grab the info variables below.
#Grab other file information that isn't encoded in a tag, but instead usually
#in the file header. Mutagen breaks that out into a separate "info" object:
info = audio_file.info
if hasattr(info, "sample_rate"): # Mutagen is annoying and inconsistent
metadata["sample_rate"] = info.sample_rate
if hasattr(info, "length"):
metadata["length_seconds"] = info.length
metadata["length_seconds"] = info.length
#Converting the length in seconds (float) to a formatted time string
track_length = datetime.timedelta(seconds=info.length)
metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length)
@ -77,12 +79,12 @@ class MetadataAnalyzer(Analyzer):
if hasattr(info, "bitrate"):
metadata["bit_rate"] = info.bitrate
# Use the mutagen to get the MIME type, if it has one. This is more reliable and
# consistent for certain types of MP3s or MPEG files than the MIMEs returned by magic.
if audio_file.mime:
metadata["mime"] = audio_file.mime[0]
#Try to get the number of channels if mutagen can...
try:
#Special handling for getting the # of channels from MP3s. It's in the "mode" field
@ -97,13 +99,13 @@ class MetadataAnalyzer(Analyzer):
except (AttributeError, KeyError):
#If mutagen can't figure out the number of channels, we'll just leave it out...
pass
#Try to extract the number of tracks on the album if we can (the "track total")
try:
track_number = audio_file["tracknumber"]
if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh
if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh
track_number = track_number[0]
track_number_tokens = track_number
track_number_tokens = track_number
if '/' in track_number:
track_number_tokens = track_number.split('/')
track_number = track_number_tokens[0]
@ -118,7 +120,7 @@ class MetadataAnalyzer(Analyzer):
pass
#We normalize the mutagen tags slightly here, so in case mutagen changes,
#we find the
#we find the
mutagen_to_airtime_mapping = {
'title': 'track_title',
'artist': 'artist_name',
@ -153,13 +155,13 @@ class MetadataAnalyzer(Analyzer):
# Some tags are returned as lists because there could be multiple values.
# This is unusual so we're going to always just take the first item in the list.
if isinstance(metadata[airtime_tag], list):
if metadata[airtime_tag]:
if metadata[airtime_tag]:
metadata[airtime_tag] = metadata[airtime_tag][0]
else: # Handle empty lists
metadata[airtime_tag] = ""
except KeyError:
continue
continue
return metadata
@ -174,7 +176,7 @@ class MetadataAnalyzer(Analyzer):
track_length = datetime.timedelta(seconds=length_seconds)
metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length)
metadata["length_seconds"] = length_seconds
metadata["cueout"] = metadata["length"]
metadata["cueout"] = metadata["length"]
except wave.Error as ex:
logging.error("Invalid WAVE file: {}".format(str(ex)))
raise

14
python_apps/airtime_analyzer/airtime_analyzer/replaygain_analyzer.py
View file

@ -1,12 +1,13 @@
import subprocess
import logging
from .analyzer import Analyzer
import re
class ReplayGainAnalyzer(Analyzer):
''' This class extracts the ReplayGain using a tool from the python-rgain package. '''
REPLAYGAIN_EXECUTABLE = 'replaygain' # From the python-rgain package
REPLAYGAIN_EXECUTABLE = 'replaygain' # From the rgain3 python package
@staticmethod
def analyze(filename, metadata):
@ -19,17 +20,16 @@ class ReplayGainAnalyzer(Analyzer):
'''
command = [ReplayGainAnalyzer.REPLAYGAIN_EXECUTABLE, '-d', filename]
try:
results = subprocess.check_output(command, stderr=subprocess.STDOUT, close_fds=True)
filename_token = "%s: " % filename
rg_pos = results.find(filename_token, results.find("Calculating Replay Gain information")) + len(filename_token)
db_pos = results.find(" dB", rg_pos)
replaygain = results[rg_pos:db_pos]
results = subprocess.check_output(command, stderr=subprocess.STDOUT,
close_fds=True, text=True)
gain_match = r'Calculating Replay Gain information \.\.\.(?:\n|.)*?:([\d.-]*) dB'
replaygain = re.search(gain_match, results).group(1)
metadata['replay_gain'] = float(replaygain)
except OSError as e: # replaygain was not found
logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have python-rgain installed?"))
except subprocess.CalledProcessError as e: # replaygain returned an error code
logging.warn("%s %s %s", e.cmd, e.message, e.returncode)
logging.warn("%s %s %s", e.cmd, e.output, e.returncode)
except Exception as e:
logging.warn(e)