From d5cacf4011995f68ad4e7e933b40294515c89f77 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 14 Nov 2012 14:43:10 -0500 Subject: [PATCH] moved replay gain to pypo for better saas performance --- .../media-monitor2/media/update/__init__.py | 0 .../media-monitor2/media/update/replaygain.py | 152 ------------------ .../media/update/replaygainupdater.py | 82 ---------- python_apps/media-monitor2/mm2.py | 3 +- python_apps/pypo/pypocli.py | 5 + 5 files changed, 6 insertions(+), 236 deletions(-) delete mode 100644 python_apps/media-monitor2/media/update/__init__.py delete mode 100644 python_apps/media-monitor2/media/update/replaygain.py delete mode 100644 python_apps/media-monitor2/media/update/replaygainupdater.py diff --git a/python_apps/media-monitor2/media/update/__init__.py b/python_apps/media-monitor2/media/update/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/python_apps/media-monitor2/media/update/replaygain.py b/python_apps/media-monitor2/media/update/replaygain.py deleted file mode 100644 index 5af7cd4a1..000000000 --- a/python_apps/media-monitor2/media/update/replaygain.py +++ /dev/null @@ -1,152 +0,0 @@ -from subprocess import Popen, PIPE -import re -import os -import sys -import shutil -import tempfile -import logging - - -logger = logging.getLogger() - -def get_process_output(command): - """ - Run subprocess and return stdout - """ - logger.debug(command) - p = Popen(command, shell=True, stdout=PIPE) - return p.communicate()[0].strip() - -def run_process(command): - """ - Run subprocess and return "return code" - """ - p = Popen(command, shell=True) - return os.waitpid(p.pid, 0)[1] - -def get_mime_type(file_path): - """ - Attempts to get the mime type but will return prematurely if the process - takes longer than 5 seconds. Note that this function should only be called - for files which do not have a mp3/ogg/flac extension. - """ - - return get_process_output("timeout 5 file -b --mime-type %s" % file_path) - -def duplicate_file(file_path): - """ - Makes a duplicate of the file and returns the path of this duplicate file. - """ - fsrc = open(file_path, 'r') - fdst = tempfile.NamedTemporaryFile(delete=False) - - logger.info("Copying %s to %s" % (file_path, fdst.name)) - - shutil.copyfileobj(fsrc, fdst) - - fsrc.close() - fdst.close() - - return fdst.name - -def get_file_type(file_path): - file_type = None - if re.search(r'mp3$', file_path, re.IGNORECASE): - file_type = 'mp3' - elif re.search(r'og(g|a)$', file_path, re.IGNORECASE): - file_type = 'vorbis' - elif re.search(r'flac$', file_path, re.IGNORECASE): - file_type = 'flac' - else: - mime_type = get_mime_type(file_path) - if 'mpeg' in mime_type: - file_type = 'mp3' - elif 'ogg' in mime_type: - file_type = 'vorbis' - elif 'flac' in mime_type: - file_type = 'flac' - - return file_type - - -def calculate_replay_gain(file_path): - """ - This function accepts files of type mp3/ogg/flac and returns a calculated - ReplayGain value in dB. - If the value cannot be calculated for some reason, then we default to 0 - (Unity Gain). - - http://wiki.hydrogenaudio.org/index.php?title=ReplayGain_1.0_specification - """ - - try: - """ - Making a duplicate is required because the ReplayGain extraction utilities we use - make unwanted modifications to the file. - """ - - search = None - temp_file_path = duplicate_file(file_path) - - file_type = get_file_type(file_path) - nice_level = '15' - - if file_type: - if file_type == 'mp3': - if run_process("which mp3gain > /dev/null") == 0: - command = 'nice -n %s mp3gain -q "%s" 2> /dev/null' \ - % (nice_level, temp_file_path) - out = get_process_output(command) - search = re.search(r'Recommended "Track" dB change: (.*)', \ - out) - else: - logger.warn("mp3gain not found") - elif file_type == 'vorbis': - command = "which vorbisgain > /dev/null && which ogginfo > \ - /dev/null" - if run_process(command) == 0: - command = 'nice -n %s vorbisgain -q -f "%s" 2>/dev/null \ - >/dev/null' % (nice_level,temp_file_path) - run_process(command) - - out = get_process_output('ogginfo "%s"' % temp_file_path) - search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out) - else: - logger.warn("vorbisgain/ogginfo not found") - elif file_type == 'flac': - if run_process("which metaflac > /dev/null") == 0: - - command = 'nice -n %s metaflac --add-replay-gain "%s"' \ - % (nice_level, temp_file_path) - run_process(command) - - command = 'nice -n %s metaflac \ - --show-tag=REPLAYGAIN_TRACK_GAIN "%s"' \ - % (nice_level, temp_file_path) - - out = get_process_output(command) - search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out) - else: logger.warn("metaflac not found") - - except Exception, e: - logger.error(str(e)) - finally: - #no longer need the temp, file simply remove it. - try: os.remove(temp_file_path) - except: pass - - replay_gain = 0 - if search: - matches = search.groups() - if len(matches) == 1: - replay_gain = matches[0] - else: - logger.warn("Received more than 1 match in: '%s'" % str(matches)) - - return replay_gain - - -# Example of running from command line: -# python replay_gain.py /path/to/filename.mp3 -if __name__ == "__main__": - print calculate_replay_gain(sys.argv[1]) diff --git a/python_apps/media-monitor2/media/update/replaygainupdater.py b/python_apps/media-monitor2/media/update/replaygainupdater.py deleted file mode 100644 index 2f52c0a23..000000000 --- a/python_apps/media-monitor2/media/update/replaygainupdater.py +++ /dev/null @@ -1,82 +0,0 @@ -from threading import Thread - -import traceback -import os -import time -import logging - -from media.update import replaygain - -class ReplayGainUpdater(Thread): - """ - The purpose of the class is to query the server for a list of files which - do not have a ReplayGain value calculated. This class will iterate over the - list calculate the values, update the server and repeat the process until - the server reports there are no files left. - - This class will see heavy activity right after a 2.1->2.2 upgrade since 2.2 - introduces ReplayGain normalization. A fresh install of Airtime 2.2 will - see this class not used at all since a file imported in 2.2 will - automatically have its ReplayGain value calculated. - """ - - @staticmethod - def start_reply_gain(apc): - me = ReplayGainUpdater(apc) - me.daemon = True - me.start() - - def __init__(self,apc): - Thread.__init__(self) - self.api_client = apc - self.logger = logging.getLogger() - - def main(self): - raw_response = self.api_client.list_all_watched_dirs() - if 'dirs' not in raw_response: - self.logger.error("Could not get a list of watched directories \ - with a dirs attribute. Printing full request:") - self.logger.error( raw_response ) - return - - directories = raw_response['dirs'] - - for dir_id, dir_path in directories.iteritems(): - try: - # keep getting few rows at a time for current music_dir (stor - # or watched folder). - total = 0 - while True: - # return a list of pairs where the first value is the - # file's database row id and the second value is the - # filepath - files = self.api_client.get_files_without_replay_gain_value(dir_id) - processed_data = [] - for f in files: - full_path = os.path.join(dir_path, f['fp']) - processed_data.append((f['id'], replaygain.calculate_replay_gain(full_path))) - - try: - self.api_client.update_replay_gain_values(processed_data) - except Exception as e: self.unexpected_exception(e) - - if len(files) == 0: break - self.logger.info("Processed: %d songs" % total) - - except Exception, e: - self.logger.error(e) - self.logger.debug(traceback.format_exc()) - def run(self): - try: - while True: - self.logger.info("Runnning replaygain updater") - self.main() - # Sleep for 5 minutes in case new files have been added - time.sleep(60 * 5) - except Exception, e: - self.logger.error('ReplayGainUpdater Exception: %s', traceback.format_exc()) - self.logger.error(e) - -if __name__ == "__main__": - rgu = ReplayGainUpdater() - rgu.main() diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 0a3acfabd..2705ef0ee 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -15,7 +15,6 @@ from media.monitor.airtime import AirtimeNotifier, \ AirtimeMessageReceiver from media.monitor.watchersyncer import WatchSyncer from media.monitor.eventdrainer import EventDrainer -from media.update.replaygainupdater import ReplayGainUpdater from std_err_override import LogWriter import media.monitor.pure as mmp @@ -95,7 +94,7 @@ def main(global_config, api_client_config, log_config, apiclient = apc.AirtimeApiClient.create_right_config(log=log, config_path=api_client_config) - ReplayGainUpdater.start_reply_gain(apiclient) + #ReplayGainUpdater.start_reply_gain(apiclient) manager = Manager() diff --git a/python_apps/pypo/pypocli.py b/python_apps/pypo/pypocli.py index 1b51a13f8..ea8950d41 100644 --- a/python_apps/pypo/pypocli.py +++ b/python_apps/pypo/pypocli.py @@ -24,6 +24,8 @@ from recorder import Recorder from listenerstat import ListenerStat from pypomessagehandler import PypoMessageHandler +from media.update.replaygainupdater import ReplayGainUpdater + from configobj import ConfigObj # custom imports @@ -174,6 +176,9 @@ if __name__ == '__main__': sys.exit() api_client = api_client.AirtimeApiClient() + + ReplayGainUpdater.start_reply_gain(api_client) + api_client.register_component("pypo") pypoFetch_q = Queue()