From fa3302cebcda0d26f1a14239ee558bd68ea9e7a2 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Mon, 5 Nov 2012 11:52:11 -0500 Subject: [PATCH 01/68] CC-4665: ReplayGain does not work for FLAC files -fixed --- python_apps/media-monitor2/media/update/replaygain.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/update/replaygain.py b/python_apps/media-monitor2/media/update/replaygain.py index a630a2655..5239bf1e2 100644 --- a/python_apps/media-monitor2/media/update/replaygain.py +++ b/python_apps/media-monitor2/media/update/replaygain.py @@ -105,7 +105,16 @@ def calculate_replay_gain(file_path): logger.warn("vorbisgain/ogginfo not found") elif file_type == 'flac': if run_process("which metaflac > /dev/null") == 0: - out = get_process_output('nice -n %s metaflac --show-tag=REPLAYGAIN_TRACK_GAIN "%s"' % (nice_level, temp_file_path)) + + command = 'nice -n %s metaflac --add-replay-gain "%s"' \ + % (nice_level, temp_file_path) + run_process(command) + + command = 'nice -n %s metaflac \ + --show-tag=REPLAYGAIN_TRACK_GAIN "%s"' \ + % (nice_level, temp_file_path) + + out = get_process_output() search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out) else: logger.warn("metaflac not found") From 6cd8938e476ef98a2b8d64a2e09af04bc84c6f9b Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Mon, 5 Nov 2012 11:52:28 -0500 Subject: [PATCH 02/68] don't break 80 character line limit --- .../media-monitor2/media/update/replaygain.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/python_apps/media-monitor2/media/update/replaygain.py b/python_apps/media-monitor2/media/update/replaygain.py index 5239bf1e2..0a9d8a224 100644 --- a/python_apps/media-monitor2/media/update/replaygain.py +++ b/python_apps/media-monitor2/media/update/replaygain.py @@ -71,8 +71,10 @@ def get_file_type(file_path): def calculate_replay_gain(file_path): """ - This function accepts files of type mp3/ogg/flac and returns a calculated ReplayGain value in dB. - If the value cannot be calculated for some reason, then we default to 0 (Unity Gain). + This function accepts files of type mp3/ogg/flac and returns a calculated + ReplayGain value in dB. + If the value cannot be calculated for some reason, then we default to 0 + (Unity Gain). http://wiki.hydrogenaudio.org/index.php?title=ReplayGain_1.0_specification """ @@ -92,13 +94,21 @@ def calculate_replay_gain(file_path): if file_type: if file_type == 'mp3': if run_process("which mp3gain > /dev/null") == 0: - out = get_process_output('nice -n %s mp3gain -q "%s" 2> /dev/null' % (nice_level, temp_file_path)) - search = re.search(r'Recommended "Track" dB change: (.*)', out) + command = 'nice -n %s mp3gain -q "%s" 2> /dev/null' \ + % (nice_level, temp_file_path) + out = get_process_output() + search = re.search(r'Recommended "Track" dB change: (.*)', \ + out) else: logger.warn("mp3gain not found") elif file_type == 'vorbis': - if run_process("which vorbisgain > /dev/null && which ogginfo > /dev/null") == 0: - run_process('nice -n %s vorbisgain -q -f "%s" 2>/dev/null >/dev/null' % (nice_level,temp_file_path)) + command = "which vorbisgain > /dev/null && which ogginfo > \ + /dev/null" + if run_process() == 0: + command = 'nice -n %s vorbisgain -q -f "%s" 2>/dev/null \ + >/dev/null' % (nice_level,temp_file_path) + run_process(command) + out = get_process_output('ogginfo "%s"' % temp_file_path) search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out) else: From ce313a5b22ca0cd3f8a1dbb3678661671b148642 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 5 Nov 2012 13:29:10 -0500 Subject: [PATCH 03/68] docstring formatting --- .../media-monitor2/media/monitor/manager.py | 98 +++++++------------ 1 file changed, 34 insertions(+), 64 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/manager.py b/python_apps/media-monitor2/media/monitor/manager.py index 548590f38..2c5512ec6 100644 --- a/python_apps/media-monitor2/media/monitor/manager.py +++ b/python_apps/media-monitor2/media/monitor/manager.py @@ -13,11 +13,10 @@ import media.monitor.pure as mmp class ManagerTimeout(threading.Thread,Loggable): - """ - The purpose of this class is to flush the organize directory every 3 - secnods. This used to be just a work around for cc-4235 but recently - became a permanent solution because it's "cheap" and reliable - """ + """ The purpose of this class is to flush the organize directory + every 3 secnods. This used to be just a work around for cc-4235 + but recently became a permanent solution because it's "cheap" and + reliable """ def __init__(self, manager, interval=1.5): # TODO : interval should be read from config and passed here instead # of just using the hard coded value @@ -30,11 +29,9 @@ class ManagerTimeout(threading.Thread,Loggable): self.manager.flush_organize() class Manager(Loggable): - """ - An abstraction over media monitors core pyinotify functions. These - include adding watched,store, organize directories, etc. Basically - composes over WatchManager from pyinotify - """ + """ An abstraction over media monitors core pyinotify functions. + These include adding watched,store, organize directories, etc. + Basically composes over WatchManager from pyinotify """ def __init__(self): self.wm = pyinotify.WatchManager() # These two instance variables are assumed to be constant @@ -76,23 +73,19 @@ class Manager(Loggable): # through dedicated handler objects. Because we must have access to a # manager instance. Hence we must slightly break encapsulation. def watch_move(self, watch_dir, sender=None): - """ - handle 'watch move' events directly sent from listener - """ + """ handle 'watch move' events directly sent from listener """ self.logger.info("Watch dir '%s' has been renamed (hence removed)" % watch_dir) self.remove_watch_directory(normpath(watch_dir)) def watch_signal(self): - """ - Return the signal string our watch_listener is reading events from - """ + """ Return the signal string our watch_listener is reading + events from """ return self.watch_listener.signal def __remove_watch(self,path): - """ - Remove path from being watched (first will check if 'path' is watched) - """ + """ Remove path from being watched (first will check if 'path' + is watched) """ # only delete if dir is actually being watched if path in self.__wd_path: wd = self.__wd_path[path] @@ -100,10 +93,8 @@ class Manager(Loggable): del(self.__wd_path[path]) def __add_watch(self,path,listener): - """ - Start watching 'path' using 'listener'. First will check if directory - is being watched before adding another watch - """ + """ Start watching 'path' using 'listener'. First will check if + directory is being watched before adding another watch """ self.logger.info("Attempting to add listener to path '%s'" % path) self.logger.info( 'Listener: %s' % str(listener) ) @@ -114,9 +105,8 @@ class Manager(Loggable): if wd: self.__wd_path[path] = wd.values()[0] def __create_organizer(self, target_path, recorded_path): - """ - creates an organizer at new destination path or modifies the old one - """ + """ creates an organizer at new destination path or modifies the + old one """ # TODO : find a proper fix for the following hack # We avoid creating new instances of organize because of the way # it interacts with pydispatch. We must be careful to never have @@ -134,23 +124,17 @@ class Manager(Loggable): recorded_path=recorded_path) def get_problem_files_path(self): - """ - returns the path where problem files should go - """ + """ returns the path where problem files should go """ return self.organize['problem_files_path'] def set_problem_files_path(self, new_path): - """ - Set the path where problem files should go - """ + """ Set the path where problem files should go """ self.organize['problem_files_path'] = new_path self.organize['problem_handler'] = \ ProblemFileHandler( PathChannel(signal='badfile',path=new_path) ) def get_recorded_path(self): - """ - returns the path of the recorded directory - """ + """ returns the path of the recorded directory """ return self.organize['recorded_path'] def set_recorded_path(self, new_path): @@ -160,17 +144,14 @@ class Manager(Loggable): self.__add_watch(new_path, self.watch_listener) def get_organize_path(self): - """ - returns the current path that is being watched for organization - """ + """ returns the current path that is being watched for + organization """ return self.organize['organize_path'] def set_organize_path(self, new_path): - """ - sets the organize path to be new_path. Under the current scheme there - is only one organize path but there is no reason why more cannot be - supported - """ + """ sets the organize path to be new_path. Under the current + scheme there is only one organize path but there is no reason + why more cannot be supported """ # if we are already organizing a particular directory we remove the # watch from it first before organizing another directory self.__remove_watch(self.organize['organize_path']) @@ -188,19 +169,15 @@ class Manager(Loggable): return self.organize['imported_path'] def set_imported_path(self,new_path): - """ - set the directory where organized files go to. - """ + """ set the directory where organized files go to. """ self.__remove_watch(self.organize['imported_path']) self.organize['imported_path'] = new_path self.__create_organizer( new_path, self.organize['recorded_path']) self.__add_watch(new_path, self.watch_listener) def change_storage_root(self, store): - """ - hooks up all the directories for you. Problem, recorded, imported, - organize. - """ + """ hooks up all the directories for you. Problem, recorded, + imported, organize. """ store_paths = mmp.expand_storage(store) # First attempt to make sure that all paths exist before adding any # watches @@ -217,18 +194,14 @@ class Manager(Loggable): mmp.create_dir(p) def has_watch(self, path): - """ - returns true if the path is being watched or not. Any kind of watch: - organize, store, watched. - """ + """ returns true if the path is being watched or not. Any kind + of watch: organize, store, watched. """ return path in self.__wd_path def add_watch_directory(self, new_dir): - """ - adds a directory to be "watched". "watched" directories are + """ adds a directory to be "watched". "watched" directories are those that are being monitored by media monitor for airtime in - this context and not directories pyinotify calls watched - """ + this context and not directories pyinotify calls watched """ if self.has_watch(new_dir): self.logger.info("Cannot add '%s' to watched directories. It's \ already being watched" % new_dir) @@ -237,9 +210,8 @@ class Manager(Loggable): self.__add_watch(new_dir, self.watch_listener) def remove_watch_directory(self, watch_dir): - """ - removes a directory from being "watched". Undoes add_watch_directory - """ + """ removes a directory from being "watched". Undoes + add_watch_directory """ if self.has_watch(watch_dir): self.logger.info("Removing watched directory: '%s'", watch_dir) self.__remove_watch(watch_dir) @@ -250,9 +222,7 @@ class Manager(Loggable): self.logger.info( self.__wd_path ) def loop(self): - """ - block until we receive pyinotify events - """ + """ block until we receive pyinotify events """ notifier = pyinotify.Notifier(self.wm) notifier.coalesce_events() notifier.loop() From c287e11bee8dc50a3129d5bef3d17b12755386e0 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 5 Nov 2012 14:02:55 -0500 Subject: [PATCH 04/68] moved setting up unicde into it's own routine --- python_apps/api_clients/api_client.py | 1 + python_apps/media-monitor2/mm2.py | 34 +++++++++++++++------------ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index ac93aa4c6..633993903 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -389,3 +389,4 @@ class AirtimeApiClient(object): """ self.logger.info( self.notify_webstream_data.req( _post_data={'data':data}, media_id=str(media_id)).retry(5)) + diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 2abace71c..dc9cd160a 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -24,6 +24,24 @@ import media.monitor.pure as mmp from api_clients import api_client as apc +def setup_global(log): + """ setup unicode and other stuff """ + log.info("Attempting to set the locale...") + try: + mmp.configure_locale(mmp.get_system_locale()) + except FailedToSetLocale as e: + log.info("Failed to set the locale...") + sys.exit(1) + except FailedToObtainLocale as e: + log.info("Failed to obtain the locale form the default path: \ + '/etc/default/locale'") + sys.exit(1) + except Exception as e: + log.info("Failed to set the locale for unknown reason. \ + Logging exception.") + log.info(str(e)) + + def main(global_config, api_client_config, log_config, index_create_attempt=False): @@ -72,21 +90,7 @@ def main(global_config, api_client_config, log_config, if not os.path.exists(config['index_path']): log.info("Index file does not exist. Terminating") - log.info("Attempting to set the locale...") - - try: - mmp.configure_locale(mmp.get_system_locale()) - except FailedToSetLocale as e: - log.info("Failed to set the locale...") - sys.exit(1) - except FailedToObtainLocale as e: - log.info("Failed to obtain the locale form the default path: \ - '/etc/default/locale'") - sys.exit(1) - except Exception as e: - log.info("Failed to set the locale for unknown reason. \ - Logging exception.") - log.info(str(e)) + setup_global(log) watch_syncer = WatchSyncer(signal='watch', chunking_number=config['chunking_number'], From 3509cfc2128b9547915f9487c978504ac6924ffb Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 5 Nov 2012 14:03:46 -0500 Subject: [PATCH 05/68] formatting --- python_apps/media-monitor2/mm2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index dc9cd160a..7e786722d 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -27,8 +27,7 @@ from api_clients import api_client as apc def setup_global(log): """ setup unicode and other stuff """ log.info("Attempting to set the locale...") - try: - mmp.configure_locale(mmp.get_system_locale()) + try: mmp.configure_locale(mmp.get_system_locale()) except FailedToSetLocale as e: log.info("Failed to set the locale...") sys.exit(1) From 83d4705bcadd079c88d2b1acccc47b72d33e83a2 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 5 Nov 2012 14:19:37 -0500 Subject: [PATCH 06/68] removed useless comments --- python_apps/media-monitor2/media/monitor/manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/manager.py b/python_apps/media-monitor2/media/monitor/manager.py index 2c5512ec6..e64d1c985 100644 --- a/python_apps/media-monitor2/media/monitor/manager.py +++ b/python_apps/media-monitor2/media/monitor/manager.py @@ -38,8 +38,6 @@ class Manager(Loggable): self.watch_channel = 'watch' self.organize_channel = 'organize' self.watch_listener = StoreWatchListener(signal = self.watch_channel) - # TODO : change this to a weak ref - # TODO : get rid of this hack once cc-4235 is fixed self.__timeout_thread = ManagerTimeout(self) self.__timeout_thread.daemon = True self.__timeout_thread.start() From a851d8dd7426b88712b2fca81d0ea81ecae0843b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 5 Nov 2012 16:04:39 -0500 Subject: [PATCH 07/68] docstring format --- .../media-monitor2/media/monitor/config.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/config.py b/python_apps/media-monitor2/media/monitor/config.py index 3a7be8eb5..b06e00a84 100644 --- a/python_apps/media-monitor2/media/monitor/config.py +++ b/python_apps/media-monitor2/media/monitor/config.py @@ -12,25 +12,21 @@ class MMConfig(object): self.cfg = ConfigObj(path) def __getitem__(self, key): - """ - We always return a copy of the config item to prevent callers from - doing any modifications through the returned objects methods - """ + """ We always return a copy of the config item to prevent + callers from doing any modifications through the returned + objects methods """ return copy.deepcopy(self.cfg[key]) def __setitem__(self, key, value): - """ - We use this method not to allow anybody to mess around with config file - any settings made should be done through MMConfig's instance methods - """ + """ We use this method not to allow anybody to mess around with + config file any settings made should be done through MMConfig's + instance methods """ raise ConfigAccessViolation(key) def save(self): self.cfg.write() def last_ran(self): - """ - Returns the last time media monitor was ran by looking at the time when - the file at 'index_path' was modified - """ + """ Returns the last time media monitor was ran by looking at + the time when the file at 'index_path' was modified """ return mmp.last_modified(self.cfg['index_path']) From 6392c69e27abe91679898e78fe1dccdf14beb875 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 5 Nov 2012 16:10:59 -0500 Subject: [PATCH 08/68] Removed unused code --- python_apps/media-monitor2/mm2.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 7e786722d..9471f698b 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -151,9 +151,6 @@ Options: --log= log config at """ -def main_loop(): - while True: pass - if __name__ == '__main__': from docopt import docopt args = docopt(__doc__,version="mm1.99") From 66138c4bafc5f831216ac935ee879d69833cb79f Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 5 Nov 2012 16:11:35 -0500 Subject: [PATCH 09/68] removed references to gevent --- python_apps/media-monitor2/mm2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 9471f698b..99d16f9f7 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -160,5 +160,4 @@ if __name__ == '__main__': sys.exit(0) print("Running mm1.99") main(args['--config'],args['--apiclient'],args['--log']) - #gevent.joinall([ gevent.spawn(main_loop) ]) From 110008e9091968837d0137e1b61a29851c248154 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 7 Nov 2012 11:29:39 -0500 Subject: [PATCH 10/68] added saas shit --- .../media-monitor2/media/saas/__init__.py | 0 .../media/saas/airtimeinstance.py | 32 +++++++++++++++++++ .../media-monitor2/media/saas/thread.py | 19 +++++++++++ 3 files changed, 51 insertions(+) create mode 100644 python_apps/media-monitor2/media/saas/__init__.py create mode 100644 python_apps/media-monitor2/media/saas/airtimeinstance.py create mode 100644 python_apps/media-monitor2/media/saas/thread.py diff --git a/python_apps/media-monitor2/media/saas/__init__.py b/python_apps/media-monitor2/media/saas/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python_apps/media-monitor2/media/saas/airtimeinstance.py b/python_apps/media-monitor2/media/saas/airtimeinstance.py new file mode 100644 index 000000000..ed37caa41 --- /dev/null +++ b/python_apps/media-monitor2/media/saas/airtimeinstance.py @@ -0,0 +1,32 @@ +import os + +from media.monitor.exceptions import NoConfigFile +from media.monitor.pure import LazyProperty +from media.monitor.config import MMConfig +from api_cients import AirtimeApiClient + +class AirtimeInstance(object): + """ AirtimeInstance is a class that abstracts away every airtime + instance by providing all the necessary objects required to interact + with the instance. ApiClient, configs, root_directory """ + + def __init__(self,name, root_path, config_paths): + """ name is an internal name only """ + for cfg in ['api_client','media_monitor', 'logging']: + if cfg not in config_paths: raise NoConfigFile(config_paths) + elif not os.path.exists(config_paths[cfg]): + raise NoConfigFile(config_paths[cfg]) + self.name = name + self.config_paths = config_paths + self.root_path = root_path + + def __str__(self): + return "%s,%s(%s)" % (self.name, self.root_path, self.config_paths) + + @LazyProperty + def api_client(self): + return AirtimeApiClient(config_path=self.config_paths['api_client']) + + @LazyProperty + def mm_config(self): + return MMConfig(self.config_paths['media_monitor']) diff --git a/python_apps/media-monitor2/media/saas/thread.py b/python_apps/media-monitor2/media/saas/thread.py new file mode 100644 index 000000000..b71c4a8fb --- /dev/null +++ b/python_apps/media-monitor2/media/saas/thread.py @@ -0,0 +1,19 @@ +import threading + +tc = threading.local() + +class InstanceThread(threading.Thread): + def __init__(self,user, *args, **kwargs): + super(InstanceThread, self).__init__(*args, **kwargs) + self._user = user + + def run(self): + tc._user = self._user + + def user(self): + return tc._user + +class InstanceInheritingThread(threading.Thread): + def __init__(self, *args, **kwargs): + super(InstanceInheritingThread, self).__init__(*args, **kwargs) + self.user = tc._user From 882a515caa4101ab009d16de1ef65f7cbcbc9027 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 7 Nov 2012 17:44:55 -0500 Subject: [PATCH 11/68] refactored threads which know the user they belong to --- python_apps/media-monitor2/media/saas/thread.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/python_apps/media-monitor2/media/saas/thread.py b/python_apps/media-monitor2/media/saas/thread.py index b71c4a8fb..489ad35a8 100644 --- a/python_apps/media-monitor2/media/saas/thread.py +++ b/python_apps/media-monitor2/media/saas/thread.py @@ -2,18 +2,16 @@ import threading tc = threading.local() -class InstanceThread(threading.Thread): +class HasUser(object): + def user(self): + return self._user + +class InstanceThread(threading.Thread, HasUser): def __init__(self,user, *args, **kwargs): super(InstanceThread, self).__init__(*args, **kwargs) self._user = user - - def run(self): - tc._user = self._user - def user(self): - return tc._user - -class InstanceInheritingThread(threading.Thread): +class InstanceInheritingThread(threading.Thread, HasUser): def __init__(self, *args, **kwargs): + self._user = threading.current_thread().user() super(InstanceInheritingThread, self).__init__(*args, **kwargs) - self.user = tc._user From 9e7b8a6b28409b5a27db32c2f1420e6bab481917 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 7 Nov 2012 17:45:08 -0500 Subject: [PATCH 12/68] added tests --- .../media-monitor2/tests/test_thread.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 python_apps/media-monitor2/tests/test_thread.py diff --git a/python_apps/media-monitor2/tests/test_thread.py b/python_apps/media-monitor2/tests/test_thread.py new file mode 100644 index 000000000..1638a60e3 --- /dev/null +++ b/python_apps/media-monitor2/tests/test_thread.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +import unittest +import time +from media.saas.thread import InstanceThread, InstanceInheritingThread + +# ugly but necessary for 2.7 +signal = False +signal2 = False + +class TestInstanceThread(unittest.TestCase): + def test_user_inject(self): + global signal + signal = False + u = "rudi" + class T(InstanceThread): + def run(me): + global signal + super(T, me).run() + signal = True + self.assertEquals(u, me.user()) + t = T(u, name="test_user_inject") + t.daemon = True + t.start() + time.sleep(0.2) + self.assertTrue(signal) + + def test_inheriting_thread(utest): + global signal2 + u = "testing..." + + class TT(InstanceInheritingThread): + def run(self): + global signal2 + utest.assertEquals(self.user(), u) + signal2 = True + + class T(InstanceThread): + def run(self): + super(T, self).run() + child_thread = TT(name="child thread") + child_thread.daemon = True + child_thread.start() + + parent_thread = T(u, name="Parent instance thread") + parent_thread.daemon = True + parent_thread.start() + + time.sleep(0.2) + utest.assertTrue(signal2) + + def test_different_user(utest): + u1, u2 = "ru", "di" + class T(InstanceThread): + def run(self): + super(T, self).run() + + for u in [u1, u2]: + t = T(u) + t.daemon = True + t.start() + utest.assertEquals(t.user(), u) + + +if __name__ == '__main__': unittest.main() From 13f59be21de8fa41a2b20f0b7626e4ec1e6dc0dc Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 7 Nov 2012 18:16:33 -0500 Subject: [PATCH 13/68] typo --- python_apps/media-monitor2/media/saas/airtimeinstance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/saas/airtimeinstance.py b/python_apps/media-monitor2/media/saas/airtimeinstance.py index ed37caa41..293fc84c2 100644 --- a/python_apps/media-monitor2/media/saas/airtimeinstance.py +++ b/python_apps/media-monitor2/media/saas/airtimeinstance.py @@ -3,7 +3,7 @@ import os from media.monitor.exceptions import NoConfigFile from media.monitor.pure import LazyProperty from media.monitor.config import MMConfig -from api_cients import AirtimeApiClient +from api_clients import AirtimeApiClient class AirtimeInstance(object): """ AirtimeInstance is a class that abstracts away every airtime From 7e5ec6505b142a466296795c2b909705a23474c4 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 7 Nov 2012 18:16:48 -0500 Subject: [PATCH 14/68] added tests for instance --- .../media-monitor2/tests/test_instance.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 python_apps/media-monitor2/tests/test_instance.py diff --git a/python_apps/media-monitor2/tests/test_instance.py b/python_apps/media-monitor2/tests/test_instance.py new file mode 100644 index 000000000..badaadf07 --- /dev/null +++ b/python_apps/media-monitor2/tests/test_instance.py @@ -0,0 +1,21 @@ +import unittest +from copy import deepcopy +from media.saas.airtimeinstance import AirtimeInstance, NoConfigFile + +class TestAirtimeInstance(unittest.TestCase): + def setUp(self): + self.cfg = { + 'api_client' : 'tests/test_instance.py', + 'media_monitor' : 'tests/test_instance.py', + 'logging' : 'tests/test_instance.py', + } + + def test_init_good(self): + AirtimeInstance("/root", self.cfg) + self.assertTrue(True) + + def test_init_bad(self): + cfg = deepcopy(self.cfg) + cfg['api_client'] = 'bs' + with self.assertRaises(NoConfigFile): + AirtimeInstance("/root", cfg) From ed00089a1a6df7ba6439ead4b600ecdb7e9e2acb Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 7 Nov 2012 22:53:39 -0500 Subject: [PATCH 15/68] added user level method to get current user of thread --- python_apps/media-monitor2/media/saas/thread.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor2/media/saas/thread.py b/python_apps/media-monitor2/media/saas/thread.py index 489ad35a8..f2e5ae28d 100644 --- a/python_apps/media-monitor2/media/saas/thread.py +++ b/python_apps/media-monitor2/media/saas/thread.py @@ -1,10 +1,12 @@ import threading -tc = threading.local() +class UserlessThread(Exception): + def __str__(): + return "Current thread: %s is not an instance of InstanceThread \ + of InstanceInheritingThread" % str(threading.current_thread()) class HasUser(object): - def user(self): - return self._user + def user(self): return self._user class InstanceThread(threading.Thread, HasUser): def __init__(self,user, *args, **kwargs): @@ -15,3 +17,7 @@ class InstanceInheritingThread(threading.Thread, HasUser): def __init__(self, *args, **kwargs): self._user = threading.current_thread().user() super(InstanceInheritingThread, self).__init__(*args, **kwargs) + +def user(): + try: return threading.current_thread().user() + except AttributeError: raise UserlessThread() From 55f0462946a7e4050050d1572c619780a8e81c51 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 7 Nov 2012 23:21:08 -0500 Subject: [PATCH 16/68] removed useless line of code --- python_apps/media-monitor2/media/saas/thread.py | 2 ++ python_apps/media-monitor2/mm2.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/saas/thread.py b/python_apps/media-monitor2/media/saas/thread.py index f2e5ae28d..0b4ae079e 100644 --- a/python_apps/media-monitor2/media/saas/thread.py +++ b/python_apps/media-monitor2/media/saas/thread.py @@ -21,3 +21,5 @@ class InstanceInheritingThread(threading.Thread, HasUser): def user(): try: return threading.current_thread().user() except AttributeError: raise UserlessThread() + +def apc(): return user().api_client diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 99d16f9f7..b246a9491 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -124,8 +124,6 @@ def main(global_config, api_client_config, log_config, airtime_receiver.new_watch({ 'directory':watch_dir }, restart=True) else: log.info("Failed to add watch on %s" % str(watch_dir)) - bs = Bootstrapper( db=sdb, watch_signal='watch' ) - ed = EventDrainer(airtime_notifier.connection, interval=float(config['rmq_event_wait'])) From c222ac10f74e5793889d3c0d35ceb2f821a11e33 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 7 Nov 2012 23:21:48 -0500 Subject: [PATCH 17/68] typo --- python_apps/media-monitor2/mm2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index b246a9491..8c1c35abb 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -48,7 +48,7 @@ def main(global_config, api_client_config, log_config, if not os.path.exists(cfg): raise NoConfigFile(cfg) # MMConfig is a proxy around ConfigObj instances. it does not allow # itself users of MMConfig instances to modify any config options - # directly through the dictionary. Users of this object muse use the + # directly through the dictionary. Users of this object must use the # correct methods designated for modification try: config = MMConfig(global_config) except NoConfigFile as e: From eaf0baa6b6501c88774ed9aa0e24d99be0531ca0 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 7 Nov 2012 23:22:25 -0500 Subject: [PATCH 18/68] removed useless code --- python_apps/media-monitor2/mm2.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 8c1c35abb..9d12ed32d 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -5,7 +5,6 @@ import logging import logging.config from media.monitor.manager import Manager -from media.monitor.bootstrap import Bootstrapper from media.monitor.log import get_logger, setup_logging from media.monitor.config import MMConfig from media.monitor.toucher import ToucherThread @@ -23,7 +22,6 @@ from std_err_override import LogWriter import media.monitor.pure as mmp from api_clients import api_client as apc - def setup_global(log): """ setup unicode and other stuff """ log.info("Attempting to set the locale...") @@ -41,7 +39,6 @@ def setup_global(log): log.info(str(e)) - def main(global_config, api_client_config, log_config, index_create_attempt=False): for cfg in [global_config, api_client_config]: @@ -100,8 +97,6 @@ def main(global_config, api_client_config, log_config, ReplayGainUpdater.start_reply_gain(apiclient) - sdb = AirtimeDB(apiclient) - manager = Manager() airtime_receiver = AirtimeMessageReceiver(config,manager) From 350d5dd6204ec7cec4d0e98bf6fdc8b9738b4e59 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Sat, 10 Nov 2012 09:24:26 -0500 Subject: [PATCH 19/68] refactored logging setup when mm is setup --- python_apps/media-monitor2/mm2.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 9d12ed32d..f78768201 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -22,6 +22,17 @@ from std_err_override import LogWriter import media.monitor.pure as mmp from api_clients import api_client as apc +def setup_logger(log_config, logpath): + logging.config.fileConfig(log_config) + #need to wait for Python 2.7 for this.. + #logging.captureWarnings(True) + logger = logging.getLogger() + LogWriter.override_std_err(logger) + logfile = unicode(logpath) + setup_logging(logfile) + log = get_logger() + return log + def setup_global(log): """ setup unicode and other stuff """ log.info("Attempting to set the locale...") @@ -55,18 +66,8 @@ def main(global_config, api_client_config, log_config, except Exception as e: print("Unknown error reading configuration file: '%s'" % global_config) print(str(e)) - - - logging.config.fileConfig(log_config) - - #need to wait for Python 2.7 for this.. - #logging.captureWarnings(True) - - logger = logging.getLogger() - LogWriter.override_std_err(logger) - logfile = unicode( config['logpath'] ) - setup_logging(logfile) - log = get_logger() + + log = setup_logger( log_config, config['logpath'] ) if not index_create_attempt: if not os.path.exists(config['index_path']): From a727f2bb1ecb632fa797ea6bb58ab5e834ba2b09 Mon Sep 17 00:00:00 2001 From: denise Date: Mon, 12 Nov 2012 14:28:50 -0500 Subject: [PATCH 20/68] CC-4638: Stream Settings -> Apache error when setting master source port to same port Airtime is running on -fixed --- .../application/forms/LiveStreamingPreferences.php | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/airtime_mvc/application/forms/LiveStreamingPreferences.php b/airtime_mvc/application/forms/LiveStreamingPreferences.php index c5b9d9aca..5a5c6fa49 100644 --- a/airtime_mvc/application/forms/LiveStreamingPreferences.php +++ b/airtime_mvc/application/forms/LiveStreamingPreferences.php @@ -150,13 +150,15 @@ class Application_Form_LiveStreamingPreferences extends Zend_Form_SubForm if ($master_harbor_input_port == $dj_harbor_input_port && $master_harbor_input_port != "") { $element = $this->getElement("dj_harbor_input_port"); $element->addError("You cannot use same port as Master DJ port."); + $isValid = false; } if ($master_harbor_input_port != "") { if (is_numeric($master_harbor_input_port)) { if ($master_harbor_input_port != Application_Model_StreamSetting::getMasterLiveStreamPort()) { $sock = socket_create(AF_INET, SOCK_STREAM, SOL_TCP); - $res = socket_bind($sock, 0, $master_harbor_input_port); - if (!$res) { + try { + socket_bind($sock, 0, $master_harbor_input_port); + } catch (Exception $e) { $element = $this->getElement("master_harbor_input_port"); $element->addError("Port '$master_harbor_input_port' is not available."); $isValid = false; @@ -171,8 +173,9 @@ class Application_Form_LiveStreamingPreferences extends Zend_Form_SubForm if (is_numeric($dj_harbor_input_port)) { if ($dj_harbor_input_port != Application_Model_StreamSetting::getDjLiveStreamPort()) { $sock = socket_create(AF_INET, SOCK_STREAM, SOL_TCP); - $res = socket_bind($sock, 0, $dj_harbor_input_port); - if (!$res) { + try { + socket_bind($sock, 0, $dj_harbor_input_port); + } catch (Exception $e) { $element = $this->getElement("dj_harbor_input_port"); $element->addError("Port '$dj_harbor_input_port' is not available."); $isValid = false; From 2b2a1db055970e3675e2d1b76997366f5dd54611 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 13 Nov 2012 17:31:30 -0500 Subject: [PATCH 21/68] CC-2401: Shuffle option in playlists - done --- .../controllers/PlaylistController.php | 28 +++++++++++++++++-- airtime_mvc/application/models/Playlist.php | 27 ++++++++++++++++-- .../views/scripts/playlist/playlist.phtml | 3 ++ airtime_mvc/public/js/airtime/library/spl.js | 20 +++++++++++++ .../js/airtime/playlist/smart_blockbuilder.js | 2 +- 5 files changed, 75 insertions(+), 5 deletions(-) diff --git a/airtime_mvc/application/controllers/PlaylistController.php b/airtime_mvc/application/controllers/PlaylistController.php index 0d2ead443..7283bba48 100644 --- a/airtime_mvc/application/controllers/PlaylistController.php +++ b/airtime_mvc/application/controllers/PlaylistController.php @@ -25,6 +25,7 @@ class PlaylistController extends Zend_Controller_Action ->addActionContext('smart-block-generate', 'json') ->addActionContext('smart-block-shuffle', 'json') ->addActionContext('get-block-info', 'json') + ->addActionContext('shuffle', 'json') ->initContext(); } @@ -72,7 +73,6 @@ class PlaylistController extends Zend_Controller_Action $isBlock = true; $viewPath = 'playlist/smart-block.phtml'; } - if (isset($obj)) { $formatter = new LengthFormatter($obj->getLength()); $this->view->length = $formatter->format(); @@ -94,7 +94,11 @@ class PlaylistController extends Zend_Controller_Action } else { $this->view->obj = $obj; $this->view->id = $obj->getId(); - $this->view->html = $this->view->render($viewPath); + if ($isJson) { + return $this->view->html = $this->view->render($viewPath); + } else { + $this->view->html = $this->view->render($viewPath); + } unset($this->view->obj); } } else { @@ -537,6 +541,26 @@ class PlaylistController extends Zend_Controller_Action $this->playlistUnknownError($e); } } + + public function shuffleAction() + { + $request = $this->getRequest(); + $params = $request->getPost(); + try { + $pl = new Application_Model_Playlist($params['obj_id']); + $result = $pl->shuffle(); + + if ($result['result'] == 0) { + die(json_encode(array("result"=>0, "html"=>$this->createFullResponse($pl, true)))); + } else { + die(json_encode($result)); + } + } catch (PlaylistNotFoundException $e) { + $this->playlistNotFound('block', true); + } catch (Exception $e) { + $this->playlistUnknownError($e); + } + } public function getBlockInfoAction() { diff --git a/airtime_mvc/application/models/Playlist.php b/airtime_mvc/application/models/Playlist.php index 8abf649d1..4e9562c72 100644 --- a/airtime_mvc/application/models/Playlist.php +++ b/airtime_mvc/application/models/Playlist.php @@ -387,8 +387,8 @@ SQL; } if (isset($obj)) { - if (($obj instanceof CcFiles && $obj->visible()) - || $obj instanceof CcWebstream || + if (($obj instanceof CcFiles && $obj->visible()) + || $obj instanceof CcWebstream || $obj instanceof CcBlock) { $entry = $this->plItem; @@ -933,6 +933,29 @@ SQL; { CcPlaylistcontentsQuery::create()->findByDbPlaylistId($this->id)->delete(); } + + public function shuffle() + { + $sql = <<$this->id)); + $maxPosition = $out[0]['max']; + + $map = range(0, $maxPosition); + shuffle($map); + + $currentPos = implode(',', array_keys($map)); + $sql = "UPDATE cc_playlistcontents SET position = CASE position "; + foreach ($map as $current => $after) { + $sql .= sprintf("WHEN %d THEN %d ", $current, $after); + } + $sql .= "END WHERE position IN ($currentPos) and playlist_id=:p1"; + + Application_Common_Database::prepareAndExecute($sql, array("p1"=>$this->id)); + $result['result'] = 0; + return $result; + } } // class Playlist diff --git a/airtime_mvc/application/views/scripts/playlist/playlist.phtml b/airtime_mvc/application/views/scripts/playlist/playlist.phtml index 6f962b97b..d287abaec 100644 --- a/airtime_mvc/application/views/scripts/playlist/playlist.phtml +++ b/airtime_mvc/application/views/scripts/playlist/playlist.phtml @@ -16,6 +16,9 @@ if (isset($this->obj)) { obj)) : ?> +
+ +
diff --git a/airtime_mvc/public/js/airtime/library/spl.js b/airtime_mvc/public/js/airtime/library/spl.js index 21a11218b..b5a808986 100644 --- a/airtime_mvc/public/js/airtime/library/spl.js +++ b/airtime_mvc/public/js/airtime/library/spl.js @@ -637,6 +637,26 @@ var AIRTIME = (function(AIRTIME){ $fs.addClass("closed"); } }); + + + $pl.on("click", 'button[id="playlist_shuffle_button"]', function(){ + obj_id = $('input[id="obj_id"]').val(); + url = "/Playlist/shuffle"; + enableLoadingIcon(); + $.post(url, {format: "json", obj_id: obj_id}, function(data){ + var json = $.parseJSON(data) + + if (json.error !== undefined) { + alert(json.error); + } + AIRTIME.playlist.fnOpenPlaylist(json); + if (json.result == "0") { + $pl.find('.success').text('Playlist shuffled'); + $pl.find('.success').show(); + } + disableLoadingIcon(); + }); + }) $pl.on("click", "#webstream_save", function(){ //get all fields and POST to server diff --git a/airtime_mvc/public/js/airtime/playlist/smart_blockbuilder.js b/airtime_mvc/public/js/airtime/playlist/smart_blockbuilder.js index f963ab9cd..c472448aa 100644 --- a/airtime_mvc/public/js/airtime/playlist/smart_blockbuilder.js +++ b/airtime_mvc/public/js/airtime/playlist/smart_blockbuilder.js @@ -351,7 +351,7 @@ function setupUI() { * It is only active if playlist is not empty */ var plContents = $('#spl_sortable').children(); - var shuffleButton = $('button[id="shuffle_button"]'); + var shuffleButton = $('button[id="shuffle_button"], button[id="playlist_shuffle_button"]'); if (!plContents.hasClass('spl_empty')) { if (shuffleButton.hasClass('ui-state-disabled')) { From 93ff2ce9f6c4c324945487ec0f413dfe5abca392 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 13 Nov 2012 17:32:42 -0500 Subject: [PATCH 22/68] removed extra import --- python_apps/media-monitor2/mm2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index f78768201..0a3acfabd 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -8,7 +8,6 @@ from media.monitor.manager import Manager from media.monitor.log import get_logger, setup_logging from media.monitor.config import MMConfig from media.monitor.toucher import ToucherThread -from media.monitor.syncdb import AirtimeDB from media.monitor.exceptions import FailedToObtainLocale, \ FailedToSetLocale, \ NoConfigFile From f1effc37a98a2606ce8d67ebdc875bf0a4f986c1 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 14 Nov 2012 11:51:00 -0500 Subject: [PATCH 23/68] removed dependency of mm from replaygain --- .../media-monitor2/media/update/replaygainupdater.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor2/media/update/replaygainupdater.py b/python_apps/media-monitor2/media/update/replaygainupdater.py index df48f09de..2f52c0a23 100644 --- a/python_apps/media-monitor2/media/update/replaygainupdater.py +++ b/python_apps/media-monitor2/media/update/replaygainupdater.py @@ -3,12 +3,11 @@ from threading import Thread import traceback import os import time +import logging from media.update import replaygain -from media.monitor.log import Loggable - -class ReplayGainUpdater(Thread, Loggable): +class ReplayGainUpdater(Thread): """ The purpose of the class is to query the server for a list of files which do not have a ReplayGain value calculated. This class will iterate over the @@ -30,6 +29,7 @@ class ReplayGainUpdater(Thread, Loggable): def __init__(self,apc): Thread.__init__(self) self.api_client = apc + self.logger = logging.getLogger() def main(self): raw_response = self.api_client.list_all_watched_dirs() From d5cacf4011995f68ad4e7e933b40294515c89f77 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 14 Nov 2012 14:43:10 -0500 Subject: [PATCH 24/68] moved replay gain to pypo for better saas performance --- .../media-monitor2/media/update/__init__.py | 0 .../media-monitor2/media/update/replaygain.py | 152 ------------------ .../media/update/replaygainupdater.py | 82 ---------- python_apps/media-monitor2/mm2.py | 3 +- python_apps/pypo/pypocli.py | 5 + 5 files changed, 6 insertions(+), 236 deletions(-) delete mode 100644 python_apps/media-monitor2/media/update/__init__.py delete mode 100644 python_apps/media-monitor2/media/update/replaygain.py delete mode 100644 python_apps/media-monitor2/media/update/replaygainupdater.py diff --git a/python_apps/media-monitor2/media/update/__init__.py b/python_apps/media-monitor2/media/update/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/python_apps/media-monitor2/media/update/replaygain.py b/python_apps/media-monitor2/media/update/replaygain.py deleted file mode 100644 index 5af7cd4a1..000000000 --- a/python_apps/media-monitor2/media/update/replaygain.py +++ /dev/null @@ -1,152 +0,0 @@ -from subprocess import Popen, PIPE -import re -import os -import sys -import shutil -import tempfile -import logging - - -logger = logging.getLogger() - -def get_process_output(command): - """ - Run subprocess and return stdout - """ - logger.debug(command) - p = Popen(command, shell=True, stdout=PIPE) - return p.communicate()[0].strip() - -def run_process(command): - """ - Run subprocess and return "return code" - """ - p = Popen(command, shell=True) - return os.waitpid(p.pid, 0)[1] - -def get_mime_type(file_path): - """ - Attempts to get the mime type but will return prematurely if the process - takes longer than 5 seconds. Note that this function should only be called - for files which do not have a mp3/ogg/flac extension. - """ - - return get_process_output("timeout 5 file -b --mime-type %s" % file_path) - -def duplicate_file(file_path): - """ - Makes a duplicate of the file and returns the path of this duplicate file. - """ - fsrc = open(file_path, 'r') - fdst = tempfile.NamedTemporaryFile(delete=False) - - logger.info("Copying %s to %s" % (file_path, fdst.name)) - - shutil.copyfileobj(fsrc, fdst) - - fsrc.close() - fdst.close() - - return fdst.name - -def get_file_type(file_path): - file_type = None - if re.search(r'mp3$', file_path, re.IGNORECASE): - file_type = 'mp3' - elif re.search(r'og(g|a)$', file_path, re.IGNORECASE): - file_type = 'vorbis' - elif re.search(r'flac$', file_path, re.IGNORECASE): - file_type = 'flac' - else: - mime_type = get_mime_type(file_path) - if 'mpeg' in mime_type: - file_type = 'mp3' - elif 'ogg' in mime_type: - file_type = 'vorbis' - elif 'flac' in mime_type: - file_type = 'flac' - - return file_type - - -def calculate_replay_gain(file_path): - """ - This function accepts files of type mp3/ogg/flac and returns a calculated - ReplayGain value in dB. - If the value cannot be calculated for some reason, then we default to 0 - (Unity Gain). - - http://wiki.hydrogenaudio.org/index.php?title=ReplayGain_1.0_specification - """ - - try: - """ - Making a duplicate is required because the ReplayGain extraction utilities we use - make unwanted modifications to the file. - """ - - search = None - temp_file_path = duplicate_file(file_path) - - file_type = get_file_type(file_path) - nice_level = '15' - - if file_type: - if file_type == 'mp3': - if run_process("which mp3gain > /dev/null") == 0: - command = 'nice -n %s mp3gain -q "%s" 2> /dev/null' \ - % (nice_level, temp_file_path) - out = get_process_output(command) - search = re.search(r'Recommended "Track" dB change: (.*)', \ - out) - else: - logger.warn("mp3gain not found") - elif file_type == 'vorbis': - command = "which vorbisgain > /dev/null && which ogginfo > \ - /dev/null" - if run_process(command) == 0: - command = 'nice -n %s vorbisgain -q -f "%s" 2>/dev/null \ - >/dev/null' % (nice_level,temp_file_path) - run_process(command) - - out = get_process_output('ogginfo "%s"' % temp_file_path) - search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out) - else: - logger.warn("vorbisgain/ogginfo not found") - elif file_type == 'flac': - if run_process("which metaflac > /dev/null") == 0: - - command = 'nice -n %s metaflac --add-replay-gain "%s"' \ - % (nice_level, temp_file_path) - run_process(command) - - command = 'nice -n %s metaflac \ - --show-tag=REPLAYGAIN_TRACK_GAIN "%s"' \ - % (nice_level, temp_file_path) - - out = get_process_output(command) - search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out) - else: logger.warn("metaflac not found") - - except Exception, e: - logger.error(str(e)) - finally: - #no longer need the temp, file simply remove it. - try: os.remove(temp_file_path) - except: pass - - replay_gain = 0 - if search: - matches = search.groups() - if len(matches) == 1: - replay_gain = matches[0] - else: - logger.warn("Received more than 1 match in: '%s'" % str(matches)) - - return replay_gain - - -# Example of running from command line: -# python replay_gain.py /path/to/filename.mp3 -if __name__ == "__main__": - print calculate_replay_gain(sys.argv[1]) diff --git a/python_apps/media-monitor2/media/update/replaygainupdater.py b/python_apps/media-monitor2/media/update/replaygainupdater.py deleted file mode 100644 index 2f52c0a23..000000000 --- a/python_apps/media-monitor2/media/update/replaygainupdater.py +++ /dev/null @@ -1,82 +0,0 @@ -from threading import Thread - -import traceback -import os -import time -import logging - -from media.update import replaygain - -class ReplayGainUpdater(Thread): - """ - The purpose of the class is to query the server for a list of files which - do not have a ReplayGain value calculated. This class will iterate over the - list calculate the values, update the server and repeat the process until - the server reports there are no files left. - - This class will see heavy activity right after a 2.1->2.2 upgrade since 2.2 - introduces ReplayGain normalization. A fresh install of Airtime 2.2 will - see this class not used at all since a file imported in 2.2 will - automatically have its ReplayGain value calculated. - """ - - @staticmethod - def start_reply_gain(apc): - me = ReplayGainUpdater(apc) - me.daemon = True - me.start() - - def __init__(self,apc): - Thread.__init__(self) - self.api_client = apc - self.logger = logging.getLogger() - - def main(self): - raw_response = self.api_client.list_all_watched_dirs() - if 'dirs' not in raw_response: - self.logger.error("Could not get a list of watched directories \ - with a dirs attribute. Printing full request:") - self.logger.error( raw_response ) - return - - directories = raw_response['dirs'] - - for dir_id, dir_path in directories.iteritems(): - try: - # keep getting few rows at a time for current music_dir (stor - # or watched folder). - total = 0 - while True: - # return a list of pairs where the first value is the - # file's database row id and the second value is the - # filepath - files = self.api_client.get_files_without_replay_gain_value(dir_id) - processed_data = [] - for f in files: - full_path = os.path.join(dir_path, f['fp']) - processed_data.append((f['id'], replaygain.calculate_replay_gain(full_path))) - - try: - self.api_client.update_replay_gain_values(processed_data) - except Exception as e: self.unexpected_exception(e) - - if len(files) == 0: break - self.logger.info("Processed: %d songs" % total) - - except Exception, e: - self.logger.error(e) - self.logger.debug(traceback.format_exc()) - def run(self): - try: - while True: - self.logger.info("Runnning replaygain updater") - self.main() - # Sleep for 5 minutes in case new files have been added - time.sleep(60 * 5) - except Exception, e: - self.logger.error('ReplayGainUpdater Exception: %s', traceback.format_exc()) - self.logger.error(e) - -if __name__ == "__main__": - rgu = ReplayGainUpdater() - rgu.main() diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 0a3acfabd..2705ef0ee 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -15,7 +15,6 @@ from media.monitor.airtime import AirtimeNotifier, \ AirtimeMessageReceiver from media.monitor.watchersyncer import WatchSyncer from media.monitor.eventdrainer import EventDrainer -from media.update.replaygainupdater import ReplayGainUpdater from std_err_override import LogWriter import media.monitor.pure as mmp @@ -95,7 +94,7 @@ def main(global_config, api_client_config, log_config, apiclient = apc.AirtimeApiClient.create_right_config(log=log, config_path=api_client_config) - ReplayGainUpdater.start_reply_gain(apiclient) + #ReplayGainUpdater.start_reply_gain(apiclient) manager = Manager() diff --git a/python_apps/pypo/pypocli.py b/python_apps/pypo/pypocli.py index 1b51a13f8..ea8950d41 100644 --- a/python_apps/pypo/pypocli.py +++ b/python_apps/pypo/pypocli.py @@ -24,6 +24,8 @@ from recorder import Recorder from listenerstat import ListenerStat from pypomessagehandler import PypoMessageHandler +from media.update.replaygainupdater import ReplayGainUpdater + from configobj import ConfigObj # custom imports @@ -174,6 +176,9 @@ if __name__ == '__main__': sys.exit() api_client = api_client.AirtimeApiClient() + + ReplayGainUpdater.start_reply_gain(api_client) + api_client.register_component("pypo") pypoFetch_q = Queue() From 2e59eca1317f97f3f8e65e7e965a2dbf4dd92908 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 14 Nov 2012 14:43:33 -0500 Subject: [PATCH 25/68] moved files to pypo dir --- python_apps/pypo/media/__init__.py | 0 python_apps/pypo/media/update/__init__.py | 0 python_apps/pypo/media/update/replaygain.py | 152 ++++++++++++++++++ .../pypo/media/update/replaygainupdater.py | 82 ++++++++++ 4 files changed, 234 insertions(+) create mode 100644 python_apps/pypo/media/__init__.py create mode 100644 python_apps/pypo/media/update/__init__.py create mode 100644 python_apps/pypo/media/update/replaygain.py create mode 100644 python_apps/pypo/media/update/replaygainupdater.py diff --git a/python_apps/pypo/media/__init__.py b/python_apps/pypo/media/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python_apps/pypo/media/update/__init__.py b/python_apps/pypo/media/update/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python_apps/pypo/media/update/replaygain.py b/python_apps/pypo/media/update/replaygain.py new file mode 100644 index 000000000..5af7cd4a1 --- /dev/null +++ b/python_apps/pypo/media/update/replaygain.py @@ -0,0 +1,152 @@ +from subprocess import Popen, PIPE +import re +import os +import sys +import shutil +import tempfile +import logging + + +logger = logging.getLogger() + +def get_process_output(command): + """ + Run subprocess and return stdout + """ + logger.debug(command) + p = Popen(command, shell=True, stdout=PIPE) + return p.communicate()[0].strip() + +def run_process(command): + """ + Run subprocess and return "return code" + """ + p = Popen(command, shell=True) + return os.waitpid(p.pid, 0)[1] + +def get_mime_type(file_path): + """ + Attempts to get the mime type but will return prematurely if the process + takes longer than 5 seconds. Note that this function should only be called + for files which do not have a mp3/ogg/flac extension. + """ + + return get_process_output("timeout 5 file -b --mime-type %s" % file_path) + +def duplicate_file(file_path): + """ + Makes a duplicate of the file and returns the path of this duplicate file. + """ + fsrc = open(file_path, 'r') + fdst = tempfile.NamedTemporaryFile(delete=False) + + logger.info("Copying %s to %s" % (file_path, fdst.name)) + + shutil.copyfileobj(fsrc, fdst) + + fsrc.close() + fdst.close() + + return fdst.name + +def get_file_type(file_path): + file_type = None + if re.search(r'mp3$', file_path, re.IGNORECASE): + file_type = 'mp3' + elif re.search(r'og(g|a)$', file_path, re.IGNORECASE): + file_type = 'vorbis' + elif re.search(r'flac$', file_path, re.IGNORECASE): + file_type = 'flac' + else: + mime_type = get_mime_type(file_path) + if 'mpeg' in mime_type: + file_type = 'mp3' + elif 'ogg' in mime_type: + file_type = 'vorbis' + elif 'flac' in mime_type: + file_type = 'flac' + + return file_type + + +def calculate_replay_gain(file_path): + """ + This function accepts files of type mp3/ogg/flac and returns a calculated + ReplayGain value in dB. + If the value cannot be calculated for some reason, then we default to 0 + (Unity Gain). + + http://wiki.hydrogenaudio.org/index.php?title=ReplayGain_1.0_specification + """ + + try: + """ + Making a duplicate is required because the ReplayGain extraction utilities we use + make unwanted modifications to the file. + """ + + search = None + temp_file_path = duplicate_file(file_path) + + file_type = get_file_type(file_path) + nice_level = '15' + + if file_type: + if file_type == 'mp3': + if run_process("which mp3gain > /dev/null") == 0: + command = 'nice -n %s mp3gain -q "%s" 2> /dev/null' \ + % (nice_level, temp_file_path) + out = get_process_output(command) + search = re.search(r'Recommended "Track" dB change: (.*)', \ + out) + else: + logger.warn("mp3gain not found") + elif file_type == 'vorbis': + command = "which vorbisgain > /dev/null && which ogginfo > \ + /dev/null" + if run_process(command) == 0: + command = 'nice -n %s vorbisgain -q -f "%s" 2>/dev/null \ + >/dev/null' % (nice_level,temp_file_path) + run_process(command) + + out = get_process_output('ogginfo "%s"' % temp_file_path) + search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out) + else: + logger.warn("vorbisgain/ogginfo not found") + elif file_type == 'flac': + if run_process("which metaflac > /dev/null") == 0: + + command = 'nice -n %s metaflac --add-replay-gain "%s"' \ + % (nice_level, temp_file_path) + run_process(command) + + command = 'nice -n %s metaflac \ + --show-tag=REPLAYGAIN_TRACK_GAIN "%s"' \ + % (nice_level, temp_file_path) + + out = get_process_output(command) + search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out) + else: logger.warn("metaflac not found") + + except Exception, e: + logger.error(str(e)) + finally: + #no longer need the temp, file simply remove it. + try: os.remove(temp_file_path) + except: pass + + replay_gain = 0 + if search: + matches = search.groups() + if len(matches) == 1: + replay_gain = matches[0] + else: + logger.warn("Received more than 1 match in: '%s'" % str(matches)) + + return replay_gain + + +# Example of running from command line: +# python replay_gain.py /path/to/filename.mp3 +if __name__ == "__main__": + print calculate_replay_gain(sys.argv[1]) diff --git a/python_apps/pypo/media/update/replaygainupdater.py b/python_apps/pypo/media/update/replaygainupdater.py new file mode 100644 index 000000000..2f52c0a23 --- /dev/null +++ b/python_apps/pypo/media/update/replaygainupdater.py @@ -0,0 +1,82 @@ +from threading import Thread + +import traceback +import os +import time +import logging + +from media.update import replaygain + +class ReplayGainUpdater(Thread): + """ + The purpose of the class is to query the server for a list of files which + do not have a ReplayGain value calculated. This class will iterate over the + list calculate the values, update the server and repeat the process until + the server reports there are no files left. + + This class will see heavy activity right after a 2.1->2.2 upgrade since 2.2 + introduces ReplayGain normalization. A fresh install of Airtime 2.2 will + see this class not used at all since a file imported in 2.2 will + automatically have its ReplayGain value calculated. + """ + + @staticmethod + def start_reply_gain(apc): + me = ReplayGainUpdater(apc) + me.daemon = True + me.start() + + def __init__(self,apc): + Thread.__init__(self) + self.api_client = apc + self.logger = logging.getLogger() + + def main(self): + raw_response = self.api_client.list_all_watched_dirs() + if 'dirs' not in raw_response: + self.logger.error("Could not get a list of watched directories \ + with a dirs attribute. Printing full request:") + self.logger.error( raw_response ) + return + + directories = raw_response['dirs'] + + for dir_id, dir_path in directories.iteritems(): + try: + # keep getting few rows at a time for current music_dir (stor + # or watched folder). + total = 0 + while True: + # return a list of pairs where the first value is the + # file's database row id and the second value is the + # filepath + files = self.api_client.get_files_without_replay_gain_value(dir_id) + processed_data = [] + for f in files: + full_path = os.path.join(dir_path, f['fp']) + processed_data.append((f['id'], replaygain.calculate_replay_gain(full_path))) + + try: + self.api_client.update_replay_gain_values(processed_data) + except Exception as e: self.unexpected_exception(e) + + if len(files) == 0: break + self.logger.info("Processed: %d songs" % total) + + except Exception, e: + self.logger.error(e) + self.logger.debug(traceback.format_exc()) + def run(self): + try: + while True: + self.logger.info("Runnning replaygain updater") + self.main() + # Sleep for 5 minutes in case new files have been added + time.sleep(60 * 5) + except Exception, e: + self.logger.error('ReplayGainUpdater Exception: %s', traceback.format_exc()) + self.logger.error(e) + +if __name__ == "__main__": + rgu = ReplayGainUpdater() + rgu.main() From fcfa5cceeee3c53d7c381f7ae67c71dc56fa4d45 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 14 Nov 2012 14:49:28 -0500 Subject: [PATCH 26/68] removed useless comment --- python_apps/media-monitor2/mm2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 2705ef0ee..bee3d7818 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -94,8 +94,6 @@ def main(global_config, api_client_config, log_config, apiclient = apc.AirtimeApiClient.create_right_config(log=log, config_path=api_client_config) - #ReplayGainUpdater.start_reply_gain(apiclient) - manager = Manager() airtime_receiver = AirtimeMessageReceiver(config,manager) From dd139e5028c193edd4ff99d1f05ed5dd3e04a432 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 14 Nov 2012 16:50:34 -0500 Subject: [PATCH 27/68] added default value for exception class parameter --- python_apps/media-monitor2/media/monitor/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/monitor/exceptions.py b/python_apps/media-monitor2/media/monitor/exceptions.py index b7b834920..f7d022fb1 100644 --- a/python_apps/media-monitor2/media/monitor/exceptions.py +++ b/python_apps/media-monitor2/media/monitor/exceptions.py @@ -23,7 +23,7 @@ class FailedToObtainLocale(Exception): class CouldNotCreateIndexFile(Exception): """exception whenever index file cannot be created""" - def __init__(self, path, cause): + def __init__(self, path, cause=None): self.path = path self.cause = cause def __str__(self): return "Failed to create touch file '%s'" % self.path From ce4dbf00280e9ec56ba2cc1f62d0f57014e56c3b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Wed, 14 Nov 2012 22:06:09 -0500 Subject: [PATCH 28/68] added launcher that should replace most of mm2.py --- .../media-monitor2/media/saas/launcher.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 python_apps/media-monitor2/media/saas/launcher.py diff --git a/python_apps/media-monitor2/media/saas/launcher.py b/python_apps/media-monitor2/media/saas/launcher.py new file mode 100644 index 000000000..fd7bacdc7 --- /dev/null +++ b/python_apps/media-monitor2/media/saas/launcher.py @@ -0,0 +1,75 @@ +import os + +from media.saas.thread import InstanceThread, user, apc +from media.monitor.log import Loggable +from media.monitor.exceptions import CouldNotCreateIndexFile +from media.monitor.toucher import ToucherThread +from media.monitor.airtime import AirtimeNotifier, \ + AirtimeMessageReceiver +from media.monitor.watchersyncer import WatchSyncer +from media.monitor.eventdrainer import EventDrainer +from media.monitor.manager import Manager + +class MM2(InstanceThread, Loggable): + + def index_create(self, index_create_attempt=False): + config = user().mm_config + if not index_create_attempt: + if not os.path.exists(config['index_path']): + self.log.info("Attempting to create index file:...") + try: + with open(config['index_path'], 'w') as f: f.write(" ") + except Exception as e: + self.log.info("Failed to create index file with exception: %s" \ + % str(e)) + else: + self.log.info("Created index file, reloading configuration:") + self.index_create(index_create_attempt=True) + else: + self.log.info("Already tried to create index. Will not try again ") + + if not os.path.exists(config['index_path']): + raise CouldNotCreateIndexFile(config['index_path']) + + def run(self): + self.index_create_attempt() + manager = Manager() + apiclient = apc() + config = user().mm_config + watch_syncer = WatchSyncer(signal='watch', + chunking_number=config['chunking_number'], + timeout=config['request_max_wait']) + airtime_receiver = AirtimeMessageReceiver(config,manager) + airtime_notifier = AirtimeNotifier(config, airtime_receiver) + + store = apiclient.setup_media_monitor() + + self.log.info( + "Initing with the following airtime response:%s" % str(store)) + + airtime_receiver.change_storage({ 'directory':store[u'stor'] }) + + for watch_dir in store[u'watched_dirs']: + if not os.path.exists(watch_dir): + # Create the watch_directory here + try: os.makedirs(watch_dir) + except Exception: + self.log.error("Could not create watch directory: '%s' \ + (given from the database)." % watch_dir) + if os.path.exists(watch_dir): + airtime_receiver.new_watch({ 'directory':watch_dir }, restart=True) + else: self.log.info("Failed to add watch on %s" % str(watch_dir)) + + ed = EventDrainer(airtime_notifier.connection, + interval=float(config['rmq_event_wait'])) + + # Launch the toucher that updates the last time when the script was + # ran every n seconds. + # TODO : verify that this does not interfere with bootstrapping because the + # toucher thread might update the last_ran variable too fast + tt = ToucherThread(path=config['index_path'], + interval=int(config['touch_interval'])) + + apiclient.register_component('media-monitor') + + return manager.loop() From 7ed1f08e07f882c808cc5964ec4694345233d1ae Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 12:10:52 -0500 Subject: [PATCH 29/68] Added new launch script for mm. --- python_apps/media-monitor2/mm2.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index bee3d7818..4f36965a3 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -16,6 +16,8 @@ from media.monitor.airtime import AirtimeNotifier, \ from media.monitor.watchersyncer import WatchSyncer from media.monitor.eventdrainer import EventDrainer from std_err_override import LogWriter +from media.saas.launcher import MM2 +from media.saas.airtimeinstance import AirtimeInstance import media.monitor.pure as mmp from api_clients import api_client as apc @@ -47,8 +49,22 @@ def setup_global(log): Logging exception.") log.info(str(e)) +def main(global_config, api_client_config, log_config): + cfg = { + 'api_client' : api_client_config, + 'media_monitor' : global_config, + 'logging' : log_config, + } + ai = AirtimeInstance('hosted_install', '/', cfg) + log = setup_logger( log_config, ai.mm_config['logpath'] ) + setup_global(log) + apc.AirtimeApiClient.create_right_config(log=log, + config_path=api_client_config) + apc.AirtimeApiClient(api_client_config) + mm = MM2(ai) + mm.start() -def main(global_config, api_client_config, log_config, +def main2(global_config, api_client_config, log_config, index_create_attempt=False): for cfg in [global_config, api_client_config]: if not os.path.exists(cfg): raise NoConfigFile(cfg) From ab35263869cbc9cbc7353b000fa5af25f66bcebd Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 12:11:18 -0500 Subject: [PATCH 30/68] inject apc properly into AirtimeNotifier --- python_apps/media-monitor2/media/monitor/airtime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/airtime.py b/python_apps/media-monitor2/media/monitor/airtime.py index b920ecd65..11436009a 100644 --- a/python_apps/media-monitor2/media/monitor/airtime.py +++ b/python_apps/media-monitor2/media/monitor/airtime.py @@ -15,7 +15,7 @@ from media.monitor.exceptions import DirectoryIsNotListed from media.monitor.bootstrap import Bootstrapper from media.monitor.listeners import FileMediator -from api_clients import api_client as apc +from media.saas.thread import apc class AirtimeNotifier(Loggable): """ @@ -98,7 +98,7 @@ class AirtimeMessageReceiver(Loggable): if (not directory_id) and (not directory): raise ValueError("You must provide either directory_id or \ directory") - sdb = AirtimeDB(apc.AirtimeApiClient.create_right_config()) + sdb = AirtimeDB(apc()) if directory : directory = os.path.normpath(directory) if directory_id == None : directory_id = sdb.to_id(directory) if directory == None : directory = sdb.to_directory(directory_id) From 15f4212360c4ec9eabe9ae148431b5ac0bd76766 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 12:11:48 -0500 Subject: [PATCH 31/68] make ManagerTimeout provide the correct AirtimeInstance object. --- python_apps/media-monitor2/media/monitor/manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/manager.py b/python_apps/media-monitor2/media/monitor/manager.py index e64d1c985..89141157e 100644 --- a/python_apps/media-monitor2/media/monitor/manager.py +++ b/python_apps/media-monitor2/media/monitor/manager.py @@ -1,5 +1,4 @@ import pyinotify -import threading import time from pydispatch import dispatcher @@ -9,10 +8,11 @@ from media.monitor.log import Loggable from media.monitor.listeners import StoreWatchListener, OrganizeListener from media.monitor.handler import ProblemFileHandler from media.monitor.organizer import Organizer +from media.saas.thread import InstanceInheritingThread import media.monitor.pure as mmp -class ManagerTimeout(threading.Thread,Loggable): +class ManagerTimeout(InstanceInheritingThread,Loggable): """ The purpose of this class is to flush the organize directory every 3 secnods. This used to be just a work around for cc-4235 but recently became a permanent solution because it's "cheap" and @@ -20,7 +20,7 @@ class ManagerTimeout(threading.Thread,Loggable): def __init__(self, manager, interval=1.5): # TODO : interval should be read from config and passed here instead # of just using the hard coded value - threading.Thread.__init__(self) + super(ManagerTimeout, self).__init__() self.manager = manager self.interval = interval def run(self): From fa66f33ffa74c89b3b0a3ca878320d4307a2ec1a Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 12:12:19 -0500 Subject: [PATCH 32/68] changed RequestSync to use injected apc and injected it in ThreadedRequestedSync. --- python_apps/media-monitor2/media/monitor/request.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/request.py b/python_apps/media-monitor2/media/monitor/request.py index a49c6bb25..e2c5fb5ef 100644 --- a/python_apps/media-monitor2/media/monitor/request.py +++ b/python_apps/media-monitor2/media/monitor/request.py @@ -1,14 +1,12 @@ # -*- coding: utf-8 -*- -import threading - from media.monitor.exceptions import BadSongFile from media.monitor.log import Loggable -import api_clients.api_client as ac +from media.saas.thread import apc, InstanceInheritingThread -class ThreadedRequestSync(threading.Thread, Loggable): +class ThreadedRequestSync(InstanceInheritingThread, Loggable): def __init__(self, rs): - threading.Thread.__init__(self) + super(ThreadedRequestSync, self).__init__() self.rs = rs self.daemon = True self.start() @@ -22,7 +20,7 @@ class RequestSync(Loggable): for some number of times """ @classmethod def create_with_api_client(cls, watcher, requests): - apiclient = ac.AirtimeApiClient.create_right_config() + apiclient = apc() self = cls(watcher, requests, apiclient) return self From 5073ced732bc0d1cfcfd512a7cf4a6d6d8dbec90 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 12:12:47 -0500 Subject: [PATCH 33/68] Made timeout watcher provide AirtimeInstnace --- python_apps/media-monitor2/media/monitor/watchersyncer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index d2df9ed3a..a913bb506 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import threading import time import copy @@ -9,15 +8,16 @@ from media.monitor.exceptions import BadSongFile from media.monitor.eventcontractor import EventContractor from media.monitor.events import EventProxy from media.monitor.request import ThreadedRequestSync, RequestSync +from media.saas.thread import InstanceInheritingThread -class TimeoutWatcher(threading.Thread,Loggable): +class TimeoutWatcher(InstanceInheritingThread,Loggable): """ The job of this thread is to keep an eye on WatchSyncer and force a request whenever the requests go over time out """ def __init__(self, watcher, timeout=5): self.logger.info("Created timeout thread...") - threading.Thread.__init__(self) + super(TimeoutWatcher, self).__init__() self.watcher = watcher self.timeout = timeout From 4ac3efe228df28c0442ba9ed2b62e782d1ac1839 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 12:13:33 -0500 Subject: [PATCH 34/68] renamed typo log to logger. --- .../media-monitor2/media/saas/launcher.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/python_apps/media-monitor2/media/saas/launcher.py b/python_apps/media-monitor2/media/saas/launcher.py index fd7bacdc7..cd35f712e 100644 --- a/python_apps/media-monitor2/media/saas/launcher.py +++ b/python_apps/media-monitor2/media/saas/launcher.py @@ -16,23 +16,23 @@ class MM2(InstanceThread, Loggable): config = user().mm_config if not index_create_attempt: if not os.path.exists(config['index_path']): - self.log.info("Attempting to create index file:...") + self.logger.info("Attempting to create index file:...") try: with open(config['index_path'], 'w') as f: f.write(" ") except Exception as e: - self.log.info("Failed to create index file with exception: %s" \ + self.logger.info("Failed to create index file with exception: %s" \ % str(e)) else: - self.log.info("Created index file, reloading configuration:") + self.logger.info("Created index file, reloading configuration:") self.index_create(index_create_attempt=True) else: - self.log.info("Already tried to create index. Will not try again ") + self.logger.info("Already tried to create index. Will not try again ") if not os.path.exists(config['index_path']): raise CouldNotCreateIndexFile(config['index_path']) def run(self): - self.index_create_attempt() + self.index_create() manager = Manager() apiclient = apc() config = user().mm_config @@ -44,7 +44,7 @@ class MM2(InstanceThread, Loggable): store = apiclient.setup_media_monitor() - self.log.info( + self.logger.info( "Initing with the following airtime response:%s" % str(store)) airtime_receiver.change_storage({ 'directory':store[u'stor'] }) @@ -54,11 +54,11 @@ class MM2(InstanceThread, Loggable): # Create the watch_directory here try: os.makedirs(watch_dir) except Exception: - self.log.error("Could not create watch directory: '%s' \ + self.logger.error("Could not create watch directory: '%s' \ (given from the database)." % watch_dir) if os.path.exists(watch_dir): airtime_receiver.new_watch({ 'directory':watch_dir }, restart=True) - else: self.log.info("Failed to add watch on %s" % str(watch_dir)) + else: self.logger.info("Failed to add watch on %s" % str(watch_dir)) ed = EventDrainer(airtime_notifier.connection, interval=float(config['rmq_event_wait'])) @@ -72,4 +72,4 @@ class MM2(InstanceThread, Loggable): apiclient.register_component('media-monitor') - return manager.loop() + manager.loop() From 8a7535774c508a4877c3e787b6eba1ae2185692d Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 12:13:52 -0500 Subject: [PATCH 35/68] fixed import typo --- python_apps/media-monitor2/media/saas/airtimeinstance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/saas/airtimeinstance.py b/python_apps/media-monitor2/media/saas/airtimeinstance.py index 293fc84c2..2cb2176c5 100644 --- a/python_apps/media-monitor2/media/saas/airtimeinstance.py +++ b/python_apps/media-monitor2/media/saas/airtimeinstance.py @@ -3,7 +3,7 @@ import os from media.monitor.exceptions import NoConfigFile from media.monitor.pure import LazyProperty from media.monitor.config import MMConfig -from api_clients import AirtimeApiClient +from api_clients.api_client import AirtimeApiClient class AirtimeInstance(object): """ AirtimeInstance is a class that abstracts away every airtime From 0b0e96f49c719fba879d33b7241e9ab939b21e34 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 12:46:57 -0500 Subject: [PATCH 36/68] removed old runner code --- python_apps/media-monitor2/mm2.py | 82 ------------------------------- 1 file changed, 82 deletions(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 4f36965a3..53e5f3c17 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -64,88 +64,6 @@ def main(global_config, api_client_config, log_config): mm = MM2(ai) mm.start() -def main2(global_config, api_client_config, log_config, - index_create_attempt=False): - for cfg in [global_config, api_client_config]: - if not os.path.exists(cfg): raise NoConfigFile(cfg) - # MMConfig is a proxy around ConfigObj instances. it does not allow - # itself users of MMConfig instances to modify any config options - # directly through the dictionary. Users of this object must use the - # correct methods designated for modification - try: config = MMConfig(global_config) - except NoConfigFile as e: - print("Cannot run mediamonitor2 without configuration file.") - print("Current config path: '%s'" % global_config) - sys.exit(1) - except Exception as e: - print("Unknown error reading configuration file: '%s'" % global_config) - print(str(e)) - - log = setup_logger( log_config, config['logpath'] ) - - if not index_create_attempt: - if not os.path.exists(config['index_path']): - log.info("Attempting to create index file:...") - try: - with open(config['index_path'], 'w') as f: f.write(" ") - except Exception as e: - log.info("Failed to create index file with exception: %s" \ - % str(e)) - else: - log.info("Created index file, reloading configuration:") - main( global_config, api_client_config, log_config, - index_create_attempt=True ) - else: - log.info("Already tried to create index. Will not try again ") - - if not os.path.exists(config['index_path']): - log.info("Index file does not exist. Terminating") - - setup_global(log) - - watch_syncer = WatchSyncer(signal='watch', - chunking_number=config['chunking_number'], - timeout=config['request_max_wait']) - - apiclient = apc.AirtimeApiClient.create_right_config(log=log, - config_path=api_client_config) - - manager = Manager() - - airtime_receiver = AirtimeMessageReceiver(config,manager) - airtime_notifier = AirtimeNotifier(config, airtime_receiver) - - store = apiclient.setup_media_monitor() - - log.info("Initing with the following airtime response:%s" % str(store)) - - airtime_receiver.change_storage({ 'directory':store[u'stor'] }) - - for watch_dir in store[u'watched_dirs']: - if not os.path.exists(watch_dir): - # Create the watch_directory here - try: os.makedirs(watch_dir) - except Exception as e: - log.error("Could not create watch directory: '%s' \ - (given from the database)." % watch_dir) - if os.path.exists(watch_dir): - airtime_receiver.new_watch({ 'directory':watch_dir }, restart=True) - else: log.info("Failed to add watch on %s" % str(watch_dir)) - - ed = EventDrainer(airtime_notifier.connection, - interval=float(config['rmq_event_wait'])) - - # Launch the toucher that updates the last time when the script was - # ran every n seconds. - # TODO : verify that this does not interfere with bootstrapping because the - # toucher thread might update the last_ran variable too fast - tt = ToucherThread(path=config['index_path'], - interval=int(config['touch_interval'])) - - apiclient.register_component('media-monitor') - - return manager.loop() - __doc__ = """ Usage: mm2.py --config= --apiclient= --log= From 57b13607c5e5309c9249ebb640dd9ee3167f4b8c Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 14:22:42 -0500 Subject: [PATCH 37/68] removed crap hacks --- python_apps/media-monitor2/mm2.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 53e5f3c17..67d3eaad2 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -4,23 +4,14 @@ import os import logging import logging.config -from media.monitor.manager import Manager from media.monitor.log import get_logger, setup_logging -from media.monitor.config import MMConfig -from media.monitor.toucher import ToucherThread from media.monitor.exceptions import FailedToObtainLocale, \ - FailedToSetLocale, \ - NoConfigFile -from media.monitor.airtime import AirtimeNotifier, \ - AirtimeMessageReceiver -from media.monitor.watchersyncer import WatchSyncer -from media.monitor.eventdrainer import EventDrainer + FailedToSetLocale from std_err_override import LogWriter from media.saas.launcher import MM2 from media.saas.airtimeinstance import AirtimeInstance import media.monitor.pure as mmp -from api_clients import api_client as apc def setup_logger(log_config, logpath): logging.config.fileConfig(log_config) @@ -58,9 +49,6 @@ def main(global_config, api_client_config, log_config): ai = AirtimeInstance('hosted_install', '/', cfg) log = setup_logger( log_config, ai.mm_config['logpath'] ) setup_global(log) - apc.AirtimeApiClient.create_right_config(log=log, - config_path=api_client_config) - apc.AirtimeApiClient(api_client_config) mm = MM2(ai) mm.start() From 975508a549f77bb0ecbee1f67a90624bb32de5ef Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 14:24:17 -0500 Subject: [PATCH 38/68] added TODO --- python_apps/media-monitor2/media/monitor/manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python_apps/media-monitor2/media/monitor/manager.py b/python_apps/media-monitor2/media/monitor/manager.py index 89141157e..d2c0a5f64 100644 --- a/python_apps/media-monitor2/media/monitor/manager.py +++ b/python_apps/media-monitor2/media/monitor/manager.py @@ -29,6 +29,8 @@ class ManagerTimeout(InstanceInheritingThread,Loggable): self.manager.flush_organize() class Manager(Loggable): + # NOTE : this massive class is a source of many problems of mm and + # is in dire need of breaking up and refactoring. """ An abstraction over media monitors core pyinotify functions. These include adding watched,store, organize directories, etc. Basically composes over WatchManager from pyinotify """ From c43dbe42a84cae391da01efe88c1bc9763a2fbbb Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 14:27:09 -0500 Subject: [PATCH 39/68] refactored creating an instance into its own function --- python_apps/media-monitor2/mm2.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 67d3eaad2..a7718812f 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -49,8 +49,10 @@ def main(global_config, api_client_config, log_config): ai = AirtimeInstance('hosted_install', '/', cfg) log = setup_logger( log_config, ai.mm_config['logpath'] ) setup_global(log) - mm = MM2(ai) - mm.start() + run_instance(ai) + +def run_instance(airtime_intance): + MM2(airtime_intance).start() __doc__ = """ Usage: From 11a43eb891c72dcc6d3a5e1297770ac4a83f1236 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 14:49:55 -0500 Subject: [PATCH 40/68] Generalized a lot of launching logic into the saas launcher away from the hosted installation script: mm2.py. --- .../media-monitor2/media/saas/launcher.py | 32 ++++++++++++++- python_apps/media-monitor2/mm2.py | 39 ++++--------------- 2 files changed, 38 insertions(+), 33 deletions(-) diff --git a/python_apps/media-monitor2/media/saas/launcher.py b/python_apps/media-monitor2/media/saas/launcher.py index cd35f712e..822307a0b 100644 --- a/python_apps/media-monitor2/media/saas/launcher.py +++ b/python_apps/media-monitor2/media/saas/launcher.py @@ -1,7 +1,11 @@ -import os +import os, sys + +from media.monitor.exceptions import FailedToObtainLocale, \ + FailedToSetLocale from media.saas.thread import InstanceThread, user, apc from media.monitor.log import Loggable +import media.monitor.pure as mmp from media.monitor.exceptions import CouldNotCreateIndexFile from media.monitor.toucher import ToucherThread from media.monitor.airtime import AirtimeNotifier, \ @@ -9,6 +13,7 @@ from media.monitor.airtime import AirtimeNotifier, \ from media.monitor.watchersyncer import WatchSyncer from media.monitor.eventdrainer import EventDrainer from media.monitor.manager import Manager +from media.saas.airtimeinstance import AirtimeInstance class MM2(InstanceThread, Loggable): @@ -73,3 +78,28 @@ class MM2(InstanceThread, Loggable): apiclient.register_component('media-monitor') manager.loop() + +def launch_instance(name, root, global_cfg, apc_cfg, log_cfg): + cfg = { + 'api_client' : apc_cfg, + 'media_monitor' : global_cfg, + 'logging' : log_cfg, + } + ai = AirtimeInstance('name', 'root', cfg) + MM2(ai).start() + +def setup_global(log): + """ setup unicode and other stuff """ + log.info("Attempting to set the locale...") + try: mmp.configure_locale(mmp.get_system_locale()) + except FailedToSetLocale as e: + log.info("Failed to set the locale...") + sys.exit(1) + except FailedToObtainLocale as e: + log.info("Failed to obtain the locale form the default path: \ + '/etc/default/locale'") + sys.exit(1) + except Exception as e: + log.info("Failed to set the locale for unknown reason. \ + Logging exception.") + log.info(str(e)) diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index a7718812f..3e8c43e72 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -5,13 +5,10 @@ import logging import logging.config from media.monitor.log import get_logger, setup_logging -from media.monitor.exceptions import FailedToObtainLocale, \ - FailedToSetLocale from std_err_override import LogWriter -from media.saas.launcher import MM2 +from media.saas.launcher import setup_global, launch_instance from media.saas.airtimeinstance import AirtimeInstance - -import media.monitor.pure as mmp +from media.monitor.config import MMConfig def setup_logger(log_config, logpath): logging.config.fileConfig(log_config) @@ -24,35 +21,13 @@ def setup_logger(log_config, logpath): log = get_logger() return log -def setup_global(log): - """ setup unicode and other stuff """ - log.info("Attempting to set the locale...") - try: mmp.configure_locale(mmp.get_system_locale()) - except FailedToSetLocale as e: - log.info("Failed to set the locale...") - sys.exit(1) - except FailedToObtainLocale as e: - log.info("Failed to obtain the locale form the default path: \ - '/etc/default/locale'") - sys.exit(1) - except Exception as e: - log.info("Failed to set the locale for unknown reason. \ - Logging exception.") - log.info(str(e)) - def main(global_config, api_client_config, log_config): - cfg = { - 'api_client' : api_client_config, - 'media_monitor' : global_config, - 'logging' : log_config, - } - ai = AirtimeInstance('hosted_install', '/', cfg) - log = setup_logger( log_config, ai.mm_config['logpath'] ) + """ function to run hosted install """ + mm_config = MMConfig(global_config) + log = setup_logger( log_config, mm_config['logpath'] ) setup_global(log) - run_instance(ai) - -def run_instance(airtime_intance): - MM2(airtime_intance).start() + launch_instance('hosted_install', '/', global_config, api_client_config, + log_config) __doc__ = """ Usage: From 350c6e2f94a1faa62f8afc5f18037a7f4390ef31 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 15:02:26 -0500 Subject: [PATCH 41/68] moved logging logic to launcher --- .../media-monitor2/media/saas/launcher.py | 16 ++++++++++++++++ python_apps/media-monitor2/mm2.py | 19 +------------------ 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/python_apps/media-monitor2/media/saas/launcher.py b/python_apps/media-monitor2/media/saas/launcher.py index 822307a0b..07d3a65cf 100644 --- a/python_apps/media-monitor2/media/saas/launcher.py +++ b/python_apps/media-monitor2/media/saas/launcher.py @@ -1,8 +1,13 @@ import os, sys +import logging +import logging.config from media.monitor.exceptions import FailedToObtainLocale, \ FailedToSetLocale +from media.monitor.log import get_logger, setup_logging +from std_err_override import LogWriter + from media.saas.thread import InstanceThread, user, apc from media.monitor.log import Loggable import media.monitor.pure as mmp @@ -103,3 +108,14 @@ def setup_global(log): log.info("Failed to set the locale for unknown reason. \ Logging exception.") log.info(str(e)) + +def setup_logger(log_config, logpath): + logging.config.fileConfig(log_config) + #need to wait for Python 2.7 for this.. + #logging.captureWarnings(True) + logger = logging.getLogger() + LogWriter.override_std_err(logger) + logfile = unicode(logpath) + setup_logging(logfile) + log = get_logger() + return log diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 3e8c43e72..4a0402fff 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -1,26 +1,9 @@ # -*- coding: utf-8 -*- import sys import os -import logging -import logging.config - -from media.monitor.log import get_logger, setup_logging -from std_err_override import LogWriter -from media.saas.launcher import setup_global, launch_instance -from media.saas.airtimeinstance import AirtimeInstance +from media.saas.launcher import setup_global, launch_instance, setup_logger from media.monitor.config import MMConfig -def setup_logger(log_config, logpath): - logging.config.fileConfig(log_config) - #need to wait for Python 2.7 for this.. - #logging.captureWarnings(True) - logger = logging.getLogger() - LogWriter.override_std_err(logger) - logfile = unicode(logpath) - setup_logging(logfile) - log = get_logger() - return log - def main(global_config, api_client_config, log_config): """ function to run hosted install """ mm_config = MMConfig(global_config) From 9c74b0ea2c37f2151e03516ea86ec6de849fc0ea Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 15:08:18 -0500 Subject: [PATCH 42/68] removed bastard injection from AirtimeApiClient --- python_apps/api_clients/api_client.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index eedb03e90..f9c85c0a9 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -127,24 +127,6 @@ class RequestProvider(object): class AirtimeApiClient(object): - - # This is a little hacky fix so that I don't have to pass the config object - # everywhere where AirtimeApiClient needs to be initialized - default_config = None - # the purpose of this custom constructor is to remember which config file - # it was called with. So that after the initial call: - # AirtimeApiClient.create_right_config('/path/to/config') - # All subsequence calls to create_right_config will be with that config - # file - @staticmethod - def create_right_config(log=None,config_path=None): - if config_path: AirtimeApiClient.default_config = config_path - elif (not AirtimeApiClient.default_config): - raise ValueError("Cannot slip config_path attribute when it has \ - never been passed yet") - return AirtimeApiClient( logger=None, - config_path=AirtimeApiClient.default_config ) - def __init__(self, logger=None,config_path='/etc/airtime/api_client.cfg'): if logger is None: self.logger = logging else: self.logger = logger From 88231a19c35cea2a140367100d0698d1a05d1f34 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Thu, 15 Nov 2012 15:08:49 -0500 Subject: [PATCH 43/68] added baby mm launcher --- python_apps/media-monitor2/baby.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 python_apps/media-monitor2/baby.py diff --git a/python_apps/media-monitor2/baby.py b/python_apps/media-monitor2/baby.py new file mode 100644 index 000000000..71ddbdcfa --- /dev/null +++ b/python_apps/media-monitor2/baby.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +from media.saas.launcher import setup_logger, setup_global + +def autoscan_instances(log_config): + print("privet mang") + +def main(log_config, log_path): + log = setup_logger(log_config, log_path) + setup_global(log) + autoscan_instances(log_config) + +if __name__ == '__main__': main('shen','sheni') From e12e31153710f971d52ccf47c1eaf0f38381409c Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 16 Nov 2012 00:09:33 -0500 Subject: [PATCH 44/68] Added some routines to baby mm. --- python_apps/media-monitor2/baby.py | 40 +++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/python_apps/media-monitor2/baby.py b/python_apps/media-monitor2/baby.py index 71ddbdcfa..5997d9481 100644 --- a/python_apps/media-monitor2/baby.py +++ b/python_apps/media-monitor2/baby.py @@ -1,12 +1,40 @@ # -*- coding: utf-8 -*- -from media.saas.launcher import setup_logger, setup_global +import re +from media.saas.launcher import setup_logger, setup_global, MM2 +from media.saas.airtimeinstance import AirtimeInstance +from os.path import isdir, join, abspath +from os import listdir -def autoscan_instances(log_config): - print("privet mang") +def list_dirs(d): return (x for x in listdir(d) if isdir(x)) -def main(log_config, log_path): +def filter_instance(d): return bool(re.match('.+/\d+$',d)) + +def get_name(p): return re.match('.+/(\d+)$',p).group(1) + +def filter_instances(l): return (x for x in l if filter_instance(l)) + +def autoscan_instances(main_cfg): + root = main_cfg['instance_root'] + instances = [] + for instance_machine in list_dirs(root): + instance_machine = join(root, instance_machine) + for instance_root in filter_instances(list_dirs(instance_machine)): + full_path = abspath(join(root,instance_root)) + ai = AirtimeInstance.root_make(get_name(full_path), full_path) + instances.append(ai) + return instances + +def main(main_cfg): + log_config, log_path = main_cfg['log_config'], main_cfg['log_path'] log = setup_logger(log_config, log_path) setup_global(log) - autoscan_instances(log_config) + for instance in autoscan_instances(main_cfg): + MM2(instance).start() -if __name__ == '__main__': main('shen','sheni') +if __name__ == '__main__': + fake = { + 'log_config' : '', + 'log_path' : '', + 'instance_root' : '' + } + main(fake) From 26ad6dae74238740ed2a235694a6901b3117b183 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 16 Nov 2012 00:09:56 -0500 Subject: [PATCH 45/68] Added a convenience constructor for an AirtimeInstance --- .../media-monitor2/media/saas/airtimeinstance.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/saas/airtimeinstance.py b/python_apps/media-monitor2/media/saas/airtimeinstance.py index 2cb2176c5..0e74de042 100644 --- a/python_apps/media-monitor2/media/saas/airtimeinstance.py +++ b/python_apps/media-monitor2/media/saas/airtimeinstance.py @@ -10,9 +10,17 @@ class AirtimeInstance(object): instance by providing all the necessary objects required to interact with the instance. ApiClient, configs, root_directory """ + @classmethod + def root_make(cls, name, root): + cfg = { + 'api_client' : join(root, 'etc/airtime/api_client.cfg'), + 'media_monitor' : join(root, 'etc/airtime/media-monitor.cfg'), + } + return cls(name, root, cfg) + def __init__(self,name, root_path, config_paths): """ name is an internal name only """ - for cfg in ['api_client','media_monitor', 'logging']: + for cfg in ['api_client','media_monitor']: if cfg not in config_paths: raise NoConfigFile(config_paths) elif not os.path.exists(config_paths[cfg]): raise NoConfigFile(config_paths[cfg]) From 757a561975698d998bf66153ee6143cb167cc8b7 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 16 Nov 2012 00:10:24 -0500 Subject: [PATCH 46/68] added missing import --- python_apps/media-monitor2/media/saas/airtimeinstance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_apps/media-monitor2/media/saas/airtimeinstance.py b/python_apps/media-monitor2/media/saas/airtimeinstance.py index 0e74de042..e0d474c33 100644 --- a/python_apps/media-monitor2/media/saas/airtimeinstance.py +++ b/python_apps/media-monitor2/media/saas/airtimeinstance.py @@ -1,4 +1,5 @@ import os +from os.path import join from media.monitor.exceptions import NoConfigFile from media.monitor.pure import LazyProperty From 7527b366d0189378f63d7aebf46ba654a8923589 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 16 Nov 2012 00:10:40 -0500 Subject: [PATCH 47/68] removed instance specific logging --- python_apps/media-monitor2/media/saas/launcher.py | 3 +-- python_apps/media-monitor2/mm2.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/python_apps/media-monitor2/media/saas/launcher.py b/python_apps/media-monitor2/media/saas/launcher.py index 07d3a65cf..d617419d3 100644 --- a/python_apps/media-monitor2/media/saas/launcher.py +++ b/python_apps/media-monitor2/media/saas/launcher.py @@ -84,11 +84,10 @@ class MM2(InstanceThread, Loggable): manager.loop() -def launch_instance(name, root, global_cfg, apc_cfg, log_cfg): +def launch_instance(name, root, global_cfg, apc_cfg): cfg = { 'api_client' : apc_cfg, 'media_monitor' : global_cfg, - 'logging' : log_cfg, } ai = AirtimeInstance('name', 'root', cfg) MM2(ai).start() diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 4a0402fff..7c447c060 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -9,8 +9,7 @@ def main(global_config, api_client_config, log_config): mm_config = MMConfig(global_config) log = setup_logger( log_config, mm_config['logpath'] ) setup_global(log) - launch_instance('hosted_install', '/', global_config, api_client_config, - log_config) + launch_instance('hosted_install', '/', global_config, api_client_config) __doc__ = """ Usage: From cc66ac4a836df1b22bf3ba2276c6258dc3b0dc88 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 16 Nov 2012 11:14:18 -0500 Subject: [PATCH 48/68] typo --- python_apps/media-monitor2/media/saas/launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/saas/launcher.py b/python_apps/media-monitor2/media/saas/launcher.py index d617419d3..76e62be8f 100644 --- a/python_apps/media-monitor2/media/saas/launcher.py +++ b/python_apps/media-monitor2/media/saas/launcher.py @@ -89,7 +89,7 @@ def launch_instance(name, root, global_cfg, apc_cfg): 'api_client' : apc_cfg, 'media_monitor' : global_cfg, } - ai = AirtimeInstance('name', 'root', cfg) + ai = AirtimeInstance(name, root, cfg) MM2(ai).start() def setup_global(log): From a50995cd9b7130d2a92b7464d599d311252644bf Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 16 Nov 2012 11:14:31 -0500 Subject: [PATCH 49/68] formatting + clearer name. --- python_apps/media-monitor2/baby.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/python_apps/media-monitor2/baby.py b/python_apps/media-monitor2/baby.py index 5997d9481..f52d83639 100644 --- a/python_apps/media-monitor2/baby.py +++ b/python_apps/media-monitor2/baby.py @@ -28,13 +28,12 @@ def main(main_cfg): log_config, log_path = main_cfg['log_config'], main_cfg['log_path'] log = setup_logger(log_config, log_path) setup_global(log) - for instance in autoscan_instances(main_cfg): - MM2(instance).start() + for instance in autoscan_instances(main_cfg): MM2(instance).start() if __name__ == '__main__': - fake = { - 'log_config' : '', - 'log_path' : '', + default = { + 'log_config' : '', + 'log_path' : '', 'instance_root' : '' } - main(fake) + main(default) From aeb449ea76fa6bdd5ad6b72d60a91375dfcdfe39 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 16 Nov 2012 11:15:49 -0500 Subject: [PATCH 50/68] import formatting --- .../media-monitor2/media/saas/launcher.py | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/python_apps/media-monitor2/media/saas/launcher.py b/python_apps/media-monitor2/media/saas/launcher.py index 76e62be8f..435b8729f 100644 --- a/python_apps/media-monitor2/media/saas/launcher.py +++ b/python_apps/media-monitor2/media/saas/launcher.py @@ -2,23 +2,20 @@ import os, sys import logging import logging.config -from media.monitor.exceptions import FailedToObtainLocale, \ - FailedToSetLocale - -from media.monitor.log import get_logger, setup_logging -from std_err_override import LogWriter - -from media.saas.thread import InstanceThread, user, apc -from media.monitor.log import Loggable import media.monitor.pure as mmp -from media.monitor.exceptions import CouldNotCreateIndexFile -from media.monitor.toucher import ToucherThread -from media.monitor.airtime import AirtimeNotifier, \ - AirtimeMessageReceiver -from media.monitor.watchersyncer import WatchSyncer -from media.monitor.eventdrainer import EventDrainer -from media.monitor.manager import Manager -from media.saas.airtimeinstance import AirtimeInstance + +from media.monitor.exceptions import FailedToObtainLocale, FailedToSetLocale +from media.monitor.log import get_logger, setup_logging +from std_err_override import LogWriter +from media.saas.thread import InstanceThread, user, apc +from media.monitor.log import Loggable +from media.monitor.exceptions import CouldNotCreateIndexFile +from media.monitor.toucher import ToucherThread +from media.monitor.airtime import AirtimeNotifier, AirtimeMessageReceiver +from media.monitor.watchersyncer import WatchSyncer +from media.monitor.eventdrainer import EventDrainer +from media.monitor.manager import Manager +from media.saas.airtimeinstance import AirtimeInstance class MM2(InstanceThread, Loggable): From e22162cc87b804b01a3365934d30bb948c8af9c1 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 16 Nov 2012 12:29:06 -0500 Subject: [PATCH 51/68] added initialization message --- python_apps/media-monitor2/media/saas/launcher.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python_apps/media-monitor2/media/saas/launcher.py b/python_apps/media-monitor2/media/saas/launcher.py index 435b8729f..c394da98b 100644 --- a/python_apps/media-monitor2/media/saas/launcher.py +++ b/python_apps/media-monitor2/media/saas/launcher.py @@ -51,6 +51,8 @@ class MM2(InstanceThread, Loggable): store = apiclient.setup_media_monitor() + self.logger.info("initializing mm with directories: %s" % str(store)) + self.logger.info( "Initing with the following airtime response:%s" % str(store)) From 11214d754dc6f0157a79781089b97d38a0f2987f Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 16 Nov 2012 12:31:10 -0500 Subject: [PATCH 52/68] docstring formatting --- .../media-monitor2/media/monitor/syncdb.py | 45 +++++++------------ 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/syncdb.py b/python_apps/media-monitor2/media/monitor/syncdb.py index cc8abc294..4dacf54ac 100644 --- a/python_apps/media-monitor2/media/monitor/syncdb.py +++ b/python_apps/media-monitor2/media/monitor/syncdb.py @@ -11,11 +11,10 @@ class AirtimeDB(Loggable): if reload_now: self.reload_directories() def reload_directories(self): - """ - this is the 'real' constructor, should be called if you ever want the - class reinitialized. there's not much point to doing it yourself - however, you should just create a new AirtimeDB instance. - """ + """ this is the 'real' constructor, should be called if you ever + want the class reinitialized. there's not much point to doing + it yourself however, you should just create a new AirtimeDB + instance. """ # dirs_setup is a dict with keys: # u'watched_dirs' and u'stor' which point to lists of corresponding # dirs @@ -42,15 +41,11 @@ class AirtimeDB(Loggable): dirs_setup[u'watched_dirs'] ]) def to_id(self, directory): - """ - directory path -> id - """ + """ directory path -> id """ return self.dir_to_id[ directory ] def to_directory(self, dir_id): - """ - id -> directory path - """ + """ id -> directory path """ return self.id_to_dir[ dir_id ] def storage_path(self) : return self.base_storage @@ -60,37 +55,31 @@ class AirtimeDB(Loggable): def recorded_path(self) : return self.storage_paths['recorded'] def list_watched(self): - """ - returns all watched directories as a list - """ + """ returns all watched directories as a list """ return list(self.watched_directories) def list_storable_paths(self): - """ - returns a list of all the watched directories in the datatabase. - (Includes the imported directory and the recorded directory) - """ + """ returns a list of all the watched directories in the + datatabase. (Includes the imported directory and the recorded + directory) """ l = self.list_watched() l.append(self.import_path()) l.append(self.recorded_path()) return l def dir_id_get_files(self, dir_id, all_files=True): - """ - Get all files in a directory with id dir_id - """ + """ Get all files in a directory with id dir_id """ base_dir = self.id_to_dir[ dir_id ] return set(( os.path.join(base_dir,p) for p in self.apc.list_all_db_files( dir_id, all_files ) )) def directory_get_files(self, directory, all_files=True): - """ - returns all the files(recursively) in a directory. a directory is an - "actual" directory path instead of its id. This is super hacky because - you create one request for the recorded directory and one for the - imported directory even though they're the same dir in the database so - you get files for both dirs in 1 request... - """ + """ returns all the files(recursively) in a directory. a + directory is an "actual" directory path instead of its id. This + is super hacky because you create one request for the recorded + directory and one for the imported directory even though they're + the same dir in the database so you get files for both dirs in 1 + request... """ normal_dir = os.path.normpath(unicode(directory)) if normal_dir not in self.dir_to_id: raise NoDirectoryInAirtime( normal_dir, self.dir_to_id ) From cf6e1d66eeefd52538560578022a34099a7b5ba0 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 16 Nov 2012 12:36:58 -0500 Subject: [PATCH 53/68] removed direct usage of api client to AirtimeDB which encapsulates that behaviour --- python_apps/media-monitor2/media/saas/launcher.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/saas/launcher.py b/python_apps/media-monitor2/media/saas/launcher.py index c394da98b..5092895ae 100644 --- a/python_apps/media-monitor2/media/saas/launcher.py +++ b/python_apps/media-monitor2/media/saas/launcher.py @@ -15,6 +15,7 @@ from media.monitor.airtime import AirtimeNotifier, AirtimeMessageReceiver from media.monitor.watchersyncer import WatchSyncer from media.monitor.eventdrainer import EventDrainer from media.monitor.manager import Manager +from media.monitor.syncdb import AirtimeDB from media.saas.airtimeinstance import AirtimeInstance class MM2(InstanceThread, Loggable): @@ -49,7 +50,12 @@ class MM2(InstanceThread, Loggable): airtime_receiver = AirtimeMessageReceiver(config,manager) airtime_notifier = AirtimeNotifier(config, airtime_receiver) - store = apiclient.setup_media_monitor() + + adb = AirtimeDB(apiclient) + store = { + u'stor' : adb.storage_path(), + u'watched_dirs' : adb.list_watched(), + } self.logger.info("initializing mm with directories: %s" % str(store)) From bc9f7c4c9595f41c5ccf83b53b113330de9efd8b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 16 Nov 2012 12:51:07 -0500 Subject: [PATCH 54/68] removed hardcoding of path received from MM for SaaS. --- python_apps/media-monitor2/media/monitor/syncdb.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/syncdb.py b/python_apps/media-monitor2/media/monitor/syncdb.py index 4dacf54ac..ba0b04fca 100644 --- a/python_apps/media-monitor2/media/monitor/syncdb.py +++ b/python_apps/media-monitor2/media/monitor/syncdb.py @@ -15,12 +15,16 @@ class AirtimeDB(Loggable): want the class reinitialized. there's not much point to doing it yourself however, you should just create a new AirtimeDB instance. """ + + saas = user().root + # dirs_setup is a dict with keys: # u'watched_dirs' and u'stor' which point to lists of corresponding # dirs dirs_setup = self.apc.setup_media_monitor() - dirs_setup[u'stor'] = normpath( dirs_setup[u'stor'] ) - dirs_setup[u'watched_dirs'] = map(normpath, dirs_setup[u'watched_dirs']) + dirs_setup[u'stor'] = normpath( join(saas, dirs_setup[u'stor'] ) ) + dirs_setup[u'watched_dirs'] = map(lambda p: normpath(join(saas,p)), + dirs_setup[u'watched_dirs']) dirs_with_id = dict([ (k,normpath(v)) for k,v in self.apc.list_all_watched_dirs()['dirs'].iteritems() ]) From 57b2762e5c20c965da5b1dd3c6f4eb72eaa90871 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 16 Nov 2012 12:51:23 -0500 Subject: [PATCH 55/68] cleaned up imports --- python_apps/media-monitor2/media/monitor/syncdb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/syncdb.py b/python_apps/media-monitor2/media/monitor/syncdb.py index ba0b04fca..012d4947a 100644 --- a/python_apps/media-monitor2/media/monitor/syncdb.py +++ b/python_apps/media-monitor2/media/monitor/syncdb.py @@ -2,7 +2,8 @@ import os from media.monitor.log import Loggable from media.monitor.exceptions import NoDirectoryInAirtime -from os.path import normpath +from media.saas.thread import user +from os.path import normpath, join import media.monitor.pure as mmp class AirtimeDB(Loggable): @@ -74,7 +75,7 @@ class AirtimeDB(Loggable): def dir_id_get_files(self, dir_id, all_files=True): """ Get all files in a directory with id dir_id """ base_dir = self.id_to_dir[ dir_id ] - return set(( os.path.join(base_dir,p) for p in + return set(( join(base_dir,p) for p in self.apc.list_all_db_files( dir_id, all_files ) )) def directory_get_files(self, directory, all_files=True): From 1cbe2fb1a1a5ad5c318e1986abb9524617953adb Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 16 Nov 2012 12:59:17 -0500 Subject: [PATCH 56/68] typo --- python_apps/media-monitor2/media/monitor/syncdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/media/monitor/syncdb.py b/python_apps/media-monitor2/media/monitor/syncdb.py index 012d4947a..786fb1a39 100644 --- a/python_apps/media-monitor2/media/monitor/syncdb.py +++ b/python_apps/media-monitor2/media/monitor/syncdb.py @@ -17,7 +17,7 @@ class AirtimeDB(Loggable): it yourself however, you should just create a new AirtimeDB instance. """ - saas = user().root + saas = user().root_path # dirs_setup is a dict with keys: # u'watched_dirs' and u'stor' which point to lists of corresponding From c784df7f6e0a8e4bdc7bea96d33f6ce2d04cf80b Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 19 Nov 2012 11:35:37 -0500 Subject: [PATCH 57/68] Added comments describing config options --- python_apps/media-monitor2/baby.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor2/baby.py b/python_apps/media-monitor2/baby.py index f52d83639..983e53dcd 100644 --- a/python_apps/media-monitor2/baby.py +++ b/python_apps/media-monitor2/baby.py @@ -32,8 +32,8 @@ def main(main_cfg): if __name__ == '__main__': default = { - 'log_config' : '', - 'log_path' : '', - 'instance_root' : '' + 'log_config' : '', # config for log + 'log_path' : '', # where to log + 'instance_root' : '' # root dir of all instances } main(default) From 647832b0a05917f137e8556d33306bb50e846bf9 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 19 Nov 2012 16:03:25 -0500 Subject: [PATCH 58/68] added comments to document config options --- python_apps/media-monitor2/baby.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor2/baby.py b/python_apps/media-monitor2/baby.py index 983e53dcd..3479d5546 100644 --- a/python_apps/media-monitor2/baby.py +++ b/python_apps/media-monitor2/baby.py @@ -32,8 +32,9 @@ def main(main_cfg): if __name__ == '__main__': default = { - 'log_config' : '', # config for log - 'log_path' : '', # where to log - 'instance_root' : '' # root dir of all instances + 'log_path' : join(root, 'test.log'), # config for log + 'log_config' : join(root, 'configs/logging.cfg'), # where to log + # root dir of all instances + 'instance_root' : join(root, 'saas_stub') } main(default) From d595d1112fd536665481dc33ba77c68161e776bc Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 19 Nov 2012 16:03:58 -0500 Subject: [PATCH 59/68] added routine to verify config exists in "baby" media monitor" --- python_apps/media-monitor2/baby.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python_apps/media-monitor2/baby.py b/python_apps/media-monitor2/baby.py index 3479d5546..5dc19349d 100644 --- a/python_apps/media-monitor2/baby.py +++ b/python_apps/media-monitor2/baby.py @@ -24,8 +24,12 @@ def autoscan_instances(main_cfg): instances.append(ai) return instances +def verify_exists(p): + if not exists(p): raise Exception("%s must exist" % p) + def main(main_cfg): log_config, log_path = main_cfg['log_config'], main_cfg['log_path'] + verify_exists(log_config) log = setup_logger(log_config, log_path) setup_global(log) for instance in autoscan_instances(main_cfg): MM2(instance).start() From dc8315fef9f6a2262d09c53763aa4b400e90dbbb Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 19 Nov 2012 16:04:22 -0500 Subject: [PATCH 60/68] added more detailed output when launching baby mm --- python_apps/media-monitor2/baby.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/baby.py b/python_apps/media-monitor2/baby.py index 5dc19349d..ca22a0693 100644 --- a/python_apps/media-monitor2/baby.py +++ b/python_apps/media-monitor2/baby.py @@ -32,7 +32,10 @@ def main(main_cfg): verify_exists(log_config) log = setup_logger(log_config, log_path) setup_global(log) - for instance in autoscan_instances(main_cfg): MM2(instance).start() + for instance in autoscan_instances(main_cfg): + print("Launching instance: %s" % str(instance)) + MM2(instance).start() + print("Launched all instances") if __name__ == '__main__': default = { From 8dd0d80f68343802a9ff3aba8069dfe3f3f7a4e5 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 19 Nov 2012 16:04:43 -0500 Subject: [PATCH 61/68] fixed typo that was causing bug --- python_apps/media-monitor2/baby.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/baby.py b/python_apps/media-monitor2/baby.py index ca22a0693..c422df2ce 100644 --- a/python_apps/media-monitor2/baby.py +++ b/python_apps/media-monitor2/baby.py @@ -11,7 +11,7 @@ def filter_instance(d): return bool(re.match('.+/\d+$',d)) def get_name(p): return re.match('.+/(\d+)$',p).group(1) -def filter_instances(l): return (x for x in l if filter_instance(l)) +def filter_instances(l): return (x for x in l if filter_instance(x)) def autoscan_instances(main_cfg): root = main_cfg['instance_root'] From 9d04204389578846b29cb2e5ff744fad4fd2f0f6 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 19 Nov 2012 16:05:26 -0500 Subject: [PATCH 62/68] fixed bug where path was only relative. --- python_apps/media-monitor2/baby.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor2/baby.py b/python_apps/media-monitor2/baby.py index c422df2ce..cb66f7465 100644 --- a/python_apps/media-monitor2/baby.py +++ b/python_apps/media-monitor2/baby.py @@ -2,10 +2,10 @@ import re from media.saas.launcher import setup_logger, setup_global, MM2 from media.saas.airtimeinstance import AirtimeInstance -from os.path import isdir, join, abspath +from os.path import isdir, join, abspath, exists from os import listdir -def list_dirs(d): return (x for x in listdir(d) if isdir(x)) +def list_dirs(d): return (x for x in listdir(d) if isdir(join(d,x))) def filter_instance(d): return bool(re.match('.+/\d+$',d)) From cd7d292aa54aae582601e8637746796036f2ed56 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 19 Nov 2012 16:05:52 -0500 Subject: [PATCH 63/68] fixed faulty regex. --- python_apps/media-monitor2/baby.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/baby.py b/python_apps/media-monitor2/baby.py index cb66f7465..ad8b2b150 100644 --- a/python_apps/media-monitor2/baby.py +++ b/python_apps/media-monitor2/baby.py @@ -7,7 +7,7 @@ from os import listdir def list_dirs(d): return (x for x in listdir(d) if isdir(join(d,x))) -def filter_instance(d): return bool(re.match('.+/\d+$',d)) +def filter_instance(d): return bool(re.match('.+\d+$',d)) def get_name(p): return re.match('.+/(\d+)$',p).group(1) From e46da35d3b1a58b31f671cfbd5b554dfa6ce1f7c Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 19 Nov 2012 16:06:19 -0500 Subject: [PATCH 64/68] made root path a variable to remove duplication. --- python_apps/media-monitor2/baby.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_apps/media-monitor2/baby.py b/python_apps/media-monitor2/baby.py index ad8b2b150..5302fc22d 100644 --- a/python_apps/media-monitor2/baby.py +++ b/python_apps/media-monitor2/baby.py @@ -38,6 +38,7 @@ def main(main_cfg): print("Launched all instances") if __name__ == '__main__': + root = '/home/rudi/reps/Airtime/python_apps/media-monitor2' default = { 'log_path' : join(root, 'test.log'), # config for log 'log_config' : join(root, 'configs/logging.cfg'), # where to log From 6101e9dfe3b4acbeb89453b997aaa3c6b8f69696 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 19 Nov 2012 16:06:36 -0500 Subject: [PATCH 65/68] fixed bug where only part of the path was joined. --- python_apps/media-monitor2/baby.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor2/baby.py b/python_apps/media-monitor2/baby.py index 5302fc22d..f9eb74d6f 100644 --- a/python_apps/media-monitor2/baby.py +++ b/python_apps/media-monitor2/baby.py @@ -19,7 +19,7 @@ def autoscan_instances(main_cfg): for instance_machine in list_dirs(root): instance_machine = join(root, instance_machine) for instance_root in filter_instances(list_dirs(instance_machine)): - full_path = abspath(join(root,instance_root)) + full_path = abspath(join(instance_machine,instance_root)) ai = AirtimeInstance.root_make(get_name(full_path), full_path) instances.append(ai) return instances From 4acb7d92dbd79b9ee84d92e201069e50ad361e9f Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 19 Nov 2012 16:07:26 -0500 Subject: [PATCH 66/68] removed hardcoded signals in the code. --- .../media-monitor2/media/monitor/bootstrap.py | 3 ++- .../media-monitor2/media/monitor/events.py | 3 ++- .../media-monitor2/media/monitor/handler.py | 9 +++++---- .../media-monitor2/media/monitor/listeners.py | 20 +++++++++---------- .../media-monitor2/media/monitor/manager.py | 17 ++++++++-------- .../media-monitor2/media/monitor/organizer.py | 5 +++-- .../media/monitor/watchersyncer.py | 4 ++-- .../media-monitor2/media/saas/launcher.py | 4 ++-- 8 files changed, 35 insertions(+), 30 deletions(-) diff --git a/python_apps/media-monitor2/media/monitor/bootstrap.py b/python_apps/media-monitor2/media/monitor/bootstrap.py index 0a13ea0b4..6e685c964 100644 --- a/python_apps/media-monitor2/media/monitor/bootstrap.py +++ b/python_apps/media-monitor2/media/monitor/bootstrap.py @@ -2,6 +2,7 @@ import os from pydispatch import dispatcher from media.monitor.events import NewFile, DeleteFile, ModifyFile from media.monitor.log import Loggable +from media.saas.thread import getsig import media.monitor.pure as mmp class Bootstrapper(Loggable): @@ -16,7 +17,7 @@ class Bootstrapper(Loggable): watch_signal - the signals should send events for every file on. """ self.db = db - self.watch_signal = watch_signal + self.watch_signal = getsig(watch_signal) def flush_all(self, last_ran): """ diff --git a/python_apps/media-monitor2/media/monitor/events.py b/python_apps/media-monitor2/media/monitor/events.py index 8fb373704..ac16b3746 100644 --- a/python_apps/media-monitor2/media/monitor/events.py +++ b/python_apps/media-monitor2/media/monitor/events.py @@ -8,6 +8,7 @@ from media.monitor.pure import LazyProperty from media.monitor.metadata import Metadata from media.monitor.log import Loggable from media.monitor.exceptions import BadSongFile +from media.saas.thread import getsig class PathChannel(object): """ @@ -15,7 +16,7 @@ class PathChannel(object): used as a named tuple """ def __init__(self, signal, path): - self.signal = signal + self.signal = getsig(signal) self.path = path # TODO : Move this to it's file. Also possible unsingleton and use it as a diff --git a/python_apps/media-monitor2/media/monitor/handler.py b/python_apps/media-monitor2/media/monitor/handler.py index dd1d3843a..c67a437ef 100644 --- a/python_apps/media-monitor2/media/monitor/handler.py +++ b/python_apps/media-monitor2/media/monitor/handler.py @@ -3,6 +3,7 @@ from pydispatch import dispatcher import abc from media.monitor.log import Loggable +from media.saas.thread import getsig import media.monitor.pure as mmp # Defines the handle interface @@ -21,10 +22,10 @@ class ReportHandler(Handles): """ __metaclass__ = abc.ABCMeta def __init__(self, signal, weak=False): - self.signal = signal - self.report_signal = "badfile" + self.signal = getsig(signal) + self.report_signal = getsig("badfile") def dummy(sender, event): self.handle(sender,event) - dispatcher.connect(dummy, signal=signal, sender=dispatcher.Any, + dispatcher.connect(dummy, signal=self.signal, sender=dispatcher.Any, weak=weak) def report_problem_file(self, event, exception=None): @@ -38,7 +39,7 @@ class ProblemFileHandler(Handles, Loggable): """ def __init__(self, channel, **kwargs): self.channel = channel - self.signal = self.channel.signal + self.signal = getsig(self.channel.signal) self.problem_dir = self.channel.path def dummy(sender, event, exception): self.handle(sender, event, exception) diff --git a/python_apps/media-monitor2/media/monitor/listeners.py b/python_apps/media-monitor2/media/monitor/listeners.py index b33a5c1a9..081d0c4a4 100644 --- a/python_apps/media-monitor2/media/monitor/listeners.py +++ b/python_apps/media-monitor2/media/monitor/listeners.py @@ -9,7 +9,7 @@ from media.monitor.events import OrganizeFile, NewFile, MoveFile, DeleteFile, \ DeleteDir, EventRegistry, MoveDir,\ DeleteDirWatch from media.monitor.log import Loggable, get_logger - +from media.saas.thread import getsig # Note: Because of the way classes that inherit from pyinotify.ProcessEvent # interact with constructors. you should only instantiate objects from them # using keyword arguments. For example: @@ -45,7 +45,7 @@ class BaseListener(object): def __str__(self): return "Listener(%s), Signal(%s)" % \ (self.__class__.__name__, self. signal) - def my_init(self, signal): self.signal = signal + def my_init(self, signal): self.signal = getsig(signal) class OrganizeListener(BaseListener, pyinotify.ProcessEvent, Loggable): def process_IN_CLOSE_WRITE(self, event): @@ -66,14 +66,14 @@ class OrganizeListener(BaseListener, pyinotify.ProcessEvent, Loggable): self.logger.info("Bootstrapping: File in 'organize' directory: \ '%s'" % f) if not mmp.file_locked(f): - dispatcher.send(signal=self.signal, sender=self, + dispatcher.send(signal=getsig(self.signal), sender=self, event=OrganizeFile(f)) flushed += 1 #self.logger.info("Flushed organized directory with %d files" % flushed) @IncludeOnly(mmp.supported_extensions) def process_to_organize(self, event): - dispatcher.send(signal=self.signal, sender=self, + dispatcher.send(signal=getsig(self.signal), sender=self, event=OrganizeFile(event)) class StoreWatchListener(BaseListener, Loggable, pyinotify.ProcessEvent): @@ -101,14 +101,14 @@ class StoreWatchListener(BaseListener, Loggable, pyinotify.ProcessEvent): def delete_watch_dir(self, event): e = DeleteDirWatch(event) - dispatcher.send(signal='watch_move', sender=self, event=e) - dispatcher.send(signal=self.signal, sender=self, event=e) + dispatcher.send(signal=getsig('watch_move'), sender=self, event=e) + dispatcher.send(signal=getsig(self.signal), sender=self, event=e) @mediate_ignored @IncludeOnly(mmp.supported_extensions) def process_create(self, event): evt = NewFile(event) - dispatcher.send(signal=self.signal, sender=self, event=evt) + dispatcher.send(signal=getsig(self.signal), sender=self, event=evt) return evt @mediate_ignored @@ -117,13 +117,13 @@ class StoreWatchListener(BaseListener, Loggable, pyinotify.ProcessEvent): evt = None if event.dir : evt = DeleteDir(event) else : evt = DeleteFile(event) - dispatcher.send(signal=self.signal, sender=self, event=evt) + dispatcher.send(signal=getsig(self.signal), sender=self, event=evt) return evt @mediate_ignored def process_delete_dir(self, event): evt = DeleteDir(event) - dispatcher.send(signal=self.signal, sender=self, event=evt) + dispatcher.send(signal=getsig(self.signal), sender=self, event=evt) return evt def flush_events(self, path): @@ -138,6 +138,6 @@ class StoreWatchListener(BaseListener, Loggable, pyinotify.ProcessEvent): added = 0 for f in mmp.walk_supported(path, clean_empties=False): added += 1 - dispatcher.send( signal=self.signal, sender=self, event=NewFile(f) ) + dispatcher.send( signal=getsig(self.signal), sender=self, event=NewFile(f) ) self.logger.info( "Flushed watch directory. added = %d" % added ) diff --git a/python_apps/media-monitor2/media/monitor/manager.py b/python_apps/media-monitor2/media/monitor/manager.py index d2c0a5f64..1fc7b695b 100644 --- a/python_apps/media-monitor2/media/monitor/manager.py +++ b/python_apps/media-monitor2/media/monitor/manager.py @@ -8,7 +8,7 @@ from media.monitor.log import Loggable from media.monitor.listeners import StoreWatchListener, OrganizeListener from media.monitor.handler import ProblemFileHandler from media.monitor.organizer import Organizer -from media.saas.thread import InstanceInheritingThread +from media.saas.thread import InstanceInheritingThread, getsig import media.monitor.pure as mmp @@ -37,8 +37,8 @@ class Manager(Loggable): def __init__(self): self.wm = pyinotify.WatchManager() # These two instance variables are assumed to be constant - self.watch_channel = 'watch' - self.organize_channel = 'organize' + self.watch_channel = getsig('watch') + self.organize_channel = getsig('organize') self.watch_listener = StoreWatchListener(signal = self.watch_channel) self.__timeout_thread = ManagerTimeout(self) self.__timeout_thread.daemon = True @@ -54,11 +54,11 @@ class Manager(Loggable): self.organize_channel), } def dummy(sender, event): self.watch_move( event.path, sender=sender ) - dispatcher.connect(dummy, signal='watch_move', sender=dispatcher.Any, - weak=False) + dispatcher.connect(dummy, signal=getsig('watch_move'), + sender=dispatcher.Any, weak=False) def subwatch_add(sender, directory): self.__add_watch(directory, self.watch_listener) - dispatcher.connect(subwatch_add, signal='add_subwatch', + dispatcher.connect(subwatch_add, signal=getsig('add_subwatch'), sender=dispatcher.Any, weak=False) # A private mapping path => watch_descriptor # we use the same dictionary for organize, watch, store wd events. @@ -81,7 +81,7 @@ class Manager(Loggable): def watch_signal(self): """ Return the signal string our watch_listener is reading events from """ - return self.watch_listener.signal + return getsig(self.watch_listener.signal) def __remove_watch(self,path): """ Remove path from being watched (first will check if 'path' @@ -131,7 +131,8 @@ class Manager(Loggable): """ Set the path where problem files should go """ self.organize['problem_files_path'] = new_path self.organize['problem_handler'] = \ - ProblemFileHandler( PathChannel(signal='badfile',path=new_path) ) + ProblemFileHandler( PathChannel(signal=getsig('badfile'), + path=new_path) ) def get_recorded_path(self): """ returns the path of the recorded directory """ diff --git a/python_apps/media-monitor2/media/monitor/organizer.py b/python_apps/media-monitor2/media/monitor/organizer.py index ce1849b90..6d72bee4b 100644 --- a/python_apps/media-monitor2/media/monitor/organizer.py +++ b/python_apps/media-monitor2/media/monitor/organizer.py @@ -7,6 +7,7 @@ from media.monitor.exceptions import BadSongFile from media.monitor.events import OrganizeFile from pydispatch import dispatcher from os.path import dirname +from media.saas.thread import getsig import os.path class Organizer(ReportHandler,Loggable): @@ -36,7 +37,7 @@ class Organizer(ReportHandler,Loggable): self.channel = channel self.target_path = target_path self.recorded_path = recorded_path - super(Organizer, self).__init__(signal=self.channel, weak=False) + super(Organizer, self).__init__(signal=getsig(self.channel), weak=False) def handle(self, sender, event): """ Intercept events where a new file has been added to the @@ -63,7 +64,7 @@ class Organizer(ReportHandler,Loggable): def new_dir_watch(d): # TODO : rewrite as return lambda : dispatcher.send(... def cb(): - dispatcher.send(signal="add_subwatch", sender=self, + dispatcher.send(signal=getsig("add_subwatch"), sender=self, directory=d) return cb diff --git a/python_apps/media-monitor2/media/monitor/watchersyncer.py b/python_apps/media-monitor2/media/monitor/watchersyncer.py index a913bb506..558759c89 100644 --- a/python_apps/media-monitor2/media/monitor/watchersyncer.py +++ b/python_apps/media-monitor2/media/monitor/watchersyncer.py @@ -8,7 +8,7 @@ from media.monitor.exceptions import BadSongFile from media.monitor.eventcontractor import EventContractor from media.monitor.events import EventProxy from media.monitor.request import ThreadedRequestSync, RequestSync -from media.saas.thread import InstanceInheritingThread +from media.saas.thread import InstanceInheritingThread, getsig class TimeoutWatcher(InstanceInheritingThread,Loggable): """ @@ -52,7 +52,7 @@ class WatchSyncer(ReportHandler,Loggable): tc = TimeoutWatcher(self, self.timeout) tc.daemon = True tc.start() - super(WatchSyncer, self).__init__(signal=signal) + super(WatchSyncer, self).__init__(signal=getsig(signal)) def handle(self, sender, event): """ diff --git a/python_apps/media-monitor2/media/saas/launcher.py b/python_apps/media-monitor2/media/saas/launcher.py index 5092895ae..e80f46034 100644 --- a/python_apps/media-monitor2/media/saas/launcher.py +++ b/python_apps/media-monitor2/media/saas/launcher.py @@ -7,7 +7,7 @@ import media.monitor.pure as mmp from media.monitor.exceptions import FailedToObtainLocale, FailedToSetLocale from media.monitor.log import get_logger, setup_logging from std_err_override import LogWriter -from media.saas.thread import InstanceThread, user, apc +from media.saas.thread import InstanceThread, user, apc, getsig from media.monitor.log import Loggable from media.monitor.exceptions import CouldNotCreateIndexFile from media.monitor.toucher import ToucherThread @@ -44,7 +44,7 @@ class MM2(InstanceThread, Loggable): manager = Manager() apiclient = apc() config = user().mm_config - watch_syncer = WatchSyncer(signal='watch', + watch_syncer = WatchSyncer(signal=getsig('watch'), chunking_number=config['chunking_number'], timeout=config['request_max_wait']) airtime_receiver = AirtimeMessageReceiver(config,manager) From a995129df59f0f1545dc08f883b52b9bfa727e73 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 19 Nov 2012 16:07:40 -0500 Subject: [PATCH 67/68] added general getsig routine to get the correct signals from anywhere in the code. --- python_apps/media-monitor2/media/saas/airtimeinstance.py | 7 +++++++ python_apps/media-monitor2/media/saas/thread.py | 2 ++ 2 files changed, 9 insertions(+) diff --git a/python_apps/media-monitor2/media/saas/airtimeinstance.py b/python_apps/media-monitor2/media/saas/airtimeinstance.py index e0d474c33..fb701b432 100644 --- a/python_apps/media-monitor2/media/saas/airtimeinstance.py +++ b/python_apps/media-monitor2/media/saas/airtimeinstance.py @@ -6,6 +6,9 @@ from media.monitor.pure import LazyProperty from media.monitor.config import MMConfig from api_clients.api_client import AirtimeApiClient +# poor man's phantom types... +class SignalString(str): pass + class AirtimeInstance(object): """ AirtimeInstance is a class that abstracts away every airtime instance by providing all the necessary objects required to interact @@ -29,6 +32,10 @@ class AirtimeInstance(object): self.config_paths = config_paths self.root_path = root_path + def signal(self, sig): + if isinstance(sig, SignalString): return sig + else: return SignalString("%s_%s" % (self.name, sig)) + def __str__(self): return "%s,%s(%s)" % (self.name, self.root_path, self.config_paths) diff --git a/python_apps/media-monitor2/media/saas/thread.py b/python_apps/media-monitor2/media/saas/thread.py index 0b4ae079e..49a3acd6f 100644 --- a/python_apps/media-monitor2/media/saas/thread.py +++ b/python_apps/media-monitor2/media/saas/thread.py @@ -23,3 +23,5 @@ def user(): except AttributeError: raise UserlessThread() def apc(): return user().api_client + +def getsig(s): return user().signal(s) From a28c9d9e278175dea91ebe76769fd38286a06fba Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 19 Nov 2012 16:46:47 -0500 Subject: [PATCH 68/68] added untracked configs --- .../media-monitor2/configs/airtime.conf | 32 +++++ .../media-monitor2/configs/api_client.cfg | 126 ++++++++++++++++++ .../media-monitor2/configs/logging.cfg | 32 +++++ .../media-monitor2/configs/media-monitor.cfg | 31 +++++ 4 files changed, 221 insertions(+) create mode 100755 python_apps/media-monitor2/configs/airtime.conf create mode 100755 python_apps/media-monitor2/configs/api_client.cfg create mode 100755 python_apps/media-monitor2/configs/logging.cfg create mode 100755 python_apps/media-monitor2/configs/media-monitor.cfg diff --git a/python_apps/media-monitor2/configs/airtime.conf b/python_apps/media-monitor2/configs/airtime.conf new file mode 100755 index 000000000..f2749ab74 --- /dev/null +++ b/python_apps/media-monitor2/configs/airtime.conf @@ -0,0 +1,32 @@ +[database] +host = localhost +dbname = airtime +dbuser = airtime +dbpass = airtime + +[rabbitmq] +host = 127.0.0.1 +port = 5672 +user = guest +password = guest +vhost = / + +[general] +api_key = I6EUOJM0D1EIGSMZ9T70 +web_server_user = www-data +airtime_dir = /usr/share/airtime +base_url = localhost +base_port = 80 +base_dir = '' + +;How many hours ahead of time should Airtime playout engine (PYPO) +;cache scheduled media files. +cache_ahead_hours = 1 + +[monit] +monit_user = guest +monit_password = airtime + +[soundcloud] +connection_retries = 3 +time_between_retries = 60 diff --git a/python_apps/media-monitor2/configs/api_client.cfg b/python_apps/media-monitor2/configs/api_client.cfg new file mode 100755 index 000000000..c7b00417c --- /dev/null +++ b/python_apps/media-monitor2/configs/api_client.cfg @@ -0,0 +1,126 @@ +bin_dir = "/usr/lib/airtime/api_clients" + +############################# +## Common +############################# + +# Value needed to access the API +api_key = 'I6EUOJM0D1EIGSMZ9T70' + +# Path to the base of the API +api_base = 'api' + +# URL to get the version number of the server API +version_url = 'version/api_key/%%api_key%%' + +#URL to register a components IP Address with the central web server +register_component = 'register-component/format/json/api_key/%%api_key%%/component/%%component%%' + +# Hostname +host = 'localhost' +base_port = 80 +base_dir = '' + +############################# +## Config for Media Monitor +############################# + +# URL to setup the media monitor +media_setup_url = 'media-monitor-setup/format/json/api_key/%%api_key%%' + +# Tell Airtime the file id associated with a show instance. +upload_recorded = 'upload-recorded/format/json/api_key/%%api_key%%/fileid/%%fileid%%/showinstanceid/%%showinstanceid%%' + +# URL to tell Airtime to update file's meta data +update_media_url = 'reload-metadata/format/json/api_key/%%api_key%%/mode/%%mode%%' + +# URL to tell Airtime we want a listing of all files it knows about +list_all_db_files = 'list-all-files/format/json/api_key/%%api_key%%/dir_id/%%dir_id%%/all/%%all%%' + +# URL to tell Airtime we want a listing of all dirs its watching (including the stor dir) +list_all_watched_dirs = 'list-all-watched-dirs/format/json/api_key/%%api_key%%' + +# URL to tell Airtime we want to add watched directory +add_watched_dir = 'add-watched-dir/format/json/api_key/%%api_key%%/path/%%path%%' + +# URL to tell Airtime we want to add watched directory +remove_watched_dir = 'remove-watched-dir/format/json/api_key/%%api_key%%/path/%%path%%' + +# URL to tell Airtime we want to add watched directory +set_storage_dir = 'set-storage-dir/format/json/api_key/%%api_key%%/path/%%path%%' + +# URL to tell Airtime about file system mount change +update_fs_mount = 'update-file-system-mount/format/json/api_key/%%api_key%%' + +# URL to commit multiple updates from media monitor at the same time + +reload_metadata_group = 'reload-metadata-group/format/json/api_key/%%api_key%%' + +# URL to tell Airtime about file system mount change +handle_watched_dir_missing = 'handle-watched-dir-missing/format/json/api_key/%%api_key%%/dir/%%dir%%' + +############################# +## Config for Recorder +############################# + +# URL to get the schedule of shows set to record +show_schedule_url = 'recorded-shows/format/json/api_key/%%api_key%%' + +# URL to upload the recorded show's file to Airtime +upload_file_url = 'upload-file/format/json/api_key/%%api_key%%' + +# URL to commit multiple updates from media monitor at the same time + +#number of retries to upload file if connection problem +upload_retries = 3 + +#time to wait between attempts to upload file if connection problem (in seconds) +upload_wait = 60 + +################################################################################ +# Uncomment *one of the sets* of values from the API clients below, and comment +# out all the others. +################################################################################ + +############################# +## Config for Pypo +############################# + +# Schedule export path. +# %%from%% - starting date/time in the form YYYY-MM-DD-hh-mm +# %%to%% - starting date/time in the form YYYY-MM-DD-hh-mm +export_url = 'schedule/api_key/%%api_key%%' + +get_media_url = 'get-media/file/%%file%%/api_key/%%api_key%%' + +# Update whether a schedule group has begun playing. +update_item_url = 'notify-schedule-group-play/api_key/%%api_key%%/schedule_id/%%schedule_id%%' + +# Update whether an audio clip is currently playing. +update_start_playing_url = 'notify-media-item-start-play/api_key/%%api_key%%/media_id/%%media_id%%/' + +# URL to tell Airtime we want to get stream setting +get_stream_setting = 'get-stream-setting/format/json/api_key/%%api_key%%/' + +#URL to update liquidsoap status +update_liquidsoap_status = 'update-liquidsoap-status/format/json/api_key/%%api_key%%/msg/%%msg%%/stream_id/%%stream_id%%/boot_time/%%boot_time%%' + +#URL to check live stream auth +check_live_stream_auth = 'check-live-stream-auth/format/json/api_key/%%api_key%%/username/%%username%%/password/%%password%%/djtype/%%djtype%%' + +#URL to update source status +update_source_status = 'update-source-status/format/json/api_key/%%api_key%%/sourcename/%%sourcename%%/status/%%status%%' + +get_bootstrap_info = 'get-bootstrap-info/format/json/api_key/%%api_key%%' + +get_files_without_replay_gain = 'get-files-without-replay-gain/api_key/%%api_key%%/dir_id/%%dir_id%%' + +update_replay_gain_value = 'update-replay-gain-value/api_key/%%api_key%%' + +notify_webstream_data = 'notify-webstream-data/api_key/%%api_key%%/media_id/%%media_id%%/format/json' + +notify_liquidsoap_started = 'rabbitmq-do-push/api_key/%%api_key%%/format/json' + +get_stream_parameters = 'get-stream-parameters/api_key/%%api_key%%/format/json' + +push_stream_stats = 'push-stream-stats/api_key/%%api_key%%/format/json' diff --git a/python_apps/media-monitor2/configs/logging.cfg b/python_apps/media-monitor2/configs/logging.cfg new file mode 100755 index 000000000..ea24f69e0 --- /dev/null +++ b/python_apps/media-monitor2/configs/logging.cfg @@ -0,0 +1,32 @@ +[loggers] +keys= root,notifier,metadata + +[handlers] +keys=fileOutHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=DEBUG +handlers=fileOutHandler + +[logger_notifier] +level=DEBUG +handlers=fileOutHandler +qualname=notifier + +[logger_metadata] +level=DEBUG +handlers=fileOutHandler +qualname=metadata + +[handler_fileOutHandler] +class=logging.handlers.RotatingFileHandler +level=DEBUG +formatter=simpleFormatter +args=("/var/log/airtime/media-monitor/media-monitor.log", 'a', 10000000, 5,) + +[formatter_simpleFormatter] +format=%(asctime)s %(levelname)s - [%(threadName)s] [%(filename)s : %(funcName)s()] : LINE %(lineno)d - %(message)s +datefmt= diff --git a/python_apps/media-monitor2/configs/media-monitor.cfg b/python_apps/media-monitor2/configs/media-monitor.cfg new file mode 100755 index 000000000..b1167f56b --- /dev/null +++ b/python_apps/media-monitor2/configs/media-monitor.cfg @@ -0,0 +1,31 @@ +api_client = "airtime" + +# where the binary files live +bin_dir = '/usr/lib/airtime/media-monitor' + +# where the logging files live +log_dir = '/var/log/airtime/media-monitor' + + +############################################ +# RabbitMQ settings # +############################################ +rabbitmq_host = 'localhost' +rabbitmq_user = 'guest' +rabbitmq_password = 'guest' +rabbitmq_vhost = '/' + +############################################ +# Media-Monitor preferences # +############################################ +check_filesystem_events = 5 #how long to queue up events performed on the files themselves. +check_airtime_events = 30 #how long to queue metadata input from airtime. + +# MM2 only: +touch_interval = 5 +chunking_number = 450 +request_max_wait = 3.0 +rmq_event_wait = 0.1 +logpath = '/var/log/airtime/media-monitor/media-monitor.log' +index_path = '/var/tmp/airtime/media-monitor/last_index' +