From 331386f74fb2d6b4c078aac273043994d113dd5b Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Mon, 4 Feb 2013 16:05:58 -0500 Subject: [PATCH] CC-4910: Need to properly handle non-200 http status codes in api client -fixed --- python_apps/api_clients/api_client.py | 128 +++++++----------- .../airtimemediamonitorbootstrap.py | 8 +- .../media-monitor2/media/monitor/syncdb.py | 46 ++++--- .../media-monitor2/media/saas/launcher.py | 10 +- .../generate_liquidsoap_cfg.py | 13 +- python_apps/pypo/listenerstat.py | 5 +- .../pypo/media/update/replaygainupdater.py | 14 +- python_apps/pypo/pypocli.py | 9 +- python_apps/pypo/pypofetch.py | 30 ++-- python_apps/pypo/recorder.py | 10 +- utils/airtime-import/airtime-import.py | 37 ++--- utils/airtime-silan/airtime-silan.py | 2 +- 12 files changed, 156 insertions(+), 156 deletions(-) diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index a3dc3f41a..e8dbbd16a 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -146,7 +146,6 @@ class AirtimeApiClient(object): sys.exit(1) def __get_airtime_version(self): - # TODO : maybe fix this function to drop an exception? try: return self.services.version_url()[u'version'] except Exception: return -1 @@ -154,18 +153,18 @@ class AirtimeApiClient(object): logger = self.logger version = self.__get_airtime_version() # logger.info('Airtime version found: ' + str(version)) - if (version == -1): + if version == -1: if (verbose): logger.info('Unable to get Airtime version number.\n') return False - elif (version[0:3] != AIRTIME_VERSION[0:3]): - if (verbose): + elif version[0:3] != AIRTIME_VERSION[0:3]: + if verbose: logger.info('Airtime version found: ' + str(version)) logger.info('pypo is at version ' + AIRTIME_VERSION + ' and is not compatible with this version of Airtime.\n') return False else: - if (verbose): + if verbose: logger.info('Airtime version: ' + str(version)) logger.info('pypo is at version ' + AIRTIME_VERSION + ' and is compatible with this version of Airtime.') return True @@ -193,14 +192,6 @@ class AirtimeApiClient(object): self.logger.error(str(e)) return None - # TODO : get this routine out of here it doesn't belong at all here - def get_liquidsoap_data(self, pkey, schedule): - playlist = schedule[pkey] - data = dict() - try: data["schedule_id"] = playlist['id'] - except Exception: data["schedule_id"] = 0 - return data - def get_shows_to_record(self): try: return self.services.show_schedule_url() @@ -259,12 +250,12 @@ class AirtimeApiClient(object): url = url.replace("%%api_key%%", self.config["api_key"]) return url + """ + Caller of this method needs to catch any exceptions such as + ValueError thrown by json.loads or URLError by urllib2.urlopen + """ def setup_media_monitor(self): - try: - return self.services.media_setup_url() - except Exception, e: - #TODO - self.logger.info(str(e)) + return self.services.media_setup_url() def send_media_monitor_requests(self, action_list, dry=False): """ @@ -323,37 +314,40 @@ class AirtimeApiClient(object): self.logger.error("Could not find index 'files' in dictionary: %s", str(response)) return [] - + """ + Caller of this method needs to catch any exceptions such as + ValueError thrown by json.loads or URLError by urllib2.urlopen + """ def list_all_watched_dirs(self): - try: - return self.services.list_all_watched_dirs() - except Exception, e: - #TODO - self.logger.error(str(e)) + return self.services.list_all_watched_dirs() + """ + Caller of this method needs to catch any exceptions such as + ValueError thrown by json.loads or URLError by urllib2.urlopen + """ def add_watched_dir(self, path): - try: - return self.services.add_watched_dir(path=base64.b64encode(path)) - except Exception, e: - #TODO - self.logger.error(str(e)) + return self.services.add_watched_dir(path=base64.b64encode(path)) + """ + Caller of this method needs to catch any exceptions such as + ValueError thrown by json.loads or URLError by urllib2.urlopen + """ def remove_watched_dir(self, path): - try: - return self.services.remove_watched_dir(path=base64.b64encode(path)) - except Exception, e: - #TODO - self.logger.error(str(e)) + return self.services.remove_watched_dir(path=base64.b64encode(path)) + """ + Caller of this method needs to catch any exceptions such as + ValueError thrown by json.loads or URLError by urllib2.urlopen + """ def set_storage_dir(self, path): return self.services.set_storage_dir(path=base64.b64encode(path)) + """ + Caller of this method needs to catch any exceptions such as + ValueError thrown by json.loads or URLError by urllib2.urlopen + """ def get_stream_setting(self): - logger = self.logger - try: return self.services.get_stream_setting() - except Exception, e: - logger.error("Exception: %s", e) - return None + return self.services.get_stream_setting() def register_component(self, component): """ Purpose of this method is to contact the server with a "Hey its @@ -361,11 +355,7 @@ class AirtimeApiClient(object): (component = media-monitor, pypo etc.) ip address, and later use it to query monit via monit's http service, or download log files via a http server. """ - try: - return self.services.register_component(component=component) - except Exception, e: - #TODO - self.logger.error(str(e)) + return self.services.register_component(component=component) def notify_liquidsoap_status(self, msg, stream_id, time): logger = self.logger @@ -388,11 +378,7 @@ class AirtimeApiClient(object): def get_bootstrap_info(self): """ Retrieve infomations needed on bootstrap time """ - try: - return self.services.get_bootstrap_info() - except Exception, e: - #TODO - self.logger.error(str(e)) + return self.services.get_bootstrap_info() def get_files_without_replay_gain_value(self, dir_id): """ @@ -404,8 +390,8 @@ class AirtimeApiClient(object): try: return self.services.get_files_without_replay_gain(dir_id=dir_id) except Exception, e: - #TODO self.logger.error(str(e)) + return [] def get_files_without_silan_value(self): """ @@ -416,20 +402,16 @@ class AirtimeApiClient(object): try: return self.services.get_files_without_silan_value() except Exception, e: - #TODO self.logger.error(str(e)) + return [] def update_replay_gain_values(self, pairs): """ 'pairs' is a list of pairs in (x, y), where x is the file's database row id and y is the file's replay_gain value in dB """ - try: - self.logger.debug(self.services.update_replay_gain_value( - _post_data={'data': json.dumps(pairs)})) - except Exception, e: - #TODO - self.logger.error(str(e)) + self.logger.debug(self.services.update_replay_gain_value( + _post_data={'data': json.dumps(pairs)})) def update_cue_values_by_silan(self, pairs): @@ -437,11 +419,7 @@ class AirtimeApiClient(object): 'pairs' is a list of pairs in (x, y), where x is the file's database row id and y is the file's cue values in dB """ - try: - print self.services.update_cue_values_by_silan(_post_data={'data': json.dumps(pairs)}) - except Exception, e: - #TODO - self.logger.error(str(e)) + return self.services.update_cue_values_by_silan(_post_data={'data': json.dumps(pairs)}) def notify_webstream_data(self, data, media_id): @@ -449,30 +427,18 @@ class AirtimeApiClient(object): Update the server with the latest metadata we've received from the external webstream """ - try: - self.logger.info( self.services.notify_webstream_data.req( - _post_data={'data':data}, media_id=str(media_id)).retry(5)) - except Exception, e: - #TODO - self.logger.error(str(e)) + self.logger.info( self.services.notify_webstream_data.req( + _post_data={'data':data}, media_id=str(media_id)).retry(5)) def get_stream_parameters(self): - try: - response = self.services.get_stream_parameters() - self.logger.debug(response) - return response - except Exception, e: - #TODO - self.logger.error(str(e)) + response = self.services.get_stream_parameters() + self.logger.debug(response) + return response def push_stream_stats(self, data): # TODO : users of this method should do their own error handling - try: - response = self.services.push_stream_stats(_post_data={'data': json.dumps(data)}) - return response - except Exception, e: - #TODO - self.logger.error(str(e)) + response = self.services.push_stream_stats(_post_data={'data': json.dumps(data)}) + return response def update_stream_setting_table(self, data): try: diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py b/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py index 7a57a74e6..02b6cf2cf 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py @@ -58,18 +58,18 @@ class AirtimeMediaMonitorBootstrap(): """ returns the path and its corresponding database row idfor all watched directories. Also returns the Stor directory, which can be identified by its row id (always has value of "1") - + Return type is a dictionary similar to: {"1":"/srv/airtime/stor/"} """ def get_list_of_watched_dirs(self): json = self.api_client.list_all_watched_dirs() - + try: return json["dirs"] except KeyError as e: self.logger.error("Could not find index 'dirs' in dictionary: %s", str(json)) - self.logger.error(e) + self.logger.error(str(e)) return {} """ @@ -94,7 +94,7 @@ class AirtimeMediaMonitorBootstrap(): db_known_files_set = set() files = self.list_db_files(dir_id) - + for f in files: db_known_files_set.add(f) diff --git a/python_apps/media-monitor2/media/monitor/syncdb.py b/python_apps/media-monitor2/media/monitor/syncdb.py index 786fb1a39..dd837b2a1 100644 --- a/python_apps/media-monitor2/media/monitor/syncdb.py +++ b/python_apps/media-monitor2/media/monitor/syncdb.py @@ -19,31 +19,35 @@ class AirtimeDB(Loggable): saas = user().root_path - # dirs_setup is a dict with keys: - # u'watched_dirs' and u'stor' which point to lists of corresponding - # dirs - dirs_setup = self.apc.setup_media_monitor() - dirs_setup[u'stor'] = normpath( join(saas, dirs_setup[u'stor'] ) ) - dirs_setup[u'watched_dirs'] = map(lambda p: normpath(join(saas,p)), - dirs_setup[u'watched_dirs']) - dirs_with_id = dict([ (k,normpath(v)) for k,v in - self.apc.list_all_watched_dirs()['dirs'].iteritems() ]) + try: + # dirs_setup is a dict with keys: + # u'watched_dirs' and u'stor' which point to lists of corresponding + # dirs + dirs_setup = self.apc.setup_media_monitor() + dirs_setup[u'stor'] = normpath( join(saas, dirs_setup[u'stor'] ) ) + dirs_setup[u'watched_dirs'] = map(lambda p: normpath(join(saas,p)), + dirs_setup[u'watched_dirs']) + dirs_with_id = dict([ (k,normpath(v)) for k,v in + self.apc.list_all_watched_dirs()['dirs'].iteritems() ]) - self.id_to_dir = dirs_with_id - self.dir_to_id = dict([ (v,k) for k,v in dirs_with_id.iteritems() ]) + self.id_to_dir = dirs_with_id + self.dir_to_id = dict([ (v,k) for k,v in dirs_with_id.iteritems() ]) - self.base_storage = dirs_setup[u'stor'] - self.storage_paths = mmp.expand_storage( self.base_storage ) - self.base_id = self.dir_to_id[self.base_storage] + self.base_storage = dirs_setup[u'stor'] + self.storage_paths = mmp.expand_storage( self.base_storage ) + self.base_id = self.dir_to_id[self.base_storage] - # hack to get around annoying schema of airtime db - self.dir_to_id[ self.recorded_path() ] = self.base_id - self.dir_to_id[ self.import_path() ] = self.base_id + # hack to get around annoying schema of airtime db + self.dir_to_id[ self.recorded_path() ] = self.base_id + self.dir_to_id[ self.import_path() ] = self.base_id + + # We don't know from the x_to_y dict which directory is watched or + # store... + self.watched_directories = set([ os.path.normpath(p) for p in + dirs_setup[u'watched_dirs'] ]) + except Exception, e: + self.logger.info(str(e)) - # We don't know from the x_to_y dict which directory is watched or - # store... - self.watched_directories = set([ os.path.normpath(p) for p in - dirs_setup[u'watched_dirs'] ]) def to_id(self, directory): """ directory path -> id """ diff --git a/python_apps/media-monitor2/media/saas/launcher.py b/python_apps/media-monitor2/media/saas/launcher.py index 6aacb8e40..83a972311 100644 --- a/python_apps/media-monitor2/media/saas/launcher.py +++ b/python_apps/media-monitor2/media/saas/launcher.py @@ -85,7 +85,15 @@ class MM2(InstanceThread, Loggable): ToucherThread(path=user().touch_file_path(), interval=int(config['touch_interval'])) - apiclient.register_component('media-monitor') + success = False + while not success: + try: + apiclient.register_component('media-monitor') + success = True + except Exception, e: + self.logger.error(str(e)) + import time + time.sleep(10) manager.loop() diff --git a/python_apps/pypo/liquidsoap_scripts/generate_liquidsoap_cfg.py b/python_apps/pypo/liquidsoap_scripts/generate_liquidsoap_cfg.py index 20d99976f..b123763e2 100644 --- a/python_apps/pypo/liquidsoap_scripts/generate_liquidsoap_cfg.py +++ b/python_apps/pypo/liquidsoap_scripts/generate_liquidsoap_cfg.py @@ -26,13 +26,10 @@ def generate_liquidsoap_config(ss): logging.basicConfig(format='%(message)s') ac = AirtimeApiClient(logging.getLogger()) -ss = ac.get_stream_setting() - -if ss is not None: - try: - generate_liquidsoap_config(ss) - except Exception, e: - logging.error(e) -else: +try: + ss = ac.get_stream_setting() + generate_liquidsoap_config(ss) +except Exception, e: + logging.error(str(e)) print "Unable to connect to the Airtime server." sys.exit(1) diff --git a/python_apps/pypo/listenerstat.py b/python_apps/pypo/listenerstat.py index e6a272703..18dbd4f1e 100644 --- a/python_apps/pypo/listenerstat.py +++ b/python_apps/pypo/listenerstat.py @@ -103,7 +103,10 @@ class ListenerStat(Thread): self.update_listener_stat_error(v["mount"], 'OK') except Exception, e: self.logger.error('Exception: %s', e) - self.update_listener_stat_error(v["mount"], str(e)) + try: + self.update_listener_stat_error(v["mount"], str(e)) + except Exception, e: + self.logger.error('Exception: %s', e) return stats diff --git a/python_apps/pypo/media/update/replaygainupdater.py b/python_apps/pypo/media/update/replaygainupdater.py index 2f52c0a23..42f218ac5 100644 --- a/python_apps/pypo/media/update/replaygainupdater.py +++ b/python_apps/pypo/media/update/replaygainupdater.py @@ -67,15 +67,15 @@ class ReplayGainUpdater(Thread): self.logger.error(e) self.logger.debug(traceback.format_exc()) def run(self): - try: - while True: - self.logger.info("Runnning replaygain updater") + while True: + try: + self.logger.info("Running replaygain updater") self.main() # Sleep for 5 minutes in case new files have been added - time.sleep(60 * 5) - except Exception, e: - self.logger.error('ReplayGainUpdater Exception: %s', traceback.format_exc()) - self.logger.error(e) + except Exception, e: + self.logger.error('ReplayGainUpdater Exception: %s', traceback.format_exc()) + self.logger.error(e) + time.sleep(60 * 5) if __name__ == "__main__": rgu = ReplayGainUpdater() diff --git a/python_apps/pypo/pypocli.py b/python_apps/pypo/pypocli.py index 93a1718aa..42cef00cc 100644 --- a/python_apps/pypo/pypocli.py +++ b/python_apps/pypo/pypocli.py @@ -179,7 +179,14 @@ if __name__ == '__main__': ReplayGainUpdater.start_reply_gain(api_client) - api_client.register_component("pypo") + success = False + while not success: + try: + api_client.register_component('pypo') + success = True + except Exception, e: + logger.error(str(e)) + time.sleep(10) pypoFetch_q = Queue() recorder_q = Queue() diff --git a/python_apps/pypo/pypofetch.py b/python_apps/pypo/pypofetch.py index 2d6ad0fb5..3e029e8f5 100644 --- a/python_apps/pypo/pypofetch.py +++ b/python_apps/pypo/pypofetch.py @@ -206,23 +206,25 @@ class PypoFetch(Thread): """ def set_bootstrap_variables(self): self.logger.debug('Getting information needed on bootstrap from Airtime') - info = self.api_client.get_bootstrap_info() - if info is None: + try: + info = self.api_client.get_bootstrap_info() + except Exception, e: self.logger.error('Unable to get bootstrap info.. Exiting pypo...') - else: - self.logger.debug('info:%s', info) - commands = [] - for k, v in info['switch_status'].iteritems(): - commands.append(self.switch_source_temp(k, v)) + self.logger.error(str(e)) - stream_format = info['stream_label'] - station_name = info['station_name'] - fade = info['transition_fade'] + self.logger.debug('info:%s', info) + commands = [] + for k, v in info['switch_status'].iteritems(): + commands.append(self.switch_source_temp(k, v)) - commands.append(('vars.stream_metadata_type %s\n' % stream_format).encode('utf-8')) - commands.append(('vars.station_name %s\n' % station_name).encode('utf-8')) - commands.append(('vars.default_dj_fade %s\n' % fade).encode('utf-8')) - PypoFetch.telnet_send(self.logger, self.telnet_lock, commands) + stream_format = info['stream_label'] + station_name = info['station_name'] + fade = info['transition_fade'] + + commands.append(('vars.stream_metadata_type %s\n' % stream_format).encode('utf-8')) + commands.append(('vars.station_name %s\n' % station_name).encode('utf-8')) + commands.append(('vars.default_dj_fade %s\n' % fade).encode('utf-8')) + PypoFetch.telnet_send(self.logger, self.telnet_lock, commands) def restart_liquidsoap(self): diff --git a/python_apps/pypo/recorder.py b/python_apps/pypo/recorder.py index fbabc1b57..70d99144d 100644 --- a/python_apps/pypo/recorder.py +++ b/python_apps/pypo/recorder.py @@ -189,9 +189,17 @@ class Recorder(Thread): self.server_timezone = '' self.queue = q self.loops = 0 - self.api_client.register_component("show-recorder") self.logger.info("RecorderFetch: init complete") + success = False + while not success: + try: + self.api_client.register_component('show-recorder') + success = True + except Exception, e: + self.logger.error(str(e)) + time.sleep(10) + def handle_message(self): if not self.queue.empty(): message = self.queue.get() diff --git a/utils/airtime-import/airtime-import.py b/utils/airtime-import/airtime-import.py index 2b3ce03cd..97d4c4e35 100644 --- a/utils/airtime-import/airtime-import.py +++ b/utils/airtime-import/airtime-import.py @@ -75,15 +75,16 @@ def format_dir_string(path): return path def helper_get_stor_dir(): - res = api_client.list_all_watched_dirs() - if(res is None): + try: + res = api_client.list_all_watched_dirs() + except Exception, e: return res + + if(res['dirs']['1'][-1] != '/'): + out = res['dirs']['1']+'/' + return out else: - if(res['dirs']['1'][-1] != '/'): - out = res['dirs']['1']+'/' - return out - else: - return res['dirs']['1'] + return res['dirs']['1'] def checkOtherOption(args): for i in args: @@ -162,8 +163,9 @@ def WatchAddAction(option, opt, value, parser): path = apc.encode_to(path, 'utf-8') if(os.path.isdir(path)): #os.chmod(path, 0765) - res = api_client.add_watched_dir(path) - if(res is None): + try: + res = api_client.add_watched_dir(path) + except Exception, e: exit("Unable to connect to the server.") # sucess if(res['msg']['code'] == 0): @@ -179,8 +181,9 @@ def WatchListAction(option, opt, value, parser): errorIfMultipleOption(parser.rargs) if(len(parser.rargs) > 0): raise OptionValueError("This option doesn't take any arguments.") - res = api_client.list_all_watched_dirs() - if(res is None): + try: + res = api_client.list_all_watched_dirs() + except Exception, e: exit("Unable to connect to the Airtime server.") dirs = res["dirs"].items() # there will be always 1 which is storage folder @@ -204,8 +207,9 @@ def WatchRemoveAction(option, opt, value, parser): path = currentDir+path path = apc.encode_to(path, 'utf-8') if(os.path.isdir(path)): - res = api_client.remove_watched_dir(path) - if(res is None): + try: + res = api_client.remove_watched_dir(path) + except Exception, e: exit("Unable to connect to the Airtime server.") # sucess if(res['msg']['code'] == 0): @@ -249,10 +253,11 @@ def StorageSetAction(option, opt, value, parser): path = currentDir+path path = apc.encode_to(path, 'utf-8') if(os.path.isdir(path)): - res = api_client.set_storage_dir(path) - if(res is None): + try: + res = api_client.set_storage_dir(path) + except Exception, e: exit("Unable to connect to the Airtime server.") - # sucess + # success if(res['msg']['code'] == 0): print "Successfully set storage folder to %s" % path else: diff --git a/utils/airtime-silan/airtime-silan.py b/utils/airtime-silan/airtime-silan.py index d5f691698..bd8a0af4d 100644 --- a/utils/airtime-silan/airtime-silan.py +++ b/utils/airtime-silan/airtime-silan.py @@ -68,7 +68,7 @@ try: subtotal += total total = 0 try: - api_client.update_cue_values_by_silan(processed_data) + print api_client.update_cue_values_by_silan(processed_data) except Exception ,e: print e print traceback.format_exc()