From 17087fc035c74d2e65b4a1ac1d37d8588c192c2f Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 5 Nov 2012 18:08:47 -0500 Subject: [PATCH 01/10] Added better error message. not much else can be done --- utils/airtime-import/airtime-import.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/airtime-import/airtime-import.py b/utils/airtime-import/airtime-import.py index c77a0f21e..221f644f9 100644 --- a/utils/airtime-import/airtime-import.py +++ b/utils/airtime-import/airtime-import.py @@ -168,6 +168,8 @@ def WatchAddAction(option, opt, value, parser): print "%s added to watched folder list successfully" % path else: print "Adding a watched folder failed: %s" % res['msg']['error'] + print "This error most likely caused by wrong permissions" + print "Try fixing this error by chmodding the parent directory(ies)" else: print "Given path is not a directory: %s" % path From cc5acf3c9e7786fc8fab1197c180942440d62124 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Wed, 30 Jan 2013 14:34:06 -0500 Subject: [PATCH 02/10] improved logging for boolean vars --- airtime_mvc/application/logging/Logging.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airtime_mvc/application/logging/Logging.php b/airtime_mvc/application/logging/Logging.php index dfe7e1fbe..65a8f6dc2 100644 --- a/airtime_mvc/application/logging/Logging.php +++ b/airtime_mvc/application/logging/Logging.php @@ -32,6 +32,8 @@ class Logging { { if (is_array($p_msg) || is_object($p_msg)) { return print_r($p_msg, true); + } else if (is_bool($p_msg)) { + return $p_msg ? "true" : "false"; } else { return $p_msg; } From 52d96241daedaa98e2833686cae9f6b01046811f Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Wed, 30 Jan 2013 14:40:29 -0500 Subject: [PATCH 03/10] CC-4754: Replay gain modifier -fix regression --- airtime_mvc/application/controllers/PreferenceController.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airtime_mvc/application/controllers/PreferenceController.php b/airtime_mvc/application/controllers/PreferenceController.php index 00b0fa6ca..e68e6c603 100644 --- a/airtime_mvc/application/controllers/PreferenceController.php +++ b/airtime_mvc/application/controllers/PreferenceController.php @@ -273,7 +273,8 @@ class PreferenceController extends Zend_Controller_Action Application_Model_Preference::SetEnableReplayGain($values["enableReplayGain"]); Application_Model_Preference::setReplayGainModifier($values["replayGainModifier"]); $md = array('schedule' => Application_Model_Schedule::getSchedule()); - Application_Model_RabbitMq::PushSchedule(); + Application_Model_RabbitMq::SendMessageToPypo("update_schedule", $md); + //Application_Model_RabbitMq::PushSchedule(); } if (!Application_Model_Preference::GetMasterDjConnectionUrlOverride()) { From 3553ebc2ee3bde9cd7c8352f322b7848d31e65c0 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Wed, 30 Jan 2013 18:21:46 -0500 Subject: [PATCH 04/10] fix upgrade script so that cue out of files is not 00:00:00 by default --- install_minimal/upgrades/airtime-2.3.0/data/upgrade.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install_minimal/upgrades/airtime-2.3.0/data/upgrade.sql b/install_minimal/upgrades/airtime-2.3.0/data/upgrade.sql index c2d176b23..cd34a28bb 100644 --- a/install_minimal/upgrades/airtime-2.3.0/data/upgrade.sql +++ b/install_minimal/upgrades/airtime-2.3.0/data/upgrade.sql @@ -15,6 +15,8 @@ INSERT INTO cc_stream_setting ("keyname", "value", "type") VALUES ('s3_admin_pas UPDATE cc_music_dirs SET directory = directory || '/' where id in (select id from cc_music_dirs where substr(directory, length(directory)) != '/'); UPDATE cc_files SET filepath = substring(filepath from 2) where id in (select id from cc_files where substring(filepath from 1 for 1) = '/'); +UPDATE cc_files SET cueout = length where cueout = '00:00:00'; + INSERT INTO cc_pref("keystr", "valstr") VALUES('locale', 'en_CA'); INSERT INTO cc_pref("subjid", "keystr", "valstr") VALUES(1, 'user_locale', 'en_CA'); From 617e1049aab8a763fbbfda4355bc05353015f164 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Wed, 30 Jan 2013 23:31:24 -0500 Subject: [PATCH 05/10] fix changelog mistakes --- changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog b/changelog index 9aa22c634..aff42ec52 100644 --- a/changelog +++ b/changelog @@ -1,6 +1,7 @@ 2.3.0 - Jan 21st, 2013 * New features - * Localization (Brazilian, Chinese, Czech, English, French, German, Italian, Korean, Portugese, Russian, Spanish) + * Localization (Chinese, Czech, English, French, German, Italian, Korean, + Portuguese, Russian, Spanish) * User management page for non-admin users * Listener statistics (Icecast/Shoutcast) * Airtime no longer requires Apache document root From 8adb4b259c031ff7107a8b30ec20e8dec185fe2c Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Thu, 31 Jan 2013 12:33:14 -0500 Subject: [PATCH 06/10] CC-4905: Unit tests for Pypo --- python_apps/pypo/pypocli.py | 2 +- python_apps/pypo/pypofetch.py | 3 +- python_apps/pypo/pypopush.py | 20 ++++++++++++-- python_apps/pypo/tests/run_tests.sh | 18 ++++++++++++ python_apps/pypo/tests/test_modify_cue_in.py | 29 ++++++++++++++++++++ 5 files changed, 67 insertions(+), 5 deletions(-) create mode 100755 python_apps/pypo/tests/run_tests.sh create mode 100644 python_apps/pypo/tests/test_modify_cue_in.py diff --git a/python_apps/pypo/pypocli.py b/python_apps/pypo/pypocli.py index ea8950d41..93a1718aa 100644 --- a/python_apps/pypo/pypocli.py +++ b/python_apps/pypo/pypocli.py @@ -176,7 +176,7 @@ if __name__ == '__main__': sys.exit() api_client = api_client.AirtimeApiClient() - + ReplayGainUpdater.start_reply_gain(api_client) api_client.register_component("pypo") diff --git a/python_apps/pypo/pypofetch.py b/python_apps/pypo/pypofetch.py index b0cbdc69a..e073efda9 100644 --- a/python_apps/pypo/pypofetch.py +++ b/python_apps/pypo/pypofetch.py @@ -18,7 +18,8 @@ from std_err_override import LogWriter from configobj import ConfigObj # configure logging -logging.config.fileConfig("logging.cfg") +logging_cfg = os.path.join(os.path.dirname(__file__), "logging.cfg") +logging.config.fileConfig(logging_cfg) logger = logging.getLogger() LogWriter.override_std_err(logger) diff --git a/python_apps/pypo/pypopush.py b/python_apps/pypo/pypopush.py index 6fedfab1b..f438b3bb1 100644 --- a/python_apps/pypo/pypopush.py +++ b/python_apps/pypo/pypopush.py @@ -9,6 +9,7 @@ import logging.config import telnetlib import calendar import math +import os from pypofetch import PypoFetch from Queue import Empty @@ -21,7 +22,8 @@ from configobj import ConfigObj # configure logging -logging.config.fileConfig("logging.cfg") +logging_cfg = os.path.join(os.path.dirname(__file__), "logging.cfg") +logging.config.fileConfig(logging_cfg) logger = logging.getLogger() LogWriter.override_std_err(logger) @@ -249,7 +251,7 @@ class PypoPush(Thread): self.start_web_stream_buffer(current_item) self.start_web_stream(current_item) if is_file(current_item): - self.modify_cue_point(file_chain[0]) + file_chain = self.modify_first_link_cue_point(file_chain) self.push_to_liquidsoap(file_chain) #we've changed the queue, so let's refetch it liquidsoap_queue_approx = self.get_queue_items_from_liquidsoap() @@ -279,7 +281,7 @@ class PypoPush(Thread): chain_to_push = file_chain[problem_at_iteration:] if len(chain_to_push) > 0: - self.modify_cue_point(chain_to_push[0]) + chain_to_push = self.modify_first_link_cue_point(chain_to_push) self.push_to_liquidsoap(chain_to_push) @@ -363,6 +365,18 @@ class PypoPush(Thread): original_cue_in_td = timedelta(seconds=float(link['cue_in'])) link['cue_in'] = self.date_interval_to_seconds(original_cue_in_td) + diff_sec + def modify_first_link_cue_point(self, chain): + if not len(chain): + return [] + + first_link = chain[0] + + self.modify_cue_point(first_link) + if float(first_link['cue_in']) >= float(first_link['cue_out']): + chain = chain [1:] + + return chain + """ Returns two chains, original chain and current_chain. current_chain is a subset of original_chain but can also be equal to original chain. diff --git a/python_apps/pypo/tests/run_tests.sh b/python_apps/pypo/tests/run_tests.sh new file mode 100755 index 000000000..830a9bb85 --- /dev/null +++ b/python_apps/pypo/tests/run_tests.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +which py.test +pytest_exist=$? + +if [ "$pytest_exist" != "0" ]; then + echo "Need to have py.test installed. Exiting..." + exit 1 +fi + +SCRIPT=`readlink -f $0` +# Absolute directory this script is in +SCRIPTPATH=`dirname $SCRIPT` + +export PYTHONPATH=$PYTHONPATH:$SCRIPTPATH/..:$SCRIPTPATH/../.. + +py.test + diff --git a/python_apps/pypo/tests/test_modify_cue_in.py b/python_apps/pypo/tests/test_modify_cue_in.py new file mode 100644 index 000000000..d5049bde5 --- /dev/null +++ b/python_apps/pypo/tests/test_modify_cue_in.py @@ -0,0 +1,29 @@ +from pypopush import PypoPush +from threading import Lock +from Queue import Queue + +import datetime + +pypoPush_q = Queue() +telnet_lock = Lock() + +pp = PypoPush(pypoPush_q, telnet_lock) + +def test_modify_cue_in(): + + link = pp.modify_first_link_cue_point([]) + assert len(link) == 0 + + min_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes = 1) + link = [{"start":min_ago.strftime("%Y-%m-%d-%H-%M-%S"), + "cue_in":"0", "cue_out":"30"}] + link = pp.modify_first_link_cue_point(link) + assert len(link) == 0 + + link = [{"start":min_ago.strftime("%Y-%m-%d-%H-%M-%S"), + "cue_in":"0", "cue_out":"70"}] + link = pp.modify_first_link_cue_point(link) + assert len(link) == 1 + + + From f145ede01b99823a96554cad3b5e58bd186315c5 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Thu, 31 Jan 2013 17:45:57 -0500 Subject: [PATCH 07/10] CC-4905: Unit tests for Pypo -whitespace removal --- python_apps/pypo/tests/test_modify_cue_in.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python_apps/pypo/tests/test_modify_cue_in.py b/python_apps/pypo/tests/test_modify_cue_in.py index d5049bde5..da17fd53f 100644 --- a/python_apps/pypo/tests/test_modify_cue_in.py +++ b/python_apps/pypo/tests/test_modify_cue_in.py @@ -10,7 +10,6 @@ telnet_lock = Lock() pp = PypoPush(pypoPush_q, telnet_lock) def test_modify_cue_in(): - link = pp.modify_first_link_cue_point([]) assert len(link) == 0 @@ -25,5 +24,3 @@ def test_modify_cue_in(): link = pp.modify_first_link_cue_point(link) assert len(link) == 1 - - From 149b744ba7d775b447f0d4bbf6bea84d422792cc Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 1 Feb 2013 01:40:48 -0500 Subject: [PATCH 08/10] remove using self in a static method --- python_apps/pypo/pypofetch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/pypo/pypofetch.py b/python_apps/pypo/pypofetch.py index e073efda9..2d6ad0fb5 100644 --- a/python_apps/pypo/pypofetch.py +++ b/python_apps/pypo/pypofetch.py @@ -136,7 +136,7 @@ class PypoFetch(Thread): try: lock.acquire() tn = telnetlib.Telnet(LS_HOST, LS_PORT) - self.logger.info(command) + logger.info(command) tn.write(command) tn.write('exit\n') tn.read_all() From 1f4cfa8f65c2a0651a555eae31028d80bc8da162 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Sat, 2 Feb 2013 08:50:44 -0500 Subject: [PATCH 09/10] fix pypo not auto restarting --- python_apps/pypo/monit-airtime-playout.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/pypo/monit-airtime-playout.cfg b/python_apps/pypo/monit-airtime-playout.cfg index 5b096c72a..453f4efec 100644 --- a/python_apps/pypo/monit-airtime-playout.cfg +++ b/python_apps/pypo/monit-airtime-playout.cfg @@ -5,5 +5,5 @@ check process airtime-playout with pidfile "/var/run/airtime-playout.pid" - start program = "/etc/init.d/airtime-playout monit-restart" with timeout 5 seconds + start program = "/etc/init.d/airtime-playout start" with timeout 5 seconds stop program = "/etc/init.d/airtime-playout stop" From 34eee60bee638e5600785f423179ca68ef1521ad Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Sun, 3 Feb 2013 00:40:41 -0500 Subject: [PATCH 10/10] CC-4910: Need to properly handle non-200 http status codes in api client -wrap all service calls in exception --- python_apps/api_clients/api_client.py | 138 +++++++++++++----- .../liquidsoap_scripts/liquidsoap_auth.py | 5 +- 2 files changed, 108 insertions(+), 35 deletions(-) diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index b28a4ca5a..a3dc3f41a 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -80,23 +80,19 @@ class ApiRequest(object): if logger is None: self.logger = logging else: self.logger = logger def __call__(self,_post_data=None, **kwargs): - # TODO : get rid of god damn urllib and replace everything with - # grequests or requests at least final_url = self.url.params(**kwargs).url() if _post_data is not None: _post_data = urllib.urlencode(_post_data) try: req = urllib2.Request(final_url, _post_data) response = urllib2.urlopen(req).read() except Exception, e: - self.logger.error('Exception: %s', e) import traceback - top = traceback.format_exc() - self.logger.error("traceback: %s", top) - response = "" + self.logger.error('Exception: %s', e) + self.logger.error("traceback: %s", traceback.format_exc()) + raise # Ghetto hack for now because we don't the content type we are getting # (Pointless to look at mime since it's not being set correctly always) - try: return json.loads(response) - except ValueError: return response + return json.loads(response) def req(self, *args, **kwargs): self.__req = lambda : self(*args, **kwargs) @@ -182,13 +178,20 @@ class AirtimeApiClient(object): except: return (False, None) def notify_liquidsoap_started(self): - return self.services.notify_liquidsoap_started() + try: + self.services.notify_liquidsoap_started() + except Exception, e: + self.logger.error(str(e)) def notify_media_item_start_playing(self, media_id): """ This is a callback from liquidsoap, we use this to notify about the currently playing *song*. We get passed a JSON string which we handed to liquidsoap in get_liquidsoap_data(). """ - return self.services.update_start_playing_url(media_id=media_id) + try: + return self.services.update_start_playing_url(media_id=media_id) + except Exception, e: + self.logger.error(str(e)) + return None # TODO : get this routine out of here it doesn't belong at all here def get_liquidsoap_data(self, pkey, schedule): @@ -199,7 +202,11 @@ class AirtimeApiClient(object): return data def get_shows_to_record(self): - return self.services.show_schedule_url() + try: + return self.services.show_schedule_url() + except Exception, e: + self.logger.error(str(e)) + return None def upload_recorded_show(self, data, headers): logger = self.logger @@ -235,8 +242,12 @@ class AirtimeApiClient(object): return response def check_live_stream_auth(self, username, password, dj_type): - return self.services.check_live_stream_auth( - username=username, password=password, djtype=dj_type) + try: + return self.services.check_live_stream_auth( + username=username, password=password, djtype=dj_type) + except Exception, e: + self.logger.error(str(e)) + return {} def construct_url(self,config_action_key): """Constructs the base url for every request""" @@ -249,7 +260,11 @@ class AirtimeApiClient(object): return url def setup_media_monitor(self): - return self.services.media_setup_url() + try: + return self.services.media_setup_url() + except Exception, e: + #TODO + self.logger.info(str(e)) def send_media_monitor_requests(self, action_list, dry=False): """ @@ -310,13 +325,25 @@ class AirtimeApiClient(object): return [] def list_all_watched_dirs(self): - return self.services.list_all_watched_dirs() + try: + return self.services.list_all_watched_dirs() + except Exception, e: + #TODO + self.logger.error(str(e)) def add_watched_dir(self, path): - return self.services.add_watched_dir(path=base64.b64encode(path)) + try: + return self.services.add_watched_dir(path=base64.b64encode(path)) + except Exception, e: + #TODO + self.logger.error(str(e)) def remove_watched_dir(self, path): - return self.services.remove_watched_dir(path=base64.b64encode(path)) + try: + return self.services.remove_watched_dir(path=base64.b64encode(path)) + except Exception, e: + #TODO + self.logger.error(str(e)) def set_storage_dir(self, path): return self.services.set_storage_dir(path=base64.b64encode(path)) @@ -334,7 +361,11 @@ class AirtimeApiClient(object): (component = media-monitor, pypo etc.) ip address, and later use it to query monit via monit's http service, or download log files via a http server. """ - return self.services.register_component(component=component) + try: + return self.services.register_component(component=component) + except Exception, e: + #TODO + self.logger.error(str(e)) def notify_liquidsoap_status(self, msg, stream_id, time): logger = self.logger @@ -343,6 +374,7 @@ class AirtimeApiClient(object): self.services.update_liquidsoap_status.req(msg=encoded_msg, stream_id=stream_id, boot_time=time).retry(5) except Exception, e: + #TODO logger.error("Exception: %s", e) def notify_source_status(self, sourcename, status): @@ -351,11 +383,16 @@ class AirtimeApiClient(object): return self.services.update_source_status.req(sourcename=sourcename, status=status).retry(5) except Exception, e: + #TODO logger.error("Exception: %s", e) def get_bootstrap_info(self): - """ Retrive infomations needed on bootstrap time """ - return self.services.get_bootstrap_info() + """ Retrieve infomations needed on bootstrap time """ + try: + return self.services.get_bootstrap_info() + except Exception, e: + #TODO + self.logger.error(str(e)) def get_files_without_replay_gain_value(self, dir_id): """ @@ -364,7 +401,11 @@ class AirtimeApiClient(object): to this file is the return value. """ #http://localhost/api/get-files-without-replay-gain/dir_id/1 - return self.services.get_files_without_replay_gain(dir_id=dir_id) + try: + return self.services.get_files_without_replay_gain(dir_id=dir_id) + except Exception, e: + #TODO + self.logger.error(str(e)) def get_files_without_silan_value(self): """ @@ -372,22 +413,35 @@ class AirtimeApiClient(object): calculated. This list of files is downloaded into a file and the path to this file is the return value. """ - return self.services.get_files_without_silan_value() + try: + return self.services.get_files_without_silan_value() + except Exception, e: + #TODO + self.logger.error(str(e)) def update_replay_gain_values(self, pairs): """ 'pairs' is a list of pairs in (x, y), where x is the file's database row id and y is the file's replay_gain value in dB """ - self.logger.debug(self.services.update_replay_gain_value( - _post_data={'data': json.dumps(pairs)})) + try: + self.logger.debug(self.services.update_replay_gain_value( + _post_data={'data': json.dumps(pairs)})) + except Exception, e: + #TODO + self.logger.error(str(e)) + def update_cue_values_by_silan(self, pairs): """ 'pairs' is a list of pairs in (x, y), where x is the file's database row id and y is the file's cue values in dB """ - print self.services.update_cue_values_by_silan(_post_data={'data': json.dumps(pairs)}) + try: + print self.services.update_cue_values_by_silan(_post_data={'data': json.dumps(pairs)}) + except Exception, e: + #TODO + self.logger.error(str(e)) def notify_webstream_data(self, data, media_id): @@ -395,19 +449,35 @@ class AirtimeApiClient(object): Update the server with the latest metadata we've received from the external webstream """ - self.logger.info( self.services.notify_webstream_data.req( - _post_data={'data':data}, media_id=str(media_id)).retry(5)) + try: + self.logger.info( self.services.notify_webstream_data.req( + _post_data={'data':data}, media_id=str(media_id)).retry(5)) + except Exception, e: + #TODO + self.logger.error(str(e)) def get_stream_parameters(self): - response = self.services.get_stream_parameters() - self.logger.debug(response) - return response + try: + response = self.services.get_stream_parameters() + self.logger.debug(response) + return response + except Exception, e: + #TODO + self.logger.error(str(e)) def push_stream_stats(self, data): # TODO : users of this method should do their own error handling - response = self.services.push_stream_stats(_post_data={'data': json.dumps(data)}) - return response + try: + response = self.services.push_stream_stats(_post_data={'data': json.dumps(data)}) + return response + except Exception, e: + #TODO + self.logger.error(str(e)) def update_stream_setting_table(self, data): - response = self.services.update_stream_setting_table(_post_data={'data': json.dumps(data)}) - return response + try: + response = self.services.update_stream_setting_table(_post_data={'data': json.dumps(data)}) + return response + except Exception, e: + #TODO + self.logger.error(str(e)) diff --git a/python_apps/pypo/liquidsoap_scripts/liquidsoap_auth.py b/python_apps/pypo/liquidsoap_scripts/liquidsoap_auth.py index 862231cc2..838898afb 100644 --- a/python_apps/pypo/liquidsoap_scripts/liquidsoap_auth.py +++ b/python_apps/pypo/liquidsoap_scripts/liquidsoap_auth.py @@ -15,4 +15,7 @@ elif dj_type == '--dj': response = api_clients.check_live_stream_auth(username, password, source_type) -print response['msg'] +if 'msg' in response: + print response['msg'] +else: + print False