From 7ade83ae7460552fbdcd0857a4ad432fb49cd72f Mon Sep 17 00:00:00 2001 From: "paul.baranowski" Date: Thu, 18 Nov 2010 18:00:13 -0500 Subject: [PATCH] Fixes to the liquidsoap scheduler and the API. Added a test() function to api_client. Fixed bug with the config file. Added line numbers to the logging. --- 3rd_party/pypo/api_clients/api_client.py | 102 +++++++++++++++++++++-- 3rd_party/pypo/config.cfg | 22 ++--- 3rd_party/pypo/logging.cfg | 2 +- 3rd_party/pypo/pypo_cli.py | 57 ++++++------- api/get_media.php | 5 +- backend/Schedule.php | 2 +- backend/StoredFile.php | 12 ++- backend/Transport.php | 2 +- backend/tests/AllTests.php | 9 +- backend/tests/StoredFileTests.php | 6 -- 10 files changed, 151 insertions(+), 68 deletions(-) diff --git a/3rd_party/pypo/api_clients/api_client.py b/3rd_party/pypo/api_clients/api_client.py index c4c179643..fc14bd648 100644 --- a/3rd_party/pypo/api_clients/api_client.py +++ b/3rd_party/pypo/api_clients/api_client.py @@ -7,6 +7,8 @@ import urllib import logging from util import json import os +from urlparse import urlparse + def api_client_factory(config): if config["api_client"] == "campcaster": @@ -28,7 +30,9 @@ class ApiClientInterface: # Required. # This is the main method you need to implement when creating a new API client. - def get_schedule(self): + # start and end are for testing purposes. + # start and end are strings in the format YYYY-DD-MM-hh-mm-ss + def get_schedule(self, start=None, end=None): return 0, [] # Required. @@ -49,6 +53,12 @@ class ApiClientInterface: def generate_range_dp(self): nil + # Put here whatever tests you want to run to make sure your API is working + def test(self): + nil + + #def get_media_type(self, playlist): + # nil class CampcasterApiClient(ApiClientInterface): @@ -65,6 +75,7 @@ class CampcasterApiClient(ApiClientInterface): logger.debug("Trying to contact %s", url) response = urllib.urlopen(url) data = response.read() + logger.debug("Data: %s", data) response_json = json.read(data) version = response_json['version'] logger.debug("Campcaster Version %s detected", version) @@ -94,6 +105,27 @@ class CampcasterApiClient(ApiClientInterface): return version + + def test(self): + logger = logging.getLogger("CampcasterApiClient.test") + status, items = self.get_schedule('2010-01-01-00-00-00', '2011-01-01-00-00-00') + #print items + schedule = items["playlists"] + logger.debug("Number of playlists found: %s", str(len(schedule))) + count = 1 + for pkey in sorted(schedule.iterkeys()): + logger.debug("Playlist #%s",str(count)) + count+=1 + #logger.info("found playlist at %s", pkey) + #print pkey + playlist = schedule[pkey] + for item in playlist["medias"]: + filename = urlparse(item["uri"]) + filename = filename.query[5:] + #print filename + self.get_media(item["uri"], filename) + + def check_version(self): version = self.__get_campcaster_version() if (version == 0): @@ -110,7 +142,7 @@ class CampcasterApiClient(ApiClientInterface): print 'pypo is compatible with this version of Campcaster.' print - def get_schedule(self): + def get_schedule(self, start=None, end=None): logger = logging.getLogger("CampcasterApiClient.get_schedule") """ @@ -118,12 +150,17 @@ class CampcasterApiClient(ApiClientInterface): (seconds are ignored, just here for consistency) """ tnow = time.localtime(time.time()) - tstart = time.localtime(time.time() - 3600 * int(self.config["cache_for"])) - tend = time.localtime(time.time() + 3600 * int(self.config["prepare_ahead"])) - + if (not start): + tstart = time.localtime(time.time() - 3600 * int(self.config["cache_for"])) + start = "%04d-%02d-%02d-%02d-%02d" % (tstart[0], tstart[1], tstart[2], tstart[3], tstart[4]) + + if (not end): + tend = time.localtime(time.time() + 3600 * int(self.config["prepare_ahead"])) + end = "%04d-%02d-%02d-%02d-%02d" % (tend[0], tend[1], tend[2], tend[3], tend[4]) + range = {} - range['start'] = "%04d-%02d-%02d-%02d-%02d" % (tstart[0], tstart[1], tstart[2], tstart[3], tstart[4]) - range['end'] = "%04d-%02d-%02d-%02d-%02d" % (tend[0], tend[1], tend[2], tend[3], tend[4]) + range['start'] = start + range['end'] = end # Construct the URL export_url = self.config["base_url"] + self.config["api_base"] + self.config["export_url"] @@ -138,6 +175,7 @@ class CampcasterApiClient(ApiClientInterface): status = 0 try: response_json = urllib.urlopen(export_url).read() + logger.debug("%s", response_json) response = json.read(response_json) logger.info("export status %s", response['check']) status = response['check'] @@ -152,7 +190,9 @@ class CampcasterApiClient(ApiClientInterface): try: src = src + "&api_key=" + self.config["api_key"] - urllib.urlretrieve(src, dst, False) + # check if file exists already before downloading again + filename, headers = urllib.urlretrieve(src, dst) + logger.info("downloaded %s to %s", src, dst) except Exception, e: logger.error("%s", e) @@ -222,7 +262,8 @@ class CampcasterApiClient(ApiClientInterface): return response - + + ################################################################################ # OpenBroadcast API Client ################################################################################ @@ -301,6 +342,49 @@ class ObpApiClient(): return obp_version + + def get_schedule(self, start=None, end=None): + logger = logging.getLogger("CampcasterApiClient.get_schedule") + + """ + calculate start/end time range (format: YYYY-DD-MM-hh-mm-ss,YYYY-DD-MM-hh-mm-ss) + (seconds are ignored, just here for consistency) + """ + tnow = time.localtime(time.time()) + if (not start): + tstart = time.localtime(time.time() - 3600 * int(self.config["cache_for"])) + start = "%04d-%02d-%02d-%02d-%02d" % (tstart[0], tstart[1], tstart[2], tstart[3], tstart[4]) + + if (not end): + tend = time.localtime(time.time() + 3600 * int(self.config["prepare_ahead"])) + end = "%04d-%02d-%02d-%02d-%02d" % (tend[0], tend[1], tend[2], tend[3], tend[4]) + + range = {} + range['start'] = start + range['end'] = end + + # Construct the URL + export_url = self.config["base_url"] + self.config["api_base"] + self.config["export_url"] + + # Insert the start and end times into the URL + export_url = export_url.replace('%%api_key%%', self.config["api_key"]) + export_url = export_url.replace('%%from%%', range['start']) + export_url = export_url.replace('%%to%%', range['end']) + logger.info("export from %s", export_url) + + response = "" + status = 0 + try: + response_json = urllib.urlopen(export_url).read() + logger.debug("%s", response_json) + response = json.read(response_json) + logger.info("export status %s", response['check']) + status = response['check'] + except Exception, e: + print e + + return status, response + def get_media(self, src, dest): try: diff --git a/3rd_party/pypo/config.cfg b/3rd_party/pypo/config.cfg index c812d4acc..1cd5a8bde 100644 --- a/3rd_party/pypo/config.cfg +++ b/3rd_party/pypo/config.cfg @@ -31,37 +31,37 @@ base_url = 'http://localhost/' # Generic Config - if you are creating a new API client, define these values # ################################################################################ # Path to the base of the API -api_base = '' +#api_base = '' # URL to get the version number of the API -version_url = '' +#version_url = '' # Schedule export path. # %%from%% - starting date/time in the form YYYY-MM-DD-hh-mm # %%to%% - starting date/time in the form YYYY-MM-DD-hh-mm -export_url = '' +#export_url = '' # Update whether an item has been played. # %%item_id%% # %%played%% -update_item_url = '' +#update_item_url = '' # Update whether an item is currently playing. -update_start_playing_url = '' +#update_start_playing_url = '' # ??? -generate_range_url = '' +#generate_range_url = '' ##################### # Campcaster Config # ##################### api_base = 'campcaster/' -version_url = 'schedule/api_version.php?api_key=%%api_key%%' -export_url = 'schedule/schedule.php?from=%%from%%&to=%%to%%&api_key=%%api_key%%' -update_item_url = 'schedule/schedule.php?item_id=%%item_id%%&played=%%played%%' -update_start_playing_url = 'schedule/update_start_playing.php?playlist_type=%%playlist_type%%&export_source=%%export_source%%&media_id=%%media_id%%&playlist_id=%%playlist_id%%&transmission_id=%%transmission_id%%' -generate_range_url = 'schedule/generate_range_dp.php' +version_url = 'api/api_version.php?api_key=%%api_key%%' +export_url = 'api/schedule.php?from=%%from%%&to=%%to%%&api_key=%%api_key%%' +update_item_url = 'api/schedule.php?item_id=%%item_id%%&played=%%played%%' +update_start_playing_url = 'api/update_start_playing.php?playlist_type=%%playlist_type%%&export_source=%%export_source%%&media_id=%%media_id%%&playlist_id=%%playlist_id%%&transmission_id=%%transmission_id%%' +generate_range_url = 'api/generate_range_dp.php' ############## diff --git a/3rd_party/pypo/logging.cfg b/3rd_party/pypo/logging.cfg index da0800dd1..46f306e82 100644 --- a/3rd_party/pypo/logging.cfg +++ b/3rd_party/pypo/logging.cfg @@ -45,7 +45,7 @@ args=("/tmp/sessionlog_null.log",) [formatter_simpleFormatter] -format=%(asctime)s %(levelname)s - [%(name)s] - %(message)s +format=%(asctime)s %(levelname)s - [%(filename)s : %(funcName)s() : line %(lineno)d] - %(message)s datefmt= diff --git a/3rd_party/pypo/pypo_cli.py b/3rd_party/pypo/pypo_cli.py index 3839acd09..a499c24b9 100755 --- a/3rd_party/pypo/pypo_cli.py +++ b/3rd_party/pypo/pypo_cli.py @@ -57,6 +57,11 @@ from util import * from api_clients import * PYPO_VERSION = '0.2' +PYPO_MEDIA_SKIP = 1 +PYPO_MEDIA_LIVE_SESSION = 2 +PYPO_MEDIA_STREAM = 3 +PYPO_MEDIA_FILE_URL = 4 +PYPO_MEDIA_FILE_LOCAL = 5 # Set up command-line options parser = OptionParser() @@ -68,6 +73,7 @@ parser = OptionParser(usage=usage) # Options parser.add_option("-v", "--compat", help="Check compatibility with server API version", default=False, action="store_true", dest="check_compat") +parser.add_option("-t", "--test", help="Do a test to make sure everything is working properly.", default=False, action="store_true", dest="test") parser.add_option("-f", "--fetch-scheduler", help="Fetch from scheduler - scheduler (loop, interval in config file)", default=False, action="store_true", dest="fetch_scheduler") parser.add_option("-p", "--push-scheduler", help="Push scheduler to Liquidsoap (loop, interval in config file)", default=False, action="store_true", dest="push_scheduler") @@ -107,14 +113,8 @@ except Exception, e: #TIME = time.localtime(time.time()) TIME = (2010, 6, 26, 15, 33, 23, 2, 322, 0) -# to help with debugging - get the current line number -def lineno(): - """Returns the current function name and line number in our program.""" - return "File " +inspect.currentframe().f_code.co_filename + " / Line " + str(inspect.currentframe().f_back.f_lineno) + ": " - class Global: def __init__(self): - #print '# global initialisation' print def selfcheck(self): @@ -122,14 +122,6 @@ class Global: self.api_client = api_client.api_client_factory(config) self.api_client.check_version() - """ - Uncomment the following lines to let pypo check if - liquidsoap is running. (checks for a telnet server) - """ -# while self.status.check_ls(LS_HOST, LS_PORT) == 0: -# print 'Unable to connect to liquidsoap. Is it up and running?' -# time.sleep(2) - """ @@ -147,6 +139,9 @@ class Playout: # set initial state self.range_updated = False + + def test_api(self): + self.api_client.test() """ Fetching part of pypo @@ -182,7 +177,7 @@ class Playout: try: self.generate_range_dp() except Exception, e: - logger.error(lineno() + "%s", e) + logger.error("%s", e) # get schedule try: @@ -190,19 +185,19 @@ class Playout: logger.warning("failed to read from export url") time.sleep(1) - except Exception, e: logger.error(lineno() +"%s", e) + except Exception, e: logger.error("%s", e) # prepare the playlists if CUE_STYLE == 'pre': try: self.prepare_playlists_cue(self.export_source) - except Exception, e: logger.error(lineno() + "%s", e) + except Exception, e: logger.error("%s", e) elif CUE_STYLE == 'otf': try: self.prepare_playlists(self.export_source) - except Exception, e: logger.error(lineno() + "%s", e) + except Exception, e: logger.error("%s", e) # cleanup try: self.cleanup(self.export_source) - except Exception, e: logger.error(lineno() + "%s", e) + except Exception, e: logger.error("%s", e) logger.info("fetch loop completed") @@ -244,7 +239,7 @@ class Playout: schedule_file.close() except Exception, e: - print lineno() + e + logger.critical("Exception %s", e) status = 0 return status @@ -269,8 +264,11 @@ class Playout: except Exception, e: logger.error("%s", e) schedule = None - - #for pkey in schedule: + + # Dont do anything if schedule is empty + if (not schedule): + return + try: for pkey in sorted(schedule.iterkeys()): logger.info("found playlist at %s", pkey) @@ -295,8 +293,10 @@ class Playout: # TODO: maybe a bit more modular.. silence_file = self.file_dir + 'basic/silence.mp3' - - if int(playlist['played']) == 1: + + mediaType = api_client.get_media_type(playlist) + if (mediaType == PYPO_MEDIA_SKIP): + #if int(playlist['played']) == 1: logger.info("playlist %s already played / sent to liquidsoap, so will ignore it", pkey) elif int(playlist['subtype']) == 5: @@ -568,7 +568,7 @@ class Playout: for dir in d: timestamp = os.path.getmtime(os.path.join(r, dir)) - logger.debug('Folder "Age": %s - %s', round((((now - offset) - timestamp) / 60), 2), os.path.join(r, dir)) + logger.debug( 'Folder "Age": %s - %s', round((((now - offset) - timestamp) / 60), 2), os.path.join(r, dir)) if now - offset > timestamp: try: @@ -944,11 +944,14 @@ if __name__ == '__main__': run = True while run == True: - logger = logging.getLogger("pypo") loops = 0 + if options.test: + po.test_api() + sys.exit() + while options.fetch_scheduler: try: po.fetch('scheduler') except Exception, e: @@ -973,7 +976,6 @@ while run == True: while options.push_scheduler: - po.push('scheduler') try: po.push('scheduler') @@ -988,7 +990,6 @@ while run == True: while options.push_daypart: - po.push('daypart') try: po.push('daypart') diff --git a/api/get_media.php b/api/get_media.php index 27011d026..8b0ee9e04 100644 --- a/api/get_media.php +++ b/api/get_media.php @@ -20,12 +20,13 @@ if (PEAR::isError($CC_DBC)) { } $CC_DBC->setFetchMode(DB_FETCHMODE_ASSOC); -$file_id = $_GET["file_id"]; +$filename = $_GET["file"]; +$file_id = substr($filename, 0, strpos($filename, ".")); if (ctype_alnum($file_id) && strlen($file_id) == 32) { $media = StoredFile::RecallByGunid($file_id); if ($media != null && !PEAR::isError($media)) { //var_dump($media); - $filepath = $media->getRealFileName(); + $filepath = $media->getRealFilePath(); if(!is_file($filepath)) { header($_SERVER["SERVER_PROTOCOL"]." 404 Not Found"); diff --git a/backend/Schedule.php b/backend/Schedule.php index 9953697f1..5d120ab01 100644 --- a/backend/Schedule.php +++ b/backend/Schedule.php @@ -418,7 +418,7 @@ class Schedule { } $result = array(); - $result['status'] = array('range' => $range_dt, 'version' => 0.2); + $result['status'] = array('range' => $range_dt, 'version' => "0.2"); $result['playlists'] = $playlists; $result['check'] = 1; diff --git a/backend/StoredFile.php b/backend/StoredFile.php index fdd18fc80..31a7e20ba 100644 --- a/backend/StoredFile.php +++ b/backend/StoredFile.php @@ -953,7 +953,7 @@ class StoredFile { $values = array( "id" => $p_nid, "filename" => $p_src->name, - "filepath" => $p_src->getRealFileName(), + "filepath" => $p_src->getRealFilePath(), "filetype" => $p_src->getType() ); $storedFile = StoredFile::Insert($values); @@ -1028,7 +1028,7 @@ class StoredFile { */ public function accessRawMediaData($p_parent='0') { - $realFname = $this->getRealFileName(); + $realFname = $this->getRealFilePath(); $ext = $this->getFileExtension(); $res = BasicStor::bsAccess($realFname, $ext, $this->gunid, 'access', $p_parent); if (PEAR::isError($res)) { @@ -1679,13 +1679,17 @@ class StoredFile { // return $resDir; // } + public function getRealFileName() + { + return $this->gunid.".".$this->getFileExtension(); + } /** * Get real filename of raw media data * * @return string */ - public function getRealFileName() + public function getRealFilePath() { return $this->filepath; } @@ -1697,7 +1701,7 @@ class StoredFile { { global $CC_CONFIG; return "http://".$CC_CONFIG["storageUrlHost"] - ."api/get_media.php?file_id={$this->gunid}"; + .$CC_CONFIG["apiPath"]."get_media.php?file={$this->getRealFileName()}"; } /** diff --git a/backend/Transport.php b/backend/Transport.php index 31405e26f..e5ee6f548 100644 --- a/backend/Transport.php +++ b/backend/Transport.php @@ -300,7 +300,7 @@ class Transport return $mdtrec; } // handle raw media file: - $fpath = $storedFile->getRealFileName(); + $fpath = $storedFile->getRealFilePath(); if (PEAR::isError($fpath)) { return $fpath; } diff --git a/backend/tests/AllTests.php b/backend/tests/AllTests.php index 30316e933..5f1de6e7d 100644 --- a/backend/tests/AllTests.php +++ b/backend/tests/AllTests.php @@ -7,17 +7,16 @@ require_once(dirname(__FILE__).'/../../conf.php'); require_once('DB.php'); require_once('PHPUnit.php'); require_once 'StoredFileTests.php'; -//require_once 'SchedulerTests.php'; +require_once 'SchedulerTests.php'; //require_once 'SchedulerExportTests.php'; require_once 'PlaylistTests.php'; //$suite = new PHPUnit_TestSuite("PlayListTests"); -$suite = new PHPUnit_TestSuite("StoredFileTest"); //$suite = new PHPUnit_TestSuite("SchedulerTests"); -//$suite->addTestSuite("BasicStorTest"); -//$suite->addTestSuite("SchedulerTests"); -//$suite->addTestSuite("SchedulerExportTests"); +$suite = new PHPUnit_TestSuite("StoredFileTest"); $suite->addTestSuite("PlaylistTests"); +$suite->addTestSuite("SchedulerTests"); +//$suite->addTestSuite("SchedulerExportTests"); $result = PHPUnit::run($suite); echo $result->toString(); diff --git a/backend/tests/StoredFileTests.php b/backend/tests/StoredFileTests.php index d5fea7ee3..e2ebcca08 100644 --- a/backend/tests/StoredFileTests.php +++ b/backend/tests/StoredFileTests.php @@ -83,11 +83,5 @@ class StoredFileTest extends PHPUnit_TestCase { } - function testFoo() { - // Add a file - $values = array("filepath" => dirname(__FILE__)."/test10001.mp3"); - $this->storedFile = StoredFile::Insert($values, false); - - } } ?> \ No newline at end of file