From f0f033b4fb8bc9e5343b5156e9754d880ed1b142 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Thu, 12 Jul 2012 17:58:29 -0400 Subject: [PATCH] CC-430: Audio normalization (Replaygain Support) --- .../application/controllers/ApiController.php | 365 ++++++------------ airtime_mvc/application/models/StoredFile.php | 18 + .../api/get-files-without-replay-gain.phtml | 8 + python_apps/api_clients/api_client.cfg | 2 + python_apps/api_clients/api_client.py | 199 ++++++---- .../airtimefilemonitor/replaygain.py | 88 +++-- .../airtimefilemonitor/replaygainupdater.py | 78 ++++ 7 files changed, 401 insertions(+), 357 deletions(-) create mode 100644 airtime_mvc/application/views/scripts/api/get-files-without-replay-gain.phtml create mode 100644 python_apps/media-monitor/airtimefilemonitor/replaygainupdater.py diff --git a/airtime_mvc/application/controllers/ApiController.php b/airtime_mvc/application/controllers/ApiController.php index f6bd83ad7..8acb09dea 100644 --- a/airtime_mvc/application/controllers/ApiController.php +++ b/airtime_mvc/application/controllers/ApiController.php @@ -31,8 +31,23 @@ class ApiController extends Zend_Controller_Action ->addActionContext('check-live-stream-auth', 'json') ->addActionContext('update-source-status', 'json') ->addActionContext('get-bootstrap-info', 'json') + ->addActionContext('get-files-without-replay-gain', 'json') ->initContext(); } + + public function checkAuth() + { + global $CC_CONFIG; + + $api_key = $this->_getParam('api_key'); + + if (!in_array($api_key, $CC_CONFIG["apiKey"]) && + is_null(Zend_Auth::getInstance()->getStorage()->read())) { + header('HTTP/1.0 401 Unauthorized'); + print 'You are not allowed to access this resource.'; + exit; + } + } public function indexAction() { @@ -51,20 +66,12 @@ class ApiController extends Zend_Controller_Action */ public function versionAction() { - global $CC_CONFIG; - + $this->checkAuth(); + // disable the view and the layout $this->view->layout()->disableLayout(); $this->_helper->viewRenderer->setNoRender(true); - $api_key = $this->_getParam('api_key'); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } $jsonStr = json_encode(array("version"=>Application_Model_Preference::GetAirtimeVersion())); echo $jsonStr; } @@ -100,23 +107,12 @@ class ApiController extends Zend_Controller_Action */ public function getMediaAction() { - global $CC_CONFIG; + $this->checkAuth(); // disable the view and the layout $this->view->layout()->disableLayout(); $this->_helper->viewRenderer->setNoRender(true); - $api_key = $this->_getParam('api_key'); - - if(!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - Logging::log("401 Unauthorized"); - return; - } - $fileID = $this->_getParam("file"); $file_id = substr($fileID, 0, strpos($fileID, ".")); @@ -186,6 +182,8 @@ class ApiController extends Zend_Controller_Action */ function smartReadFile($location, $mimeType = 'audio/mp3') { + $this->checkAuth(); + $size= filesize($location); $time= date('r', filemtime($location)); @@ -343,43 +341,24 @@ class ApiController extends Zend_Controller_Action public function scheduleAction() { - global $CC_CONFIG; + $this->checkAuth(); // disable the view and the layout $this->view->layout()->disableLayout(); $this->_helper->viewRenderer->setNoRender(true); - $api_key = $this->_getParam('api_key'); - - if(!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource. '; - exit; - } - $data = Application_Model_Schedule::GetScheduledPlaylists(); echo json_encode($data, JSON_FORCE_OBJECT); } public function notifyMediaItemStartPlayAction() { - global $CC_CONFIG; + $this->checkAuth(); // disable the view and the layout $this->view->layout()->disableLayout(); $this->_helper->viewRenderer->setNoRender(true); - $api_key = $this->_getParam('api_key'); - if(!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } - $schedule_group_id = $this->_getParam("schedule_id"); $media_id = $this->_getParam("media_id"); $result = Application_Model_Schedule::UpdateMediaPlayedStatus($media_id); @@ -388,16 +367,7 @@ class ApiController extends Zend_Controller_Action public function recordedShowsAction() { - global $CC_CONFIG; - - $api_key = $this->_getParam('api_key'); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } + $this->checkAuth(); $today_timestamp = date("Y-m-d H:i:s"); $now = new DateTime($today_timestamp); @@ -422,16 +392,7 @@ class ApiController extends Zend_Controller_Action public function uploadFileAction() { - global $CC_CONFIG; - - $api_key = $this->_getParam('api_key'); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } + $this->checkAuth(); $upload_dir = ini_get("upload_tmp_dir"); $tempFilePath = Application_Model_StoredFile::uploadFile($upload_dir); @@ -447,16 +408,7 @@ class ApiController extends Zend_Controller_Action public function uploadRecordedAction() { - global $CC_CONFIG; - - $api_key = $this->_getParam('api_key'); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } + $this->checkAuth(); //this file id is the recording for this show instance. $show_instance_id = $this->_getParam('showinstanceid'); @@ -520,21 +472,12 @@ class ApiController extends Zend_Controller_Action } public function mediaMonitorSetupAction() { - global $CC_CONFIG; + $this->checkAuth(); // disable the view and the layout $this->view->layout()->disableLayout(); $this->_helper->viewRenderer->setNoRender(true); - $api_key = $this->_getParam('api_key'); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } - $this->view->stor = Application_Model_MusicDir::getStorDir()->getDirectory(); $watchedDirs = Application_Model_MusicDir::getWatchedDirs(); @@ -546,17 +489,9 @@ class ApiController extends Zend_Controller_Action } public function reloadMetadataAction() { - global $CC_CONFIG; + $this->checkAuth(); $request = $this->getRequest(); - $api_key = $request->getParam('api_key'); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } $mode = $request->getParam('mode'); $params = $request->getParams(); @@ -650,34 +585,18 @@ class ApiController extends Zend_Controller_Action } public function listAllFilesAction() { - global $CC_CONFIG; + $this->checkAuth(); $request = $this->getRequest(); - $api_key = $request->getParam('api_key'); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } $dir_id = $request->getParam('dir_id'); $this->view->files = Application_Model_StoredFile::listAllFiles($dir_id); } public function listAllWatchedDirsAction() { - global $CC_CONFIG; + $this->checkAuth(); $request = $this->getRequest(); - $api_key = $request->getParam('api_key'); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } $result = array(); @@ -694,91 +613,47 @@ class ApiController extends Zend_Controller_Action } public function addWatchedDirAction() { - global $CC_CONFIG; + $this->checkAuth(); $request = $this->getRequest(); - $api_key = $request->getParam('api_key'); $path = base64_decode($request->getParam('path')); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } - $this->view->msg = Application_Model_MusicDir::addWatchedDir($path); } public function removeWatchedDirAction() { - global $CC_CONFIG; + $this->checkAuth(); $request = $this->getRequest(); - $api_key = $request->getParam('api_key'); $path = base64_decode($request->getParam('path')); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } - $this->view->msg = Application_Model_MusicDir::removeWatchedDir($path); } public function setStorageDirAction() { - global $CC_CONFIG; + $this->checkAuth(); $request = $this->getRequest(); - $api_key = $request->getParam('api_key'); $path = base64_decode($request->getParam('path')); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } - $this->view->msg = Application_Model_MusicDir::setStorDir($path); } public function getStreamSettingAction() { - global $CC_CONFIG; + $this->checkAuth(); $request = $this->getRequest(); - $api_key = $request->getParam('api_key'); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } $info = Application_Model_StreamSetting::getStreamSetting(); $this->view->msg = $info; } public function statusAction() { - global $CC_CONFIG; - + $this->checkAuth(); + $request = $this->getRequest(); - $api_key = $request->getParam('api_key'); $getDiskInfo = $request->getParam('diskinfo') == "true"; - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } - $status = array( "platform"=>Application_Model_Systemstatus::GetPlatformInfo(), "airtime_version"=>Application_Model_Preference::GetAirtimeVersion(), @@ -844,92 +719,76 @@ class ApiController extends Zend_Controller_Action // handles addition/deletion of mount point which watched dirs reside public function updateFileSystemMountAction(){ - global $CC_CONFIG; + $this->checkAuth(); $request = $this->getRequest(); - $api_key = $request->getParam('api_key'); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } $params = $request->getParams(); $added_list = empty($params['added_dir'])?array():explode(',',$params['added_dir']); $removed_list = empty($params['removed_dir'])?array():explode(',',$params['removed_dir']); // get all watched dirs - $watched_dirs = Application_Model_MusicDir::getWatchedDirs(null,null); + $watched_dirs = Application_Model_MusicDir::getWatchedDirs(null, null); - foreach( $added_list as $ad){ - $ad .= '/'; - foreach( $watched_dirs as $dir ){ - $dirPath = $dir->getDirectory(); + foreach ($added_list as $ad) { + $ad .= '/'; + foreach ($watched_dirs as $dir) { + $dirPath = $dir->getDirectory(); - // if mount path itself was watched - if($dirPath == $ad){ - Application_Model_MusicDir::addWatchedDir($dirPath, false); - } + // if mount path itself was watched + if ($dirPath == $ad) { + Application_Model_MusicDir::addWatchedDir($dirPath, false); + } else if(substr($dirPath, 0, strlen($ad)) === $ad && $dir->getExistsFlag() == false) { // if dir contains any dir in removed_list( if watched dir resides on new mounted path ) - else if(substr($dirPath, 0, strlen($ad)) === $ad && $dir->getExistsFlag() == false){ - Application_Model_MusicDir::addWatchedDir($dirPath, false); - } + Application_Model_MusicDir::addWatchedDir($dirPath, false); + } else if (substr($ad, 0, strlen($dirPath)) === $dirPath) { // is new mount point within the watched dir? // pyinotify doesn't notify anyhing in this case, so we add this mount point as // watched dir - else if(substr($ad, 0, strlen($dirPath)) === $dirPath){ - // bypass nested loop check - Application_Model_MusicDir::addWatchedDir($ad, false, true); - } + // bypass nested loop check + Application_Model_MusicDir::addWatchedDir($ad, false, true); } } - foreach( $removed_list as $rd){ - $rd .= '/'; - foreach( $watched_dirs as $dir ){ - $dirPath = $dir->getDirectory(); - // if dir contains any dir in removed_list( if watched dir resides on new mounted path ) - if(substr($dirPath, 0, strlen($rd)) === $rd && $dir->getExistsFlag() == true){ - Application_Model_MusicDir::removeWatchedDir($dirPath, false); - } + } + + foreach( $removed_list as $rd) { + $rd .= '/'; + foreach ($watched_dirs as $dir) { + $dirPath = $dir->getDirectory(); + // if dir contains any dir in removed_list( if watched dir resides on new mounted path ) + if (substr($dirPath, 0, strlen($rd)) === $rd && $dir->getExistsFlag() == true) { + Application_Model_MusicDir::removeWatchedDir($dirPath, false); + } else if (substr($rd, 0, strlen($dirPath)) === $dirPath) { // is new mount point within the watched dir? // pyinotify doesn't notify anyhing in this case, so we walk through all files within // this watched dir in DB and mark them deleted. // In case of h) of use cases, due to pyinotify behaviour of noticing mounted dir, we need to // compare agaisnt all files in cc_files table - else if(substr($rd, 0, strlen($dirPath)) === $dirPath ){ - $watchDir = Application_Model_MusicDir::getDirByPath($rd); - // get all the files that is under $dirPath - $files = Application_Model_StoredFile::listAllFiles($dir->getId(), true); - foreach($files as $f){ - // if the file is from this mount - if(substr( $f->getFilePath(),0,strlen($rd) ) === $rd){ - $f->delete(); - } - } - if($watchDir){ - Application_Model_MusicDir::removeWatchedDir($rd, false); + + $watchDir = Application_Model_MusicDir::getDirByPath($rd); + // get all the files that is under $dirPath + $files = Application_Model_StoredFile::listAllFiles($dir->getId(), true); + foreach ($files as $f) { + // if the file is from this mount + if (substr( $f->getFilePath(),0,strlen($rd) ) === $rd) { + $f->delete(); } } + + if($watchDir) { + Application_Model_MusicDir::removeWatchedDir($rd, false); + } } } - + } } // handles case where watched dir is missing - public function handleWatchedDirMissingAction(){ - global $CC_CONFIG; + public function handleWatchedDirMissingAction() + { + $this->checkAuth(); $request = $this->getRequest(); - $api_key = $request->getParam('api_key'); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } $dir = base64_decode($request->getParam('dir')); Application_Model_MusicDir::removeWatchedDir($dir, false); @@ -938,24 +797,18 @@ class ApiController extends Zend_Controller_Action /* This action is for use by our dev scripts, that make * a change to the database and we want rabbitmq to send * out a message to pypo that a potential change has been made. */ - public function rabbitmqDoPushAction(){ - global $CC_CONFIG; + public function rabbitmqDoPushAction() + { + $this->checkAuth(); $request = $this->getRequest(); - $api_key = $request->getParam('api_key'); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } Logging::log("Notifying RabbitMQ to send message to pypo"); Application_Model_RabbitMq::PushSchedule(); } - public function getBootstrapInfoAction(){ + public function getBootstrapInfoAction() + { $live_dj = Application_Model_Preference::GetSourceSwitchStatus('live_dj'); $master_dj = Application_Model_Preference::GetSourceSwitchStatus('master_dj'); $scheduled_play = Application_Model_Preference::GetSourceSwitchStatus('scheduled_play'); @@ -968,36 +821,29 @@ class ApiController extends Zend_Controller_Action } /* This is used but Liquidsoap to check authentication of live streams*/ - public function checkLiveStreamAuthAction(){ - global $CC_CONFIG; + public function checkLiveStreamAuthAction() + { + $this->checkAuth(); $request = $this->getRequest(); - $api_key = $request->getParam('api_key'); $username = $request->getParam('username'); $password = $request->getParam('password'); $djtype = $request->getParam('djtype'); - if (!in_array($api_key, $CC_CONFIG["apiKey"]) && - is_null(Zend_Auth::getInstance()->getStorage()->read())) - { - header('HTTP/1.0 401 Unauthorized'); - print 'You are not allowed to access this resource.'; - exit; - } - - if($djtype == 'master'){ + if ($djtype == 'master') { //check against master - if($username == Application_Model_Preference::GetLiveSteamMasterUsername() && $password == Application_Model_Preference::GetLiveSteamMasterPassword()){ + if ($username == Application_Model_Preference::GetLiveSteamMasterUsername() + && $password == Application_Model_Preference::GetLiveSteamMasterPassword()) { $this->view->msg = true; - }else{ + } else { $this->view->msg = false; } - }elseif($djtype == "dj"){ + } elseif ($djtype == "dj") { //check against show dj auth $showInfo = Application_Model_Show::GetCurrentShow(); // there is current playing show - if(isset($showInfo[0]['id'])){ + if (isset($showInfo[0]['id'])) { $current_show_id = $showInfo[0]['id']; $CcShow = CcShowQuery::create()->findPK($current_show_id); @@ -1010,31 +856,46 @@ class ApiController extends Zend_Controller_Action $hosts_ids = $show->getHostsIds(); // check against hosts auth - if($CcShow->getDbLiveStreamUsingAirtimeAuth()){ - foreach( $hosts_ids as $host){ + if ($CcShow->getDbLiveStreamUsingAirtimeAuth()) { + foreach ($hosts_ids as $host) { $h = new Application_Model_User($host['subjs_id']); - if($username == $h->getLogin() && md5($password) == $h->getPassword()){ + if($username == $h->getLogin() && md5($password) == $h->getPassword()) { $this->view->msg = true; return; } } } // check against custom auth - if($CcShow->getDbLiveStreamUsingCustomAuth()){ - if($username == $custom_user && $password == $custom_pass){ + if ($CcShow->getDbLiveStreamUsingCustomAuth()) { + if ($username == $custom_user && $password == $custom_pass) { $this->view->msg = true; - }else{ + } else { $this->view->msg = false; } - } - else{ + } else { $this->view->msg = false; } - }else{ + } else { // no show is currently playing $this->view->msg = false; } } } + + /* This action is for use by our dev scripts, that make + * a change to the database and we want rabbitmq to send + * out a message to pypo that a potential change has been made. */ + public function getFilesWithoutReplayGainAction() + { + $this->checkAuth(); + + // disable the view and the layout + $this->view->layout()->disableLayout(); + $dir_id = $this->_getParam('dir_id'); + + //connect to db and get get sql + $this->view->rows = Application_Model_StoredFile::listAllFiles2($dir_id, 0); + + } } diff --git a/airtime_mvc/application/models/StoredFile.php b/airtime_mvc/application/models/StoredFile.php index b6699bce1..8dd20cbd8 100644 --- a/airtime_mvc/application/models/StoredFile.php +++ b/airtime_mvc/application/models/StoredFile.php @@ -960,6 +960,24 @@ Logging::log("getting media! - 2"); return $results; } + + //TODO: MERGE THIS FUNCTION AND "listAllFiles" -MK + public static function listAllFiles2($dir_id=null, $limit=null) + { + $con = Propel::getConnection(); + + $sql = "SELECT id, filepath as fp" + ." FROM CC_FILES" + ." WHERE directory = $dir_id" + ." AND file_exists = 'TRUE'"; + + if (!is_null($limit) && is_int($limit)){ + $sql .= " LIMIT $limit"; + } + + $rows = $con->query($sql, PDO::FETCH_ASSOC); + return $rows; + } /* Gets number of tracks uploaded to * Soundcloud in the last 24 hours diff --git a/airtime_mvc/application/views/scripts/api/get-files-without-replay-gain.phtml b/airtime_mvc/application/views/scripts/api/get-files-without-replay-gain.phtml new file mode 100644 index 000000000..9d38517bd --- /dev/null +++ b/airtime_mvc/application/views/scripts/api/get-files-without-replay-gain.phtml @@ -0,0 +1,8 @@ +rows as $row) { + echo json_encode($row)."\n"; +} diff --git a/python_apps/api_clients/api_client.cfg b/python_apps/api_clients/api_client.cfg index 2ff113b9c..a3c04be94 100644 --- a/python_apps/api_clients/api_client.cfg +++ b/python_apps/api_clients/api_client.cfg @@ -109,3 +109,5 @@ update_source_status = 'update-source-status/format/json/api_key/%%api_key%%/sou get_bootstrap_info = 'get-bootstrap-info/format/json/api_key/%%api_key%%' +get-files-without-replay-gain = 'get-files-without-replay-gain/api_key/%%api_key%%/dir_id/%%dir_id%%'' + diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index 1d006b2fe..f3a5ad215 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -54,11 +54,11 @@ class AirTimeApiClient(): except Exception, e: self.logger.error('Error loading config file: %s', e) sys.exit(1) - + def get_response_from_server(self, url): logger = self.logger successful_response = False - + while not successful_response: try: response = urllib2.urlopen(url).read() @@ -68,22 +68,58 @@ class AirTimeApiClient(): except Exception, e: logger.error('Couldn\'t connect to remote server. Is it running?') logger.error("%s" % e) - + if not successful_response: logger.error("Error connecting to server, waiting 5 seconds and trying again.") time.sleep(5) - - return response - - def __get_airtime_version(self, verbose = True): + return response + + def get_response_into_file(self, url, block=True): + """ + This function will query the server and download its response directly + into a temporary file. This is useful in the situation where the response + from the server can be huge and we don't want to store it into memory (potentially + causing Python to use hundreds of MB's of memory). By writing into a file we can + then open this file later, and read data a little bit at a time and be very mem + efficient. + + The return value of this function is the path of the temporary file. Unless specified using + block = False, this function will block until a successful HTTP 200 response is received. + """ + + logger = self.logger + successful_response = False + + while not successful_response: + try: + path = urllib.urlretrieve(url)[0] + successful_response = True + except IOError, e: + logger.error('Error Authenticating with remote server: %s', e) + if not block: + raise + except Exception, e: + logger.error('Couldn\'t connect to remote server. Is it running?') + logger.error("%s" % e) + if not block: + raise + + if not successful_response: + logger.error("Error connecting to server, waiting 5 seconds and trying again.") + time.sleep(5) + + return path + + + + def __get_airtime_version(self): logger = self.logger url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["version_url"]) logger.debug("Trying to contact %s", url) url = url.replace("%%api_key%%", self.config["api_key"]) version = -1 - response = None try: data = self.get_response_from_server(url) logger.debug("Data: %s", data) @@ -98,13 +134,13 @@ class AirTimeApiClient(): def test(self): logger = self.logger - status, items = self.get_schedule('2010-01-01-00-00-00', '2011-01-01-00-00-00') + items = self.get_schedule()[1] schedule = items["playlists"] logger.debug("Number of playlists found: %s", str(len(schedule))) count = 1 for pkey in sorted(schedule.iterkeys()): - logger.debug("Playlist #%s",str(count)) - count+=1 + logger.debug("Playlist #%s", str(count)) + count += 1 playlist = schedule[pkey] for item in playlist["medias"]: filename = urlparse(item["uri"]) @@ -112,9 +148,9 @@ class AirTimeApiClient(): self.get_media(item["uri"], filename) - def is_server_compatible(self, verbose = True): + def is_server_compatible(self, verbose=True): logger = self.logger - version = self.__get_airtime_version(verbose) + version = self.__get_airtime_version() if (version == -1): if (verbose): logger.info('Unable to get Airtime version number.\n') @@ -122,16 +158,16 @@ class AirTimeApiClient(): elif (version[0:3] != AIRTIME_VERSION[0:3]): if (verbose): logger.info('Airtime version found: ' + str(version)) - logger.info('pypo is at version ' +AIRTIME_VERSION+' and is not compatible with this version of Airtime.\n') + logger.info('pypo is at version ' + AIRTIME_VERSION + ' and is not compatible with this version of Airtime.\n') return False else: if (verbose): logger.info('Airtime version: ' + str(version)) - logger.info('pypo is at version ' +AIRTIME_VERSION+' and is compatible with this version of Airtime.') + logger.info('pypo is at version ' + AIRTIME_VERSION + ' and is compatible with this version of Airtime.') return True - def get_schedule(self, start=None, end=None): + def get_schedule(self): logger = self.logger # Construct the URL @@ -160,7 +196,7 @@ class AirTimeApiClient(): logger.info("try to download from %s to %s", src, dst) src = src.replace("%%api_key%%", self.config["api_key"]) # check if file exists already before downloading again - filename, headers = urllib.urlretrieve(src, dst) + headers = urllib.urlretrieve(src, dst)[1] logger.info(headers) except Exception, e: logger.error("%s", e) @@ -180,7 +216,7 @@ class AirTimeApiClient(): url = url.replace("%%schedule_id%%", str(schedule_id)) logger.debug(url) url = url.replace("%%api_key%%", self.config["api_key"]) - + response = self.get_response_from_server(url) response = json.loads(response) logger.info("API-Status %s", response['status']) @@ -192,12 +228,11 @@ class AirTimeApiClient(): return response def get_liquidsoap_data(self, pkey, schedule): - logger = self.logger playlist = schedule[pkey] data = dict() try: data["schedule_id"] = playlist['id'] - except Exception, e: + except Exception: data["schedule_id"] = 0 return data @@ -232,7 +267,7 @@ class AirTimeApiClient(): url = url.replace("%%api_key%%", self.config["api_key"]) for i in range(0, retries): - logger.debug("Upload attempt: %s", i+1) + logger.debug("Upload attempt: %s", i + 1) try: request = urllib2.Request(url, data, headers) @@ -252,27 +287,29 @@ class AirTimeApiClient(): time.sleep(retries_wait) return response - + def check_live_stream_auth(self, username, password, dj_type): - #logger = logging.getLogger() + """ + TODO: Why are we using print statements here? Possibly use logger that + is directed to stdout. -MK + """ + response = '' try: url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["check_live_stream_auth"]) - + url = url.replace("%%api_key%%", self.config["api_key"]) url = url.replace("%%username%%", username) url = url.replace("%%djtype%%", dj_type) url = url.replace("%%password%%", password) - + response = self.get_response_from_server(url) response = json.loads(response) except Exception, e: - import traceback - top = traceback.format_exc() print "Exception: %s", e - print "traceback: %s", top + print "traceback: %s", traceback.format_exc() response = None - + return response def setup_media_monitor(self): @@ -282,7 +319,7 @@ class AirTimeApiClient(): try: url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["media_setup_url"]) url = url.replace("%%api_key%%", self.config["api_key"]) - + response = self.get_response_from_server(url) response = json.loads(response) logger.info("Connected to Airtime Server. Json Media Storage Dir: %s", response) @@ -299,9 +336,9 @@ class AirTimeApiClient(): url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["update_media_url"]) url = url.replace("%%api_key%%", self.config["api_key"]) url = url.replace("%%mode%%", mode) - + md = convert_dict_value_to_utf8(md) - + data = urllib.urlencode(md) req = urllib2.Request(url, data) @@ -320,10 +357,8 @@ class AirTimeApiClient(): logger.info("associate recorded %s", response) except Exception, e: response = None - import traceback - top = traceback.format_exc() logger.error('Exception: %s', e) - logger.error("traceback: %s", top) + logger.error("traceback: %s", traceback.format_exc()) return response @@ -338,7 +373,7 @@ class AirTimeApiClient(): url = url.replace("%%api_key%%", self.config["api_key"]) url = url.replace("%%dir_id%%", dir_id) - + response = self.get_response_from_server(url) response = json.loads(response) except Exception, e: @@ -353,7 +388,7 @@ class AirTimeApiClient(): url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["list_all_watched_dirs"]) url = url.replace("%%api_key%%", self.config["api_key"]) - + response = self.get_response_from_server(url) response = json.loads(response) except Exception, e: @@ -369,7 +404,7 @@ class AirTimeApiClient(): url = url.replace("%%api_key%%", self.config["api_key"]) url = url.replace("%%path%%", base64.b64encode(path)) - + response = self.get_response_from_server(url) response = json.loads(response) except Exception, e: @@ -385,7 +420,7 @@ class AirTimeApiClient(): url = url.replace("%%api_key%%", self.config["api_key"]) url = url.replace("%%path%%", base64.b64encode(path)) - + response = self.get_response_from_server(url) response = json.loads(response) except Exception, e: @@ -409,13 +444,13 @@ class AirTimeApiClient(): logger.error("Exception: %s", e) return response - + def get_stream_setting(self): logger = self.logger try: url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["get_stream_setting"]) - - url = url.replace("%%api_key%%", self.config["api_key"]) + + url = url.replace("%%api_key%%", self.config["api_key"]) response = self.get_response_from_server(url) response = json.loads(response) except Exception, e: @@ -434,42 +469,42 @@ class AirTimeApiClient(): logger = self.logger try: url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["register_component"]) - + url = url.replace("%%api_key%%", self.config["api_key"]) url = url.replace("%%component%%", component) self.get_response_from_server(url) except Exception, e: logger.error("Exception: %s", e) - + def notify_liquidsoap_status(self, msg, stream_id, time): logger = self.logger try: url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["update_liquidsoap_status"]) - + url = url.replace("%%api_key%%", self.config["api_key"]) msg = msg.replace('/', ' ') encoded_msg = urllib.quote(msg, '') url = url.replace("%%msg%%", encoded_msg) url = url.replace("%%stream_id%%", stream_id) url = url.replace("%%boot_time%%", time) - - response = self.get_response_from_server(url) + + self.get_response_from_server(url) except Exception, e: logger.error("Exception: %s", e) - + def notify_source_status(self, sourcename, status): logger = self.logger try: url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["update_source_status"]) - + url = url.replace("%%api_key%%", self.config["api_key"]) url = url.replace("%%sourcename%%", sourcename) url = url.replace("%%status%%", status) - - response = self.get_response_from_server(url) + + self.get_response_from_server(url) except Exception, e: logger.error("Exception: %s", e) - + """ This function updates status of mounted file system information on airtime """ @@ -477,26 +512,24 @@ class AirTimeApiClient(): logger = self.logger try: url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["update_fs_mount"]) - + url = url.replace("%%api_key%%", self.config["api_key"]) added_data_string = string.join(added_dir, ',') removed_data_string = string.join(removed_dir, ',') - - map = [("added_dir", added_data_string),("removed_dir",removed_data_string)] - + + map = [("added_dir", added_data_string), ("removed_dir", removed_data_string)] + data = urllib.urlencode(map) - + req = urllib2.Request(url, data) response = self.get_response_from_server(req) - + logger.info("update file system mount: %s", json.loads(response)) except Exception, e: - import traceback - top = traceback.format_exc() logger.error('Exception: %s', e) - logger.error("traceback: %s", top) - + logger.error("traceback: %s", traceback.format_exc()) + """ When watched dir is missing(unplugged or something) on boot up, this function will get called and will call appropriate function on Airtime. @@ -505,18 +538,16 @@ class AirTimeApiClient(): logger = self.logger try: url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["handle_watched_dir_missing"]) - + url = url.replace("%%api_key%%", self.config["api_key"]) url = url.replace("%%dir%%", base64.b64encode(dir)) - + response = self.get_response_from_server(url) logger.info("update file system mount: %s", json.loads(response)) except Exception, e: - import traceback - top = traceback.format_exc() logger.error('Exception: %s', e) - logger.error("traceback: %s", top) - + logger.error("traceback: %s", traceback.format_exc()) + """ Retrive infomations needed on bootstrap time """ @@ -524,17 +555,37 @@ class AirTimeApiClient(): logger = self.logger try: url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["get_bootstrap_info"]) - + url = url.replace("%%api_key%%", self.config["api_key"]) - + response = self.get_response_from_server(url) response = json.loads(response) logger.info("Bootstrap info retrieved %s", response) except Exception, e: response = None - import traceback - top = traceback.format_exc() logger.error('Exception: %s', e) - logger.error("traceback: %s", top) + logger.error("traceback: %s", traceback.format_exc()) return response - + + def get_files_without_replay_gain_value(self, dir_id): + """ + Download a list of files that need to have their ReplayGain value calculated. This list + of files is downloaded into a file and the path to this file is the return value. + """ + + #http://localhost/api/get-files-without-replay-gain/dir_id/1 + + logger = self.logger + try: + url = "http://%(base_url)s:%(base_port)s/%(api_base)s/%(get-files-without-replay-gain)s/" % (self.config) + url = url.replace("%%api_key%%", self.config["api_key"]) + url = url.replace("%%dir_id%%", dir_id) + + file_path = self.get_response_into_file(url) + except Exception, e: + file_path = None + logger.error('Exception: %s', e) + logger.error("traceback: %s", traceback.format_exc()) + + return file_path + diff --git a/python_apps/media-monitor/airtimefilemonitor/replaygain.py b/python_apps/media-monitor/airtimefilemonitor/replaygain.py index 8207deaa8..bcb9cf6a7 100644 --- a/python_apps/media-monitor/airtimefilemonitor/replaygain.py +++ b/python_apps/media-monitor/airtimefilemonitor/replaygain.py @@ -2,6 +2,8 @@ from subprocess import Popen, PIPE import re import os import sys +import shutil +import tempfile def get_process_output(command): """ @@ -26,45 +28,69 @@ def get_mime_type(file_path): return get_process_output("timeout 5 file -b --mime-type %s" % file_path) +def duplicate_file(file_path): + """ + Makes a duplicate of the file and returns the path of this duplicate file. + """ + fsrc = open(file_path, 'r') + fdst = tempfile.NamedTemporaryFile(delete=False) + + print "Copying %s to %s" % (file_path, fdst.name) + + shutil.copyfileobj(fsrc, fdst) + + fsrc.close() + fdst.close() + + return fdst.name + def calculate_replay_gain(file_path): """ This function accepts files of type mp3/ogg/flac and returns a calculated ReplayGain value in dB. If the value cannot be calculated for some reason, then we default to 0 (Unity Gain). - - TODO: - Currently some of the subprocesses called will actually insert metadata into the file itself, - which we do *not* want as this changes the file's hash. Need to make a copy of the file before - we run this function. - + http://wiki.hydrogenaudio.org/index.php?title=ReplayGain_1.0_specification """ - search = None - if re.search(r'mp3$', file_path, re.IGNORECASE) or get_mime_type(file_path) == "audio/mpeg": - if run_process("which mp3gain > /dev/null") == 0: - out = get_process_output('mp3gain -q "%s" 2> /dev/null' % file_path) - search = re.search(r'Recommended "Track" dB change: (.*)', out) + try: + """ + Making a duplicate is required because the ReplayGain extraction utilities we use + make unwanted modifications to the file. + """ + + search = None + temp_file_path = duplicate_file(file_path) + + if re.search(r'mp3$', temp_file_path, re.IGNORECASE) or get_mime_type(temp_file_path) == "audio/mpeg": + if run_process("which mp3gain > /dev/null") == 0: + out = get_process_output('mp3gain -q "%s" 2> /dev/null' % temp_file_path) + search = re.search(r'Recommended "Track" dB change: (.*)', out) + else: + print "mp3gain not found" + #Log warning + elif re.search(r'ogg$', temp_file_path, re.IGNORECASE) or get_mime_type(temp_file_path) == "application/ogg": + if run_process("which vorbisgain > /dev/null && which ogginfo > /dev/null") == 0: + run_process('vorbisgain -q -f "%s" 2>/dev/null >/dev/null' % temp_file_path) + out = get_process_output('ogginfo "%s"' % temp_file_path) + search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out) + else: + print "vorbisgain/ogginfo not found" + #Log warning + elif re.search(r'flac$', temp_file_path, re.IGNORECASE) or get_mime_type(temp_file_path) == "audio/x-flac": + if run_process("which metaflac > /dev/null") == 0: + out = get_process_output('metaflac --show-tag=REPLAYGAIN_TRACK_GAIN "%s"' % temp_file_path) + search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out) + else: + print "metaflac not found" + #Log warning else: - print "mp3gain not found" - #Log warning - elif re.search(r'ogg$', file_path, re.IGNORECASE) or get_mime_type(file_path) == "application/ogg": - if run_process("which vorbisgain > /dev/null && which ogginfo > /dev/null") == 0: - run_process('vorbisgain -q -f "%s" 2>/dev/null >/dev/null' % file_path) - out = get_process_output('ogginfo "%s"' % file_path) - search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out) - else: - print "vorbisgain/ogginfo not found" - #Log warning - elif re.search(r'flac$', file_path, re.IGNORECASE) or get_mime_type(file_path) == "audio/x-flac": - if run_process("which metaflac > /dev/null") == 0: - out = get_process_output('metaflac --show-tag=REPLAYGAIN_TRACK_GAIN "%s"' % file_path) - search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out) - else: - print "metaflac not found" - #Log warning - else: - pass - #Log unknown file type. + pass + #Log unknown file type. + + #no longer need the temp, file simply remove it. + os.remove(temp_file_path) + except Exception, e: + print e replay_gain = 0 if search: diff --git a/python_apps/media-monitor/airtimefilemonitor/replaygainupdater.py b/python_apps/media-monitor/airtimefilemonitor/replaygainupdater.py new file mode 100644 index 000000000..48d73f58f --- /dev/null +++ b/python_apps/media-monitor/airtimefilemonitor/replaygainupdater.py @@ -0,0 +1,78 @@ +from threading import Thread + +import traceback +import os +import logging +import json + +from api_clients import api_client +import replaygain + + +class ReplayGainUpdater(Thread): + """ + The purpose of the class is to query the server for a list of files which do not have a ReplayGain + value calculated. This class will iterate over the list calculate the values, update the server and + repeat the process until the the server reports there are no files left. + + This class will see heavy activity right after a 2.1->2.2 upgrade since 2.2 introduces ReplayGain + normalization. A fresh install of Airtime 2.2 will see this class not used at all since a file + imported in 2.2 will automatically have its ReplayGain value calculated. + """ + + def __init__(self, logger): + Thread.__init__(self) + self.logger = logger + self.api_client = api_client.AirTimeApiClient() + + def main(self): + + #TODO + directories = self.api_client.list_all_watched_dirs()['dirs'] + + for dir_id, dir_path in directories.iteritems(): + try: + processed_data = [] + + #keep getting 100 rows at a time for current music_dir (stor or watched folder). + #When we get a response with 0 rows, then we will set response to True. + finished = False + + while not finished: + # return a list of pairs where the first value is the file's database row id + # and the second value is the filepath + file_path = self.api_client.get_files_without_replay_gain_value(dir_id) + print "temp file saved to %s" % file_path + + num_lines = 0 + + with open(file_path) as f: + for line in f: + num_lines += 1 + data = json.loads(line.strip()) + track_path = os.path.join(dir_path, data['fp']) + processed_data.append((data['id'], replaygain.calculate_replay_gain(track_path))) + + if num_lines == 0: + finished = True + + os.remove(file_path) + + #send data here + pass + except Exception, e: + print e + + def run(self): + try: self.main() + except Exception, e: + self.logger.error('ReplayGainUpdater Exception: %s', traceback.format_exc()) + self.logger.error(e) + +if __name__ == "__main__": + try: + rgu = ReplayGainUpdater(logging) + print rgu.main() + except Exception, e: + print e + print traceback.format_exc()