From d9813f21a2c4181c968e416147206033e9675f25 Mon Sep 17 00:00:00 2001 From: James Moon Date: Tue, 10 May 2011 14:10:46 -0700 Subject: [PATCH 01/14] CC-2209:Put database install stuff into separate script Note: Please carefully review the changes as I'm not sure if I did it correctly. I tested it and it installed fine in both cases(without an option and with option -n followed by db-install script. --- install/airtime-db-install.php | 23 +++++++++++++++++++++++ install/airtime-install.php | 23 +++++++++++++++-------- 2 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 install/airtime-db-install.php diff --git a/install/airtime-db-install.php b/install/airtime-db-install.php new file mode 100644 index 000000000..18db57f3b --- /dev/null +++ b/install/airtime-db-install.php @@ -0,0 +1,23 @@ + 'Displays usage information.', 'overwrite|o' => 'Overwrite any existing config files.', - 'preserve|p' => 'Keep any existing config files.' + 'preserve|p' => 'Keep any existing config files.', + 'no-db|n' => 'Turn off database install.' ) ); $opts->parse(); @@ -34,6 +35,10 @@ if (isset($opts->h)) { echo $opts->getUsageMessage(); exit; } +$db_install = true; +if (isset($opts->n)){ + $db_install = false; +} $overwrite = false; if (isset($opts->o)) { @@ -72,17 +77,21 @@ require_once(AirtimeInstall::GetAirtimeSrcDir().'/application/configs/conf.php') echo "* Airtime Version: ".AIRTIME_VERSION.PHP_EOL; +if ($db_install) { + //echo PHP_EOL."*** Database Installation ***".PHP_EOL; -AirtimeInstall::CreateDatabaseUser(); +/* AirtimeInstall::CreateDatabaseUser(); -AirtimeInstall::CreateDatabase(); + AirtimeInstall::CreateDatabase(); -AirtimeInstall::DbConnect(true); + AirtimeInstall::DbConnect(true); -AirtimeInstall::InstallPostgresScriptingLanguage(); + AirtimeInstall::InstallPostgresScriptingLanguage(); -AirtimeInstall::CreateDatabaseTables(); + AirtimeInstall::CreateDatabaseTables();*/ + require( 'airtime-db-install.php' ); +} AirtimeInstall::InstallStorageDirectory(); @@ -98,7 +107,5 @@ system("python ".__DIR__."/../python_apps/pypo/install/pypo-install.py"); echo PHP_EOL."*** Recorder Installation ***".PHP_EOL; system("python ".__DIR__."/../python_apps/show-recorder/install/recorder-install.py"); -AirtimeInstall::SetAirtimeVersion(AIRTIME_VERSION); - echo "******************************* Install Complete *******************************".PHP_EOL; From c5ce86bd407d09cac559738090ba4232c5158b4e Mon Sep 17 00:00:00 2001 From: Naomi Date: Mon, 25 Apr 2011 12:49:01 -0400 Subject: [PATCH 02/14] CC-1799 : Live Studio Playout from media library (pytagsfs) set up daemontools process for the media monitor script. --- .../application/controllers/ApiController.php | 13 ++ install/airtime-install.php | 3 + install/airtime-uninstall.php | 4 + install/include/AirtimeIni.php | 13 +- python_apps/api_clients/api_client.py | 21 +++ python_apps/pytag-fs/MediaMonitor.cfg | 8 +- python_apps/pytag-fs/MediaMonitor.py | 18 ++- .../pytag-fs/airtime-media-monitor-start | 16 +++ .../pytag-fs/airtime-media-monitor-stop | 27 ++++ .../media-monitor-daemontools-logger.sh | 2 + .../install/media-monitor-daemontools.sh | 16 +++ .../pytag-fs/install/media-monitor-install.py | 125 ++++++++++++++++++ .../install/media-monitor-uninstall.py | 58 ++++++++ 13 files changed, 315 insertions(+), 9 deletions(-) create mode 100755 python_apps/pytag-fs/airtime-media-monitor-start create mode 100755 python_apps/pytag-fs/airtime-media-monitor-stop create mode 100755 python_apps/pytag-fs/install/media-monitor-daemontools-logger.sh create mode 100755 python_apps/pytag-fs/install/media-monitor-daemontools.sh create mode 100755 python_apps/pytag-fs/install/media-monitor-install.py create mode 100755 python_apps/pytag-fs/install/media-monitor-uninstall.py diff --git a/airtime_mvc/application/controllers/ApiController.php b/airtime_mvc/application/controllers/ApiController.php index 7bd4397ee..7e0597214 100644 --- a/airtime_mvc/application/controllers/ApiController.php +++ b/airtime_mvc/application/controllers/ApiController.php @@ -316,5 +316,18 @@ class ApiController extends Zend_Controller_Action $this->view->id = $file->getId(); } + + public function reloadMetadataAction() { + + global $CC_CONFIG; + + $api_key = $this->_getParam('api_key'); + if (!in_array($api_key, $CC_CONFIG["apiKey"])) + { + header('HTTP/1.0 401 Unauthorized'); + print 'You are not allowed to access this resource.'; + exit; + } + } } diff --git a/install/airtime-install.php b/install/airtime-install.php index 51ff49f81..2349b2fa7 100644 --- a/install/airtime-install.php +++ b/install/airtime-install.php @@ -98,6 +98,9 @@ system("python ".__DIR__."/../python_apps/pypo/install/pypo-install.py"); echo PHP_EOL."*** Recorder Installation ***".PHP_EOL; system("python ".__DIR__."/../python_apps/show-recorder/install/recorder-install.py"); +echo PHP_EOL."*** Media Monitor Installation ***".PHP_EOL; +system("python ".__DIR__."/../python_apps/pytag-fs/install/media-monitor-install.py"); + AirtimeInstall::SetAirtimeVersion(AIRTIME_VERSION); echo "******************************* Install Complete *******************************".PHP_EOL; diff --git a/install/airtime-uninstall.php b/install/airtime-uninstall.php index 3da55df11..7513c874a 100644 --- a/install/airtime-uninstall.php +++ b/install/airtime-uninstall.php @@ -85,6 +85,10 @@ echo PHP_EOL."*** Uninstalling Show Recorder ***".PHP_EOL; $command = "python ".__DIR__."/../python_apps/show-recorder/install/recorder-uninstall.py"; system($command); +echo PHP_EOL."*** Uninstalling Media Monitor ***".PHP_EOL; +$command = "python ".__DIR__."/../python_apps/pytag-fs/install/media-monitor-uninstall.py"; +system($command); + #Disabled as this should be a manual process #AirtimeIni::RemoveIniFiles(); diff --git a/install/include/AirtimeIni.php b/install/include/AirtimeIni.php index 930b6260d..70c059af2 100644 --- a/install/include/AirtimeIni.php +++ b/install/include/AirtimeIni.php @@ -26,13 +26,15 @@ class AirtimeIni const CONF_FILE_PYPO = "/etc/airtime/pypo.cfg"; const CONF_FILE_RECORDER = "/etc/airtime/recorder.cfg"; const CONF_FILE_LIQUIDSOAP = "/etc/airtime/liquidsoap.cfg"; + const CONF_FILE_MEDIAMONITOR = "/etc/airtime/MediaMonitor.cfg"; public static function IniFilesExist() { $configFiles = array(AirtimeIni::CONF_FILE_AIRTIME, AirtimeIni::CONF_FILE_PYPO, AirtimeIni::CONF_FILE_RECORDER, - AirtimeIni::CONF_FILE_LIQUIDSOAP); + AirtimeIni::CONF_FILE_LIQUIDSOAP, + AirtimeIni::CONF_FILE_MEDIAMONITOR); $exist = false; foreach ($configFiles as $conf) { if (file_exists($conf)) { @@ -72,6 +74,10 @@ class AirtimeIni echo "Could not copy liquidsoap.cfg to /etc/airtime/. Exiting."; exit(1); } + if (!copy(__DIR__."/../../python_apps/pytag-fs/MediaMonitor.cfg", AirtimeIni::CONF_FILE_MEDIAMONITOR)){ + echo "Could not copy MediaMonitor.cfg to /etc/airtime/. Exiting."; + exit(1); + } } /** @@ -96,6 +102,10 @@ class AirtimeIni unlink(AirtimeIni::CONF_FILE_LIQUIDSOAP); } + if (file_exists(AirtimeIni::CONF_FILE_MEDIAMONITOR)){ + unlink(AirtimeIni::CONF_FILE_MEDIAMONITOR); + } + if (file_exists("etc/airtime")){ rmdir("/etc/airtime/"); } @@ -171,6 +181,7 @@ class AirtimeIni AirtimeIni::UpdateIniValue(AirtimeIni::CONF_FILE_AIRTIME, 'airtime_dir', AirtimeInstall::CONF_DIR_WWW); AirtimeIni::UpdateIniValue(AirtimeIni::CONF_FILE_PYPO, 'api_key', "'$api_key'"); AirtimeIni::UpdateIniValue(AirtimeIni::CONF_FILE_RECORDER, 'api_key', "'$api_key'"); + AirtimeIni::UpdateIniValue(AirtimeIni::CONF_FILE_MEDIAMONITOR, 'api_key', "'$api_key'"); AirtimeIni::UpdateIniValue(AirtimeInstall::CONF_DIR_WWW.'/build/build.properties', 'project.home', AirtimeInstall::CONF_DIR_WWW); } } diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index f551ebc5f..197e67aae 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -97,6 +97,9 @@ class ApiClientInterface: def upload_recorded_show(self): pass + + def update_media_metadata(self): + pass # Put here whatever tests you want to run to make sure your API is working def test(self): @@ -346,6 +349,24 @@ class AirTimeApiClient(ApiClientInterface): time.sleep(retries_wait) return response + + def update_media_metadata(self): + logger = logging.getLogger() + response = None + try: + url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["show_schedule_url"]) + #url = self.config["base_url"] + self.config["api_base"] + self.config["show_schedule_url"] + logger.debug(url) + url = url.replace("%%api_key%%", self.config["api_key"]) + + response = urllib.urlopen(url) + response = json.loads(response.read()) + logger.info("shows %s", response) + + except Exception, e: + logger.error("Exception: %s", e) + + return response diff --git a/python_apps/pytag-fs/MediaMonitor.cfg b/python_apps/pytag-fs/MediaMonitor.cfg index e4610b720..d32a6d4b1 100644 --- a/python_apps/pytag-fs/MediaMonitor.cfg +++ b/python_apps/pytag-fs/MediaMonitor.cfg @@ -7,11 +7,8 @@ base_port = 80 # where the binary files live bin_dir = '/usr/lib/airtime/media-monitor' -# base path to store recordered shows at -base_recorded_files = '/var/tmp/airtime/show-recorder/' - # where the logging files live -log_dir = '/var/log/airtime/show-recorder' +log_dir = '/var/log/airtime/media-monitor' # Value needed to access the API api_key = 'AAA' @@ -24,6 +21,3 @@ version_url = 'version/api_key/%%api_key%%' # URL to get the schedule of shows set to record show_schedule_url = 'recorded-shows/format/json/api_key/%%api_key%%' - -# URL to upload the recorded show's file to Airtime -upload_file_url = 'upload-recorded/format/json/api_key/%%api_key%%' diff --git a/python_apps/pytag-fs/MediaMonitor.py b/python_apps/pytag-fs/MediaMonitor.py index 2bdf148aa..c84cc3abf 100644 --- a/python_apps/pytag-fs/MediaMonitor.py +++ b/python_apps/pytag-fs/MediaMonitor.py @@ -1,7 +1,21 @@ +#!/usr/local/bin/python +import urllib +import urllib2 +import logging +import logging.config +import json +import time +import datetime import os +import sys + +from configobj import ConfigObj + import pyinotify from pyinotify import WatchManager, Notifier, ThreadedNotifier, EventsCodes, ProcessEvent +from api_clients import api_client + # configure logging try: logging.config.fileConfig("logging.cfg") @@ -11,7 +25,7 @@ except Exception, e: # loading config file try: - config = ConfigObj('/etc/airtime/recorder.cfg') + config = ConfigObj('/etc/airtime/MediaMonitor.cfg') except Exception, e: print 'Error loading config file: ', e sys.exit() @@ -42,6 +56,8 @@ class PTmp(ProcessEvent): if __name__ == '__main__': + print 'Media Monitor' + try: notifier = Notifier(wm, PTmp(), read_freq=2, timeout=1) notifier.coalesce_events() diff --git a/python_apps/pytag-fs/airtime-media-monitor-start b/python_apps/pytag-fs/airtime-media-monitor-start new file mode 100755 index 000000000..084b2b1ad --- /dev/null +++ b/python_apps/pytag-fs/airtime-media-monitor-start @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import sys + +if os.geteuid() != 0: + print "Please run this as root." + sys.exit(1) + +try: + print "Starting daemontool script recorder" + os.system("svc -u /etc/service/recorder") + +except Exception, e: + print "exception:" + str(e) diff --git a/python_apps/pytag-fs/airtime-media-monitor-stop b/python_apps/pytag-fs/airtime-media-monitor-stop new file mode 100755 index 000000000..b51b1ba6a --- /dev/null +++ b/python_apps/pytag-fs/airtime-media-monitor-stop @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import sys +import subprocess + +if os.geteuid() != 0: + print "Please run this as root." + sys.exit(1) + +try: + print "Stopping daemontool script recorder" + + p1 = subprocess.Popen(["ps", "aux"], stdout=subprocess.PIPE) + p2 = subprocess.Popen(["awk", "/recorder.py/ && !/awk/ {print $2}"], stdin=p1.stdout, stdout=subprocess.PIPE) + recorder_pid = p2.communicate()[0].strip(" \n\r\t") + if (len(recorder_pid) > 0): + os.system("svc -d /etc/service/recorder 1>/dev/null 2>&1") + os.system("svc -d /etc/service/recorder/log 1>/dev/null 2>&1") + os.system("kill -2 %s" % recorder_pid) + print "Success." + else: + print "Not Running." + +except Exception, e: + print "exception:" + str(e) diff --git a/python_apps/pytag-fs/install/media-monitor-daemontools-logger.sh b/python_apps/pytag-fs/install/media-monitor-daemontools-logger.sh new file mode 100755 index 000000000..6819f837d --- /dev/null +++ b/python_apps/pytag-fs/install/media-monitor-daemontools-logger.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exec setuidgid pypo multilog t /var/log/airtime/media-monitor/main diff --git a/python_apps/pytag-fs/install/media-monitor-daemontools.sh b/python_apps/pytag-fs/install/media-monitor-daemontools.sh new file mode 100755 index 000000000..0f6fa39e7 --- /dev/null +++ b/python_apps/pytag-fs/install/media-monitor-daemontools.sh @@ -0,0 +1,16 @@ +#!/bin/sh +media_monitor_user="pypo" + +# Location of pypo_cli.py Python script +media_monitor_path="/usr/lib/airtime/media-monitor/" +media_monitor_script="MediaMonitor.py" + +api_client_path="/usr/lib/airtime/pypo/" +cd ${media_monitor_path} + +echo "*** Daemontools: starting daemon" +exec 2>&1 +# Note the -u when calling python! we need it to get unbuffered binary stdout and stderr + +sudo PYTHONPATH=${api_client_path} -u ${media_monitor_user} python -u ${media_monitor_path}${media_monitor_script} +# EOF diff --git a/python_apps/pytag-fs/install/media-monitor-install.py b/python_apps/pytag-fs/install/media-monitor-install.py new file mode 100755 index 000000000..f0bb2adc0 --- /dev/null +++ b/python_apps/pytag-fs/install/media-monitor-install.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import time +import os +import traceback +from optparse import * +import sys +import time +import datetime +import logging +import logging.config +import shutil +import string +import platform +from configobj import ConfigObj +from subprocess import Popen, PIPE, STDOUT + +if os.geteuid() != 0: + print "Please run this as root." + sys.exit(1) + +PATH_INI_FILE = '/etc/airtime/MediaMonitor.cfg' + +def create_path(path): + if not (os.path.exists(path)): + print "Creating directory " + path + os.makedirs(path) + +def create_user(username): + print "Checking for user "+username + p = Popen('id '+username, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) + output = p.stdout.read() + if (output[0:3] != "uid"): + # Make the pypo user + print "Creating user "+username + os.system("adduser --system --quiet --group --shell /bin/bash "+username) + + #set pypo password + p = os.popen('/usr/bin/passwd pypo 1>/dev/null 2>&1', 'w') + p.write('pypo\n') + p.write('pypo\n') + p.close() + else: + print "User already exists." + #add pypo to audio group + os.system("adduser " + username + " audio 1>/dev/null 2>&1") + +def copy_dir(src_dir, dest_dir): + if (os.path.exists(dest_dir)) and (dest_dir != "/"): + print "Removing old directory "+dest_dir + shutil.rmtree(dest_dir) + if not (os.path.exists(dest_dir)): + print "Copying directory "+os.path.realpath(src_dir)+" to "+os.path.realpath(dest_dir) + shutil.copytree(src_dir, dest_dir) + +def get_current_script_dir(): + current_script_dir = os.path.realpath(__file__) + index = current_script_dir.rindex('/') + #print current_script_dir[0:index] + return current_script_dir[0:index] + + +try: + # load config file + try: + config = ConfigObj(PATH_INI_FILE) + except Exception, e: + print 'Error loading config file: ', e + sys.exit() + + current_script_dir = get_current_script_dir() + print "Checking and removing any existing media monitor processes" + os.system("python %s/media-monitor-uninstall.py 1>/dev/null 2>&1"% current_script_dir) + time.sleep(5) + + # Create users + create_user("pypo") + + print "Creating log directories" + create_path(config["log_dir"]) + os.system("chmod -R 755 " + config["log_dir"]) + os.system("chown -R pypo:pypo "+config["log_dir"]) + + copy_dir("%s/.."%current_script_dir, config["bin_dir"]) + + print "Setting permissions" + os.system("chmod -R 755 "+config["bin_dir"]) + os.system("chown -R pypo:pypo "+config["bin_dir"]) + + print "Creating symbolic links" + os.system("rm -f /usr/bin/airtime-media-monitor-start") + os.system("ln -s "+config["bin_dir"]+"/airtime-media-monitor-start /usr/bin/") + os.system("rm -f /usr/bin/airtime-media-monitor-stop") + os.system("ln -s "+config["bin_dir"]+"/airtime-media-monitor-stop /usr/bin/") + + print "Installing recorder daemon" + create_path("/etc/service/media-monitor") + create_path("/etc/service/media-monitor/log") + shutil.copy("%s/media-monitor-daemontools.sh"%current_script_dir, "/etc/service/media-monitor/run") + shutil.copy("%s/media-monitor-daemontools-logger.sh"%current_script_dir, "/etc/service/media-monitor/log/run") + os.system("chmod -R 755 /etc/service/media-monitor") + os.system("chown -R pypo:pypo /etc/service/media-monitor") + + print "Waiting for processes to start..." + time.sleep(5) + os.system("python /usr/bin/airtime-media-monitor-start") + time.sleep(2) + + found = True + + p = Popen('svstat /etc/service/media-monitor', shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) + output = p.stdout.read() + if (output.find("unable to open supervise/ok: file does not exist") >= 0): + found = False + print output + + if not found: + print "Media monitor install has completed, but daemontools is not running, please make sure you have it installed and then reboot." +except Exception, e: + print "exception:" + str(e) + sys.exit(1) + + + diff --git a/python_apps/pytag-fs/install/media-monitor-uninstall.py b/python_apps/pytag-fs/install/media-monitor-uninstall.py new file mode 100755 index 000000000..e0a5ca262 --- /dev/null +++ b/python_apps/pytag-fs/install/media-monitor-uninstall.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import sys +import time +from configobj import ConfigObj + +if os.geteuid() != 0: + print "Please run this as root." + sys.exit(1) + +PATH_INI_FILE = '/etc/airtime/MediaMonitor.cfg' + +def remove_path(path): + os.system("rm -rf " + path) + +def remove_user(username): + os.system("killall -u %s 1>/dev/null 2>&1" % username) + + #allow all process to be completely closed before we attempt to delete user + print "Waiting for processes to close..." + time.sleep(5) + + os.system("deluser --remove-home " + username + " 1>/dev/null 2>&1") + +def get_current_script_dir(): + current_script_dir = os.path.realpath(__file__) + index = current_script_dir.rindex('/') + return current_script_dir[0:index] + +try: + # load config file + try: + config = ConfigObj(PATH_INI_FILE) + except Exception, e: + print 'Error loading config file: ', e + sys.exit() + + os.system("python /usr/bin/airtime-media-monitor-stop") + + print "Removing log directories" + remove_path(config["log_dir"]) + + print "Removing symlinks" + os.system("rm -f /usr/bin/airtime-media-monitor-start") + os.system("rm -f /usr/bin/airtime-media-monitor-stop") + + print "Removing application files" + remove_path(config["bin_dir"]) + + print "Removing daemontool script media-monitor" + remove_path("rm -rf /etc/service/media-monitor") + + remove_user("pypo") + print "Uninstall complete." +except Exception, e: + print "exception:" + str(e) From a2b44689b4daed43d81fbfa277b8a8a2aa72cc65 Mon Sep 17 00:00:00 2001 From: Naomi Date: Mon, 25 Apr 2011 17:34:58 -0400 Subject: [PATCH 03/14] CC-1799 : Live Studio Playout from media library (pytagsfs) sending a file's metadata to airtime when it is modified --- .../application/controllers/ApiController.php | 11 +++++-- python_apps/api_clients/api_client.py | 14 ++++---- python_apps/pytag-fs/MediaMonitor.cfg | 4 +-- python_apps/pytag-fs/MediaMonitor.py | 32 ++++++++++++++++--- 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/airtime_mvc/application/controllers/ApiController.php b/airtime_mvc/application/controllers/ApiController.php index 7e0597214..307f116ee 100644 --- a/airtime_mvc/application/controllers/ApiController.php +++ b/airtime_mvc/application/controllers/ApiController.php @@ -10,6 +10,7 @@ class ApiController extends Zend_Controller_Action $context->addActionContext('version', 'json') ->addActionContext('recorded-shows', 'json') ->addActionContext('upload-recorded', 'json') + ->addActionContext('reload-metadata', 'json') ->initContext(); } @@ -121,7 +122,7 @@ class ApiController extends Zend_Controller_Action "nextShow"=>Show_DAL::GetNextShows($timeNow, 5), "timezone"=> date("T"), "timezoneOffset"=> date("Z")); - + //echo json_encode($result); header("Content-type: text/javascript"); echo $_GET['callback'].'('.json_encode($result).')'; @@ -314,11 +315,11 @@ class ApiController extends Zend_Controller_Action } } - $this->view->id = $file->getId(); + $this->view->id = $file->getId(); } public function reloadMetadataAction() { - + global $CC_CONFIG; $api_key = $this->_getParam('api_key'); @@ -328,6 +329,10 @@ class ApiController extends Zend_Controller_Action print 'You are not allowed to access this resource.'; exit; } + + $md = $this->_getParam('md'); + + $this->view->response = $md; } } diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index 197e67aae..167ddccae 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -98,7 +98,7 @@ class ApiClientInterface: def upload_recorded_show(self): pass - def update_media_metadata(self): + def update_media_metadata(self, md): pass # Put here whatever tests you want to run to make sure your API is working @@ -350,18 +350,20 @@ class AirTimeApiClient(ApiClientInterface): return response - def update_media_metadata(self): + def update_media_metadata(self, md): logger = logging.getLogger() response = None try: - url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["show_schedule_url"]) - #url = self.config["base_url"] + self.config["api_base"] + self.config["show_schedule_url"] + url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["update_media_url"]) logger.debug(url) url = url.replace("%%api_key%%", self.config["api_key"]) + + data = urllib.urlencode(md) + req = urllib2.Request(url, data) + response = urllib2.urlopen(req) - response = urllib.urlopen(url) response = json.loads(response.read()) - logger.info("shows %s", response) + logger.info("update media %s", response) except Exception, e: logger.error("Exception: %s", e) diff --git a/python_apps/pytag-fs/MediaMonitor.cfg b/python_apps/pytag-fs/MediaMonitor.cfg index d32a6d4b1..2c22fe656 100644 --- a/python_apps/pytag-fs/MediaMonitor.cfg +++ b/python_apps/pytag-fs/MediaMonitor.cfg @@ -19,5 +19,5 @@ api_base = 'api' # URL to get the version number of the server API version_url = 'version/api_key/%%api_key%%' -# URL to get the schedule of shows set to record -show_schedule_url = 'recorded-shows/format/json/api_key/%%api_key%%' +# URL to tell Airtime to update file's meta data +update_media_url = 'reload-metadata/format/json/api_key/%%api_key%%' diff --git a/python_apps/pytag-fs/MediaMonitor.py b/python_apps/pytag-fs/MediaMonitor.py index c84cc3abf..1016afcfa 100644 --- a/python_apps/pytag-fs/MediaMonitor.py +++ b/python_apps/pytag-fs/MediaMonitor.py @@ -8,6 +8,7 @@ import time import datetime import os import sys +from subprocess import Popen, PIPE, STDOUT from configobj import ConfigObj @@ -34,20 +35,41 @@ except Exception, e: mask = pyinotify.ALL_EVENTS wm = WatchManager() -wdd = wm.add_watch('/srv/airtime/stor', mask, rec=True) +wdd = wm.add_watch('/srv/airtime/stor', mask, rec=True) + +class MediaMonitor(ProcessEvent): + + def my_init(self): + """ + Method automatically called from ProcessEvent.__init__(). Additional + keyworded arguments passed to ProcessEvent.__init__() are then + delegated to my_init(). + """ + self.api_client = api_client.api_client_factory(config) -class PTmp(ProcessEvent): def process_IN_CREATE(self, event): if event.dir : global wm wdd = wm.add_watch(event.pathname, mask, rec=True) #print wdd.keys() - + print "%s: %s" % (event.maskname, os.path.join(event.path, event.name)) def process_IN_MODIFY(self, event): if not event.dir : - print event.path + p = Popen(["pytags", event.pathname], stdout=PIPE, stderr=STDOUT) + output = p.stdout.read().decode("utf-8").strip() + print output.split("\n") + + md = {'filepath':event.pathname} + + for tag in output.split("\n")[2:] : + key,value = tag.split("=") + md[key] = value + + data = {'md': md} + + response = self.api_client.update_media_metadata(data) print "%s: %s" % (event.maskname, os.path.join(event.path, event.name)) @@ -59,7 +81,7 @@ if __name__ == '__main__': print 'Media Monitor' try: - notifier = Notifier(wm, PTmp(), read_freq=2, timeout=1) + notifier = Notifier(wm, MediaMonitor(), read_freq=2, timeout=1) notifier.coalesce_events() notifier.loop() except KeyboardInterrupt: From a3e0ab5a8e958edfdaefb9d23a2ecaf5f6d9885f Mon Sep 17 00:00:00 2001 From: Naomi Date: Tue, 26 Apr 2011 11:32:40 -0400 Subject: [PATCH 04/14] CC-1799 : Live Studio Playout from media library (pytagsfs) --- python_apps/pytag-fs/MediaMonitor.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/python_apps/pytag-fs/MediaMonitor.py b/python_apps/pytag-fs/MediaMonitor.py index 1016afcfa..1d446d211 100644 --- a/python_apps/pytag-fs/MediaMonitor.py +++ b/python_apps/pytag-fs/MediaMonitor.py @@ -31,12 +31,6 @@ except Exception, e: print 'Error loading config file: ', e sys.exit() -# watched events -mask = pyinotify.ALL_EVENTS - -wm = WatchManager() -wdd = wm.add_watch('/srv/airtime/stor', mask, rec=True) - class MediaMonitor(ProcessEvent): def my_init(self): @@ -48,18 +42,14 @@ class MediaMonitor(ProcessEvent): self.api_client = api_client.api_client_factory(config) def process_IN_CREATE(self, event): - if event.dir : - global wm - wdd = wm.add_watch(event.pathname, mask, rec=True) - #print wdd.keys() - - print "%s: %s" % (event.maskname, os.path.join(event.path, event.name)) + if not event.dir : + #This is a newly imported file. + print "%s: %s" % (event.maskname, os.path.join(event.path, event.name)) def process_IN_MODIFY(self, event): if not event.dir : p = Popen(["pytags", event.pathname], stdout=PIPE, stderr=STDOUT) output = p.stdout.read().decode("utf-8").strip() - print output.split("\n") md = {'filepath':event.pathname} @@ -81,7 +71,13 @@ if __name__ == '__main__': print 'Media Monitor' try: - notifier = Notifier(wm, MediaMonitor(), read_freq=2, timeout=1) + # watched events + mask = pyinotify.IN_CREATE | pyinotify.IN_MODIFY + + wm = WatchManager() + wdd = wm.add_watch('/srv/airtime/stor', mask, rec=True, auto_add=True) + + notifier = Notifier(wm, MediaMonitor(), read_freq=10, timeout=1) notifier.coalesce_events() notifier.loop() except KeyboardInterrupt: From 5bd82cd3d7dc5e0ff135ba608922095f0f8cf641 Mon Sep 17 00:00:00 2001 From: naomiaro Date: Wed, 11 May 2011 11:47:41 -0400 Subject: [PATCH 05/14] rebasing branch --- python_apps/pytag-fs/MediaMonitor.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/python_apps/pytag-fs/MediaMonitor.py b/python_apps/pytag-fs/MediaMonitor.py index 1d446d211..189aaad97 100644 --- a/python_apps/pytag-fs/MediaMonitor.py +++ b/python_apps/pytag-fs/MediaMonitor.py @@ -1,6 +1,4 @@ #!/usr/local/bin/python -import urllib -import urllib2 import logging import logging.config import json @@ -8,12 +6,14 @@ import time import datetime import os import sys +import hashlib + from subprocess import Popen, PIPE, STDOUT from configobj import ConfigObj import pyinotify -from pyinotify import WatchManager, Notifier, ThreadedNotifier, EventsCodes, ProcessEvent +from pyinotify import WatchManager, Notifier, ProcessEvent from api_clients import api_client @@ -46,13 +46,21 @@ class MediaMonitor(ProcessEvent): #This is a newly imported file. print "%s: %s" % (event.maskname, os.path.join(event.path, event.name)) + #event.path : /srv/airtime/stor/bd2 + #event.name : bd2aa73b58d9c8abcced989621846e99.mp3 + #event.pathname : /srv/airtime/stor/bd2/bd2aa73b58d9c8abcced989621846e99.mp3 def process_IN_MODIFY(self, event): if not event.dir : p = Popen(["pytags", event.pathname], stdout=PIPE, stderr=STDOUT) output = p.stdout.read().decode("utf-8").strip() - md = {'filepath':event.pathname} + #get md5, most likely different + f = file(event.pathname, 'rb') + m = hashlib.md5() + m.update(f.read()) + md5 = m.hexdigest() + md = {'filepath':event.pathname, 'md5':md5} for tag in output.split("\n")[2:] : key,value = tag.split("=") md[key] = value @@ -61,7 +69,7 @@ class MediaMonitor(ProcessEvent): response = self.api_client.update_media_metadata(data) - print "%s: %s" % (event.maskname, os.path.join(event.path, event.name)) + print "%s: path: %s name: %s" % (event.maskname, event.path, event.name) def process_default(self, event): print "%s: %s" % (event.maskname, os.path.join(event.path, event.name)) From 5ba7679284c83a254e2485eb9083604e145222de Mon Sep 17 00:00:00 2001 From: Naomi Date: Wed, 27 Apr 2011 13:36:30 -0400 Subject: [PATCH 06/14] CC-1799 : Live Studio Playout from media library (pytagsfs) switching to using mutagen to get audio metadata (used by pytagsfs) passing gunid of file to locate it in airtime --- python_apps/pytag-fs/MediaMonitor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python_apps/pytag-fs/MediaMonitor.py b/python_apps/pytag-fs/MediaMonitor.py index 189aaad97..b7bad1da3 100644 --- a/python_apps/pytag-fs/MediaMonitor.py +++ b/python_apps/pytag-fs/MediaMonitor.py @@ -15,6 +15,8 @@ from configobj import ConfigObj import pyinotify from pyinotify import WatchManager, Notifier, ProcessEvent +import mutagen + from api_clients import api_client # configure logging @@ -58,9 +60,11 @@ class MediaMonitor(ProcessEvent): f = file(event.pathname, 'rb') m = hashlib.md5() m.update(f.read()) - md5 = m.hexdigest() - md = {'filepath':event.pathname, 'md5':md5} + md5 = m.hexdigest() + gunid = event.name.split('.')[0] + + md = {'gunid':gunid, 'md5':md5} for tag in output.split("\n")[2:] : key,value = tag.split("=") md[key] = value From f64a1aae3426f66d2a516be3f0fa8a42d5e77b8d Mon Sep 17 00:00:00 2001 From: naomiaro Date: Wed, 27 Apr 2011 20:19:41 -0400 Subject: [PATCH 07/14] CC-1799: Live Studio Playout from media library (pytagsfs) using mutagen to get audio tags --- python_apps/pytag-fs/MediaMonitor.py | 40 +++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/python_apps/pytag-fs/MediaMonitor.py b/python_apps/pytag-fs/MediaMonitor.py index b7bad1da3..7f7fe8ad2 100644 --- a/python_apps/pytag-fs/MediaMonitor.py +++ b/python_apps/pytag-fs/MediaMonitor.py @@ -33,6 +33,11 @@ except Exception, e: print 'Error loading config file: ', e sys.exit() +""" +list of supported easy tags in mutagen version 1.20 +['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry', 'date', 'performer', 'musicbrainz_albumartistid', 'composer', 'encodedby', 'tracknumber', 'musicbrainz_albumid', 'album', 'asin', 'musicbrainz_artistid', 'mood', 'copyright', 'author', 'media', 'length', 'version', 'artistsort', 'titlesort', 'discsubtitle', 'website', 'musicip_fingerprint', 'conductor', 'compilation', 'barcode', 'performer:*', 'composersort', 'musicbrainz_discid', 'musicbrainz_albumtype', 'genre', 'isrc', 'discnumber', 'musicbrainz_trmid', 'replaygain_*_gain', 'musicip_puid', 'artist', 'title', 'bpm', 'musicbrainz_trackid', 'arranger', 'albumsort', 'replaygain_*_peak', 'organization'] +""" + class MediaMonitor(ProcessEvent): def my_init(self): @@ -43,20 +48,34 @@ class MediaMonitor(ProcessEvent): """ self.api_client = api_client.api_client_factory(config) + self.mutagen2airtime = {\ + "title": "track_title",\ + "artist": "artist_name",\ + "album": "album_title",\ + "genre": "genre",\ + "mood": "mood",\ + "tracknumber": "track_number",\ + "bpm": "bpm",\ + "organization": "label",\ + "composer": "composer",\ + "encodedby": "encoded_by",\ + "conductor": "conductor",\ + "date": "year",\ + "website": "info_url",\ + "isrc": "isrc_number",\ + "copyright": "copyright",\ + } + def process_IN_CREATE(self, event): if not event.dir : #This is a newly imported file. - print "%s: %s" % (event.maskname, os.path.join(event.path, event.name)) + print "%s: %s%s" % (event.maskname, event.path, event.name) #event.path : /srv/airtime/stor/bd2 #event.name : bd2aa73b58d9c8abcced989621846e99.mp3 #event.pathname : /srv/airtime/stor/bd2/bd2aa73b58d9c8abcced989621846e99.mp3 def process_IN_MODIFY(self, event): if not event.dir : - p = Popen(["pytags", event.pathname], stdout=PIPE, stderr=STDOUT) - output = p.stdout.read().decode("utf-8").strip() - - #get md5, most likely different f = file(event.pathname, 'rb') m = hashlib.md5() m.update(f.read()) @@ -65,9 +84,12 @@ class MediaMonitor(ProcessEvent): gunid = event.name.split('.')[0] md = {'gunid':gunid, 'md5':md5} - for tag in output.split("\n")[2:] : - key,value = tag.split("=") - md[key] = value + + file_info = mutagen.File(event.pathname, easy=True) + attrs = self.mutagen2airtime + for key in file_info.keys() : + if key in attrs : + md[attrs[key]] = file_info[key] data = {'md': md} @@ -76,7 +98,7 @@ class MediaMonitor(ProcessEvent): print "%s: path: %s name: %s" % (event.maskname, event.path, event.name) def process_default(self, event): - print "%s: %s" % (event.maskname, os.path.join(event.path, event.name)) + print "%s: %s%s" % (event.maskname, event.path, event.name) if __name__ == '__main__': From 6834ecca76de029f2f08dcbecabd5d7e04353237 Mon Sep 17 00:00:00 2001 From: Naomi Date: Thu, 28 Apr 2011 18:02:40 -0400 Subject: [PATCH 08/14] CC-1799 : Live Studio Playout from media library (pytagsfs) if a tag is edited by easytag, the change is now reflected in airtime database. --- .../application/controllers/ApiController.php | 23 ++++++++++--- airtime_mvc/application/models/StoredFile.php | 17 +++++++--- python_apps/api_clients/api_client.py | 33 +++++++++++++++---- python_apps/pytag-fs/MediaMonitor.py | 4 +-- 4 files changed, 59 insertions(+), 18 deletions(-) diff --git a/airtime_mvc/application/controllers/ApiController.php b/airtime_mvc/application/controllers/ApiController.php index 307f116ee..a6f0825b7 100644 --- a/airtime_mvc/application/controllers/ApiController.php +++ b/airtime_mvc/application/controllers/ApiController.php @@ -8,10 +8,10 @@ class ApiController extends Zend_Controller_Action /* Initialize action controller here */ $context = $this->_helper->getHelper('contextSwitch'); $context->addActionContext('version', 'json') - ->addActionContext('recorded-shows', 'json') - ->addActionContext('upload-recorded', 'json') - ->addActionContext('reload-metadata', 'json') - ->initContext(); + ->addActionContext('recorded-shows', 'json') + ->addActionContext('upload-recorded', 'json') + ->addActionContext('reload-metadata', 'json') + ->initContext(); } public function indexAction() @@ -332,7 +332,20 @@ class ApiController extends Zend_Controller_Action $md = $this->_getParam('md'); - $this->view->response = $md; + $file = StoredFile::Recall(null, $md['gunid']); + if (PEAR::isError($file) || is_null($file)) { + $this->view->response = "File not in Airtime's Database"; + return; + } + + $res = $file->replaceDbMetadata($md); + + if (PEAR::isError($res)) { + $this->view->response = "Metadata Change Failed"; + } + else { + $this->view->response = "Success!"; + } } } diff --git a/airtime_mvc/application/models/StoredFile.php b/airtime_mvc/application/models/StoredFile.php index 847e284f6..d5d4faf80 100644 --- a/airtime_mvc/application/models/StoredFile.php +++ b/airtime_mvc/application/models/StoredFile.php @@ -543,16 +543,25 @@ class StoredFile { public function replaceDbMetadata($p_values) { global $CC_CONFIG, $CC_DBC; + + $data = array(); foreach ($p_values as $category => $value) { $escapedValue = pg_escape_string($value); $columnName = $category; if (!is_null($columnName)) { - $sql = "UPDATE ".$CC_CONFIG["filesTable"] - ." SET $columnName='$escapedValue'" - ." WHERE gunid = '".$this->gunid."'"; - $CC_DBC->query($sql); + $data[] = "$columnName='$escapedValue'"; } } + + $data = join(",", $data); + $sql = "UPDATE ".$CC_CONFIG["filesTable"] + ." SET $data" + ." WHERE gunid = '".$this->gunid."'"; + $res = $CC_DBC->query($sql); + if (PEAR::isError($res)) { + $CC_DBC->query("ROLLBACK"); + return $res; + } } public function clearMetadata() diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index 167ddccae..3661a8fdf 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -29,7 +29,25 @@ def api_client_factory(config): print 'API Client "'+config["api_client"]+'" not supported. Please check your config file.' print sys.exit() - + +def recursive_urlencode(d): + def recursion(d, base=None): + pairs = [] + + for key, value in d.items(): + if hasattr(value, 'values'): + pairs += recursion(value, key) + else: + new_pair = None + if base: + new_pair = "%s[%s]=%s" % (base, urllib.quote(unicode(key)), urllib.quote(unicode(value))) + else: + new_pair = "%s=%s" % (urllib.quote(unicode(key)), urllib.quote(unicode(value))) + pairs.append(new_pair) + return pairs + + return '&'.join(recursion(d)) + class ApiClientInterface: # Implementation: optional @@ -159,7 +177,6 @@ class AirTimeApiClient(ApiClientInterface): return version - def test(self): logger = logging.getLogger() status, items = self.get_schedule('2010-01-01-00-00-00', '2011-01-01-00-00-00') @@ -355,14 +372,16 @@ class AirTimeApiClient(ApiClientInterface): response = None try: url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["update_media_url"]) - logger.debug(url) + #logger.debug(url) url = url.replace("%%api_key%%", self.config["api_key"]) + logger.debug(url) - data = urllib.urlencode(md) + data = recursive_urlencode(md) req = urllib2.Request(url, data) - response = urllib2.urlopen(req) - - response = json.loads(response.read()) + + response = urllib2.urlopen(req).read() + logger.info("update media %s", response) + response = json.loads(response) logger.info("update media %s", response) except Exception, e: diff --git a/python_apps/pytag-fs/MediaMonitor.py b/python_apps/pytag-fs/MediaMonitor.py index 7f7fe8ad2..a0ed37a90 100644 --- a/python_apps/pytag-fs/MediaMonitor.py +++ b/python_apps/pytag-fs/MediaMonitor.py @@ -84,12 +84,12 @@ class MediaMonitor(ProcessEvent): gunid = event.name.split('.')[0] md = {'gunid':gunid, 'md5':md5} - + file_info = mutagen.File(event.pathname, easy=True) attrs = self.mutagen2airtime for key in file_info.keys() : if key in attrs : - md[attrs[key]] = file_info[key] + md[attrs[key]] = file_info[key][0] data = {'md': md} From 9c46b0d396d2062d70a1a6b0254050d7963eab95 Mon Sep 17 00:00:00 2001 From: naomiaro Date: Thu, 5 May 2011 10:55:14 -0400 Subject: [PATCH 09/14] CC-1799 : Live Studio Playout from media library (pytagsfs) metadata almost working from airtime form -> python daemon. Just need to look into permissions for some stor folders. --- .../controllers/LibraryController.php | 4 ++ airtime_mvc/application/forms/EditAudioMD.php | 52 +++++++++++++- airtime_mvc/application/models/RabbitMq.php | 22 ++++++ python_apps/pytag-fs/MediaMonitor.cfg | 7 ++ python_apps/pytag-fs/MediaMonitor.py | 70 ++++++++++++++++--- 5 files changed, 144 insertions(+), 11 deletions(-) diff --git a/airtime_mvc/application/controllers/LibraryController.php b/airtime_mvc/application/controllers/LibraryController.php index 6d6c70acc..b7e15bd95 100644 --- a/airtime_mvc/application/controllers/LibraryController.php +++ b/airtime_mvc/application/controllers/LibraryController.php @@ -168,6 +168,10 @@ class LibraryController extends Zend_Controller_Action $formdata = $form->getValues(); $file->replaceDbMetadata($formdata); + $data = $formdata; + $data['filepath'] = $file->getRealFilePath(); + RabbitMq::SendFileMetaData($data); + $this->_helper->redirector('index'); } } diff --git a/airtime_mvc/application/forms/EditAudioMD.php b/airtime_mvc/application/forms/EditAudioMD.php index 6a002cd03..022745e6f 100644 --- a/airtime_mvc/application/forms/EditAudioMD.php +++ b/airtime_mvc/application/forms/EditAudioMD.php @@ -2,6 +2,24 @@ class Application_Form_EditAudioMD extends Zend_Form { + /* + "title": "track_title",\ + "artist": "artist_name",\ + "album": "album_title",\ + "genre": "genre",\ + "mood": "mood",\ + "tracknumber": "track_number",\ + "bpm": "bpm",\ + "organization": "label",\ + "composer": "composer",\ + "encodedby": "encoded_by",\ + "conductor": "conductor",\ + "date": "year",\ + "website": "info_url",\ + "isrc": "isrc_number",\ + "copyright": "copyright",\ + */ + public function init() { @@ -37,6 +55,13 @@ class Application_Form_EditAudioMD extends Zend_Form 'filters' => array('StringTrim') )); + // Add mood field + $this->addElement('text', 'track_number', array( + 'label' => 'Track:', + 'class' => 'input_text', + 'filters' => array('StringTrim') + )); + // Add genre field $this->addElement('text', 'genre', array( 'label' => 'Genre:', @@ -77,9 +102,30 @@ class Application_Form_EditAudioMD extends Zend_Form 'filters' => array('StringTrim') )); - // Add language field - $this->addElement('text', 'language', array( - 'label' => 'Language:', + // Add mood field + $this->addElement('text', 'bpm', array( + 'label' => 'BPM:', + 'class' => 'input_text', + 'filters' => array('StringTrim') + )); + + // Add mood field + $this->addElement('text', 'copyright', array( + 'label' => 'Copyright:', + 'class' => 'input_text', + 'filters' => array('StringTrim') + )); + + // Add mood field + $this->addElement('text', 'isrc_number', array( + 'label' => 'ISRC Number:', + 'class' => 'input_text', + 'filters' => array('StringTrim') + )); + + // Add mood field + $this->addElement('text', 'info_url', array( + 'label' => 'Website:', 'class' => 'input_text', 'filters' => array('StringTrim') )); diff --git a/airtime_mvc/application/models/RabbitMq.php b/airtime_mvc/application/models/RabbitMq.php index 1672dd4e2..971933f20 100644 --- a/airtime_mvc/application/models/RabbitMq.php +++ b/airtime_mvc/application/models/RabbitMq.php @@ -40,5 +40,27 @@ class RabbitMq } } + public static function SendFileMetaData($md) + { + global $CC_CONFIG; + + $conn = new AMQPConnection($CC_CONFIG["rabbitmq"]["host"], + $CC_CONFIG["rabbitmq"]["port"], + $CC_CONFIG["rabbitmq"]["user"], + $CC_CONFIG["rabbitmq"]["password"]); + $channel = $conn->channel(); + $channel->access_request($CC_CONFIG["rabbitmq"]["vhost"], false, false, true, true); + + $EXCHANGE = 'airtime-media-monitor'; + $channel->exchange_declare($EXCHANGE, 'direct', false, true); + + $data = json_encode($md); + $msg = new AMQPMessage($data, array('content_type' => 'text/plain')); + + $channel->basic_publish($msg, $EXCHANGE); + $channel->close(); + $conn->close(); + } + } diff --git a/python_apps/pytag-fs/MediaMonitor.cfg b/python_apps/pytag-fs/MediaMonitor.cfg index 2c22fe656..925bcbdb4 100644 --- a/python_apps/pytag-fs/MediaMonitor.cfg +++ b/python_apps/pytag-fs/MediaMonitor.cfg @@ -21,3 +21,10 @@ version_url = 'version/api_key/%%api_key%%' # URL to tell Airtime to update file's meta data update_media_url = 'reload-metadata/format/json/api_key/%%api_key%%' + +############################################ +# RabbitMQ settings # +############################################ +rabbitmq_host = 'localhost' +rabbitmq_user = 'guest' +rabbitmq_password = 'guest' diff --git a/python_apps/pytag-fs/MediaMonitor.py b/python_apps/pytag-fs/MediaMonitor.py index a0ed37a90..e3104b767 100644 --- a/python_apps/pytag-fs/MediaMonitor.py +++ b/python_apps/pytag-fs/MediaMonitor.py @@ -7,16 +7,19 @@ import datetime import os import sys import hashlib +import json from subprocess import Popen, PIPE, STDOUT from configobj import ConfigObj +import mutagen import pyinotify from pyinotify import WatchManager, Notifier, ProcessEvent -import mutagen - +# For RabbitMQ +from kombu.connection import BrokerConnection +from kombu.messaging import Exchange, Queue, Consumer, Producer from api_clients import api_client # configure logging @@ -38,6 +41,59 @@ list of supported easy tags in mutagen version 1.20 ['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry', 'date', 'performer', 'musicbrainz_albumartistid', 'composer', 'encodedby', 'tracknumber', 'musicbrainz_albumid', 'album', 'asin', 'musicbrainz_artistid', 'mood', 'copyright', 'author', 'media', 'length', 'version', 'artistsort', 'titlesort', 'discsubtitle', 'website', 'musicip_fingerprint', 'conductor', 'compilation', 'barcode', 'performer:*', 'composersort', 'musicbrainz_discid', 'musicbrainz_albumtype', 'genre', 'isrc', 'discnumber', 'musicbrainz_trmid', 'replaygain_*_gain', 'musicip_puid', 'artist', 'title', 'bpm', 'musicbrainz_trackid', 'arranger', 'albumsort', 'replaygain_*_peak', 'organization'] """ +def checkRabbitMQ(notifier): + try: + notifier.connection.drain_events(timeout=5) + except Exception, e: + logger = logging.getLogger('root') + logger.info("%s", e) + +class AirtimeNotifier(Notifier): + + def __init__(self, watch_manager, default_proc_fun=None, read_freq=0, threshold=0, timeout=None): + Notifier.__init__(self, watch_manager, default_proc_fun, read_freq, threshold, timeout) + + self.airtime2mutagen = {\ + "track_title": "title",\ + "artist_name": "artist",\ + "album_title": "album",\ + "genre": "genre",\ + "mood": "mood",\ + "track_number": "tracknumber",\ + "bpm": "bpm",\ + "label": "organization",\ + "composer": "composer",\ + "encoded_by": "encodedby",\ + "conductor": "conductor",\ + "year": "date",\ + "info_url": "website",\ + "isrc_number": "isrc",\ + "copyright": "copyright",\ + } + + schedule_exchange = Exchange("airtime-media-monitor", "direct", durable=True, auto_delete=True) + schedule_queue = Queue("media-monitor", exchange=schedule_exchange, key="filesystem") + self.connection = BrokerConnection(config["rabbitmq_host"], config["rabbitmq_user"], config["rabbitmq_password"], "/") + channel = self.connection.channel() + consumer = Consumer(channel, schedule_queue) + consumer.register_callback(self.handle_message) + consumer.consume() + + def handle_message(self, body, message): + # ACK the message to take it off the queue + message.ack() + + logger = logging.getLogger('root') + logger.info("Received md from RabbitMQ: " + body) + + m = json.loads(message.body) + airtime_file = mutagen.File(m['filepath'], easy=True) + del m['filepath'] + for key in m.keys() : + airtime_file[self.airtime2mutagen[key]] = m[key] + + airtime_file.save() + class MediaMonitor(ProcessEvent): def my_init(self): @@ -69,7 +125,7 @@ class MediaMonitor(ProcessEvent): def process_IN_CREATE(self, event): if not event.dir : #This is a newly imported file. - print "%s: %s%s" % (event.maskname, event.path, event.name) + print "%s: %s" % (event.maskname, event.pathname) #event.path : /srv/airtime/stor/bd2 #event.name : bd2aa73b58d9c8abcced989621846e99.mp3 @@ -98,12 +154,10 @@ class MediaMonitor(ProcessEvent): print "%s: path: %s name: %s" % (event.maskname, event.path, event.name) def process_default(self, event): - print "%s: %s%s" % (event.maskname, event.path, event.name) + print "%s: %s" % (event.maskname, event.pathname) if __name__ == '__main__': - print 'Media Monitor' - try: # watched events mask = pyinotify.IN_CREATE | pyinotify.IN_MODIFY @@ -111,9 +165,9 @@ if __name__ == '__main__': wm = WatchManager() wdd = wm.add_watch('/srv/airtime/stor', mask, rec=True, auto_add=True) - notifier = Notifier(wm, MediaMonitor(), read_freq=10, timeout=1) + notifier = AirtimeNotifier(wm, MediaMonitor(), read_freq=10, timeout=1) notifier.coalesce_events() - notifier.loop() + notifier.loop(callback=checkRabbitMQ) except KeyboardInterrupt: notifier.stop() From c398b82c680dcc71de38381f46a45b651b5d3326 Mon Sep 17 00:00:00 2001 From: naomiaro Date: Thu, 5 May 2011 11:58:45 -0400 Subject: [PATCH 10/14] CC-1799 : Live Studio Playout from media library (pytagsfs) need to add pypo to www-data group --- python_apps/pytag-fs/install/media-monitor-install.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python_apps/pytag-fs/install/media-monitor-install.py b/python_apps/pytag-fs/install/media-monitor-install.py index f0bb2adc0..8f3fb804e 100755 --- a/python_apps/pytag-fs/install/media-monitor-install.py +++ b/python_apps/pytag-fs/install/media-monitor-install.py @@ -45,6 +45,8 @@ def create_user(username): print "User already exists." #add pypo to audio group os.system("adduser " + username + " audio 1>/dev/null 2>&1") + #add pypo to www-data group + os.system("adduser " + username + " www-data 1>/dev/null 2>&1") def copy_dir(src_dir, dest_dir): if (os.path.exists(dest_dir)) and (dest_dir != "/"): From 6fbd5112dd7038332deba5651141ed30aa996516 Mon Sep 17 00:00:00 2001 From: naomiaro Date: Thu, 5 May 2011 18:12:26 -0400 Subject: [PATCH 11/14] CC-1799 - Live Studio Playout from media library (pytagsfs) dealing with programs that modify a file by using a tmp file. --- python_apps/api_clients/api_client.py | 8 ++- python_apps/pytag-fs/MediaMonitor.cfg | 6 +++ python_apps/pytag-fs/MediaMonitor.py | 72 +++++++++++++++++++++------ 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index 3661a8fdf..bf749cfad 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -372,18 +372,16 @@ class AirTimeApiClient(ApiClientInterface): response = None try: url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["update_media_url"]) - #logger.debug(url) - url = url.replace("%%api_key%%", self.config["api_key"]) logger.debug(url) - + url = url.replace("%%api_key%%", self.config["api_key"]) + data = recursive_urlencode(md) req = urllib2.Request(url, data) response = urllib2.urlopen(req).read() logger.info("update media %s", response) response = json.loads(response) - logger.info("update media %s", response) - + except Exception, e: logger.error("Exception: %s", e) diff --git a/python_apps/pytag-fs/MediaMonitor.cfg b/python_apps/pytag-fs/MediaMonitor.cfg index 925bcbdb4..5f2a79a83 100644 --- a/python_apps/pytag-fs/MediaMonitor.cfg +++ b/python_apps/pytag-fs/MediaMonitor.cfg @@ -28,3 +28,9 @@ update_media_url = 'reload-metadata/format/json/api_key/%%api_key%%' rabbitmq_host = 'localhost' rabbitmq_user = 'guest' rabbitmq_password = 'guest' + +############################################ +# Media-Monitor preferences # +############################################ +check_filesystem_events = 30 #how long to queue up events performed on the files themselves. +check_airtime_events = 30 #how long to queue metadata input from airtime. diff --git a/python_apps/pytag-fs/MediaMonitor.py b/python_apps/pytag-fs/MediaMonitor.py index e3104b767..38423fcc9 100644 --- a/python_apps/pytag-fs/MediaMonitor.py +++ b/python_apps/pytag-fs/MediaMonitor.py @@ -43,7 +43,7 @@ list of supported easy tags in mutagen version 1.20 def checkRabbitMQ(notifier): try: - notifier.connection.drain_events(timeout=5) + notifier.connection.drain_events(timeout=int(config["check_airtime_events"])) except Exception, e: logger = logging.getLogger('root') logger.info("%s", e) @@ -90,7 +90,8 @@ class AirtimeNotifier(Notifier): airtime_file = mutagen.File(m['filepath'], easy=True) del m['filepath'] for key in m.keys() : - airtime_file[self.airtime2mutagen[key]] = m[key] + if m[key] != "" : + airtime_file[self.airtime2mutagen[key]] = m[key] airtime_file.save() @@ -122,17 +123,14 @@ class MediaMonitor(ProcessEvent): "copyright": "copyright",\ } - def process_IN_CREATE(self, event): - if not event.dir : - #This is a newly imported file. - print "%s: %s" % (event.maskname, event.pathname) + self.logger = logging.getLogger('root') - #event.path : /srv/airtime/stor/bd2 - #event.name : bd2aa73b58d9c8abcced989621846e99.mp3 - #event.pathname : /srv/airtime/stor/bd2/bd2aa73b58d9c8abcced989621846e99.mp3 - def process_IN_MODIFY(self, event): - if not event.dir : - f = file(event.pathname, 'rb') + self.temp_files = {} + + def update_airtime(self, event): + self.logger.info("Updating Change to Airtime") + try: + f = open(event.pathname, 'rb') m = hashlib.md5() m.update(f.read()) @@ -151,21 +149,63 @@ class MediaMonitor(ProcessEvent): response = self.api_client.update_media_metadata(data) - print "%s: path: %s name: %s" % (event.maskname, event.path, event.name) + except Exception, e: + self.logger.info("%s", e) + + def process_IN_CREATE(self, event): + if not event.dir : + filename_info = event.name.split(".") + + #file created is a tmp file which will be modified and then moved back to the original filename. + if len(filename_info) > 2 : + self.temp_files[event.pathname] = None + #This is a newly imported file. + else : + pass + + self.logger.info("%s: %s", event.maskname, event.pathname) + + #event.path : /srv/airtime/stor/bd2 + #event.name : bd2aa73b58d9c8abcced989621846e99.mp3 + #event.pathname : /srv/airtime/stor/bd2/bd2aa73b58d9c8abcced989621846e99.mp3 + def process_IN_MODIFY(self, event): + if not event.dir : + filename_info = event.name.split(".") + + #file modified is not a tmp file. + if len(filename_info) == 2 : + self.update_airtime(event) + + self.logger.info("%s: path: %s name: %s", event.maskname, event.path, event.name) + + def process_IN_MOVED_FROM(self, event): + if event.pathname in self.temp_files : + del self.temp_files[event.pathname] + self.temp_files[event.cookie] = event.pathname + + self.logger.info("%s: %s", event.maskname, event.pathname) + + def process_IN_MOVED_TO(self, event): + if event.cookie in self.temp_files : + del self.temp_files[event.cookie] + self.update_airtime(event) + + self.logger.info("%s: %s", event.maskname, event.pathname) def process_default(self, event): - print "%s: %s" % (event.maskname, event.pathname) + self.logger.info("%s: %s", event.maskname, event.pathname) if __name__ == '__main__': try: # watched events - mask = pyinotify.IN_CREATE | pyinotify.IN_MODIFY + mask = pyinotify.IN_CREATE | pyinotify.IN_MODIFY | pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO + #mask = pyinotify.ALL_EVENTS wm = WatchManager() wdd = wm.add_watch('/srv/airtime/stor', mask, rec=True, auto_add=True) - notifier = AirtimeNotifier(wm, MediaMonitor(), read_freq=10, timeout=1) + notifier = AirtimeNotifier(wm, MediaMonitor(), read_freq=int(config["check_filesystem_events"]), timeout=1) notifier.coalesce_events() notifier.loop(callback=checkRabbitMQ) except KeyboardInterrupt: From 0beed387c57746718186787f04700bfac9890b67 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Thu, 12 May 2011 12:34:30 -0400 Subject: [PATCH 12/14] -replace sudo with setuidgid in media-monitor --- python_apps/pytag-fs/install/media-monitor-daemontools.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python_apps/pytag-fs/install/media-monitor-daemontools.sh b/python_apps/pytag-fs/install/media-monitor-daemontools.sh index 0f6fa39e7..d629ec49d 100755 --- a/python_apps/pytag-fs/install/media-monitor-daemontools.sh +++ b/python_apps/pytag-fs/install/media-monitor-daemontools.sh @@ -12,5 +12,6 @@ echo "*** Daemontools: starting daemon" exec 2>&1 # Note the -u when calling python! we need it to get unbuffered binary stdout and stderr -sudo PYTHONPATH=${api_client_path} -u ${media_monitor_user} python -u ${media_monitor_path}${media_monitor_script} +export PYTHONPATH=${api_client_path} +setuidgid ${media_monitor_user} python -u ${media_monitor_path}${media_monitor_script} # EOF From bd588f7f3ed3772b635895b64eaf1e6409cd8d82 Mon Sep 17 00:00:00 2001 From: James Moon Date: Thu, 12 May 2011 12:05:48 -0700 Subject: [PATCH 13/14] CC-2209:Put database install stuff into separate script done. --- install/airtime-uninstall.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/install/airtime-uninstall.php b/install/airtime-uninstall.php index bd95a175a..3d6e2f3cb 100644 --- a/install/airtime-uninstall.php +++ b/install/airtime-uninstall.php @@ -29,7 +29,9 @@ AirtimeInstall::UninstallPhpCode(); // still be a connection to the database and you wont be able to delete it. //------------------------------------------------------------------------ echo " * Dropping the database '".$CC_CONFIG['dsn']['database']."'...".PHP_EOL; -$command = "sudo -u postgres dropdb {$CC_CONFIG['dsn']['database']} 2> /dev/null"; + +// check if DB exists +$command = "echo \"DROP DATABASE IF EXISTS ".$CC_CONFIG['dsn']['database']."\" | sudo -u postgres psql"; @exec($command, $output, $dbDeleteFailed); //------------------------------------------------------------------------ From e2a521aa37e0250d29c6d5be459f2c90548de823 Mon Sep 17 00:00:00 2001 From: Naomi Date: Thu, 12 May 2011 19:05:08 -0400 Subject: [PATCH 14/14] show recorder daemon uses logging instead of print statements. --- python_apps/show-recorder/recorder.py | 46 ++++++++++++--------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/python_apps/show-recorder/recorder.py b/python_apps/show-recorder/recorder.py index 7e1f7ef7a..0421086c3 100644 --- a/python_apps/show-recorder/recorder.py +++ b/python_apps/show-recorder/recorder.py @@ -43,7 +43,7 @@ def getDateTimeObj(time): date = timeinfo[0].split("-") time = timeinfo[1].split(":") - return datetime.datetime(int(date[0]), int(date[1]), int(date[2]), int(time[0]), int(time[1]), int(time[2])) + return datetime.datetime(int(date[0]), int(date[1]), int(date[2]), int(time[0]), int(time[1]), int(time[2])) class ShowRecorder(Thread): @@ -55,6 +55,7 @@ class ShowRecorder(Thread): self.start_time = start_time self.filetype = filetype self.show_instance = show_instance + self.logger = logging.getLogger('root') def record_show(self): @@ -67,11 +68,9 @@ class ShowRecorder(Thread): #-ge:0.1,0.1,0,-1 args = command.split(" ") - print "starting record" - + self.logger.info("starting record") code = call(args) - - print "finishing record, return code %s" % (code) + self.logger.info("finishing record, return code %s", code) return code, filepath @@ -94,55 +93,56 @@ class ShowRecorder(Thread): if code == 0: self.upload_file(filepath) else: - print "problem recording show" + self.logger.info("problem recording show") class Record(): def __init__(self): - self.api_client = api_client.api_client_factory(config) - self.shows_to_record = {} + self.api_client = api_client.api_client_factory(config) + self.shows_to_record = {} + self.logger = logging.getLogger('root') def process_shows(self, shows): self.shows_to_record = {} - + for show in shows: show_starts = getDateTimeObj(show[u'starts']) show_end = getDateTimeObj(show[u'ends']) time_delta = show_end - show_starts - + self.shows_to_record[show[u'starts']] = [time_delta, show[u'instance_id'], show[u'name']] def check_record(self): - + tnow = datetime.datetime.now() sorted_show_keys = sorted(self.shows_to_record.keys()) - + start_time = sorted_show_keys[0] next_show = getDateTimeObj(start_time) - print next_show - print tnow + self.logger.debug("Next show %s", next_show) + self.logger.debug("Now %s", tnow) delta = next_show - tnow min_delta = datetime.timedelta(seconds=60) if delta <= min_delta: - print "sleeping %s seconds until show" % (delta.seconds) + self.logger.debug("sleeping %s seconds until show", delta.seconds) time.sleep(delta.seconds) - + show_length = self.shows_to_record[start_time][0] show_instance = self.shows_to_record[start_time][1] show_name = self.shows_to_record[start_time][2] - - show = ShowRecorder(show_instance, show_length.seconds, show_name, start_time, filetype="mp3", ) + + show = ShowRecorder(show_instance, show_length.seconds, show_name, start_time, filetype="mp3") show.start() - + #remove show from shows to record. del self.shows_to_record[start_time] - + def get_shows(self): @@ -154,7 +154,7 @@ class Record(): if len(shows): self.process_shows(shows) - self.check_record() + self.check_record() if __name__ == '__main__': @@ -165,7 +165,3 @@ if __name__ == '__main__': recorder.get_shows() time.sleep(5) - - - -