Merge branch 'devel' into 2.3.x-saas

Conflicts:
	airtime_mvc/application/controllers/PreferenceController.php
	airtime_mvc/application/forms/AddShowWhen.php
	airtime_mvc/application/forms/GeneralPreferences.php
	airtime_mvc/application/forms/LiveStreamingPreferences.php
	airtime_mvc/application/forms/SoundcloudPreferences.php
	airtime_mvc/application/forms/SupportSettings.php
	airtime_mvc/application/views/scripts/form/preferences.phtml
	airtime_mvc/application/views/scripts/form/preferences_email_server.phtml
	airtime_mvc/application/views/scripts/form/preferences_general.phtml
	airtime_mvc/application/views/scripts/form/preferences_livestream.phtml
	airtime_mvc/application/views/scripts/form/support-setting.phtml
	airtime_mvc/application/views/scripts/schedule/add-
show-form.phtml
	airtime_mvc/public/js/airtime/preferences/preferences.js
	python_apps/api_clients/api_client.py
	python_apps/pypo/listenerstat.py
This commit is contained in:
Martin Konecny 2013-01-15 13:44:44 -05:00
commit 8cd6bd9aa4
346 changed files with 48955 additions and 11856 deletions

View file

@ -38,27 +38,6 @@ def get_os_codename():
return ("unknown", "unknown")
def generate_liquidsoap_config(ss):
data = ss['msg']
fh = open('/etc/airtime/liquidsoap.cfg', 'w')
fh.write("################################################\n")
fh.write("# THIS FILE IS AUTO GENERATED. DO NOT CHANGE!! #\n")
fh.write("################################################\n")
for d in data:
buffer = d[u'keyname'] + " = "
if(d[u'type'] == 'string'):
temp = d[u'value']
buffer += '"%s"' % temp
else:
temp = d[u'value']
if(temp == ""):
temp = "0"
buffer += temp
buffer += "\n"
fh.write(api_client.encode_to(buffer))
fh.write('log_file = "/var/log/airtime/pypo-liquidsoap/<script>.log"\n')
fh.close()
PATH_INI_FILE = '/etc/airtime/pypo.cfg'
PATH_LIQUIDSOAP_BIN = '/usr/lib/airtime/pypo/bin/liquidsoap_bin'

View file

@ -407,7 +407,7 @@ end
# fade using both cross() and switch().
def input.http_restart(~id,~initial_url="http://dummy/url")
source = input.http(buffer=5.,max=15.,id=id,autostart=false,initial_url)
source = audio_to_stereo(input.http(buffer=5.,max=15.,id=id,autostart=false,initial_url))
def stopped()
"stopped" == list.hd(server.execute("#{id}.status"))

View file

@ -35,10 +35,6 @@ just_switched = ref false
%include "ls_lib.liq"
#web_stream = input.harbor("test-harbor", port=8999, password=stream_harbor_pass)
#web_stream = on_metadata(notify_stream, web_stream)
#output.dummy(fallible=true, web_stream)
queue = audio_to_stereo(id="queue_src", request.equeue(id="queue", length=0.5))
queue = cue_cut(queue)
queue = amplify(1., override="replay_gain", queue)
@ -51,7 +47,8 @@ output.dummy(fallible=true, queue)
http = input.http_restart(id="http")
http = cross_http(http_input_id="http",http)
stream_queue = http_fallback(http_input_id="http",http=http,default=queue)
output.dummy(fallible=true, http)
stream_queue = http_fallback(http_input_id="http", http=http, default=queue)
ignore(output.dummy(stream_queue, fallible=true))
@ -120,7 +117,11 @@ server.register(namespace="dynamic_source",
# fun (s) -> begin log("dynamic_source.read_stop") destroy_dynamic_source_all() end)
default = amplify(id="silence_src", 0.00001, noise())
default = rewrite_metadata([("artist","Airtime"), ("title", "offline")], default)
ref_off_air_meta = ref off_air_meta
if !ref_off_air_meta == "" then
ref_off_air_meta := "Airtime - offline"
end
default = rewrite_metadata([("title", !ref_off_air_meta)], default)
ignore(output.dummy(default, fallible=true))
master_dj_enabled = ref false

View file

@ -31,15 +31,15 @@ class ListenerStat(Thread):
return self.api_client.get_stream_parameters()
def get_icecast_xml(self, ip):
#encoded = base64.b64encode("%(admin_user)s:%(admin_password)s" % ip)
def get_stream_server_xml(self, ip, url):
encoded = base64.b64encode("%(admin_user)s:%(admin_pass)s" % ip)
#header = {"Authorization":"Basic %s" % encoded}
self.logger.debug(ip)
url = 'http://%(host)s:%(port)s/stats.xsl' % ip
self.logger.debug(url)
req = urllib2.Request(url=url)
#headers=header)
header = {"Authorization":"Basic %s" % encoded}
req = urllib2.Request(
#assuming that the icecast stats path is /admin/stats.xml
#need to fix this
url=url,
headers=header)
f = urllib2.urlopen(req)
document = f.read()
@ -48,7 +48,8 @@ class ListenerStat(Thread):
def get_icecast_stats(self, ip):
document = self.get_icecast_xml(ip)
url = 'http://%(host)s:%(port)s/admin/stats.xml' % ip
document = self.get_stream_server_xml(ip, url)
dom = xml.dom.minidom.parseString(document)
sources = dom.getElementsByTagName("source")
@ -66,6 +67,24 @@ class ListenerStat(Thread):
mount_stats = {"timestamp":timestamp, \
"num_listeners": num_listeners, \
"mount_name": mount_name}
return mount_stats
def get_shoutcast_stats(self, ip):
url = 'http://%(host)s:%(port)s/admin.cgi?sid=1&mode=viewxml' % ip
document = self.get_stream_server_xml(ip, url)
dom = xml.dom.minidom.parseString(document)
current_listeners = dom.getElementsByTagName("CURRENTLISTENERS")
timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
num_listeners = 0
if len(current_listeners):
num_listeners = self.get_node_text(current_listeners[0].childNodes)
mount_stats = {"timestamp":timestamp, \
"num_listeners": num_listeners, \
"mount_name": "shoutcast"}
return mount_stats
def get_stream_stats(self, stream_parameters):
@ -77,16 +96,27 @@ class ListenerStat(Thread):
#streams are the same server, we will still initiate 3 separate
#connections
for k, v in stream_parameters.items():
#v["admin_user"] = "admin"
#v["admin_password"] = "hackme"
if v["enable"] == 'true':
stats.append(self.get_icecast_stats(v))
try:
if v["output"] == "icecast":
stats.append(self.get_icecast_stats(v))
else:
stats.append(self.get_shoutcast_stats(v))
self.update_listener_stat_error(v["mount"], 'OK')
except Exception, e:
self.logger.error('Exception: %s', e)
self.update_listener_stat_error(v["mount"], str(e))
return stats
def push_stream_stats(self, stats):
self.api_client.push_stream_stats(stats)
def update_listener_stat_error(self, stream_id, error):
keyname = '%s_listener_stat_error' % stream_id
data = {keyname: error}
self.api_client.update_stream_setting_table(data)
def run(self):
#Wake up every 120 seconds and gather icecast statistics. Note that we
#are currently querying the server every 2 minutes for list of
@ -99,10 +129,12 @@ class ListenerStat(Thread):
stats = self.get_stream_stats(stream_parameters["stream_params"])
self.logger.debug(stats)
self.push_stream_stats(stats)
if not stats:
self.logger.error("Not able to get listener stats")
else:
self.push_stream_stats(stats)
except Exception, e:
top = traceback.format_exc()
self.logger.error('Exception: %s', top)
self.logger.error('Exception: %s', e)
time.sleep(120)

View file

View file

@ -0,0 +1,152 @@
from subprocess import Popen, PIPE
import re
import os
import sys
import shutil
import tempfile
import logging
logger = logging.getLogger()
def get_process_output(command):
"""
Run subprocess and return stdout
"""
logger.debug(command)
p = Popen(command, shell=True, stdout=PIPE)
return p.communicate()[0].strip()
def run_process(command):
"""
Run subprocess and return "return code"
"""
p = Popen(command, shell=True)
return os.waitpid(p.pid, 0)[1]
def get_mime_type(file_path):
"""
Attempts to get the mime type but will return prematurely if the process
takes longer than 5 seconds. Note that this function should only be called
for files which do not have a mp3/ogg/flac extension.
"""
return get_process_output("timeout 5 file -b --mime-type %s" % file_path)
def duplicate_file(file_path):
"""
Makes a duplicate of the file and returns the path of this duplicate file.
"""
fsrc = open(file_path, 'r')
fdst = tempfile.NamedTemporaryFile(delete=False)
logger.info("Copying %s to %s" % (file_path, fdst.name))
shutil.copyfileobj(fsrc, fdst)
fsrc.close()
fdst.close()
return fdst.name
def get_file_type(file_path):
file_type = None
if re.search(r'mp3$', file_path, re.IGNORECASE):
file_type = 'mp3'
elif re.search(r'og(g|a)$', file_path, re.IGNORECASE):
file_type = 'vorbis'
elif re.search(r'flac$', file_path, re.IGNORECASE):
file_type = 'flac'
else:
mime_type = get_mime_type(file_path)
if 'mpeg' in mime_type:
file_type = 'mp3'
elif 'ogg' in mime_type:
file_type = 'vorbis'
elif 'flac' in mime_type:
file_type = 'flac'
return file_type
def calculate_replay_gain(file_path):
"""
This function accepts files of type mp3/ogg/flac and returns a calculated
ReplayGain value in dB.
If the value cannot be calculated for some reason, then we default to 0
(Unity Gain).
http://wiki.hydrogenaudio.org/index.php?title=ReplayGain_1.0_specification
"""
try:
"""
Making a duplicate is required because the ReplayGain extraction utilities we use
make unwanted modifications to the file.
"""
search = None
temp_file_path = duplicate_file(file_path)
file_type = get_file_type(file_path)
nice_level = '15'
if file_type:
if file_type == 'mp3':
if run_process("which mp3gain > /dev/null") == 0:
command = 'nice -n %s mp3gain -q "%s" 2> /dev/null' \
% (nice_level, temp_file_path)
out = get_process_output(command)
search = re.search(r'Recommended "Track" dB change: (.*)', \
out)
else:
logger.warn("mp3gain not found")
elif file_type == 'vorbis':
command = "which vorbisgain > /dev/null && which ogginfo > \
/dev/null"
if run_process(command) == 0:
command = 'nice -n %s vorbisgain -q -f "%s" 2>/dev/null \
>/dev/null' % (nice_level,temp_file_path)
run_process(command)
out = get_process_output('ogginfo "%s"' % temp_file_path)
search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out)
else:
logger.warn("vorbisgain/ogginfo not found")
elif file_type == 'flac':
if run_process("which metaflac > /dev/null") == 0:
command = 'nice -n %s metaflac --add-replay-gain "%s"' \
% (nice_level, temp_file_path)
run_process(command)
command = 'nice -n %s metaflac \
--show-tag=REPLAYGAIN_TRACK_GAIN "%s"' \
% (nice_level, temp_file_path)
out = get_process_output(command)
search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out)
else: logger.warn("metaflac not found")
except Exception, e:
logger.error(str(e))
finally:
#no longer need the temp, file simply remove it.
try: os.remove(temp_file_path)
except: pass
replay_gain = 0
if search:
matches = search.groups()
if len(matches) == 1:
replay_gain = matches[0]
else:
logger.warn("Received more than 1 match in: '%s'" % str(matches))
return replay_gain
# Example of running from command line:
# python replay_gain.py /path/to/filename.mp3
if __name__ == "__main__":
print calculate_replay_gain(sys.argv[1])

View file

@ -0,0 +1,82 @@
from threading import Thread
import traceback
import os
import time
import logging
from media.update import replaygain
class ReplayGainUpdater(Thread):
"""
The purpose of the class is to query the server for a list of files which
do not have a ReplayGain value calculated. This class will iterate over the
list calculate the values, update the server and repeat the process until
the server reports there are no files left.
This class will see heavy activity right after a 2.1->2.2 upgrade since 2.2
introduces ReplayGain normalization. A fresh install of Airtime 2.2 will
see this class not used at all since a file imported in 2.2 will
automatically have its ReplayGain value calculated.
"""
@staticmethod
def start_reply_gain(apc):
me = ReplayGainUpdater(apc)
me.daemon = True
me.start()
def __init__(self,apc):
Thread.__init__(self)
self.api_client = apc
self.logger = logging.getLogger()
def main(self):
raw_response = self.api_client.list_all_watched_dirs()
if 'dirs' not in raw_response:
self.logger.error("Could not get a list of watched directories \
with a dirs attribute. Printing full request:")
self.logger.error( raw_response )
return
directories = raw_response['dirs']
for dir_id, dir_path in directories.iteritems():
try:
# keep getting few rows at a time for current music_dir (stor
# or watched folder).
total = 0
while True:
# return a list of pairs where the first value is the
# file's database row id and the second value is the
# filepath
files = self.api_client.get_files_without_replay_gain_value(dir_id)
processed_data = []
for f in files:
full_path = os.path.join(dir_path, f['fp'])
processed_data.append((f['id'], replaygain.calculate_replay_gain(full_path)))
try:
self.api_client.update_replay_gain_values(processed_data)
except Exception as e: self.unexpected_exception(e)
if len(files) == 0: break
self.logger.info("Processed: %d songs" % total)
except Exception, e:
self.logger.error(e)
self.logger.debug(traceback.format_exc())
def run(self):
try:
while True:
self.logger.info("Runnning replaygain updater")
self.main()
# Sleep for 5 minutes in case new files have been added
time.sleep(60 * 5)
except Exception, e:
self.logger.error('ReplayGainUpdater Exception: %s', traceback.format_exc())
self.logger.error(e)
if __name__ == "__main__":
rgu = ReplayGainUpdater()
rgu.main()

View file

@ -24,6 +24,8 @@ from recorder import Recorder
from listenerstat import ListenerStat
from pypomessagehandler import PypoMessageHandler
from media.update.replaygainupdater import ReplayGainUpdater
from configobj import ConfigObj
# custom imports
@ -174,6 +176,9 @@ if __name__ == '__main__':
sys.exit()
api_client = api_client.AirtimeApiClient()
ReplayGainUpdater.start_reply_gain(api_client)
api_client.register_component("pypo")
pypoFetch_q = Queue()

View file

@ -188,28 +188,6 @@ class PypoFetch(Thread):
self.update_liquidsoap_station_name(info['station_name'])
self.update_liquidsoap_transition_fade(info['transition_fade'])
def write_liquidsoap_config(self, setting):
fh = open('/etc/airtime/liquidsoap.cfg', 'w')
self.logger.info("Rewriting liquidsoap.cfg...")
fh.write("################################################\n")
fh.write("# THIS FILE IS AUTO GENERATED. DO NOT CHANGE!! #\n")
fh.write("################################################\n")
for k, d in setting:
buffer_str = d[u'keyname'] + " = "
if d[u'type'] == 'string':
temp = d[u'value']
buffer_str += '"%s"' % temp
else:
temp = d[u'value']
if temp == "":
temp = "0"
buffer_str += temp
buffer_str += "\n"
fh.write(api_client.encode_to(buffer_str))
fh.write("log_file = \"/var/log/airtime/pypo-liquidsoap/<script>.log\"\n");
fh.close()
def restart_liquidsoap(self):
self.telnet_lock.acquire()
@ -296,10 +274,10 @@ class PypoFetch(Thread):
dump, stream = s[u'keyname'].split('_', 1)
state_change_restart[stream] = False
# This is the case where restart is required no matter what
if (existing[s[u'keyname']] != s[u'value']):
if (existing[s[u'keyname']] != str(s[u'value'])):
self.logger.info("'Need-to-restart' state detected for %s...", s[u'keyname'])
restart = True;
elif "master_live_stream_port" in s[u'keyname'] or "master_live_stream_mp" in s[u'keyname'] or "dj_live_stream_port" in s[u'keyname'] or "dj_live_stream_mp" in s[u'keyname']:
elif "master_live_stream_port" in s[u'keyname'] or "master_live_stream_mp" in s[u'keyname'] or "dj_live_stream_port" in s[u'keyname'] or "dj_live_stream_mp" in s[u'keyname'] or "off_air_meta" in s[u'keyname']:
if (existing[s[u'keyname']] != s[u'value']):
self.logger.info("'Need-to-restart' state detected for %s...", s[u'keyname'])
restart = True;

View file

@ -209,7 +209,8 @@ class PypoPush(Thread):
else:
correct = liquidsoap_queue_approx[0]['start'] == media_item['start'] and \
liquidsoap_queue_approx[0]['row_id'] == media_item['row_id'] and \
liquidsoap_queue_approx[0]['end'] == media_item['end']
liquidsoap_queue_approx[0]['end'] == media_item['end'] and \
liquidsoap_queue_approx[0]['replay_gain'] == media_item['replay_gain']
elif is_stream(media_item):
correct = liquidsoap_stream_id == str(media_item['row_id'])

View file

@ -1,100 +0,0 @@
<?php
// Define path to application directory
define('APPLICATION_PATH', realpath(dirname(__FILE__) . '/../../../application'));
echo APPLICATION_PATH.PHP_EOL;
// Ensure library/ is on include_path
set_include_path(get_include_path() . PATH_SEPARATOR . realpath(APPLICATION_PATH . '/../library'));
set_include_path(get_include_path() . PATH_SEPARATOR . APPLICATION_PATH . '/models');
echo get_include_path().PHP_EOL;
//Controller plugins.
set_include_path(APPLICATION_PATH . get_include_path() . PATH_SEPARATOR . '/controllers/plugins');
require_once APPLICATION_PATH.'/configs/conf.php';
require_once(APPLICATION_PATH.'/../library/propel/runtime/lib/Propel.php');
require_once 'Soundcloud.php';
require_once 'Playlist.php';
require_once 'StoredFile.php';
require_once 'Schedule.php';
require_once 'Shows.php';
require_once 'User.php';
require_once 'RabbitMq.php';
require_once 'Preference.php';
//require_once APPLICATION_PATH.'/controllers/plugins/RabbitMqPlugin.php';
// Initialize Propel with the runtime configuration
Propel::init(__DIR__."/../../../application/configs/airtime-conf.php");
$playlistName = "pypo_playlist_test";
$secondsFromNow = 30;
echo " ************************************************************** \n";
echo " This script schedules a playlist to play $secondsFromNow minute(s) from now.\n";
echo " This is a utility to help you debug the scheduler.\n";
echo " ************************************************************** \n";
echo "\n";
echo "Deleting playlists with the name '$playlistName'...";
// Delete any old playlists
$pl2 = Playlist::findPlaylistByName($playlistName);
foreach ($pl2 as $playlist) {
//var_dump($playlist);
$playlist->delete();
}
echo "done.\n";
// Create a new playlist
echo "Creating new playlist '$playlistName'...";
$pl = new Playlist();
$pl->create($playlistName);
$mediaFile = Application_Model_StoredFile::findByOriginalName("Peter_Rudenko_-_Opening.mp3");
if (is_null($mediaFile)) {
echo "Adding test audio clip to the database.\n";
$v = array("filepath" => __DIR__."/../../../audio_samples/vorbis.com/Hydrate-Kenny_Beltrey.ogg");
$mediaFile = Application_Model_StoredFile::Insert($v);
}
$pl->addAudioClip($mediaFile->getId());
echo "done.\n";
//$pl2 = Playlist::findPlaylistByName("pypo_playlist_test");
//var_dump($pl2);
// Get current time
// In the format YYYY-MM-DD HH:MM:SS.nnnnnn
$startTime = date("Y-m-d H:i:s");
$endTime = date("Y-m-d H:i:s", time()+(60*60));
echo "Removing everything from the scheduler between $startTime and $endTime...";
// Check for succces
$scheduleClear = Schedule::isScheduleEmptyInRange($startTime, "01:00:00");
if (!$scheduleClear) {
echo "\nERROR: Schedule could not be cleared.\n\n";
var_dump(Schedule::getItems($startTime, $endTime));
exit;
}
echo "done.\n";
// Schedule the playlist for two minutes from now
echo "Scheduling new playlist...\n";
//$playTime = date("Y-m-d H:i:s", time()+(60*$minutesFromNow));
$playTime = date("Y-m-d H:i:s", time()+($secondsFromNow));
//$scheduleGroup = new ScheduleGroup();
//$scheduleGroup->add($playTime, null, $pl->getId());
//$show = new Application_Model_ShowInstance($showInstanceId);
//$show->scheduleShow(array($pl->getId()));
//$show->setShowStart();
//$show->setShowEnd();
echo " SUCCESS: Playlist scheduled at $playTime\n\n";

View file

@ -1,59 +0,0 @@
[loggers]
keys=root
[handlers]
keys=consoleHandler,fileHandlerERROR,fileHandlerDEBUG,nullHandler
[formatters]
keys=simpleFormatter
[logger_root]
level=DEBUG
handlers=consoleHandler,fileHandlerERROR,fileHandlerDEBUG
[logger_libs]
handlers=nullHandler
level=CRITICAL
qualname="process"
propagate=0
[handler_consoleHandler]
class=StreamHandler
level=CRITICAL
formatter=simpleFormatter
args=(sys.stdout,)
[handler_fileHandlerERROR]
class=FileHandler
level=CRITICAL
formatter=simpleFormatter
args=("./error-unit-test.log",)
[handler_fileHandlerDEBUG]
class=FileHandler
level=CRITICAL
formatter=simpleFormatter
args=("./debug-unit-test.log",)
[handler_nullHandler]
class=FileHandler
level=CRITICAL
formatter=simpleFormatter
args=("./log-null-unit-test.log",)
[formatter_simpleFormatter]
format=%(asctime)s %(levelname)s - [%(filename)s : %(funcName)s() : line %(lineno)d] - %(message)s
datefmt=
## multitail color sheme
## pyml / python
# colorscheme:pyml:www.obp.net
# cs_re:blue:\[[^ ]*\]
# cs_re:red:CRITICAL:*
# cs_re:red,black,blink:ERROR:*
# cs_re:blue:NOTICE:*
# cs_re:cyan:INFO:*
# cs_re:green:DEBUG:*

View file

@ -1,74 +0,0 @@
import time
import os
import traceback
from optparse import *
import sys
import time
import datetime
import logging
import logging.config
import shutil
import urllib
import urllib2
import pickle
import telnetlib
import random
import string
import operator
import inspect
# additional modules (should be checked)
from configobj import ConfigObj
# custom imports
from util import *
from api_clients import *
import random
import unittest
# configure logging
logging.config.fileConfig("logging-api-validator.cfg")
try:
config = ConfigObj('/etc/airtime/pypo.cfg')
except Exception, e:
print 'Error loading config file: ', e
sys.exit()
class TestApiFunctions(unittest.TestCase):
def setUp(self):
self.api_client = api_client.api_client_factory(config)
def test_is_server_compatible(self):
self.assertTrue(self.api_client.is_server_compatible(False))
def test_get_schedule(self):
status, response = self.api_client.get_schedule()
self.assertTrue(response.has_key("status"))
self.assertTrue(response.has_key("playlists"))
self.assertTrue(response.has_key("check"))
self.assertTrue(status == 1)
def test_get_media(self):
self.assertTrue(True)
def test_notify_scheduled_item_start_playing(self):
arr = dict()
arr["x"] = dict()
arr["x"]["schedule_id"]=1
response = self.api_client.notify_scheduled_item_start_playing("x", arr)
self.assertTrue(response.has_key("status"))
self.assertTrue(response.has_key("message"))
def test_notify_media_item_start_playing(self):
response = self.api_client.notify_media_item_start_playing('{"schedule_id":1}', 5)
self.assertTrue(response.has_key("status"))
self.assertTrue(response.has_key("message"))
if __name__ == '__main__':
unittest.main()