Congegni/sintonia
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch '2.2.x' into 2.2.x-saas

Browse source
This commit is contained in:
Martin Konecny 2012-11-06 16:11:51 -05:00
commit 3f7d8a0c7f
95 changed files with 6172 additions and 6094 deletions
Download patch file Download diff file Expand all files Collapse all files

92
python_apps/api_clients/api_client.py
View file

@ -20,6 +20,11 @@ import traceback
AIRTIME_VERSION = "2.2.0"
# TODO : Place these functions in some common module. Right now, media
# monitor uses the same functions and it would be better to reuse them
# instead of copy pasting them around
def to_unicode(obj, encoding='utf-8'):
if isinstance(obj, basestring):
if not isinstance(obj, unicode):
@ -39,7 +44,7 @@ def convert_dict_value_to_utf8(md):
# Airtime API Client
################################################################################
class AirtimeApiClient():
class AirtimeApiClient(object):
# This is a little hacky fix so that I don't have to pass the config object
# everywhere where AirtimeApiClient needs to be initialized
@ -422,53 +427,46 @@ class AirtimeApiClient():
def send_media_monitor_requests(self, action_list, dry=False):
"""
Send a gang of media monitor events at a time. actions_list is a list
of dictionaries where every dictionary is representing an action. Every
action dict must contain a 'mode' key that says what kind of action it
is and an optional 'is_record' key that says whether the show was
recorded or not. The value of this key does not matter, only if it's
present or not.
Send a gang of media monitor events at a time. actions_list is a
list of dictionaries where every dictionary is representing an
action. Every action dict must contain a 'mode' key that says
what kind of action it is and an optional 'is_record' key that
says whether the show was recorded or not. The value of this key
does not matter, only if it's present or not.
"""
logger = self.logger
try:
url = self.construct_url('reload_metadata_group')
# We are assuming that action_list is a list of dictionaries such
# that every dictionary represents the metadata of a file along
# with a special mode key that is the action to be executed by the
# controller.
valid_actions = []
# We could get a list of valid_actions in a much shorter way using
# filter but here we prefer a little more verbosity to help
# debugging
for action in action_list:
if not 'mode' in action:
self.logger.debug("Warning: Trying to send a request element without a 'mode'")
self.logger.debug("Here is the the request: '%s'" % str(action) )
else:
# We alias the value of is_record to true or false no
# matter what it is based on if it's absent in the action
if 'is_record' not in action:
action['is_record'] = 0
valid_actions.append(action)
# Note that we must prefix every key with: mdX where x is a number
# Is there a way to format the next line a little better? The
# parenthesis make the code almost unreadable
md_list = dict((("md%d" % i), json.dumps(convert_dict_value_to_utf8(md))) \
for i,md in enumerate(valid_actions))
# For testing we add the following "dry" parameter to tell the
# controller not to actually do any changes
if dry: md_list['dry'] = 1
self.logger.info("Pumping out %d requests..." % len(valid_actions))
data = urllib.urlencode(md_list)
req = urllib2.Request(url, data)
response = self.get_response_from_server(req)
response = json.loads(response)
return response
except ValueError: raise
except Exception, e:
logger.error('Exception: %s', e)
logger.error("traceback: %s", traceback.format_exc())
raise
url = self.construct_url('reload_metadata_group')
# We are assuming that action_list is a list of dictionaries such
# that every dictionary represents the metadata of a file along
# with a special mode key that is the action to be executed by the
# controller.
valid_actions = []
# We could get a list of valid_actions in a much shorter way using
# filter but here we prefer a little more verbosity to help
# debugging
for action in action_list:
if not 'mode' in action:
self.logger.debug("Warning: Trying to send a request element without a 'mode'")
self.logger.debug("Here is the the request: '%s'" % str(action) )
else:
# We alias the value of is_record to true or false no
# matter what it is based on if it's absent in the action
if 'is_record' not in action:
action['is_record'] = 0
valid_actions.append(action)
# Note that we must prefix every key with: mdX where x is a number
# Is there a way to format the next line a little better? The
# parenthesis make the code almost unreadable
md_list = dict((("md%d" % i), json.dumps(convert_dict_value_to_utf8(md))) \
for i,md in enumerate(valid_actions))
# For testing we add the following "dry" parameter to tell the
# controller not to actually do any changes
if dry: md_list['dry'] = 1
self.logger.info("Pumping out %d requests..." % len(valid_actions))
data = urllib.urlencode(md_list)
req = urllib2.Request(url, data)
response = self.get_response_from_server(req)
response = json.loads(response)
return response
#returns a list of all db files for a given directory in JSON format:
#{"files":["path/to/file1", "path/to/file2"]}

14
python_apps/icecast2/airtime-icecast-status.xsl
View file

@ -1,7 +1,7 @@
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<xsl:template match="@*|node()">
<xsl:copy>
<xsl:apply-templates select="@*|node()"/>
</xsl:copy>
</xsl:template>
</xsl:stylesheet>
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<xsl:template match="@*|node()">
<xsl:copy>
<xsl:apply-templates select="@*|node()"/>
</xsl:copy>
</xsl:template>
</xsl:stylesheet>

22
python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py
View file

@ -12,14 +12,14 @@ import traceback
"""
list of supported easy tags in mutagen version 1.20
['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry',
'date', 'performer', 'musicbrainz_albumartistid', 'composer', 'encodedby',
'tracknumber', 'musicbrainz_albumid', 'album', 'asin', 'musicbrainz_artistid',
'mood', 'copyright', 'author', 'media', 'length', 'version', 'artistsort',
'titlesort', 'discsubtitle', 'website', 'musicip_fingerprint', 'conductor',
'compilation', 'barcode', 'performer:*', 'composersort', 'musicbrainz_discid',
'musicbrainz_albumtype', 'genre', 'isrc', 'discnumber', 'musicbrainz_trmid',
'replaygain_*_gain', 'musicip_puid', 'artist', 'title', 'bpm', 'musicbrainz_trackid',
['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry',
'date', 'performer', 'musicbrainz_albumartistid', 'composer', 'encodedby',
'tracknumber', 'musicbrainz_albumid', 'album', 'asin', 'musicbrainz_artistid',
'mood', 'copyright', 'author', 'media', 'length', 'version', 'artistsort',
'titlesort', 'discsubtitle', 'website', 'musicip_fingerprint', 'conductor',
'compilation', 'barcode', 'performer:*', 'composersort', 'musicbrainz_discid',
'musicbrainz_albumtype', 'genre', 'isrc', 'discnumber', 'musicbrainz_trmid',
'replaygain_*_gain', 'musicip_puid', 'artist', 'title', 'bpm', 'musicbrainz_trackid',
'arranger', 'albumsort', 'replaygain_*_peak', 'organization']
"""
@ -139,7 +139,7 @@ class AirtimeMetadata:
def get_md_from_file(self, filepath):
"""
Returns None if error retrieving metadata. Otherwise returns a dictionary
representing the file's metadata
representing the file's metadata
"""
self.logger.info("getting info from filepath %s", filepath)
@ -248,9 +248,9 @@ class AirtimeMetadata:
try:
md['MDATA_KEY_BITRATE'] = getattr(file_info.info, "bitrate", 0)
md['MDATA_KEY_SAMPLERATE'] = getattr(file_info.info, "sample_rate", 0)
md['MDATA_KEY_DURATION'] = self.format_length(getattr(file_info.info, "length", 0.0))
md['MDATA_KEY_MIME'] = ""
if len(file_info.mime) > 0:
md['MDATA_KEY_MIME'] = file_info.mime[0]

200
python_apps/media-monitor2/media/metadata/definitions.py
View file

@ -1,111 +1,143 @@
# -*- coding: utf-8 -*-
import media.monitor.process as md
import media.metadata.process as md
import re
from os.path import normpath
from media.monitor.pure import format_length, file_md5
from media.monitor.pure import format_length, file_md5, is_airtime_recorded, \
no_extension_basename
with md.metadata('MDATA_KEY_DURATION') as t:
t.default(u'0.0')
t.depends('length')
t.translate(lambda k: format_length(k['length']))
defs_loaded = False
with md.metadata('MDATA_KEY_MIME') as t:
t.default(u'')
t.depends('mime')
t.translate(lambda k: k['mime'].replace('-','/'))
def is_defs_loaded():
global defs_loaded
return defs_loaded
with md.metadata('MDATA_KEY_BITRATE') as t:
t.default(u'')
t.depends('bitrate')
t.translate(lambda k: k['bitrate'])
def load_definitions():
with md.metadata('MDATA_KEY_DURATION') as t:
t.default(u'0.0')
t.depends('length')
t.translate(lambda k: format_length(k['length']))
with md.metadata('MDATA_KEY_SAMPLERATE') as t:
t.default(u'0')
t.depends('sample_rate')
t.translate(lambda k: k['sample_rate'])
with md.metadata('MDATA_KEY_MIME') as t:
t.default(u'')
t.depends('mime')
# Is this necessary?
t.translate(lambda k: k['mime'].replace('audio/vorbis','audio/ogg'))
with md.metadata('MDATA_KEY_FTYPE'):
t.depends('ftype') # i don't think this field even exists
t.default(u'audioclip')
t.translate(lambda k: k['ftype']) # but just in case
with md.metadata('MDATA_KEY_BITRATE') as t:
t.default(u'')
t.depends('bitrate')
t.translate(lambda k: k['bitrate'])
with md.metadata("MDATA_KEY_CREATOR") as t:
t.depends("artist")
# A little kludge to make sure that we have some value for when we parse
# MDATA_KEY_TITLE
t.default(u"")
t.max_length(512)
with md.metadata('MDATA_KEY_SAMPLERATE') as t:
t.default(u'0')
t.depends('sample_rate')
t.translate(lambda k: k['sample_rate'])
with md.metadata("MDATA_KEY_SOURCE") as t:
t.depends("album")
t.max_length(512)
with md.metadata('MDATA_KEY_FTYPE') as t:
t.depends('ftype') # i don't think this field even exists
t.default(u'audioclip')
t.translate(lambda k: k['ftype']) # but just in case
with md.metadata("MDATA_KEY_GENRE") as t:
t.depends("genre")
t.max_length(64)
with md.metadata("MDATA_KEY_CREATOR") as t:
t.depends("artist")
# A little kludge to make sure that we have some value for when we parse
# MDATA_KEY_TITLE
t.default(u"")
t.max_length(512)
with md.metadata("MDATA_KEY_MOOD") as t:
t.depends("mood")
t.max_length(64)
with md.metadata("MDATA_KEY_SOURCE") as t:
t.depends("album")
t.max_length(512)
with md.metadata("MDATA_KEY_TRACKNUMBER") as t:
t.depends("tracknumber")
with md.metadata("MDATA_KEY_GENRE") as t:
t.depends("genre")
t.max_length(64)
with md.metadata("MDATA_KEY_BPM") as t:
t.depends("bpm")
t.max_length(8)
with md.metadata("MDATA_KEY_MOOD") as t:
t.depends("mood")
t.max_length(64)
with md.metadata("MDATA_KEY_LABEL") as t:
t.depends("organization")
t.max_length(512)
with md.metadata("MDATA_KEY_TRACKNUMBER") as t:
t.depends("tracknumber")
with md.metadata("MDATA_KEY_COMPOSER") as t:
t.depends("composer")
t.max_length(512)
with md.metadata("MDATA_KEY_BPM") as t:
t.depends("bpm")
t.max_length(8)
with md.metadata("MDATA_KEY_ENCODER") as t:
t.depends("encodedby")
t.max_length(512)
with md.metadata("MDATA_KEY_LABEL") as t:
t.depends("organization")
t.max_length(512)
with md.metadata("MDATA_KEY_CONDUCTOR") as t:
t.depends("conductor")
t.max_length(512)
with md.metadata("MDATA_KEY_COMPOSER") as t:
t.depends("composer")
t.max_length(512)
with md.metadata("MDATA_KEY_YEAR") as t:
t.depends("date")
t.max_length(16)
with md.metadata("MDATA_KEY_ENCODER") as t:
t.depends("encodedby")
t.max_length(512)
with md.metadata("MDATA_KEY_URL") as t:
t.depends("website")
with md.metadata("MDATA_KEY_CONDUCTOR") as t:
t.depends("conductor")
t.max_length(512)
with md.metadata("MDATA_KEY_ISRC") as t:
t.depends("isrc")
t.max_length(512)
with md.metadata("MDATA_KEY_YEAR") as t:
t.depends("date")
t.max_length(16)
with md.metadata("MDATA_KEY_COPYRIGHT") as t:
t.depends("copyright")
t.max_length(512)
with md.metadata("MDATA_KEY_URL") as t:
t.depends("website")
with md.metadata("MDATA_KEY_FILEPATH") as t:
t.depends('path')
t.translate(lambda k: normpath(k['path']))
with md.metadata("MDATA_KEY_ISRC") as t:
t.depends("isrc")
t.max_length(512)
with md.metadata("MDATA_KEY_MD5") as t:
t.depends('path')
t.optional(False)
t.translate(lambda k: file_md5(k['path'], max_length=100))
with md.metadata("MDATA_KEY_COPYRIGHT") as t:
t.depends("copyright")
t.max_length(512)
# owner is handled differently by (by events.py)
with md.metadata("MDATA_KEY_ORIGINAL_PATH") as t:
t.depends('path')
t.translate(lambda k: unicode(normpath(k['path'])))
with md.metadata('MDATA_KEY_ORIGINAL_PATH') as t:
t.depends('original_path')
with md.metadata("MDATA_KEY_MD5") as t:
t.depends('path')
t.optional(False)
t.translate(lambda k: file_md5(k['path'], max_length=100))
# MDATA_KEY_TITLE is the annoying special case
with md.metadata('MDATA_KEY_TITLE') as t:
# Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is
# defaulted to "" from definitions above
t.depends('title','MDATA_KEY_CREATOR')
t.max_length(512)
# owner is handled differently by (by events.py)
with md.metadata('MDATA_KEY_LABEL') as t:
t.depends('label')
t.max_length(512)
# MDATA_KEY_TITLE is the annoying special case b/c we sometimes read it
# from file name
# must handle 3 cases:
# 1. regular case (not recorded + title is present)
# 2. title is absent (read from file)
# 3. recorded file
def tr_title(k):
#unicode_unknown = u"unknown"
new_title = u""
if is_airtime_recorded(k) or k['title'] != u"":
new_title = k['title']
else:
default_title = no_extension_basename(k['path'])
default_title = re.sub(r'__\d+\.',u'.', default_title)
# format is: track_number-title-123kbps.mp3
m = re.match(".+?-(?P<title>.+)-(\d+kbps|unknown)$", default_title)
if m: new_title = m.group('title')
else: new_title = re.sub(r'-\d+kbps$', u'', default_title)
return new_title
with md.metadata('MDATA_KEY_TITLE') as t:
# Need to know MDATA_KEY_CREATOR to know if show was recorded. Value is
# defaulted to "" from definitions above
t.depends('title','MDATA_KEY_CREATOR','path')
t.optional(False)
t.translate(tr_title)
t.max_length(512)
with md.metadata('MDATA_KEY_LABEL') as t:
t.depends('label')
t.max_length(512)

93
python_apps/media-monitor2/media/metadata/process.py
View file

@ -1,14 +1,36 @@
# -*- coding: utf-8 -*-
from contextlib import contextmanager
from media.monitor.pure import truncate_to_length, toposort
from os.path import normpath
from media.monitor.exceptions import BadSongFile
from media.monitor.log import Loggable
import media.monitor.pure as mmp
from collections import namedtuple
import mutagen
class FakeMutagen(dict):
"""
Need this fake mutagen object so that airtime_special functions
return a proper default value instead of throwing an exceptions for
files that mutagen doesn't recognize
"""
FakeInfo = namedtuple('FakeInfo','length bitrate')
def __init__(self,path):
self.path = path
self.mime = ['audio/wav']
self.info = FakeMutagen.FakeInfo(0.0, '')
dict.__init__(self)
def set_length(self,l):
old_bitrate = self.info.bitrate
self.info = FakeMutagen.FakeInfo(l, old_bitrate)
class MetadataAbsent(Exception):
def __init__(self, name): self.name = name
def __str__(self): return "Could not obtain element '%s'" % self.name
class MetadataElement(object):
class MetadataElement(Loggable):
def __init__(self,name):
self.name = name
# "Sane" defaults
@ -18,6 +40,7 @@ class MetadataElement(object):
self.__default = None
self.__is_normalized = lambda _ : True
self.__max_length = -1
self.__translator = None
def max_length(self,l):
self.__max_length = l
@ -57,31 +80,64 @@ class MetadataElement(object):
return self.__path
def __slice_deps(self, d):
"""
returns a dictionary of all the key value pairs in d that are also
present in self.__deps
"""
return dict( (k,v) for k,v in d.iteritems() if k in self.__deps)
def __str__(self):
return "%s(%s)" % (self.name, ' '.join(list(self.__deps)))
def read_value(self, path, original, running={}):
# If value is present and normalized then we don't touch it
# If value is present and normalized then we only check if it's
# normalized or not. We normalize if it's not normalized already
if self.name in original:
v = original[self.name]
if self.__is_normalized(v): return v
else: return self.__normalizer(v)
# A dictionary slice with all the dependencies and their values
# We slice out only the dependencies that are required for the metadata
# element.
dep_slice_orig = self.__slice_deps(original)
dep_slice_running = self.__slice_deps(running)
# TODO : remove this later
dep_slice_special = self.__slice_deps({'path' : path})
# We combine all required dependencies into a single dictionary
# that we will pass to the translator
full_deps = dict( dep_slice_orig.items()
+ dep_slice_running.items() )
+ dep_slice_running.items()
+ dep_slice_special.items())
# check if any dependencies are absent
if len(full_deps) != len(self.__deps) or len(self.__deps) == 0:
# note: there is no point checking the case that len(full_deps) >
# len(self.__deps) because we make sure to "slice out" any supefluous
# dependencies above.
if len(full_deps) != len(self.dependencies()) or \
len(self.dependencies()) == 0:
# If we have a default value then use that. Otherwise throw an
# exception
if self.has_default(): return self.get_default()
else: raise MetadataAbsent(self.name)
# We have all dependencies. Now for actual for parsing
def def_translate(dep):
def wrap(k):
e = [ x for x in dep ][0]
return k[e]
return wrap
# Only case where we can select a default translator
if self.__translator is None:
self.translate(def_translate(self.dependencies()))
if len(self.dependencies()) > 2: # dependencies include themselves
self.logger.info("Ignoring some dependencies in translate %s"
% self.name)
self.logger.info(self.dependencies())
r = self.__normalizer( self.__translator(full_deps) )
if self.__max_length != -1:
r = truncate_to_length(r, self.__max_length)
@ -92,24 +148,40 @@ def normalize_mutagen(path):
Consumes a path and reads the metadata using mutagen. normalizes some of
the metadata that isn't read through the mutagen hash
"""
m = mutagen.File(path, easy=True)
if not mmp.file_playable(path): raise BadSongFile(path)
try : m = mutagen.File(path, easy=True)
except Exception : raise BadSongFile(path)
if m is None: m = FakeMutagen(path)
try:
if mmp.extension(path) == 'wav':
m.set_length(mmp.read_wave_duration(path))
except Exception: raise BadSongFile(path)
md = {}
for k,v in m.iteritems():
if type(v) is list: md[k] = v[0]
if type(v) is list:
if len(v) > 0: md[k] = v[0]
else: md[k] = v
# populate special metadata values
md['length'] = getattr(m.info, u'length', 0.0)
md['length'] = getattr(m.info, 'length', 0.0)
md['bitrate'] = getattr(m.info, 'bitrate', u'')
md['sample_rate'] = getattr(m.info, 'sample_rate', 0)
md['mime'] = m.mime[0] if len(m.mime) > 0 else u''
md['path'] = path
md['path'] = normpath(path)
if 'title' not in md: md['title'] = u''
return md
class OverwriteMetadataElement(Exception):
def __init__(self, m): self.m = m
def __str__(self): return "Trying to overwrite: %s" % self.m
class MetadataReader(object):
def __init__(self):
self.clear()
def register_metadata(self,m):
if m in self.__mdata_name_map:
raise OverwriteMetadataElement(m)
self.__mdata_name_map[m.name] = m
d = dict( (name,m.dependencies()) for name,m in
self.__mdata_name_map.iteritems() )
@ -131,6 +203,9 @@ class MetadataReader(object):
if not mdata.is_optional(): raise
return normalized_metadata
def read_mutagen(self, path):
return self.read(path, normalize_mutagen(path))
global_reader = MetadataReader()
@contextmanager

3
python_apps/media-monitor2/media/monitor/events.py
View file

@ -148,6 +148,9 @@ class BaseEvent(Loggable):
owners.remove_file_owner(self.path)
return ret
except BadSongFile as e: return [e]
except Exception as e:
self.unexpected_exception(e)
return[e]
# nothing to see here, please move along
def morph_into(self, evt):

13
python_apps/media-monitor2/media/monitor/listeners.py
View file

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
import pyinotify
from pydispatch import dispatcher
from functools import wraps
import media.monitor.pure as mmp
from media.monitor.pure import IncludeOnly
@ -31,6 +32,7 @@ class FileMediator(object):
def unignore(path): FileMediator.ignored_set.remove(path)
def mediate_ignored(fn):
@wraps(fn)
def wrapped(self, event, *args,**kwargs):
event.pathname = unicode(event.pathname, "utf-8")
if FileMediator.is_ignored(event.pathname):
@ -49,15 +51,11 @@ class OrganizeListener(BaseListener, pyinotify.ProcessEvent, Loggable):
def process_IN_CLOSE_WRITE(self, event):
#self.logger.info("===> handling: '%s'" % str(event))
self.process_to_organize(event)
# got cookie
def process_IN_MOVED_TO(self, event):
#self.logger.info("===> handling: '%s'" % str(event))
self.process_to_organize(event)
def process_default(self, event):
pass
#self.logger.info("===> Not handling: '%s'" % str(event))
def flush_events(self, path):
"""
organize the whole directory at path. (pretty much by doing what
@ -67,8 +65,9 @@ class OrganizeListener(BaseListener, pyinotify.ProcessEvent, Loggable):
for f in mmp.walk_supported(path, clean_empties=True):
self.logger.info("Bootstrapping: File in 'organize' directory: \
'%s'" % f)
dispatcher.send(signal=self.signal, sender=self,
event=OrganizeFile(f))
if not mmp.file_locked(f):
dispatcher.send(signal=self.signal, sender=self,
event=OrganizeFile(f))
flushed += 1
#self.logger.info("Flushed organized directory with %d files" % flushed)

25
python_apps/media-monitor2/media/monitor/log.py
View file

@ -6,37 +6,30 @@ from media.monitor.pure import LazyProperty
appname = 'root'
def setup_logging(log_path):
"""
Setup logging by writing log to 'log_path'
"""
""" Setup logging by writing log to 'log_path' """
#logger = logging.getLogger(appname)
logging.basicConfig(filename=log_path, level=logging.DEBUG)
def get_logger():
"""
in case we want to use the common logger from a procedural interface
"""
""" in case we want to use the common logger from a procedural
interface """
return logging.getLogger()
class Loggable(object):
"""
Any class that wants to log can inherit from this class and automatically
get a logger attribute that can be used like: self.logger.info(...) etc.
"""
""" Any class that wants to log can inherit from this class and
automatically get a logger attribute that can be used like:
self.logger.info(...) etc. """
__metaclass__ = abc.ABCMeta
@LazyProperty
def logger(self): return get_logger()
def unexpected_exception(self,e):
"""
Default message for 'unexpected' exceptions
"""
""" Default message for 'unexpected' exceptions """
self.fatal_exception("'Unexpected' exception has occured:", e)
def fatal_exception(self, message, e):
"""
Prints an exception 'e' with 'message'. Also outputs the traceback.
"""
""" Prints an exception 'e' with 'message'. Also outputs the
traceback. """
self.logger.error( message )
self.logger.error( str(e) )
self.logger.error( traceback.format_exc() )

12
python_apps/media-monitor2/media/monitor/manager.py
View file

@ -18,7 +18,7 @@ class ManagerTimeout(threading.Thread,Loggable):
secnods. This used to be just a work around for cc-4235 but recently
became a permanent solution because it's "cheap" and reliable
"""
def __init__(self, manager, interval=3):
def __init__(self, manager, interval=1.5):
# TODO : interval should be read from config and passed here instead
# of just using the hard coded value
threading.Thread.__init__(self)
@ -26,7 +26,7 @@ class ManagerTimeout(threading.Thread,Loggable):
self.interval = interval
def run(self):
while True:
time.sleep(self.interval) # every 3 seconds
time.sleep(self.interval)
self.manager.flush_organize()
class Manager(Loggable):
@ -178,7 +178,7 @@ class Manager(Loggable):
# the OrganizeListener instance will walk path and dispatch an organize
# event for every file in that directory
self.organize['organize_listener'].flush_events(new_path)
self.__add_watch(new_path, self.organize['organize_listener'])
#self.__add_watch(new_path, self.organize['organize_listener'])
def flush_organize(self):
path = self.organize['organize_path']
@ -202,6 +202,12 @@ class Manager(Loggable):
organize.
"""
store_paths = mmp.expand_storage(store)
# First attempt to make sure that all paths exist before adding any
# watches
for path_type, path in store_paths.iteritems():
try: mmp.create_dir(path)
except mmp.FailedToCreateDir as e: self.unexpected_exception(e)
self.set_problem_files_path(store_paths['problem_files'])
self.set_imported_path(store_paths['imported'])
self.set_recorded_path(store_paths['recorded'])

97
python_apps/media-monitor2/media/monitor/metadata.py
View file

@ -2,15 +2,19 @@
import mutagen
import os
import copy
from collections import namedtuple
from mutagen.easymp4 import EasyMP4KeyError
from mutagen.easyid3 import EasyID3KeyError
from media.monitor.exceptions import BadSongFile, InvalidMetadataElement
from media.monitor.log import Loggable
from media.monitor.pure import format_length, truncate_to_length
from media.monitor.pure import format_length
import media.monitor.pure as mmp
# emf related stuff
from media.metadata.process import global_reader
import media.metadata.definitions as defs
defs.load_definitions()
"""
list of supported easy tags in mutagen version 1.20
['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry',
@ -43,21 +47,6 @@ airtime2mutagen = {
"MDATA_KEY_COPYRIGHT" : "copyright",
}
class FakeMutagen(dict):
"""
Need this fake mutagen object so that airtime_special functions
return a proper default value instead of throwing an exceptions for
files that mutagen doesn't recognize
"""
FakeInfo = namedtuple('FakeInfo','length bitrate')
def __init__(self,path):
self.path = path
self.mime = ['audio/wav']
self.info = FakeMutagen.FakeInfo(0.0, '')
dict.__init__(self)
def set_length(self,l):
old_bitrate = self.info.bitrate
self.info = FakeMutagen.FakeInfo(l, old_bitrate)
# Some airtime attributes are special because they must use the mutagen object
# itself to calculate the value that they need. The lambda associated with each
@ -100,6 +89,7 @@ class Metadata(Loggable):
# little bit messy. Some of the handling is in m.m.pure while the rest is
# here. Also interface is not very consistent
# TODO : what is this shit? maybe get rid of it?
@staticmethod
def fix_title(path):
# If we have no title in path we will format it
@ -110,39 +100,6 @@ class Metadata(Loggable):
m[u'title'] = new_title
m.save()
@staticmethod
def airtime_dict(d):
"""
Converts mutagen dictionary 'd' into airtime dictionary
"""
temp_dict = {}
for m_key, m_val in d.iteritems():
# TODO : some files have multiple fields for the same metadata.
# genre is one example. In that case mutagen will return a list
# of values
if isinstance(m_val, list):
# TODO : does it make more sense to just skip the element in
# this case?
if len(m_val) == 0: assign_val = ''
else: assign_val = m_val[0]
else: assign_val = m_val
temp_dict[ m_key ] = assign_val
airtime_dictionary = {}
for muta_k, muta_v in temp_dict.iteritems():
# We must check if we can actually translate the mutagen key into
# an airtime key before doing the conversion
if muta_k in mutagen2airtime:
airtime_key = mutagen2airtime[muta_k]
# Apply truncation in the case where airtime_key is in our
# truncation table
muta_v = \
truncate_to_length(muta_v, truncate_table[airtime_key])\
if airtime_key in truncate_table else muta_v
airtime_dictionary[ airtime_key ] = muta_v
return airtime_dictionary
@staticmethod
def write_unsafe(path,md):
"""
@ -157,6 +114,7 @@ class Metadata(Loggable):
if airtime_k in airtime2mutagen:
# The unicode cast here is mostly for integers that need to be
# strings
if airtime_v is None: continue
try:
song_file[ airtime2mutagen[airtime_k] ] = unicode(airtime_v)
except (EasyMP4KeyError, EasyID3KeyError) as e:
@ -170,44 +128,7 @@ class Metadata(Loggable):
# Forcing the unicode through
try : fpath = fpath.decode("utf-8")
except : pass
if not mmp.file_playable(fpath): raise BadSongFile(fpath)
try : full_mutagen = mutagen.File(fpath, easy=True)
except Exception : raise BadSongFile(fpath)
self.path = fpath
if not os.path.exists(self.path):
self.logger.info("Attempting to read metadata of file \
that does not exist. Setting metadata to {}")
self.__metadata = {}
return
# TODO : Simplify the way all of these rules are handled right now it's
# extremely unclear and needs to be refactored.
#if full_mutagen is None: raise BadSongFile(fpath)
if full_mutagen is None: full_mutagen = FakeMutagen(fpath)
self.__metadata = Metadata.airtime_dict(full_mutagen)
# Now we extra the special values that are calculated from the mutagen
# object itself:
if mmp.extension(fpath) == 'wav':
full_mutagen.set_length(mmp.read_wave_duration(fpath))
for special_key,f in airtime_special.iteritems():
try:
new_val = f(full_mutagen)
if new_val is not None:
self.__metadata[special_key] = new_val
except Exception as e:
self.logger.info("Could not get special key %s for %s" %
(special_key, fpath))
self.logger.info(str(e))
# Finally, we "normalize" all the metadata here:
self.__metadata = mmp.normalized_metadata(self.__metadata, fpath)
# Now we must load the md5:
# TODO : perhaps we shouldn't hard code how many bytes we're reading
# from the file?
self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100)
self.__metadata = global_reader.read_mutagen(fpath)
def is_recorded(self):
"""

27
python_apps/media-monitor2/media/monitor/organizer.py
View file

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
import time
import media.monitor.pure as mmp
import media.monitor.owners as owners
from media.monitor.handler import ReportHandler
@ -11,14 +10,12 @@ from os.path import dirname
import os.path
class Organizer(ReportHandler,Loggable):
"""
Organizer is responsible to to listening to OrganizeListener events
and committing the appropriate changes to the filesystem. It does
not in any interact with WatchSyncer's even when the the WatchSyncer
is a "storage directory". The "storage" directory picks up all of
its events through pyinotify. (These events are fed to it through
StoreWatchListener)
"""
""" Organizer is responsible to to listening to OrganizeListener
events and committing the appropriate changes to the filesystem.
It does not in any interact with WatchSyncer's even when the the
WatchSyncer is a "storage directory". The "storage" directory picks
up all of its events through pyinotify. (These events are fed to it
through StoreWatchListener) """
# Commented out making this class a singleton because it's just a band aid
# for the real issue. The real issue being making multiple Organizer
@ -42,11 +39,9 @@ class Organizer(ReportHandler,Loggable):
super(Organizer, self).__init__(signal=self.channel, weak=False)
def handle(self, sender, event):
"""
Intercept events where a new file has been added to the organize
directory and place it in the correct path (starting with
self.target_path)
"""
""" Intercept events where a new file has been added to the
organize directory and place it in the correct path (starting
with self.target_path) """
# Only handle this event type
assert isinstance(event, OrganizeFile), \
"Organizer can only handle OrganizeFile events.Given '%s'" % event
@ -72,13 +67,9 @@ class Organizer(ReportHandler,Loggable):
directory=d)
return cb
time.sleep(0.05)
mmp.magic_move(event.path, new_path,
after_dir_make=new_dir_watch(dirname(new_path)))
time.sleep(0.05)
# The reason we need to go around saving the owner in this ass
# backwards way is bewcause we are unable to encode the owner id
# into the file itself so that the StoreWatchListener listener can

178
python_apps/media-monitor2/media/monitor/pure.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
import copy
from subprocess import Popen, PIPE
import subprocess
import os
import math
@ -21,7 +22,6 @@ from configobj import ConfigObj
from media.monitor.exceptions import FailedToSetLocale, FailedToCreateDir
#supported_extensions = [u"mp3", u"ogg", u"oga"]
supported_extensions = [u"mp3", u"ogg", u"oga", u"flac", u"wav",
u'm4a', u'mp4']
@ -66,7 +66,6 @@ class IncludeOnly(object):
return func(moi, event, *args, **kwargs)
return _wrap
def partition(f, alist):
"""
Partition is very similar to filter except that it also returns the
@ -92,14 +91,13 @@ def is_file_supported(path):
# TODO : In the future we would like a better way to find out whether a show
# has been recorded
def is_airtime_recorded(md):
"""
Takes a metadata dictionary and returns True if it belongs to a file that
was recorded by Airtime.
"""
""" Takes a metadata dictionary and returns True if it belongs to a
file that was recorded by Airtime. """
if not 'MDATA_KEY_CREATOR' in md: return False
return md['MDATA_KEY_CREATOR'] == u'Airtime Show Recorder'
def read_wave_duration(path):
""" Read the length of .wav file (mutagen does not handle this) """
with contextlib.closing(wave.open(path,'r')) as f:
frames = f.getnframes()
rate = f.getframerate()
@ -107,9 +105,7 @@ def read_wave_duration(path):
return duration
def clean_empty_dirs(path):
"""
walks path and deletes every empty directory it finds
"""
""" walks path and deletes every empty directory it finds """
# TODO : test this function
if path.endswith('/'): clean_empty_dirs(path[0:-1])
else:
@ -154,22 +150,25 @@ def no_extension_basename(path):
else: return '.'.join(base.split(".")[0:-1])
def walk_supported(directory, clean_empties=False):
"""
A small generator wrapper around os.walk to only give us files that support
the extensions we are considering. When clean_empties is True we
recursively delete empty directories left over in directory after the walk.
"""
""" A small generator wrapper around os.walk to only give us files
that support the extensions we are considering. When clean_empties
is True we recursively delete empty directories left over in
directory after the walk. """
for root, dirs, files in os.walk(directory):
full_paths = ( os.path.join(root, name) for name in files
if is_file_supported(name) )
for fp in full_paths: yield fp
if clean_empties: clean_empty_dirs(directory)
def file_locked(path):
cmd = "lsof %s" % path
f = Popen(cmd, shell=True, stdout=PIPE).stdout
return bool(f.readlines())
def magic_move(old, new, after_dir_make=lambda : None):
"""
Moves path old to new and constructs the necessary to directories for new
along the way
"""
""" Moves path old to new and constructs the necessary to
directories for new along the way """
new_dir = os.path.dirname(new)
if not os.path.exists(new_dir): os.makedirs(new_dir)
# We need this crusty hack because anytime a directory is created we must
@ -179,18 +178,15 @@ def magic_move(old, new, after_dir_make=lambda : None):
shutil.move(old,new)
def move_to_dir(dir_path,file_path):
"""
moves a file at file_path into dir_path/basename(filename)
"""
""" moves a file at file_path into dir_path/basename(filename) """
bs = os.path.basename(file_path)
magic_move(file_path, os.path.join(dir_path, bs))
def apply_rules_dict(d, rules):
"""
Consumes a dictionary of rules that maps some keys to lambdas which it
applies to every matching element in d and returns a new dictionary with
the rules applied. If a rule returns none then it's not applied
"""
""" Consumes a dictionary of rules that maps some keys to lambdas
which it applies to every matching element in d and returns a new
dictionary with the rules applied. If a rule returns none then it's
not applied """
new_d = copy.deepcopy(d)
for k, rule in rules.iteritems():
if k in d:
@ -205,17 +201,14 @@ def default_to_f(dictionary, keys, default, condition):
return new_d
def default_to(dictionary, keys, default):
"""
Checks if the list of keys 'keys' exists in 'dictionary'. If not then it
returns a new dictionary with all those missing keys defaults to 'default'
"""
""" Checks if the list of keys 'keys' exists in 'dictionary'. If
not then it returns a new dictionary with all those missing keys
defaults to 'default' """
cnd = lambda dictionary, key: key not in dictionary
return default_to_f(dictionary, keys, default, cnd)
def remove_whitespace(dictionary):
"""
Remove values that empty whitespace in the dictionary
"""
""" Remove values that empty whitespace in the dictionary """
nd = copy.deepcopy(dictionary)
bad_keys = []
for k,v in nd.iteritems():
@ -227,6 +220,7 @@ def remove_whitespace(dictionary):
return nd
def parse_int(s):
# TODO : this function isn't used anywhere yet but it may useful for emf
"""
Tries very hard to get some sort of integer result from s. Defaults to 0
when it fails
@ -242,53 +236,6 @@ def parse_int(s):
try : return str(reduce(op.add, takewhile(lambda x: x.isdigit(), s)))
except: return None
def normalized_metadata(md, original_path):
"""
consumes a dictionary of metadata and returns a new dictionary with the
formatted meta data. We also consume original_path because we must set
MDATA_KEY_CREATOR based on in it sometimes
"""
new_md = copy.deepcopy(md)
# replace all slashes with dashes
#for k,v in new_md.iteritems(): new_md[k] = unicode(v).replace('/','-')
# Specific rules that are applied in a per attribute basis
format_rules = {
'MDATA_KEY_TRACKNUMBER' : parse_int,
'MDATA_KEY_FILEPATH' : lambda x: os.path.normpath(x),
'MDATA_KEY_BPM' : lambda x: x[0:8],
'MDATA_KEY_MIME' : lambda x: x.replace('audio/vorbis','audio/ogg'),
# Whenever 0 is reported we change it to empty
#'MDATA_KEY_BITRATE' : lambda x: '' if str(x) == '0' else x
}
new_md = remove_whitespace(new_md) # remove whitespace fields
# Format all the fields in format_rules
new_md = apply_rules_dict(new_md, format_rules)
# set filetype to audioclip by default
new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_FTYPE'],
default=u'audioclip')
# Try to parse bpm but delete the whole key if that fails
if 'MDATA_KEY_BPM' in new_md:
new_md['MDATA_KEY_BPM'] = parse_int(new_md['MDATA_KEY_BPM'])
if new_md['MDATA_KEY_BPM'] is None:
del new_md['MDATA_KEY_BPM']
if not is_airtime_recorded(new_md):
# Read title from filename if it does not exist
default_title = no_extension_basename(original_path)
default_title = re.sub(r'__\d+\.',u'.', default_title)
if re.match(".+-%s-.+$" % unicode_unknown, default_title):
default_title = u''
new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_TITLE'],
default=default_title)
new_md['MDATA_KEY_TITLE'] = re.sub(r'-\d+kbps$', u'',
new_md['MDATA_KEY_TITLE'])
# TODO : wtf is this for again?
new_md['MDATA_KEY_TITLE'] = re.sub(r'-?%s-?' % unicode_unknown, u'',
new_md['MDATA_KEY_TITLE'])
return new_md
def organized_path(old_path, root_path, orig_md):
"""
@ -348,10 +295,9 @@ def organized_path(old_path, root_path, orig_md):
# TODO : Get rid of this function and every one of its uses. We no longer use
# the md5 signature of a song for anything
def file_md5(path,max_length=100):
"""
Get md5 of file path (if it exists). Use only max_length characters to save
time and memory. Pass max_length=-1 to read the whole file (like in mm1)
"""
""" Get md5 of file path (if it exists). Use only max_length
characters to save time and memory. Pass max_length=-1 to read the
whole file (like in mm1) """
if os.path.exists(path):
with open(path, 'rb') as f:
m = hashlib.md5()
@ -367,16 +313,12 @@ def encode_to(obj, encoding='utf-8'):
return obj
def convert_dict_value_to_utf8(md):
"""
formats a dictionary to send as a request to api client
"""
""" formats a dictionary to send as a request to api client """
return dict([(item[0], encode_to(item[1], "utf-8")) for item in md.items()])
def get_system_locale(locale_path='/etc/default/locale'):
"""
Returns the configuration object for the system's default locale. Normally
requires root access.
"""
""" Returns the configuration object for the system's default
locale. Normally requires root access. """
if os.path.exists(locale_path):
try:
config = ConfigObj(locale_path)
@ -386,9 +328,7 @@ def get_system_locale(locale_path='/etc/default/locale'):
permissions issue?" % locale_path)
def configure_locale(config):
"""
sets the locale according to the system's locale.
"""
""" sets the locale according to the system's locale. """
current_locale = locale.getlocale()
if current_locale[1] is None:
default_locale = locale.getdefaultlocale()
@ -405,27 +345,21 @@ def configure_locale(config):
def fondle(path,times=None):
# TODO : write unit tests for this
"""
touch a file to change the last modified date. Beware of calling this
function on the same file from multiple threads.
"""
""" touch a file to change the last modified date. Beware of calling
this function on the same file from multiple threads. """
with file(path, 'a'): os.utime(path, times)
def last_modified(path):
"""
return the time of the last time mm2 was ran. path refers to the index file
whose date modified attribute contains this information. In the case when
the file does not exist we set this time 0 so that any files on the
filesystem were modified after it
"""
""" return the time of the last time mm2 was ran. path refers to the
index file whose date modified attribute contains this information.
In the case when the file does not exist we set this time 0 so that
any files on the filesystem were modified after it """
if os.path.exists(path): return os.path.getmtime(path)
else: return 0
def expand_storage(store):
"""
A storage directory usually consists of 4 different subdirectories. This
function returns their paths
"""
""" A storage directory usually consists of 4 different
subdirectories. This function returns their paths """
store = os.path.normpath(store)
return {
'organize' : os.path.join(store, 'organize'),
@ -435,10 +369,8 @@ def expand_storage(store):
}
def create_dir(path):
"""
will try and make sure that path exists at all costs. raises an exception
if it fails at this task.
"""
""" will try and make sure that path exists at all costs. raises an
exception if it fails at this task. """
if not os.path.exists(path):
try : os.makedirs(path)
except Exception as e : raise FailedToCreateDir(path, e)
@ -456,11 +388,10 @@ def sub_path(directory,f):
return common == normalized
def owner_id(original_path):
"""
Given 'original_path' return the file name of the of 'identifier' file.
return the id that is contained in it. If no file is found or nothing is
read then -1 is returned. File is deleted after the number has been read
"""
""" Given 'original_path' return the file name of the of
'identifier' file. return the id that is contained in it. If no file
is found or nothing is read then -1 is returned. File is deleted
after the number has been read """
fname = "%s.identifier" % original_path
owner_id = -1
try:
@ -476,9 +407,8 @@ def owner_id(original_path):
return owner_id
def file_playable(pathname):
"""
Returns True if 'pathname' is playable by liquidsoap. False otherwise.
"""
""" Returns True if 'pathname' is playable by liquidsoap. False
otherwise. """
# when there is an single apostrophe inside of a string quoted by
# apostrophes, we can only escape it by replace that apostrophe with
# '\''. This breaks the string into two, and inserts an escaped
@ -514,18 +444,14 @@ def toposort(data):
assert not data, "A cyclic dependency exists amongst %r" % data
def truncate_to_length(item, length):
"""
Truncates 'item' to 'length'
"""
""" Truncates 'item' to 'length' """
if isinstance(item, int): item = str(item)
if isinstance(item, basestring):
if len(item) > length: return item[0:length]
else: return item
def format_length(mutagen_length):
"""
Convert mutagen length to airtime length
"""
""" Convert mutagen length to airtime length """
t = float(mutagen_length)
h = int(math.floor(t / 3600))
t = t % 3600

60
python_apps/media-monitor2/media/monitor/request.py Normal file
View file

@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-
import threading
from media.monitor.exceptions import BadSongFile
from media.monitor.log import Loggable
import api_clients.api_client as ac
class ThreadedRequestSync(threading.Thread, Loggable):
def __init__(self, rs):
threading.Thread.__init__(self)
self.rs = rs
self.daemon = True
self.start()
def run(self):
self.rs.run_request()
class RequestSync(Loggable):
""" This class is responsible for making the api call to send a
request to airtime. In the process it packs the requests and retries
for some number of times """
@classmethod
def create_with_api_client(cls, watcher, requests):
apiclient = ac.AirtimeApiClient.create_right_config()
self = cls(watcher, requests, apiclient)
return self
def __init__(self, watcher, requests, apiclient):
self.watcher = watcher
self.requests = requests
self.apiclient = apiclient
def run_request(self):
self.logger.info("Attempting request with %d items." %
len(self.requests))
packed_requests = []
for request_event in self.requests:
try:
for request in request_event.safe_pack():
if isinstance(request, BadSongFile):
self.logger.info("Bad song file: '%s'" % request.path)
else: packed_requests.append(request)
except Exception as e:
self.unexpected_exception( e )
if hasattr(request_event, 'path'):
self.logger.info("Possibly related to path: '%s'" %
request_event.path)
try: self.apiclient.send_media_monitor_requests( packed_requests )
# most likely we did not get json response as we expected
except ValueError:
self.logger.info("ApiController.php probably crashed, we \
diagnose this from the fact that it did not return \
valid json")
self.logger.info("Trying again after %f seconds" %
self.request_wait)
except Exception as e: self.unexpected_exception(e)
else: self.logger.info("Request was successful")
self.watcher.flag_done() # poor man's condition variable

10
python_apps/media-monitor2/media/monitor/syncdb.py
View file

@ -53,11 +53,11 @@ class AirtimeDB(Loggable):
"""
return self.id_to_dir[ dir_id ]
def storage_path(self): return self.base_storage
def organize_path(self): return self.storage_paths['organize']
def problem_path(self): return self.storage_paths['problem_files']
def import_path(self): return self.storage_paths['imported']
def recorded_path(self): return self.storage_paths['recorded']
def storage_path(self) : return self.base_storage
def organize_path(self) : return self.storage_paths['organize']
def problem_path(self) : return self.storage_paths['problem_files']
def import_path(self) : return self.storage_paths['imported']
def recorded_path(self) : return self.storage_paths['recorded']
def list_watched(self):
"""

72
python_apps/media-monitor2/media/monitor/watchersyncer.py
View file

@ -6,69 +6,9 @@ import copy
from media.monitor.handler import ReportHandler
from media.monitor.log import Loggable
from media.monitor.exceptions import BadSongFile
from media.monitor.pure import LazyProperty
from media.monitor.eventcontractor import EventContractor
from media.monitor.events import EventProxy
import api_clients.api_client as ac
class RequestSync(threading.Thread,Loggable):
"""
This class is responsible for making the api call to send a request
to airtime. In the process it packs the requests and retries for
some number of times
"""
def __init__(self, watcher, requests):
threading.Thread.__init__(self)
self.watcher = watcher
self.requests = requests
self.retries = 1
self.request_wait = 0.3
@LazyProperty
def apiclient(self):
return ac.AirtimeApiClient.create_right_config()
def run(self):
self.logger.info("Attempting request with %d items." %
len(self.requests))
# Note that we must attach the appropriate mode to every
# response. Also Not forget to attach the 'is_record' to any
# requests that are related to recorded shows
# TODO : recorded shows aren't flagged right
# Is this retry shit even necessary? Consider getting rid of this.
packed_requests = []
for request_event in self.requests:
try:
for request in request_event.safe_pack():
if isinstance(request, BadSongFile):
self.logger.info("Bad song file: '%s'" % request.path)
else: packed_requests.append(request)
except Exception as e:
self.unexpected_exception( e )
if hasattr(request_event, 'path'):
self.logger.info("Possibly related to path: '%s'" %
request_event.path)
def make_req():
self.apiclient.send_media_monitor_requests( packed_requests )
for try_index in range(0,self.retries):
try: make_req()
# most likely we did not get json response as we expected
except ValueError:
self.logger.info("ApiController.php probably crashed, we \
diagnose this from the fact that it did not return \
valid json")
self.logger.info("Trying again after %f seconds" %
self.request_wait)
time.sleep( self.request_wait )
except Exception as e: self.unexpected_exception(e)
else:
self.logger.info("Request worked on the '%d' try" %
(try_index + 1))
break
else: self.logger.info("Failed to send request after '%d' tries..." %
self.retries)
self.watcher.flag_done()
from media.monitor.request import ThreadedRequestSync, RequestSync
class TimeoutWatcher(threading.Thread,Loggable):
"""
@ -131,8 +71,7 @@ class WatchSyncer(ReportHandler,Loggable):
#self.push_queue( event )
except BadSongFile as e:
self.fatal_exception("Received bas song file '%s'" % e.path, e)
except Exception as e:
self.unexpected_exception(e)
except Exception as e: self.unexpected_exception(e)
else:
self.logger.info("Received event that does not implement packing.\
Printing its representation:")
@ -209,8 +148,8 @@ class WatchSyncer(ReportHandler,Loggable):
requests = copy.copy(self.__queue)
def launch_request():
# Need shallow copy here
t = RequestSync(watcher=self, requests=requests)
t.start()
t = ThreadedRequestSync( RequestSync.create_with_api_client(
watcher=self, requests=requests) )
self.__current_thread = t
self.__requests.append(launch_request)
self.__reset_queue()
@ -218,7 +157,8 @@ class WatchSyncer(ReportHandler,Loggable):
def __reset_queue(self): self.__queue = []
def __del__(self):
# Ideally we would like to do a little more to ensure safe shutdown
#this destructor is completely untested and it's unclear whether
#it's even doing anything useful. consider removing it
if self.events_in_queue():
self.logger.warn("Terminating with events still in the queue...")
if self.requests_in_queue():

33
python_apps/media-monitor2/media/update/replaygain.py
View file

@ -71,8 +71,10 @@ def get_file_type(file_path):
def calculate_replay_gain(file_path):
"""
This function accepts files of type mp3/ogg/flac and returns a calculated ReplayGain value in dB.
If the value cannot be calculated for some reason, then we default to 0 (Unity Gain).
This function accepts files of type mp3/ogg/flac and returns a calculated
ReplayGain value in dB.
If the value cannot be calculated for some reason, then we default to 0
(Unity Gain).
http://wiki.hydrogenaudio.org/index.php?title=ReplayGain_1.0_specification
"""
@ -92,20 +94,37 @@ def calculate_replay_gain(file_path):
if file_type:
if file_type == 'mp3':
if run_process("which mp3gain > /dev/null") == 0:
out = get_process_output('nice -n %s mp3gain -q "%s" 2> /dev/null' % (nice_level, temp_file_path))
search = re.search(r'Recommended "Track" dB change: (.*)', out)
command = 'nice -n %s mp3gain -q "%s" 2> /dev/null' \
% (nice_level, temp_file_path)
out = get_process_output(command)
search = re.search(r'Recommended "Track" dB change: (.*)', \
out)
else:
logger.warn("mp3gain not found")
elif file_type == 'vorbis':
if run_process("which vorbisgain > /dev/null && which ogginfo > /dev/null") == 0:
run_process('nice -n %s vorbisgain -q -f "%s" 2>/dev/null >/dev/null' % (nice_level,temp_file_path))
command = "which vorbisgain > /dev/null && which ogginfo > \
/dev/null"
if run_process(command) == 0:
command = 'nice -n %s vorbisgain -q -f "%s" 2>/dev/null \
>/dev/null' % (nice_level,temp_file_path)
run_process(command)
out = get_process_output('ogginfo "%s"' % temp_file_path)
search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out)
else:
logger.warn("vorbisgain/ogginfo not found")
elif file_type == 'flac':
if run_process("which metaflac > /dev/null") == 0:
out = get_process_output('nice -n %s metaflac --show-tag=REPLAYGAIN_TRACK_GAIN "%s"' % (nice_level, temp_file_path))
command = 'nice -n %s metaflac --add-replay-gain "%s"' \
% (nice_level, temp_file_path)
run_process(command)
command = 'nice -n %s metaflac \
--show-tag=REPLAYGAIN_TRACK_GAIN "%s"' \
% (nice_level, temp_file_path)
out = get_process_output(command)
search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out)
else: logger.warn("metaflac not found")

7
python_apps/media-monitor2/mm2.py
View file

@ -59,7 +59,8 @@ def main(global_config, api_client_config, log_config,
try:
with open(config['index_path'], 'w') as f: f.write(" ")
except Exception as e:
log.info("Failed to create index file with exception: %s" % str(e))
log.info("Failed to create index file with exception: %s" \
% str(e))
else:
log.info("Created index file, reloading configuration:")
main( global_config, api_client_config, log_config,
@ -103,6 +104,9 @@ def main(global_config, api_client_config, log_config,
airtime_notifier = AirtimeNotifier(config, airtime_receiver)
store = apiclient.setup_media_monitor()
log.info("Initing with the following airtime response:%s" % str(store))
airtime_receiver.change_storage({ 'directory':store[u'stor'] })
for watch_dir in store[u'watched_dirs']:
@ -114,6 +118,7 @@ def main(global_config, api_client_config, log_config,
(given from the database)." % watch_dir)
if os.path.exists(watch_dir):
airtime_receiver.new_watch({ 'directory':watch_dir }, restart=True)
else: log.info("Failed to add watch on %s" % str(watch_dir))
bs = Bootstrapper( db=sdb, watch_signal='watch' )

4
python_apps/media-monitor2/tests/test_api_client.py
View file

@ -19,8 +19,8 @@ class TestApiClient(unittest.TestCase):
self.apc.register_component("api-client-tester")
# All of the following requests should error out in some way
self.bad_requests = [
{ 'mode' : 'dang it', 'is_record' : 0 },
{ 'mode' : 'damn frank', 'is_record' : 1 },
{ 'mode' : 'foo', 'is_record' : 0 },
{ 'mode' : 'bar', 'is_record' : 1 },
{ 'no_mode' : 'at_all' }, ]
def test_bad_requests(self):

31
python_apps/media-monitor2/tests/test_emf.py Normal file
View file

@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
import unittest
#from pprint import pprint as pp
from media.metadata.process import global_reader
from media.monitor.metadata import Metadata
import media.metadata.definitions as defs
defs.load_definitions()
class TestMMP(unittest.TestCase):
def setUp(self):
self.maxDiff = None
def metadatas(self,f):
return global_reader.read_mutagen(f), Metadata(f).extract()
def test_old_metadata(self):
path = "/home/rudi/music/Nightingale.mp3"
m = global_reader.read_mutagen(path)
self.assertTrue( len(m) > 0 )
n = Metadata(path)
self.assertEqual(n.extract(), m)
def test_recorded(self):
recorded_file = "./15:15:00-Untitled Show-256kbps.ogg"
emf, old = self.metadatas(recorded_file)
self.assertEqual(emf, old)
if __name__ == '__main__': unittest.main()

7
python_apps/media-monitor2/tests/test_metadata.py
View file

@ -26,7 +26,6 @@ class TestMetadata(unittest.TestCase):
i += 1
print("Sample metadata: '%s'" % md)
self.assertTrue( len( md.keys() ) > 0 )
self.assertTrue( 'MDATA_KEY_MD5' in md )
utf8 = md_full.utf8()
for k,v in md.iteritems():
if hasattr(utf8[k], 'decode'):
@ -42,10 +41,4 @@ class TestMetadata(unittest.TestCase):
x1 = 123456
print("Formatting '%s' to '%s'" % (x1, mmm.format_length(x1)))
def test_truncate_to_length(self):
s1 = "testing with non string literal"
s2 = u"testing with unicode literal"
self.assertEqual( len(mmm.truncate_to_length(s1, 5)), 5)
self.assertEqual( len(mmm.truncate_to_length(s2, 8)), 8)
if __name__ == '__main__': unittest.main()

44
python_apps/media-monitor2/tests/test_metadata_def.py Normal file
View file

@ -0,0 +1,44 @@
# -*- coding: utf-8 -*-
import unittest
import media.metadata.process as md
class TestMetadataDef(unittest.TestCase):
def test_simple(self):
with md.metadata('MDATA_TESTING') as t:
t.optional(True)
t.depends('ONE','TWO')
t.default('unknown')
t.translate(lambda kw: kw['ONE'] + kw['TWO'])
h = { 'ONE' : "testing", 'TWO' : "123" }
result = md.global_reader.read('test_path',h)
self.assertTrue( 'MDATA_TESTING' in result )
self.assertEqual( result['MDATA_TESTING'], 'testing123' )
h1 = { 'ONE' : 'big testing', 'two' : 'nothing' }
result1 = md.global_reader.read('bs path', h1)
self.assertEqual( result1['MDATA_TESTING'], 'unknown' )
def test_topo(self):
with md.metadata('MDATA_TESTING') as t:
t.depends('shen','sheni')
t.default('megitzda')
t.translate(lambda kw: kw['shen'] + kw['sheni'])
with md.metadata('shen') as t:
t.default('vaxo')
with md.metadata('sheni') as t:
t.default('gio')
with md.metadata('vaxo') as t:
t.depends('shevetsi')
v = md.global_reader.read('bs mang', {})
self.assertEqual(v['MDATA_TESTING'], 'vaxogio')
self.assertTrue( 'vaxo' not in v )
md.global_reader.clear()
if __name__ == '__main__': unittest.main()

70
python_apps/media-monitor2/tests/test_pure.py
View file

@ -2,7 +2,6 @@
import unittest
import os
import media.monitor.pure as mmp
from media.monitor.metadata import Metadata
class TestMMP(unittest.TestCase):
def setUp(self):
@ -34,68 +33,6 @@ class TestMMP(unittest.TestCase):
sd = mmp.default_to(dictionary=sd, keys=def_keys, default='DEF')
for k in def_keys: self.assertEqual( sd[k], 'DEF' )
def test_normalized_metadata(self):
#Recorded show test first
orig = Metadata.airtime_dict({
'date' : [u'2012-08-21'],
'tracknumber' : [u'2'],
'title' : [u'record-2012-08-21-11:29:00'],
'artist' : [u'Airtime Show Recorder']
})
orga = Metadata.airtime_dict({
'date' : [u'2012-08-21'],
'tracknumber' : [u'2'],
'artist' : [u'Airtime Show Recorder'],
'title' : [u'record-2012-08-21-11:29:00']
})
orga['MDATA_KEY_FTYPE'] = u'audioclip'
orig['MDATA_KEY_BITRATE'] = u'256000'
orga['MDATA_KEY_BITRATE'] = u'256000'
old_path = "/home/rudi/recorded/2012-08-21-11:29:00.ogg"
normalized = mmp.normalized_metadata(orig, old_path)
normalized['MDATA_KEY_BITRATE'] = u'256000'
self.assertEqual( orga, normalized )
organized_base_name = "11:29:00-record-256kbps.ogg"
base = "/srv/airtime/stor/"
organized_path = mmp.organized_path(old_path,base, normalized)
self.assertEqual(os.path.basename(organized_path), organized_base_name)
def test_normalized_metadata2(self):
"""
cc-4305
"""
orig = Metadata.airtime_dict({
'date' : [u'2012-08-27'],
'tracknumber' : [u'3'],
'title' : [u'18-11-00-Untitled Show'],
'artist' : [u'Airtime Show Recorder']
})
old_path = "/home/rudi/recorded/doesnt_really_matter.ogg"
normalized = mmp.normalized_metadata(orig, old_path)
normalized['MDATA_KEY_BITRATE'] = u'256000'
opath = mmp.organized_path(old_path, "/srv/airtime/stor/",
normalized)
# TODO : add a better test than this...
self.assertTrue( len(opath) > 0 )
def test_normalized_metadata3(self):
"""
Test the case where the metadata is empty
"""
orig = Metadata.airtime_dict({})
paths_unknown_title = [
("/testin/unknown-unknown-unknown.mp3",""),
("/testin/01-unknown-123kbps.mp3",""),
("/testin/02-unknown-140kbps.mp3",""),
("/testin/unknown-unknown-123kbps.mp3",""),
("/testin/unknown-bibimbop-unknown.mp3","bibimbop"),
]
for p,res in paths_unknown_title:
normalized = mmp.normalized_metadata(orig, p)
self.assertEqual( normalized['MDATA_KEY_TITLE'], res)
def test_file_md5(self):
p = os.path.realpath(__file__)
m1 = mmp.file_md5(p)
@ -116,6 +53,13 @@ class TestMMP(unittest.TestCase):
self.assertEqual( mmp.parse_int("123asf"), "123" )
self.assertEqual( mmp.parse_int("asdf"), None )
def test_truncate_to_length(self):
s1 = "testing with non string literal"
s2 = u"testing with unicode literal"
self.assertEqual( len(mmp.truncate_to_length(s1, 5)), 5)
self.assertEqual( len(mmp.truncate_to_length(s2, 8)), 8)
def test_owner_id(self):
start_path = "testing.mp3"
id_path = "testing.mp3.identifier"

48
python_apps/media-monitor2/tests/test_requestsync.py Normal file
View file

@ -0,0 +1,48 @@
import unittest
from mock import MagicMock
from media.monitor.request import RequestSync
class TestRequestSync(unittest.TestCase):
def apc_mock(self):
fake_apc = MagicMock()
fake_apc.send_media_monitor_requests = MagicMock()
return fake_apc
def watcher_mock(self):
fake_watcher = MagicMock()
fake_watcher.flag_done = MagicMock()
return fake_watcher
def request_mock(self):
fake_request = MagicMock()
fake_request.safe_pack = MagicMock(return_value=[])
return fake_request
def test_send_media_monitor(self):
fake_apc = self.apc_mock()
fake_requests = [ self.request_mock() for x in range(1,5) ]
fake_watcher = self.watcher_mock()
rs = RequestSync(fake_watcher, fake_requests, fake_apc)
rs.run_request()
self.assertEquals(fake_apc.send_media_monitor_requests.call_count, 1)
def test_flag_done(self):
fake_apc = self.apc_mock()
fake_requests = [ self.request_mock() for x in range(1,5) ]
fake_watcher = self.watcher_mock()
rs = RequestSync(fake_watcher, fake_requests, fake_apc)
rs.run_request()
self.assertEquals(fake_watcher.flag_done.call_count, 1)
def test_safe_pack(self):
fake_apc = self.apc_mock()
fake_requests = [ self.request_mock() for x in range(1,5) ]
fake_watcher = self.watcher_mock()
rs = RequestSync(fake_watcher, fake_requests, fake_apc)
rs.run_request()
for req in fake_requests:
self.assertEquals(req.safe_pack.call_count, 1)
if __name__ == '__main__': unittest.main()

314
python_apps/pypo/liquidsoap_scripts/library/external-todo.liq
View file

@ -1,314 +0,0 @@
# These operators need to be updated..
# Stream data from mplayer
# @category Source / Input
# @param s data URI.
# @param ~restart restart on exit.
# @param ~restart_on_error restart on exit with error.
# @param ~buffer Duration of the pre-buffered data.
# @param ~max Maximum duration of the buffered data.
def input.mplayer(~id="input.mplayer",
~restart=true,~restart_on_error=false,
~buffer=0.2,~max=10.,s) =
input.external(id=id,restart=restart,
restart_on_error=restart_on_error,
buffer=buffer,max=max,
"mplayer -really-quiet -ao pcm:file=/dev/stdout \
-vc null -vo null #{quote(s)} 2>/dev/null")
end
# Output the stream using aplay.
# Using this turns "root.sync" to false
# since aplay will do the synchronisation
# @category Source / Output
# @param ~id Output's ID
# @param ~device Alsa pcm device name
# @param ~restart_on_crash Restart external process on crash. If false, liquidsoap will stop.
# @param ~fallible Allow the child source to fail, in which case the output will be (temporarily) stopped.
# @param ~on_start Callback executed when outputting starts.
# @param ~on_stop Callback executed when outputting stops.
# @param s Source to play
def output.aplay(~id="output.aplay",~device="default",
~fallible=false,~on_start={()},~on_stop={()},
~restart_on_crash=false,s)
def aplay_p(m) =
"aplay -D #{device}"
end
log(label=id,level=3,"Setting root.sync to false")
set("root.sync",false)
output.pipe.external(id=id,
fallible=fallible,on_start=on_start,on_stop=on_stop,
restart_on_crash=restart_on_crash,
restart_on_new_track=false,
process=aplay_p,s)
end
%ifdef output.icecast.external
# Output to icecast using the lame command line encoder.
# @category Source / Output
# @param ~id Output's ID
# @param ~start Start output threads on operator initialization.
# @param ~restart Restart output after a failure. By default, liquidsoap will stop if the output failed.
# @param ~restart_delay Delay, in seconds, before attempting new connection, if restart is enabled.
# @param ~restart_on_crash Restart external process on crash. If false, liquidsoap will stop.
# @param ~restart_on_new_track Restart encoder upon new track.
# @param ~restart_encoder_delay Restart the encoder after this delay, in seconds.
# @param ~user User for shout source connection. Useful only in special cases, like with per-mountpoint users.
# @param ~lame The lame binary
# @param ~bitrate Encoder bitrate
# @param ~swap Swap audio samples. Depends on local machine's endianess and lame's version. Test this parameter if you experience garbaged mp3 audio data. On intel 32 and 64 architectures, the parameter should be "true" for lame version >= 3.98.
# @param ~dumpfile Dump stream to file, for debugging purpose. Disabled if empty.
# @param ~protocol Protocol of the streaming server: 'http' for Icecast, 'icy' for Shoutcast.
# @param ~fallible Allow the child source to fail, in which case the output will be (temporarily) stopped.
# @param ~on_start Callback executed when outputting starts.
# @param ~on_stop Callback executed when outputting stops.
# @param s The source to output
def output.icecast.lame(
~id="output.icecast.lame",~start=true,
~restart=false,~restart_delay=3,
~host="localhost",~port=8000,
~user="source",~password="hackme",
~genre="Misc",~url="http://savonet.sf.net/",
~description="Liquidsoap Radio!",~public=true,
~dumpfile="",~mount="Use [name]",
~name="Use [mount]",~protocol="http",
~lame="lame",~bitrate=128,~swap=false,
~fallible=false,~on_start={()},~on_stop={()},
~restart_on_crash=false,~restart_on_new_track=false,
~restart_encoder_delay=3600,~headers=[],s)
samplerate = get(default=44100,"frame.samplerate")
samplerate = float_of_int(samplerate) / 1000.
channels = get(default=2,"frame.channels")
swap = if swap then "-x" else "" end
mode =
if channels == 2 then
"j" # Encoding in joint stereo..
else
"m"
end
# Metadata update is set by ICY with icecast
def lame_p(m)
"#{lame} -b #{bitrate} -r --bitwidth 16 -s #{samplerate} \
--signed -m #{mode} --nores #{swap} -t - -"
end
output.icecast.external(id=id,
process=lame_p,bitrate=bitrate,start=start,
restart=restart,restart_delay=restart_delay,
host=host,port=port,user=user,password=password,
genre=genre,url=url,description=description,
public=public,dumpfile=dumpfile,restart_encoder_delay=restart_encoder_delay,
name=name,mount=mount,protocol=protocol,
header=false,restart_on_crash=restart_on_crash,
restart_on_new_track=restart_on_new_track,headers=headers,
fallible=fallible,on_start=on_start,on_stop=on_stop,
s)
end
# Output to shoutcast using the lame encoder.
# @category Source / Output
# @param ~id Output's ID
# @param ~start Start output threads on operator initialization.
# @param ~restart Restart output after a failure. By default, liquidsoap will stop if the output failed.
# @param ~restart_delay Delay, in seconds, before attempting new connection, if restart is enabled.
# @param ~restart_on_crash Restart external process on crash. If false, liquidsoap will stop.
# @param ~restart_on_new_track Restart encoder upon new track.
# @param ~restart_encoder_delay Restart the encoder after this delay, in seconds.
# @param ~user User for shout source connection. Useful only in special cases, like with per-mountpoint users.
# @param ~lame The lame binary
# @param ~bitrate Encoder bitrate
# @param ~icy_reset Reset shoutcast source buffer upon connecting (necessary for NSV).
# @param ~dumpfile Dump stream to file, for debugging purpose. Disabled if empty.
# @param ~fallible Allow the child source to fail, in which case the output will be (temporarily) stopped.
# @param ~on_start Callback executed when outputting starts.
# @param ~on_stop Callback executed when outputting stops.
# @param s The source to output
def output.shoutcast.lame(
~id="output.shoutcast.mp3",~start=true,
~restart=false,~restart_delay=3,
~host="localhost",~port=8000,
~user="source",~password="hackme",
~genre="Misc",~url="http://savonet.sf.net/",
~description="Liquidsoap Radio!",~public=true,
~dumpfile="",~name="Use [mount]",~icy_reset=true,
~lame="lame",~aim="",~icq="",~irc="",
~fallible=false,~on_start={()},~on_stop={()},
~restart_on_crash=false,~restart_on_new_track=false,
~restart_encoder_delay=3600,~bitrate=128,s) =
icy_reset = if icy_reset then "1" else "0" end
headers = [("icy-aim",aim),("icy-irc",irc),
("icy-icq",icq),("icy-reset",icy_reset)]
output.icecast.lame(
id=id, headers=headers, lame=lame,
bitrate=bitrate, start=start,
restart=restart, restart_encoder_delay=restart_encoder_delay,
host=host, port=port, user=user, password=password,
genre=genre, url=url, description=description,
public=public, dumpfile=dumpfile,
restart_on_crash=restart_on_crash,
restart_on_new_track=restart_on_new_track,
name=name, mount="/", protocol="icy",
fallible=fallible,on_start=on_start,on_stop=on_stop,
s)
end
# Output to icecast using the flac command line encoder.
# @category Source / Output
# @param ~id Output's ID
# @param ~start Start output threads on operator initialization.
# @param ~restart Restart output after a failure. By default, liquidsoap will stop if the output failed.
# @param ~restart_delay Delay, in seconds, before attempting new connection, if restart is enabled.
# @param ~restart_on_crash Restart external process on crash. If false, liquidsoap will stop.
# @param ~restart_on_new_track Restart encoder upon new track. If false, the resulting stream will have a single track.
# @param ~restart_encoder_delay Restart the encoder after this delay, in seconds.
# @param ~user User for shout source connection. Useful only in special cases, like with per-mountpoint users.
# @param ~flac The flac binary
# @param ~quality Encoder quality (0..8)
# @param ~dumpfile Dump stream to file, for debugging purpose. Disabled if empty.
# @param ~protocol Protocol of the streaming server: 'http' for Icecast, 'icy' for Shoutcast.
# @param ~fallible Allow the child source to fail, in which case the output will be (temporarily) stopped.
# @param ~on_start Callback executed when outputting starts.
# @param ~on_stop Callback executed when outputting stops.
# @param s The source to output
def output.icecast.flac(
~id="output.icecast.flac",~start=true,
~restart=false,~restart_delay=3,
~host="localhost",~port=8000,
~user="source",~password="hackme",
~genre="Misc",~url="http://savonet.sf.net/",
~description="Liquidsoap Radio!",~public=true,
~dumpfile="",~mount="Use [name]",
~name="Use [mount]",~protocol="http",
~flac="flac",~quality=6,
~restart_on_crash=false,
~restart_on_new_track=true,
~restart_encoder_delay=(-1),
~fallible=false,~on_start={()},~on_stop={()},
s)
# We will use raw format, to
# bypass input length value in WAV
# header (input length is not known)
channels = get(default=2,"frame.channels")
samplerate = get(default=44100,"frame.samplerate")
def flac_p(m)=
def option(x) =
"-T #{quote(fst(x))}=#{quote(snd(x))}"
end
m = list.map(option,m)
m = string.concat(separator=" ",m)
"#{flac} --force-raw-format --endian=little --channels=#{channels} \
--bps=16 --sample-rate=#{samplerate} --sign=signed #{m} \
-#{quality} --ogg -c -"
end
output.icecast.external(id=id,
process=flac_p,bitrate=(-1),start=start,
restart=restart,restart_delay=restart_delay,
host=host,port=port,user=user,password=password,
genre=genre,url=url,description=description,
public=public,dumpfile=dumpfile,
name=name,mount=mount,protocol=protocol,
fallible=fallible,on_start=on_start,on_stop=on_stop,
restart_on_new_track=restart_on_new_track,
format="ogg",header=false,icy_metadata=false,
restart_on_crash=restart_on_crash,
restart_encoder_delay=restart_encoder_delay,
s)
end
# Output to icecast using the aacplusenc command line encoder.
# @category Source / Output
# @param ~id Output's ID
# @param ~start Start output threads on operator initialization.
# @param ~restart Restart output after a failure. By default, liquidsoap will stop if the output failed.
# @param ~restart_delay Delay, in seconds, before attempting new connection, if restart is enabled.
# @param ~restart_on_crash Restart external process on crash. If false, liquidsoap will stop.
# @param ~restart_on_new_track Restart encoder upon new track.
# @param ~restart_encoder_delay Restart the encoder after this delay, in seconds.
# @param ~user User for shout source connection. Useful only in special cases, like with per-mountpoint users.
# @param ~aacplusenc The aacplusenc binary
# @param ~bitrate Encoder bitrate
# @param ~dumpfile Dump stream to file, for debugging purpose. Disabled if empty.
# @param ~protocol Protocol of the streaming server: 'http' for Icecast, 'icy' for Shoutcast.
# @param ~fallible Allow the child source to fail, in which case the output will be (temporarily) stopped.
# @param ~on_start Callback executed when outputting starts.
# @param ~on_stop Callback executed when outputting stops.
# @param s The source to output
def output.icecast.aacplusenc(
~id="output.icecast.aacplusenc",~start=true,
~restart=false,~restart_delay=3,
~host="localhost",~port=8000,
~user="source",~password="hackme",
~genre="Misc",~url="http://savonet.sf.net/",
~description="Liquidsoap Radio!",~public=true,
~dumpfile="",~mount="Use [name]",
~name="Use [mount]",~protocol="http",
~aacplusenc="aacplusenc",~bitrate=64,
~fallible=false,~on_start={()},~on_stop={()},
~restart_on_crash=false,~restart_on_new_track=false,
~restart_encoder_delay=3600,~headers=[],s)
# Metadata update is set by ICY with icecast
def aacplusenc_p(m)
"#{aacplusenc} - - #{bitrate}"
end
output.icecast.external(id=id,
process=aacplusenc_p,bitrate=bitrate,start=start,
restart=restart,restart_delay=restart_delay,
host=host,port=port,user=user,password=password,
genre=genre,url=url,description=description,
public=public,dumpfile=dumpfile,
name=name,mount=mount,protocol=protocol,
fallible=fallible,on_start=on_start,on_stop=on_stop,
header=true,restart_on_crash=restart_on_crash,
restart_on_new_track=restart_on_new_track,headers=headers,
restart_encoder_delay=restart_encoder_delay,format="audio/aacp",s)
end
# Output to shoutcast using the aacplusenc encoder.
# @category Source / Output
# @param ~id Output's ID
# @param ~start Start output threads on operator initialization.
# @param ~restart Restart output after a failure. By default, liquidsoap will stop if the output failed.
# @param ~restart_delay Delay, in seconds, before attempting new connection, if restart is enabled.
# @param ~restart_on_crash Restart external process on crash. If false, liquidsoap will stop.
# @param ~restart_on_new_track Restart encoder upon new track.
# @param ~restart_encoder_delay Restart the encoder after this delay, in seconds.
# @param ~user User for shout source connection. Useful only in special cases, like with per-mountpoint users.
# @param ~aacplusenc The aacplusenc binary
# @param ~bitrate Encoder bitrate
# @param ~icy_reset Reset shoutcast source buffer upon connecting (necessary for NSV).
# @param ~dumpfile Dump stream to file, for debugging purpose. Disabled if empty.
# @param ~fallible Allow the child source to fail, in which case the output will be (temporarily) stopped.
# @param ~on_start Callback executed when outputting starts.
# @param ~on_stop Callback executed when outputting stops.
# @param s The source to output
def output.shoutcast.aacplusenc(
~id="output.shoutcast.aacplusenc",~start=true,
~restart=false,~restart_delay=3,
~host="localhost",~port=8000,
~user="source",~password="hackme",
~genre="Misc",~url="http://savonet.sf.net/",
~description="Liquidsoap Radio!",~public=true,
~fallible=false,~on_start={()},~on_stop={()},
~dumpfile="",~name="Use [mount]",~icy_reset=true,
~aim="",~icq="",~irc="",~aacplusenc="aacplusenc",
~restart_on_crash=false,~restart_on_new_track=false,
~restart_encoder_delay=3600,~bitrate=64,s) =
icy_reset = if icy_reset then "1" else "0" end
headers = [("icy-aim",aim),("icy-irc",irc),
("icy-icq",icq),("icy-reset",icy_reset)]
output.icecast.aacplusenc(
id=id, headers=headers, aacplusenc=aacplusenc,
bitrate=bitrate, start=start,
restart=restart, restart_delay=restart_delay,
host=host, port=port, user=user, password=password,
genre=genre, url=url, description=description,
public=public, dumpfile=dumpfile,
fallible=fallible,on_start=on_start,on_stop=on_stop,
restart_on_crash=restart_on_crash, restart_encoder_delay=restart_encoder_delay,
restart_on_new_track=restart_on_new_track,
name=name, mount="/", protocol="icy",
s)
end
%endif

1
python_apps/pypo/liquidsoap_scripts/library/externals.liq
View file

@ -4,6 +4,7 @@
# Enable external Musepack decoder. Requires the
# mpcdec binary in the path. Does not work on
# Win32.
# @category Liquidsoap
def enable_external_mpc_decoder() =
# A list of know extensions and content-type for Musepack.
# Values from http://en.wikipedia.org/wiki/Musepack

34
python_apps/pypo/liquidsoap_scripts/library/gstreamer.liq Normal file
View file

@ -0,0 +1,34 @@
%ifdef input.gstreamer.video
# Stream from a video4linux 2 input device, such as a webcam.
# @category Source / Input
# @param ~id Force the value of the source ID.
# @param ~clock_safe Force the use of the dedicated v4l clock.
# @param ~device V4L2 device to use.
def input.v4l2(~id="",~clock_safe=true,~device="/dev/video0")
pipeline = "v4l2src device=#{device}"
input.gstreamer.video(id=id, clock_safe=clock_safe, pipeline=pipeline)
end
# Stream from a video4linux 2 input device, such as a webcam.
# @category Source / Input
# @param ~id Force the value of the source ID.
# @param ~clock_safe Force the use of the dedicated v4l clock.
# @param ~device V4L2 device to use.
def input.v4l2_with_audio(~id="",~clock_safe=true,~device="/dev/video0")
audio_pipeline = "autoaudiosrc"
video_pipeline = "v4l2src device=#{device}"
input.gstreamer.audio_video(id=id, clock_safe=clock_safe, audio_pipeline=audio_pipeline, video_pipeline=video_pipeline)
end
def gstreamer.encode_x264_avi(fname, source)
output.gstreamer.video(pipeline="videoconvert ! x264enc ! avimux ! filesink location=\"#{fname}\"", source)
end
def gstreamer.encode_jpeg_avi(fname, source)
output.gstreamer.video(pipeline="videoconvert ! jpegenc ! avimux ! filesink location=\"#{fname}\"", source)
end
def gstreamer.encode_mp3(fname, source)
output.gstreamer.audio(pipeline="audioconvert ! lamemp3enc ! filesink location=\"#{fname}\"", source)
end
%endif

16
python_apps/pypo/liquidsoap_scripts/library/http.liq
View file

@ -13,18 +13,12 @@ def http_response(~protocol="HTTP/1.1",
~headers=[],
~data="") =
status = http_codes[string_of(code)]
# Set content-length if needed and not set by the
# user.
# Set content-length and connection: close
headers =
if data != "" and
not list.mem_assoc("Content-Length",headers)
then
list.append([("Content-Length",
"#{string.length(data)}")],
headers)
else
headers
end
list.append(headers,
[("Content-Length", "#{string.length(data)}"),
("Connection", "close")])
headers = list.map(fun (x) -> "#{fst(x)}: #{snd(x)}",headers)
headers = string.concat(separator="\r\n",headers)
# If no headers are provided, we should avoid

3
python_apps/pypo/liquidsoap_scripts/library/pervasives.liq
View file

@ -4,4 +4,5 @@
%include "lastfm.liq"
%include "flows.liq"
%include "http.liq"
%include "video_text.liq"
%include "video_text.liq"
%include "gstreamer.liq"

59
python_apps/pypo/liquidsoap_scripts/library/utils.liq
View file

@ -21,10 +21,14 @@ end
# @param a Key to look for
# @param l List of pairs (key,value)
def list.mem_assoc(a,l)
v = list.assoc(a,l)
# We check for existence, since "" may indicate
# either a binding (a,"") or no binding..
list.mem((a,v),l)
def f(cur, el) =
if not cur then
fst(el) == a
else
cur
end
end
list.fold(f, false, l)
end
# Remove a pair from an associative list
@ -164,8 +168,7 @@ def out(s)
output.prefered(mksafe(s))
end
# Special track insensitive fallback that
# always skip current song before switching.
# Special track insensitive fallback that always skips current song before switching.
# @category Source / Track Processing
# @param ~input The input source
# @param f The fallback source
@ -212,14 +215,17 @@ end
# Simple crossfade.
# @category Source / Track Processing
# @param ~start_next Duration in seconds of the crossed end of track.
# @param ~fade_in Duration of the fade in for next track
# @param ~fade_out Duration of the fade out for previous track
# @param s The source to use
def crossfade(~id="",~start_next,~fade_in,~fade_out,s)
# @param ~fade_in Duration of the fade in for next track.
# @param ~fade_out Duration of the fade out for previous track.
# @param ~conservative Always prepare for a premature end-of-track.
# @param s The source to use.
def crossfade(~id="",~conservative=true,
~start_next=5.,~fade_in=3.,~fade_out=3.,
s)
s = fade.in(duration=fade_in,s)
s = fade.out(duration=fade_out,s)
fader = fun (a,b) -> add(normalize=false,[b,a])
cross(id=id,conservative=true,duration=start_next,fader,s)
cross(id=id,conservative=conservative,duration=start_next,fader,s)
end
# Append speech-synthesized tracks reading the metadata.
@ -242,8 +248,7 @@ def helium(s)
end
%endif
# Return true if process exited with 0 code.
# Command should return quickly.
# Return true if process exited with 0 code. Command should return quickly.
# @category System
# @param command Command to test
def test_process(command)
@ -277,12 +282,9 @@ def url.split(uri) =
end
end
# Register a server/telnet command to
# update a source's metadata. Returns
# a new source, which will receive the
# updated metadata. It behaves just like
# the pre-1.0 insert_metadata() operator,
# i.e. insert key1="val1",key2="val2",...
# Register a server/telnet command to update a source's metadata. Returns
# a new source, which will receive the updated metadata. The command has
# the following format: insert key1="val1",key2="val2",...
# @category Source / Track Processing
# @param ~id Force the value of the source ID.
def server.insert_metadata(~id="",s) =
@ -424,15 +426,15 @@ end
# @param ~conservative Always prepare for a premature end-of-track.
# @param ~default Transition used when no rule applies \
# (default: sequence).
# @param ~high Value, in dB, for loud sound level
# @param ~medium Value, in dB, for medium sound level
# @param ~high Value, in dB, for loud sound level.
# @param ~medium Value, in dB, for medium sound level.
# @param ~margin Margin to detect sources that have too different \
# sound level for crossing.
# @param s The input source.
def smart_crossfade (~start_next=5.,~fade_in=3.,~fade_out=3.,
~default=(fun (a,b) -> sequence([a, b])),
~high=-15., ~medium=-32., ~margin=4.,
~width=2.,~conservative=false,s)
~width=2.,~conservative=true,s)
fade.out = fade.out(type="sin",duration=fade_out)
fade.in = fade.in(type="sin",duration=fade_in)
add = fun (a,b) -> add(normalize=false,[b, a])
@ -549,7 +551,18 @@ def playlist.reloadable(~id="",~random=false,~on_done={()},uri)
if request.resolve(playlist) then
playlist = request.filename(playlist)
files = playlist.parse(playlist)
list.map(snd,files)
def file_request(el) =
meta = fst(el)
file = snd(el)
s = list.fold(fun (cur, el) ->
"#{cur},#{fst(el)}=#{string.escape(snd(el))}", "", meta)
if s == "" then
file
else
"annotate:#{s}:#{file}"
end
end
list.map(file_request,files)
else
log(label=id,"Couldn't read playlist: request resolution failed.")
[]

2
python_apps/pypo/liquidsoap_scripts/library/video_text.liq
View file

@ -1,5 +1,6 @@
%ifdef video.add_text.gd
# Add a scrolling line of text on video frames.
# @category Source / Video Processing
# @param ~id Force the value of the source ID.
# @param ~color Text color (in 0xRRGGBB format).
# @param ~cycle Cycle text.
@ -22,6 +23,7 @@ end
%ifdef video.add_text.sdl
# Add a scrolling line of text on video frames.
# @category Source / Video Processing
# @param ~id Force the value of the source ID.
# @param ~color Text color (in 0xRRGGBB format).
# @param ~cycle Cycle text.

7
python_apps/pypo/liquidsoap_scripts/ls_script.liq
View file

@ -201,9 +201,6 @@ def append_dj_inputs(master_harbor_input_port, master_harbor_input_mount_point,
dj_live = mksafe(audio_to_stereo(input.harbor(id="live_dj_harbor", dj_harbor_input_mount_point, port=dj_harbor_input_port, auth=check_dj_client,
max=40., on_connect=live_dj_connect, on_disconnect=live_dj_disconnect)))
master_dj = rewrite_metadata([("artist","Airtime"), ("title", "Master Dj")],master_dj)
dj_live = rewrite_metadata([("artist","Airtime"), ("title", "Live Dj")],dj_live)
ignore(output.dummy(master_dj, fallible=true))
ignore(output.dummy(dj_live, fallible=true))
switch(id="master_dj_switch", track_sensitive=false, transitions=[transition, transition, transition], [({!master_dj_enabled},master_dj), ({!live_dj_enabled},dj_live), ({true}, s)])
@ -211,14 +208,12 @@ def append_dj_inputs(master_harbor_input_port, master_harbor_input_mount_point,
master_dj = mksafe(audio_to_stereo(input.harbor(id="master_harbor", master_harbor_input_mount_point, port=master_harbor_input_port, auth=check_master_dj_client,
max=40., on_connect=master_dj_connect, on_disconnect=master_dj_disconnect)))
ignore(output.dummy(master_dj, fallible=true))
master_dj = rewrite_metadata([("artist","Airtime"), ("title", "Master Dj")],master_dj)
switch(id="master_dj_switch", track_sensitive=false, transitions=[transition, transition], [({!master_dj_enabled},master_dj), ({true}, s)])
elsif dj_harbor_input_port != 0 and dj_harbor_input_mount_point != "" then
dj_live = mksafe(audio_to_stereo(input.harbor(id="live_dj_harbor", dj_harbor_input_mount_point, port=dj_harbor_input_port, auth=check_dj_client,
max=40., on_connect=live_dj_connect, on_disconnect=live_dj_disconnect)))
dj_live = rewrite_metadata([("artist","Airtime"), ("title", "Live Dj")],dj_live)
ignore(output.dummy(dj_live, fallible=true))
switch(id="live_dj_switch", track_sensitive=false, transitions=[transition, transition], [({!live_dj_enabled},dj_live), ({true}, s)])
else

61
python_apps/pypo/pyponotify.py
View file

@ -1,4 +1,5 @@
# -*- coding: utf-8 -*-
import traceback
"""
Python part of radio playout (pypo)
@ -102,6 +103,24 @@ class Notify:
logger.debug('# Calling server to update webstream data #')
logger.debug('#################################################')
response = self.api_client.notify_webstream_data(data, media_id)
logger.debug("Response: " + json.dumps(response))
def run_with_options(self, options):
if options.error and options.stream_id:
self.notify_liquidsoap_status(options.error, options.stream_id, options.time)
elif options.connect and options.stream_id:
self.notify_liquidsoap_status("OK", options.stream_id, options.time)
elif options.source_name and options.source_status:
self.notify_source_status(options.source_name, options.source_status)
elif options.webstream:
self.notify_webstream_data(options.webstream, options.media_id)
elif options.media_id:
self.notify_media_start_playing(options.media_id)
elif options.liquidsoap_started:
self.notify_liquidsoap_started()
else:
logger.debug("Unrecognized option in options(%s). Doing nothing" \
% str(options))
if __name__ == '__main__':
@ -112,41 +131,9 @@ if __name__ == '__main__':
print '#########################################'
# initialize
if options.error and options.stream_id:
try:
n = Notify()
n.notify_liquidsoap_status(options.error, options.stream_id, options.time)
except Exception, e:
print e
elif options.connect and options.stream_id:
try:
n = Notify()
n.notify_liquidsoap_status("OK", options.stream_id, options.time)
except Exception, e:
print e
elif options.source_name and options.source_status:
try:
n = Notify()
n.notify_source_status(options.source_name, options.source_status)
except Exception, e:
print e
elif options.webstream:
try:
n = Notify()
n.notify_webstream_data(options.webstream, options.media_id)
except Exception, e:
print e
elif options.media_id:
try:
n = Notify()
n.notify_media_start_playing(options.media_id)
except Exception, e:
print e
elif options.liquidsoap_started:
try:
n = Notify()
n.notify_liquidsoap_started()
except Exception, e:
print e
try:
n = Notify()
n.run_with_options(options)
except Exception as e:
print( traceback.format_exc() )