Merge branch 'devel' of dev.sourcefabric.org:airtime into devel

This commit is contained in:
Martin Konecny 2012-08-21 16:55:17 -04:00
commit 15eae0698b
12 changed files with 215 additions and 147 deletions

View file

@ -59,4 +59,6 @@ class EventContractor(Loggable):
try: del self.store[evt.path]
except Exception as e:
self.unexpected_exception(e)
self.logger.info("Unregistering. Left: '%d'" % len(self.store.keys()))
# the next line is commented out because it clutters up logging real
# bad
#self.logger.info("Unregistering. Left: '%d'" % len(self.store.keys()))

View file

@ -105,10 +105,38 @@ class Metadata(Loggable):
# little bit messy. Some of the handling is in m.m.pure while the rest is
# here. Also interface is not very consistent
@staticmethod
def airtime_dict(d):
"""
Converts mutagen dictionary 'd' into airtime dictionary
"""
temp_dict = {}
for m_key, m_val in d.iteritems():
# TODO : some files have multiple fields for the same metadata.
# genre is one example. In that case mutagen will return a list
# of values
assign_val = m_val[0] if isinstance(m_val, list) else m_val
temp_dict[ m_key ] = assign_val
airtime_dictionary = {}
for muta_k, muta_v in temp_dict.iteritems():
# We must check if we can actually translate the mutagen key into
# an airtime key before doing the conversion
if muta_k in mutagen2airtime:
airtime_key = mutagen2airtime[muta_k]
# Apply truncation in the case where airtime_key is in our
# truncation table
muta_v = \
truncate_to_length(muta_v, truncate_table[airtime_key])\
if airtime_key in truncate_table else muta_v
airtime_dictionary[ airtime_key ] = muta_v
return airtime_dictionary
@staticmethod
def write_unsafe(path,md):
"""
Writes 'md' metadata into 'path' through mutagen
Writes 'md' metadata into 'path' through mutagen. Converts all
dictionary values to strings because mutagen will not write anything
else
"""
if not os.path.exists(path): raise BadSongFile(path)
song_file = mutagen.File(path, easy=True)
@ -128,37 +156,13 @@ class Metadata(Loggable):
self.path = fpath
# TODO : Simplify the way all of these rules are handled right not it's
# extremely unclear and needs to be refactored.
metadata = {}
# Load only the metadata avilable in mutagen into metdata
for k,v in full_mutagen.iteritems():
# Special handling of attributes here
if isinstance(v, list):
# TODO : some files have multiple fields for the same metadata.
# genre is one example. In that case mutagen will return a list
# of values
metadata[k] = v[0]
#if len(v) == 1: metadata[k] = v[0]
#else: raise Exception("Unknown mutagen %s:%s" % (k,str(v)))
else: metadata[k] = v
self.__metadata = {}
# Start populating a dictionary of airtime metadata in __metadata
for muta_k, muta_v in metadata.iteritems():
# We must check if we can actually translate the mutagen key into
# an airtime key before doing the conversion
if muta_k in mutagen2airtime:
airtime_key = mutagen2airtime[muta_k]
# Apply truncation in the case where airtime_key is in our
# truncation table
muta_v = \
truncate_to_length(muta_v, truncate_table[airtime_key])\
if airtime_key in truncate_table else muta_v
self.__metadata[ airtime_key ] = muta_v
self.__metadata = Metadata.airtime_dict(full_mutagen)
# Now we extra the special values that are calculated from the mutagen
# object itself:
for special_key,f in airtime_special.iteritems():
new_val = f(full_mutagen)
if new_val is not None:
self.__metadata[special_key] = f(full_mutagen)
self.__metadata[special_key] = new_val
# Finally, we "normalize" all the metadata here:
self.__metadata = mmp.normalized_metadata(self.__metadata, fpath)
# Now we must load the md5:

View file

@ -2,6 +2,7 @@
import copy
import os
import shutil
import re
import sys
import hashlib
import locale
@ -228,34 +229,33 @@ def normalized_metadata(md, original_path):
# Specific rules that are applied in a per attribute basis
format_rules = {
'MDATA_KEY_TRACKNUMBER' : parse_int,
'MDATA_KEY_BITRATE' : lambda x: str(int(x) / 1000) + "kbps",
'MDATA_KEY_FILEPATH' : lambda x: os.path.normpath(x),
'MDATA_KEY_MIME' : lambda x: x.replace('-','/'),
'MDATA_KEY_BPM' : lambda x: x[0:8],
}
new_md = remove_whitespace(new_md)
new_md = remove_whitespace(new_md) # remove whitespace fields
# Format all the fields in format_rules
new_md = apply_rules_dict(new_md, format_rules)
new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_TITLE'],
default=no_extension_basename(original_path))
new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_CREATOR',
'MDATA_KEY_SOURCE'], default=u'')
# set filetype to audioclip by default
new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_FTYPE'],
default=u'audioclip')
# In the case where the creator is 'Airtime Show Recorder' we would like to
# format the MDATA_KEY_TITLE slightly differently
# Note: I don't know why I'm doing a unicode string comparison here
# that part is copied from the original code
# Try to parse bpm but delete the whole key if that fails
if 'MDATA_KEY_BPM' in new_md:
new_md['MDATA_KEY_BPM'] = parse_int(new_md['MDATA_KEY_BPM'])
if new_md['MDATA_KEY_BPM'] is None:
del new_md['MDATA_KEY_BPM']
if is_airtime_recorded(new_md):
hour,minute,second,name = md['MDATA_KEY_TITLE'].split("-",3)
# We assume that MDATA_KEY_YEAR is always given for airtime recorded
# shows
hour,minute,second,name = new_md['MDATA_KEY_TITLE'].split("-",3)
new_md['MDATA_KEY_TITLE'] = u'%s-%s-%s:%s:%s' % \
(name, new_md['MDATA_KEY_YEAR'], hour, minute, second)
# IMPORTANT: in the original code. MDATA_KEY_FILEPATH would also
# be set to the original path of the file for airtime recorded shows
# (before it was "organized"). We will skip this procedure for now
# because it's not clear why it was done
else:
# Read title from filename if it does not exist
new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_TITLE'],
default=no_extension_basename(original_path))
return new_md
def organized_path(old_path, root_path, orig_md):
@ -274,15 +274,26 @@ def organized_path(old_path, root_path, orig_md):
if key in dictionary: return len(dictionary[key]) == 0
else: return True
# We set some metadata elements to a default "unknown" value because we use
# these fields to create a path hence they cannot be empty
# these fields to create a path hence they cannot be empty Here "normal"
# means normalized only for organized path
# MDATA_KEY_BITRATE is in bytes/second i.e. (256000) we want to turn this
# into 254kbps
normal_md = default_to_f(orig_md, path_md, unicode_unknown, default_f)
if normal_md['MDATA_KEY_BITRATE']:
formatted = str(int(normal_md['MDATA_KEY_BITRATE']) / 1000)
normal_md['MDATA_KEY_BITRATE'] = formatted + 'kbps'
else: normal_md['MDATA_KEY_BITRATE'] = unicode_unknown
if is_airtime_recorded(normal_md):
fname = u'%s-%s-%s.%s' % ( normal_md['MDATA_KEY_YEAR'],
normal_md['MDATA_KEY_TITLE'],
normal_md['MDATA_KEY_BITRATE'], ext )
yyyy, mm, _ = normal_md['MDATA_KEY_YEAR'].split('-',3)
path = os.path.join(root_path, yyyy, mm)
filepath = os.path.join(path,fname)
title_re = re.match("(?P<show>\w+)-(?P<date>\d+-\d+-\d+-\d+:\d+:\d+)$",
normal_md['MDATA_KEY_TITLE'])
show_name, = title_re.group('show'),
date = title_re.group('date').replace(':','-')
yyyy, mm, _ = normal_md['MDATA_KEY_YEAR'].split('-',2)
fname_base = '%s-%s-%s.%s' % \
(date, show_name, normal_md['MDATA_KEY_BITRATE'], ext)
filepath = os.path.join(root_path, yyyy, mm, fname_base)
elif len(normal_md['MDATA_KEY_TRACKNUMBER']) == 0:
fname = u'%s-%s.%s' % (normal_md['MDATA_KEY_TITLE'],
normal_md['MDATA_KEY_BITRATE'], ext)

View file

@ -127,6 +127,8 @@ def main(global_config, api_client_config, log_config,
tt = ToucherThread(path=config['index_path'],
interval=int(config['touch_interval']))
apiclient.register_component('media-monitor')
pyi = manager.pyinotify()
pyi.loop()

View file

@ -2,6 +2,7 @@
import unittest
import os
import media.monitor.pure as mmp
from media.monitor.metadata import Metadata
class TestMMP(unittest.TestCase):
def setUp(self):
@ -34,23 +35,32 @@ class TestMMP(unittest.TestCase):
for k in def_keys: self.assertEqual( sd[k], 'DEF' )
def test_normalized_metadata(self):
normal = mmp.normalized_metadata(self.md1,"")
self.assertTrue(hasattr(normal['MDATA_KEY_CREATOR'],'startswith'))
self.assertTrue('MDATA_KEY_CREATOR' in normal)
self.assertTrue('MDATA_KEY_SOURCE' in normal)
# Recorded show test first
orig = Metadata.airtime_dict({
'date': [u'2012-08-21'],
'tracknumber': [u'2'],
'title': [u'11-29-00-record'],
'artist': [u'Airtime Show Recorder']
})
orga = Metadata.airtime_dict({
'date': [u'2012-08-21'],
'tracknumber': [u'2'],
'artist': [u'Airtime Show Recorder'],
'title': [u'record-2012-08-21-11:29:00']
})
orga['MDATA_KEY_FTYPE'] = u'audioclip'
orig['MDATA_KEY_BITRATE'] = u'256000'
orga['MDATA_KEY_BITRATE'] = u'256kbps'
def test_organized_path(self):
o_path = '/home/rudi/throwaway/ACDC_-_Back_In_Black-sample-64kbps.ogg'
normal = mmp.normalized_metadata(self.md1,o_path)
og = mmp.organized_path(o_path,
'/home/rudi/throwaway/fucking_around/watch/',
normal)
real_path1 = \
u'/home/rudi/throwaway/fucking_around/watch/unknown/unknown/ACDC_-_Back_In_Black-sample-64kbps-64kbps.ogg'
self.assertTrue( 'unknown' in og, True )
self.assertEqual( og, real_path1 ) # TODO : fix this failure
# for recorded it should be something like this
# ./recorded/2012/07/2012-07-09-17-55-00-Untitled Show-256kbps.ogg
old_path = "/home/rudi/recorded/2012-08-21-11:29:00.ogg"
normalized = mmp.normalized_metadata(orig, old_path)
print(normalized)
self.assertEqual( orga, normalized )
organized_base_name = "2012-08-21-11-29-00-record-256kbps.ogg"
base = "/srv/airtime/stor/"
organized_path = mmp.organized_path(old_path,base, normalized)
self.assertEqual(os.path.basename(organized_path), organized_base_name)
def test_file_md5(self):
p = os.path.realpath(__file__)

View file

@ -9,6 +9,7 @@ import sys
import pytz
import signal
import math
import traceback
from configobj import ConfigObj
@ -55,16 +56,16 @@ class ShowRecorder(Thread):
def __init__ (self, show_instance, show_name, filelength, start_time):
Thread.__init__(self)
self.logger = logging.getLogger('recorder')
self.api_client = api_client(self.logger)
self.filelength = filelength
self.start_time = start_time
self.logger = logging.getLogger('recorder')
self.api_client = api_client(self.logger)
self.filelength = filelength
self.start_time = start_time
self.show_instance = show_instance
self.show_name = show_name
self.p = None
self.show_name = show_name
self.p = None
def record_show(self):
length = str(self.filelength) + ".0"
length = str(self.filelength) + ".0"
filename = self.start_time
filename = filename.replace(" ", "-")
@ -73,16 +74,18 @@ class ShowRecorder(Thread):
else:
filetype = "ogg";
filepath = "%s%s.%s" % (config["base_recorded_files"], filename, filetype)
joined_path = os.path.join(config["base_recorded_files"], filename)
filepath = "%s.%s" % (joined_path, filetype)
br = config["record_bitrate"]
sr = config["record_samplerate"]
c = config["record_channels"]
c = config["record_channels"]
ss = config["record_sample_size"]
#-f:16,2,44100
#-b:256
command = "ecasound -f:%s,%s,%s -i alsa -o %s,%s000 -t:%s" % (ss, c, sr, filepath, br, length)
command = "ecasound -f:%s,%s,%s -i alsa -o %s,%s000 -t:%s" % \
(ss, c, sr, filepath, br, length)
args = command.split(" ")
self.logger.info("starting record")
@ -144,17 +147,16 @@ class ShowRecorder(Thread):
artist = "Airtime Show Recorder"
#set some metadata for our file daemon
recorded_file = mutagen.File(filepath, easy=True)
recorded_file['title'] = name
recorded_file = mutagen.File(filepath, easy = True)
recorded_file['title'] = name
recorded_file['artist'] = artist
recorded_file['date'] = md[0]
recorded_file['date'] = md[0]
#recorded_file['date'] = md[0].split("-")[0]
#You cannot pass ints into the metadata of a file. Even tracknumber needs to be a string
recorded_file['tracknumber'] = unicode(self.show_instance)
recorded_file.save()
except Exception, e:
import traceback
top = traceback.format_exc()
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", top)
@ -180,20 +182,20 @@ class ShowRecorder(Thread):
class Recorder(Thread):
def __init__(self, q):
Thread.__init__(self)
self.logger = logging.getLogger('recorder')
self.api_client = api_client(self.logger)
self.api_client.register_component("show-recorder")
self.sr = None
self.logger = logging.getLogger('recorder')
self.api_client = api_client(self.logger)
self.sr = None
self.shows_to_record = {}
self.server_timezone = ''
self.queue = q
self.queue = q
self.loops = 0
self.api_client.register_component("show-recorder")
self.logger.info("RecorderFetch: init complete")
self.loops = 0
def handle_message(self):
if not self.queue.empty():
message = self.queue.get()
msg = json.loads(message)
msg = json.loads(message)
command = msg["event_type"]
self.logger.info("Received msg from Pypo Message Handler: %s", msg)
if command == 'cancel_recording':
@ -212,8 +214,8 @@ class Recorder(Thread):
shows = m['shows']
for show in shows:
show_starts = getDateTimeObj(show[u'starts'])
show_end = getDateTimeObj(show[u'ends'])
time_delta = show_end - show_starts
show_end = getDateTimeObj(show[u'ends'])
time_delta = show_end - show_starts
temp_shows_to_record[show[u'starts']] = [time_delta, show[u'instance_id'], show[u'name'], m['server_timezone']]
self.shows_to_record = temp_shows_to_record
@ -223,10 +225,10 @@ class Recorder(Thread):
tnow = datetime.datetime.utcnow()
sorted_show_keys = sorted(self.shows_to_record.keys())
start_time = sorted_show_keys[0]
next_show = getDateTimeObj(start_time)
start_time = sorted_show_keys[0]
next_show = getDateTimeObj(start_time)
delta = next_show - tnow
delta = next_show - tnow
s = '%s.%s' % (delta.seconds, delta.microseconds)
out = float(s)
@ -237,36 +239,35 @@ class Recorder(Thread):
return out
def start_record(self):
if len(self.shows_to_record) != 0:
try:
delta = self.get_time_till_next_show()
if delta < 5:
self.logger.debug("sleeping %s seconds until show", delta)
time.sleep(delta)
if len(self.shows_to_record) == 0: return None
try:
delta = self.get_time_till_next_show()
if delta < 5:
self.logger.debug("sleeping %s seconds until show", delta)
time.sleep(delta)
sorted_show_keys = sorted(self.shows_to_record.keys())
start_time = sorted_show_keys[0]
show_length = self.shows_to_record[start_time][0]
show_instance = self.shows_to_record[start_time][1]
show_name = self.shows_to_record[start_time][2]
server_timezone = self.shows_to_record[start_time][3]
sorted_show_keys = sorted(self.shows_to_record.keys())
start_time = sorted_show_keys[0]
show_length = self.shows_to_record[start_time][0]
show_instance = self.shows_to_record[start_time][1]
show_name = self.shows_to_record[start_time][2]
server_timezone = self.shows_to_record[start_time][3]
T = pytz.timezone(server_timezone)
start_time_on_UTC = getDateTimeObj(start_time)
start_time_on_server = start_time_on_UTC.replace(tzinfo=pytz.utc).astimezone(T)
start_time_formatted = '%(year)d-%(month)02d-%(day)02d %(hour)02d:%(min)02d:%(sec)02d' % \
{'year': start_time_on_server.year, 'month': start_time_on_server.month, 'day': start_time_on_server.day, \
'hour': start_time_on_server.hour, 'min': start_time_on_server.minute, 'sec': start_time_on_server.second}
self.sr = ShowRecorder(show_instance, show_name, show_length.seconds, start_time_formatted)
self.sr.start()
#remove show from shows to record.
del self.shows_to_record[start_time]
#self.time_till_next_show = self.get_time_till_next_show()
except Exception, e :
import traceback
top = traceback.format_exc()
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", top)
T = pytz.timezone(server_timezone)
start_time_on_UTC = getDateTimeObj(start_time)
start_time_on_server = start_time_on_UTC.replace(tzinfo=pytz.utc).astimezone(T)
start_time_formatted = '%(year)d-%(month)02d-%(day)02d %(hour)02d:%(min)02d:%(sec)02d' % \
{'year': start_time_on_server.year, 'month': start_time_on_server.month, 'day': start_time_on_server.day, \
'hour': start_time_on_server.hour, 'min': start_time_on_server.minute, 'sec': start_time_on_server.second}
self.sr = ShowRecorder(show_instance, show_name, show_length.seconds, start_time_formatted)
self.sr.start()
#remove show from shows to record.
del self.shows_to_record[start_time]
#self.time_till_next_show = self.get_time_till_next_show()
except Exception, e :
top = traceback.format_exc()
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", top)
"""
Main loop of the thread:
@ -284,6 +285,7 @@ class Recorder(Thread):
self.process_recorder_schedule(temp)
self.logger.info("Bootstrap recorder schedule received: %s", temp)
except Exception, e:
self.logger.error( traceback.format_exc() )
self.logger.error(e)
self.logger.info("Bootstrap complete: got initial copy of the schedule")
@ -305,14 +307,15 @@ class Recorder(Thread):
self.process_recorder_schedule(temp)
self.logger.info("updated recorder schedule received: %s", temp)
except Exception, e:
self.logger.error( traceback.format_exc() )
self.logger.error(e)
try: self.handle_message()
except Exception, e:
self.logger.error( traceback.format_exc() )
self.logger.error('Pypo Recorder Exception: %s', e)
time.sleep(PUSH_INTERVAL)
self.loops += 1
except Exception, e :
import traceback
top = traceback.format_exc()
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", top)