Congegni/sintonia
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merged baseUrl related conflicts

Browse source
This commit is contained in:
Rudi Grinberg 2012-10-29 11:56:20 -04:00
commit dc00528cd5
62 changed files with 1155 additions and 1234 deletions
Download patch file Download diff file Expand all files Collapse all files

1
python_apps/media-monitor2/media/monitor/events.py
View file

@ -199,6 +199,7 @@ class NewFile(BaseEvent, HasMetaData):
"""
req_dict = self.metadata.extract()
req_dict['mode'] = u'create'
req_dict['is_record'] = self.metadata.is_recorded()
self.assign_owner(req_dict)
req_dict['MDATA_KEY_FILEPATH'] = unicode( self.path )
return [req_dict]

13
python_apps/media-monitor2/media/monitor/listeners.py
View file

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
import pyinotify
from pydispatch import dispatcher
from functools import wraps
import media.monitor.pure as mmp
from media.monitor.pure import IncludeOnly
@ -31,6 +32,7 @@ class FileMediator(object):
def unignore(path): FileMediator.ignored_set.remove(path)
def mediate_ignored(fn):
@wraps(fn)
def wrapped(self, event, *args,**kwargs):
event.pathname = unicode(event.pathname, "utf-8")
if FileMediator.is_ignored(event.pathname):
@ -49,15 +51,11 @@ class OrganizeListener(BaseListener, pyinotify.ProcessEvent, Loggable):
def process_IN_CLOSE_WRITE(self, event):
#self.logger.info("===> handling: '%s'" % str(event))
self.process_to_organize(event)
# got cookie
def process_IN_MOVED_TO(self, event):
#self.logger.info("===> handling: '%s'" % str(event))
self.process_to_organize(event)
def process_default(self, event):
pass
#self.logger.info("===> Not handling: '%s'" % str(event))
def flush_events(self, path):
"""
organize the whole directory at path. (pretty much by doing what
@ -67,8 +65,9 @@ class OrganizeListener(BaseListener, pyinotify.ProcessEvent, Loggable):
for f in mmp.walk_supported(path, clean_empties=True):
self.logger.info("Bootstrapping: File in 'organize' directory: \
'%s'" % f)
dispatcher.send(signal=self.signal, sender=self,
event=OrganizeFile(f))
if not mmp.file_locked(f):
dispatcher.send(signal=self.signal, sender=self,
event=OrganizeFile(f))
flushed += 1
#self.logger.info("Flushed organized directory with %d files" % flushed)

25
python_apps/media-monitor2/media/monitor/log.py
View file

@ -6,37 +6,30 @@ from media.monitor.pure import LazyProperty
appname = 'root'
def setup_logging(log_path):
"""
Setup logging by writing log to 'log_path'
"""
""" Setup logging by writing log to 'log_path' """
#logger = logging.getLogger(appname)
logging.basicConfig(filename=log_path, level=logging.DEBUG)
def get_logger():
"""
in case we want to use the common logger from a procedural interface
"""
""" in case we want to use the common logger from a procedural
interface """
return logging.getLogger()
class Loggable(object):
"""
Any class that wants to log can inherit from this class and automatically
get a logger attribute that can be used like: self.logger.info(...) etc.
"""
""" Any class that wants to log can inherit from this class and
automatically get a logger attribute that can be used like:
self.logger.info(...) etc. """
__metaclass__ = abc.ABCMeta
@LazyProperty
def logger(self): return get_logger()
def unexpected_exception(self,e):
"""
Default message for 'unexpected' exceptions
"""
""" Default message for 'unexpected' exceptions """
self.fatal_exception("'Unexpected' exception has occured:", e)
def fatal_exception(self, message, e):
"""
Prints an exception 'e' with 'message'. Also outputs the traceback.
"""
""" Prints an exception 'e' with 'message'. Also outputs the
traceback. """
self.logger.error( message )
self.logger.error( str(e) )
self.logger.error( traceback.format_exc() )

6
python_apps/media-monitor2/media/monitor/manager.py
View file

@ -18,7 +18,7 @@ class ManagerTimeout(threading.Thread,Loggable):
secnods. This used to be just a work around for cc-4235 but recently
became a permanent solution because it's "cheap" and reliable
"""
def __init__(self, manager, interval=3):
def __init__(self, manager, interval=1.5):
# TODO : interval should be read from config and passed here instead
# of just using the hard coded value
threading.Thread.__init__(self)
@ -26,7 +26,7 @@ class ManagerTimeout(threading.Thread,Loggable):
self.interval = interval
def run(self):
while True:
time.sleep(self.interval) # every 3 seconds
time.sleep(self.interval)
self.manager.flush_organize()
class Manager(Loggable):
@ -178,7 +178,7 @@ class Manager(Loggable):
# the OrganizeListener instance will walk path and dispatch an organize
# event for every file in that directory
self.organize['organize_listener'].flush_events(new_path)
self.__add_watch(new_path, self.organize['organize_listener'])
#self.__add_watch(new_path, self.organize['organize_listener'])
def flush_organize(self):
path = self.organize['organize_path']

97
python_apps/media-monitor2/media/monitor/metadata.py
View file

@ -2,15 +2,19 @@
import mutagen
import os
import copy
from collections import namedtuple
from mutagen.easymp4 import EasyMP4KeyError
from mutagen.easyid3 import EasyID3KeyError
from media.monitor.exceptions import BadSongFile, InvalidMetadataElement
from media.monitor.log import Loggable
from media.monitor.pure import format_length, truncate_to_length
from media.monitor.pure import format_length
import media.monitor.pure as mmp
# emf related stuff
from media.metadata.process import global_reader
import media.metadata.definitions as defs
defs.load_definitions()
"""
list of supported easy tags in mutagen version 1.20
['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry',
@ -43,21 +47,6 @@ airtime2mutagen = {
"MDATA_KEY_COPYRIGHT" : "copyright",
}
class FakeMutagen(dict):
"""
Need this fake mutagen object so that airtime_special functions
return a proper default value instead of throwing an exceptions for
files that mutagen doesn't recognize
"""
FakeInfo = namedtuple('FakeInfo','length bitrate')
def __init__(self,path):
self.path = path
self.mime = ['audio/wav']
self.info = FakeMutagen.FakeInfo(0.0, '')
dict.__init__(self)
def set_length(self,l):
old_bitrate = self.info.bitrate
self.info = FakeMutagen.FakeInfo(l, old_bitrate)
# Some airtime attributes are special because they must use the mutagen object
# itself to calculate the value that they need. The lambda associated with each
@ -100,6 +89,7 @@ class Metadata(Loggable):
# little bit messy. Some of the handling is in m.m.pure while the rest is
# here. Also interface is not very consistent
# TODO : what is this shit? maybe get rid of it?
@staticmethod
def fix_title(path):
# If we have no title in path we will format it
@ -110,39 +100,6 @@ class Metadata(Loggable):
m[u'title'] = new_title
m.save()
@staticmethod
def airtime_dict(d):
"""
Converts mutagen dictionary 'd' into airtime dictionary
"""
temp_dict = {}
for m_key, m_val in d.iteritems():
# TODO : some files have multiple fields for the same metadata.
# genre is one example. In that case mutagen will return a list
# of values
if isinstance(m_val, list):
# TODO : does it make more sense to just skip the element in
# this case?
if len(m_val) == 0: assign_val = ''
else: assign_val = m_val[0]
else: assign_val = m_val
temp_dict[ m_key ] = assign_val
airtime_dictionary = {}
for muta_k, muta_v in temp_dict.iteritems():
# We must check if we can actually translate the mutagen key into
# an airtime key before doing the conversion
if muta_k in mutagen2airtime:
airtime_key = mutagen2airtime[muta_k]
# Apply truncation in the case where airtime_key is in our
# truncation table
muta_v = \
truncate_to_length(muta_v, truncate_table[airtime_key])\
if airtime_key in truncate_table else muta_v
airtime_dictionary[ airtime_key ] = muta_v
return airtime_dictionary
@staticmethod
def write_unsafe(path,md):
"""
@ -157,6 +114,7 @@ class Metadata(Loggable):
if airtime_k in airtime2mutagen:
# The unicode cast here is mostly for integers that need to be
# strings
if airtime_v is None: continue
try:
song_file[ airtime2mutagen[airtime_k] ] = unicode(airtime_v)
except (EasyMP4KeyError, EasyID3KeyError) as e:
@ -170,44 +128,7 @@ class Metadata(Loggable):
# Forcing the unicode through
try : fpath = fpath.decode("utf-8")
except : pass
if not mmp.file_playable(fpath): raise BadSongFile(fpath)
try : full_mutagen = mutagen.File(fpath, easy=True)
except Exception : raise BadSongFile(fpath)
self.path = fpath
if not os.path.exists(self.path):
self.logger.info("Attempting to read metadata of file \
that does not exist. Setting metadata to {}")
self.__metadata = {}
return
# TODO : Simplify the way all of these rules are handled right now it's
# extremely unclear and needs to be refactored.
#if full_mutagen is None: raise BadSongFile(fpath)
if full_mutagen is None: full_mutagen = FakeMutagen(fpath)
self.__metadata = Metadata.airtime_dict(full_mutagen)
# Now we extra the special values that are calculated from the mutagen
# object itself:
if mmp.extension(fpath) == 'wav':
full_mutagen.set_length(mmp.read_wave_duration(fpath))
for special_key,f in airtime_special.iteritems():
try:
new_val = f(full_mutagen)
if new_val is not None:
self.__metadata[special_key] = new_val
except Exception as e:
self.logger.info("Could not get special key %s for %s" %
(special_key, fpath))
self.logger.info(str(e))
# Finally, we "normalize" all the metadata here:
self.__metadata = mmp.normalized_metadata(self.__metadata, fpath)
# Now we must load the md5:
# TODO : perhaps we shouldn't hard code how many bytes we're reading
# from the file?
self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100)
self.__metadata = global_reader.read_mutagen(fpath)
def is_recorded(self):
"""

23
python_apps/media-monitor2/media/monitor/organizer.py
View file

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
import media.monitor.pure as mmp
import media.monitor.owners as owners
from media.monitor.handler import ReportHandler
@ -11,14 +10,12 @@ from os.path import dirname
import os.path
class Organizer(ReportHandler,Loggable):
"""
Organizer is responsible to to listening to OrganizeListener events
and committing the appropriate changes to the filesystem. It does
not in any interact with WatchSyncer's even when the the WatchSyncer
is a "storage directory". The "storage" directory picks up all of
its events through pyinotify. (These events are fed to it through
StoreWatchListener)
"""
""" Organizer is responsible to to listening to OrganizeListener
events and committing the appropriate changes to the filesystem.
It does not in any interact with WatchSyncer's even when the the
WatchSyncer is a "storage directory". The "storage" directory picks
up all of its events through pyinotify. (These events are fed to it
through StoreWatchListener) """
# Commented out making this class a singleton because it's just a band aid
# for the real issue. The real issue being making multiple Organizer
@ -42,11 +39,9 @@ class Organizer(ReportHandler,Loggable):
super(Organizer, self).__init__(signal=self.channel, weak=False)
def handle(self, sender, event):
"""
Intercept events where a new file has been added to the organize
directory and place it in the correct path (starting with
self.target_path)
"""
""" Intercept events where a new file has been added to the
organize directory and place it in the correct path (starting
with self.target_path) """
# Only handle this event type
assert isinstance(event, OrganizeFile), \
"Organizer can only handle OrganizeFile events.Given '%s'" % event

178
python_apps/media-monitor2/media/monitor/pure.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
import copy
from subprocess import Popen, PIPE
import subprocess
import os
import math
@ -21,7 +22,6 @@ from configobj import ConfigObj
from media.monitor.exceptions import FailedToSetLocale, FailedToCreateDir
#supported_extensions = [u"mp3", u"ogg", u"oga"]
supported_extensions = [u"mp3", u"ogg", u"oga", u"flac", u"wav",
u'm4a', u'mp4']
@ -66,7 +66,6 @@ class IncludeOnly(object):
return func(moi, event, *args, **kwargs)
return _wrap
def partition(f, alist):
"""
Partition is very similar to filter except that it also returns the
@ -92,14 +91,13 @@ def is_file_supported(path):
# TODO : In the future we would like a better way to find out whether a show
# has been recorded
def is_airtime_recorded(md):
"""
Takes a metadata dictionary and returns True if it belongs to a file that
was recorded by Airtime.
"""
""" Takes a metadata dictionary and returns True if it belongs to a
file that was recorded by Airtime. """
if not 'MDATA_KEY_CREATOR' in md: return False
return md['MDATA_KEY_CREATOR'] == u'Airtime Show Recorder'
def read_wave_duration(path):
""" Read the length of .wav file (mutagen does not handle this) """
with contextlib.closing(wave.open(path,'r')) as f:
frames = f.getnframes()
rate = f.getframerate()
@ -107,9 +105,7 @@ def read_wave_duration(path):
return duration
def clean_empty_dirs(path):
"""
walks path and deletes every empty directory it finds
"""
""" walks path and deletes every empty directory it finds """
# TODO : test this function
if path.endswith('/'): clean_empty_dirs(path[0:-1])
else:
@ -154,22 +150,25 @@ def no_extension_basename(path):
else: return '.'.join(base.split(".")[0:-1])
def walk_supported(directory, clean_empties=False):
"""
A small generator wrapper around os.walk to only give us files that support
the extensions we are considering. When clean_empties is True we
recursively delete empty directories left over in directory after the walk.
"""
""" A small generator wrapper around os.walk to only give us files
that support the extensions we are considering. When clean_empties
is True we recursively delete empty directories left over in
directory after the walk. """
for root, dirs, files in os.walk(directory):
full_paths = ( os.path.join(root, name) for name in files
if is_file_supported(name) )
for fp in full_paths: yield fp
if clean_empties: clean_empty_dirs(directory)
def file_locked(path):
cmd = "lsof %s" % path
f = Popen(cmd, shell=True, stdout=PIPE).stdout
return bool(f.readlines())
def magic_move(old, new, after_dir_make=lambda : None):
"""
Moves path old to new and constructs the necessary to directories for new
along the way
"""
""" Moves path old to new and constructs the necessary to
directories for new along the way """
new_dir = os.path.dirname(new)
if not os.path.exists(new_dir): os.makedirs(new_dir)
# We need this crusty hack because anytime a directory is created we must
@ -179,18 +178,15 @@ def magic_move(old, new, after_dir_make=lambda : None):
shutil.move(old,new)
def move_to_dir(dir_path,file_path):
"""
moves a file at file_path into dir_path/basename(filename)
"""
""" moves a file at file_path into dir_path/basename(filename) """
bs = os.path.basename(file_path)
magic_move(file_path, os.path.join(dir_path, bs))
def apply_rules_dict(d, rules):
"""
Consumes a dictionary of rules that maps some keys to lambdas which it
applies to every matching element in d and returns a new dictionary with
the rules applied. If a rule returns none then it's not applied
"""
""" Consumes a dictionary of rules that maps some keys to lambdas
which it applies to every matching element in d and returns a new
dictionary with the rules applied. If a rule returns none then it's
not applied """
new_d = copy.deepcopy(d)
for k, rule in rules.iteritems():
if k in d:
@ -205,17 +201,14 @@ def default_to_f(dictionary, keys, default, condition):
return new_d
def default_to(dictionary, keys, default):
"""
Checks if the list of keys 'keys' exists in 'dictionary'. If not then it
returns a new dictionary with all those missing keys defaults to 'default'
"""
""" Checks if the list of keys 'keys' exists in 'dictionary'. If
not then it returns a new dictionary with all those missing keys
defaults to 'default' """
cnd = lambda dictionary, key: key not in dictionary
return default_to_f(dictionary, keys, default, cnd)
def remove_whitespace(dictionary):
"""
Remove values that empty whitespace in the dictionary
"""
""" Remove values that empty whitespace in the dictionary """
nd = copy.deepcopy(dictionary)
bad_keys = []
for k,v in nd.iteritems():
@ -227,6 +220,7 @@ def remove_whitespace(dictionary):
return nd
def parse_int(s):
# TODO : this function isn't used anywhere yet but it may useful for emf
"""
Tries very hard to get some sort of integer result from s. Defaults to 0
when it fails
@ -242,53 +236,6 @@ def parse_int(s):
try : return str(reduce(op.add, takewhile(lambda x: x.isdigit(), s)))
except: return None
def normalized_metadata(md, original_path):
"""
consumes a dictionary of metadata and returns a new dictionary with the
formatted meta data. We also consume original_path because we must set
MDATA_KEY_CREATOR based on in it sometimes
"""
new_md = copy.deepcopy(md)
# replace all slashes with dashes
#for k,v in new_md.iteritems(): new_md[k] = unicode(v).replace('/','-')
# Specific rules that are applied in a per attribute basis
format_rules = {
'MDATA_KEY_TRACKNUMBER' : parse_int,
'MDATA_KEY_FILEPATH' : lambda x: os.path.normpath(x),
'MDATA_KEY_BPM' : lambda x: x[0:8],
'MDATA_KEY_MIME' : lambda x: x.replace('audio/vorbis','audio/ogg'),
# Whenever 0 is reported we change it to empty
#'MDATA_KEY_BITRATE' : lambda x: '' if str(x) == '0' else x
}
new_md = remove_whitespace(new_md) # remove whitespace fields
# Format all the fields in format_rules
new_md = apply_rules_dict(new_md, format_rules)
# set filetype to audioclip by default
new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_FTYPE'],
default=u'audioclip')
# Try to parse bpm but delete the whole key if that fails
if 'MDATA_KEY_BPM' in new_md:
new_md['MDATA_KEY_BPM'] = parse_int(new_md['MDATA_KEY_BPM'])
if new_md['MDATA_KEY_BPM'] is None:
del new_md['MDATA_KEY_BPM']
if not is_airtime_recorded(new_md):
# Read title from filename if it does not exist
default_title = no_extension_basename(original_path)
default_title = re.sub(r'__\d+\.',u'.', default_title)
if re.match(".+-%s-.+$" % unicode_unknown, default_title):
default_title = u''
new_md = default_to(dictionary=new_md, keys=['MDATA_KEY_TITLE'],
default=default_title)
new_md['MDATA_KEY_TITLE'] = re.sub(r'-\d+kbps$', u'',
new_md['MDATA_KEY_TITLE'])
# TODO : wtf is this for again?
new_md['MDATA_KEY_TITLE'] = re.sub(r'-?%s-?' % unicode_unknown, u'',
new_md['MDATA_KEY_TITLE'])
return new_md
def organized_path(old_path, root_path, orig_md):
"""
@ -348,10 +295,9 @@ def organized_path(old_path, root_path, orig_md):
# TODO : Get rid of this function and every one of its uses. We no longer use
# the md5 signature of a song for anything
def file_md5(path,max_length=100):
"""
Get md5 of file path (if it exists). Use only max_length characters to save
time and memory. Pass max_length=-1 to read the whole file (like in mm1)
"""
""" Get md5 of file path (if it exists). Use only max_length
characters to save time and memory. Pass max_length=-1 to read the
whole file (like in mm1) """
if os.path.exists(path):
with open(path, 'rb') as f:
m = hashlib.md5()
@ -367,16 +313,12 @@ def encode_to(obj, encoding='utf-8'):
return obj
def convert_dict_value_to_utf8(md):
"""
formats a dictionary to send as a request to api client
"""
""" formats a dictionary to send as a request to api client """
return dict([(item[0], encode_to(item[1], "utf-8")) for item in md.items()])
def get_system_locale(locale_path='/etc/default/locale'):
"""
Returns the configuration object for the system's default locale. Normally
requires root access.
"""
""" Returns the configuration object for the system's default
locale. Normally requires root access. """
if os.path.exists(locale_path):
try:
config = ConfigObj(locale_path)
@ -386,9 +328,7 @@ def get_system_locale(locale_path='/etc/default/locale'):
permissions issue?" % locale_path)
def configure_locale(config):
"""
sets the locale according to the system's locale.
"""
""" sets the locale according to the system's locale. """
current_locale = locale.getlocale()
if current_locale[1] is None:
default_locale = locale.getdefaultlocale()
@ -405,27 +345,21 @@ def configure_locale(config):
def fondle(path,times=None):
# TODO : write unit tests for this
"""
touch a file to change the last modified date. Beware of calling this
function on the same file from multiple threads.
"""
""" touch a file to change the last modified date. Beware of calling
this function on the same file from multiple threads. """
with file(path, 'a'): os.utime(path, times)
def last_modified(path):
"""
return the time of the last time mm2 was ran. path refers to the index file
whose date modified attribute contains this information. In the case when
the file does not exist we set this time 0 so that any files on the
filesystem were modified after it
"""
""" return the time of the last time mm2 was ran. path refers to the
index file whose date modified attribute contains this information.
In the case when the file does not exist we set this time 0 so that
any files on the filesystem were modified after it """
if os.path.exists(path): return os.path.getmtime(path)
else: return 0
def expand_storage(store):
"""
A storage directory usually consists of 4 different subdirectories. This
function returns their paths
"""
""" A storage directory usually consists of 4 different
subdirectories. This function returns their paths """
store = os.path.normpath(store)
return {
'organize' : os.path.join(store, 'organize'),
@ -435,10 +369,8 @@ def expand_storage(store):
}
def create_dir(path):
"""
will try and make sure that path exists at all costs. raises an exception
if it fails at this task.
"""
""" will try and make sure that path exists at all costs. raises an
exception if it fails at this task. """
if not os.path.exists(path):
try : os.makedirs(path)
except Exception as e : raise FailedToCreateDir(path, e)
@ -456,11 +388,10 @@ def sub_path(directory,f):
return common == normalized
def owner_id(original_path):
"""
Given 'original_path' return the file name of the of 'identifier' file.
return the id that is contained in it. If no file is found or nothing is
read then -1 is returned. File is deleted after the number has been read
"""
""" Given 'original_path' return the file name of the of
'identifier' file. return the id that is contained in it. If no file
is found or nothing is read then -1 is returned. File is deleted
after the number has been read """
fname = "%s.identifier" % original_path
owner_id = -1
try:
@ -476,9 +407,8 @@ def owner_id(original_path):
return owner_id
def file_playable(pathname):
"""
Returns True if 'pathname' is playable by liquidsoap. False otherwise.
"""
""" Returns True if 'pathname' is playable by liquidsoap. False
otherwise. """
# when there is an single apostrophe inside of a string quoted by
# apostrophes, we can only escape it by replace that apostrophe with
# '\''. This breaks the string into two, and inserts an escaped
@ -514,18 +444,14 @@ def toposort(data):
assert not data, "A cyclic dependency exists amongst %r" % data
def truncate_to_length(item, length):
"""
Truncates 'item' to 'length'
"""
""" Truncates 'item' to 'length' """
if isinstance(item, int): item = str(item)
if isinstance(item, basestring):
if len(item) > length: return item[0:length]
else: return item
def format_length(mutagen_length):
"""
Convert mutagen length to airtime length
"""
""" Convert mutagen length to airtime length """
t = float(mutagen_length)
h = int(math.floor(t / 3600))
t = t % 3600

60
python_apps/media-monitor2/media/monitor/request.py Normal file
View file

@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-
import threading
from media.monitor.exceptions import BadSongFile
from media.monitor.log import Loggable
import api_clients.api_client as ac
class ThreadedRequestSync(threading.Thread, Loggable):
def __init__(self, rs):
threading.Thread.__init__(self)
self.rs = rs
self.daemon = True
self.start()
def run(self):
self.rs.run_request()
class RequestSync(Loggable):
""" This class is responsible for making the api call to send a
request to airtime. In the process it packs the requests and retries
for some number of times """
@classmethod
def create_with_api_client(cls, watcher, requests):
apiclient = ac.AirtimeApiClient.create_right_config()
self = cls(watcher, requests, apiclient)
return self
def __init__(self, watcher, requests, apiclient):
self.watcher = watcher
self.requests = requests
self.apiclient = apiclient
def run_request(self):
self.logger.info("Attempting request with %d items." %
len(self.requests))
packed_requests = []
for request_event in self.requests:
try:
for request in request_event.safe_pack():
if isinstance(request, BadSongFile):
self.logger.info("Bad song file: '%s'" % request.path)
else: packed_requests.append(request)
except Exception as e:
self.unexpected_exception( e )
if hasattr(request_event, 'path'):
self.logger.info("Possibly related to path: '%s'" %
request_event.path)
try: self.apiclient.send_media_monitor_requests( packed_requests )
# most likely we did not get json response as we expected
except ValueError:
self.logger.info("ApiController.php probably crashed, we \
diagnose this from the fact that it did not return \
valid json")
self.logger.info("Trying again after %f seconds" %
self.request_wait)
except Exception as e: self.unexpected_exception(e)
else: self.logger.info("Request was successful")
self.watcher.flag_done() # poor man's condition variable

10
python_apps/media-monitor2/media/monitor/syncdb.py
View file

@ -53,11 +53,11 @@ class AirtimeDB(Loggable):
"""
return self.id_to_dir[ dir_id ]
def storage_path(self): return self.base_storage
def organize_path(self): return self.storage_paths['organize']
def problem_path(self): return self.storage_paths['problem_files']
def import_path(self): return self.storage_paths['imported']
def recorded_path(self): return self.storage_paths['recorded']
def storage_path(self) : return self.base_storage
def organize_path(self) : return self.storage_paths['organize']
def problem_path(self) : return self.storage_paths['problem_files']
def import_path(self) : return self.storage_paths['imported']
def recorded_path(self) : return self.storage_paths['recorded']
def list_watched(self):
"""

72
python_apps/media-monitor2/media/monitor/watchersyncer.py
View file

@ -6,69 +6,9 @@ import copy
from media.monitor.handler import ReportHandler
from media.monitor.log import Loggable
from media.monitor.exceptions import BadSongFile
from media.monitor.pure import LazyProperty
from media.monitor.eventcontractor import EventContractor
from media.monitor.events import EventProxy
import api_clients.api_client as ac
class RequestSync(threading.Thread,Loggable):
"""
This class is responsible for making the api call to send a request
to airtime. In the process it packs the requests and retries for
some number of times
"""
def __init__(self, watcher, requests):
threading.Thread.__init__(self)
self.watcher = watcher
self.requests = requests
self.retries = 1
self.request_wait = 0.3
@LazyProperty
def apiclient(self):
return ac.AirtimeApiClient.create_right_config()
def run(self):
self.logger.info("Attempting request with %d items." %
len(self.requests))
# Note that we must attach the appropriate mode to every
# response. Also Not forget to attach the 'is_record' to any
# requests that are related to recorded shows
# TODO : recorded shows aren't flagged right
# Is this retry shit even necessary? Consider getting rid of this.
packed_requests = []
for request_event in self.requests:
try:
for request in request_event.safe_pack():
if isinstance(request, BadSongFile):
self.logger.info("Bad song file: '%s'" % request.path)
else: packed_requests.append(request)
except Exception as e:
self.unexpected_exception( e )
if hasattr(request_event, 'path'):
self.logger.info("Possibly related to path: '%s'" %
request_event.path)
def make_req():
self.apiclient.send_media_monitor_requests( packed_requests )
for try_index in range(0,self.retries):
try: make_req()
# most likely we did not get json response as we expected
except ValueError:
self.logger.info("ApiController.php probably crashed, we \
diagnose this from the fact that it did not return \
valid json")
self.logger.info("Trying again after %f seconds" %
self.request_wait)
time.sleep( self.request_wait )
except Exception as e: self.unexpected_exception(e)
else:
self.logger.info("Request worked on the '%d' try" %
(try_index + 1))
break
else: self.logger.info("Failed to send request after '%d' tries..." %
self.retries)
self.watcher.flag_done()
from media.monitor.request import ThreadedRequestSync, RequestSync
class TimeoutWatcher(threading.Thread,Loggable):
"""
@ -131,8 +71,7 @@ class WatchSyncer(ReportHandler,Loggable):
#self.push_queue( event )
except BadSongFile as e:
self.fatal_exception("Received bas song file '%s'" % e.path, e)
except Exception as e:
self.unexpected_exception(e)
except Exception as e: self.unexpected_exception(e)
else:
self.logger.info("Received event that does not implement packing.\
Printing its representation:")
@ -209,8 +148,8 @@ class WatchSyncer(ReportHandler,Loggable):
requests = copy.copy(self.__queue)
def launch_request():
# Need shallow copy here
t = RequestSync(watcher=self, requests=requests)
t.start()
t = ThreadedRequestSync( RequestSync.create_with_api_client(
watcher=self, requests=requests) )
self.__current_thread = t
self.__requests.append(launch_request)
self.__reset_queue()
@ -218,7 +157,8 @@ class WatchSyncer(ReportHandler,Loggable):
def __reset_queue(self): self.__queue = []
def __del__(self):
# Ideally we would like to do a little more to ensure safe shutdown
#this destructor is completely untested and it's unclear whether
#it's even doing anything useful. consider removing it
if self.events_in_queue():
self.logger.warn("Terminating with events still in the queue...")
if self.requests_in_queue():