Congegni/sintonia
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

CC-2505 : Media monitor doesn't delete some tracks from the Airtime server due to weird characters in file names

Browse source 
seems to fix unicode problems I was having.
This commit is contained in:
Naomi Aro 2011-07-18 12:05:16 +02:00
parent c74af05f18
commit e369ab8dee
2 changed files with 59 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

69
python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py
View file

@ -28,7 +28,7 @@ class AirtimeNotifier(Notifier):
self.mmc = mmc self.mmc = mmc
self.wm = watch_manager self.wm = watch_manager
self.mask = pyinotify.ALL_EVENTS self.mask = pyinotify.ALL_EVENTS
while not self.init_rabbit_mq(): while not self.init_rabbit_mq():
self.logger.error("Error connecting to RabbitMQ Server. Trying again in few seconds") self.logger.error("Error connecting to RabbitMQ Server. Trying again in few seconds")
@ -95,11 +95,11 @@ class AirtimeNotifier(Notifier):
self.mmc.set_needed_file_permissions(new_storage_directory, True) self.mmc.set_needed_file_permissions(new_storage_directory, True)
self.bootstrap.sync_database_to_filesystem(new_storage_directory_id, new_storage_directory) self.bootstrap.sync_database_to_filesystem(new_storage_directory_id, new_storage_directory)
self.config.storage_directory = os.path.normpath(new_storage_directory) self.config.storage_directory = os.path.normpath(new_storage_directory)
self.config.imported_directory = os.path.normpath(new_storage_directory + '/imported') self.config.imported_directory = os.path.normpath(new_storage_directory + '/imported')
self.config.organize_directory = os.path.normpath(new_storage_directory + '/organize') self.config.organize_directory = os.path.normpath(new_storage_directory + '/organize')
self.mmc.ensure_is_dir(self.config.storage_directory) self.mmc.ensure_is_dir(self.config.storage_directory)
self.mmc.ensure_is_dir(self.config.imported_directory) self.mmc.ensure_is_dir(self.config.imported_directory)
self.mmc.ensure_is_dir(self.config.organize_directory) self.mmc.ensure_is_dir(self.config.organize_directory)
@ -113,52 +113,59 @@ class AirtimeNotifier(Notifier):
#update airtime with information about files discovered in our #update airtime with information about files discovered in our
#watched directories. Pass in a dict() object with the following #watched directories. Pass in a dict() object with the following
#attributes: #attributes:
# -filepath # -filepath
# -mode # -mode
# -data # -data
# -is_recorded_show # -is_recorded_show
def update_airtime(self, d): def update_airtime(self, d):
filepath = d['filepath']
mode = d['mode']
md = {} try:
md['MDATA_KEY_FILEPATH'] = filepath.encode("utf_8") self.logger.info("updating filepath: %s ", d['filepath'])
filepath = d['filepath']
mode = d['mode']
if 'data' in d: md = {}
file_md = d['data'] md['MDATA_KEY_FILEPATH'] = filepath
md.update(file_md)
else: if 'data' in d:
file_md = None file_md = d['data']
data = None md.update(file_md)
else:
file_md = None
data = None
if (os.path.exists(filepath) and (mode == self.config.MODE_CREATE)): if (os.path.exists(filepath) and (mode == self.config.MODE_CREATE)):
if file_md is None: if file_md is None:
mutagen = self.md_manager.get_md_from_file(filepath)
if mutagen is None:
return
md.update(mutagen)
if d['is_recorded_show']:
self.api_client.update_media_metadata(md, mode, True)
else:
self.api_client.update_media_metadata(md, mode)
elif (os.path.exists(filepath) and (mode == self.config.MODE_MODIFY)):
mutagen = self.md_manager.get_md_from_file(filepath) mutagen = self.md_manager.get_md_from_file(filepath)
if mutagen is None: if mutagen is None:
return return
md.update(mutagen) md.update(mutagen)
if d['is_recorded_show']:
self.api_client.update_media_metadata(md, mode, True)
else:
self.api_client.update_media_metadata(md, mode) self.api_client.update_media_metadata(md, mode)
elif (os.path.exists(filepath) and (mode == self.config.MODE_MODIFY)): elif (mode == self.config.MODE_MOVED):
mutagen = self.md_manager.get_md_from_file(filepath) md['MDATA_KEY_MD5'] = self.md_manager.get_md5(filepath)
if mutagen is None: self.api_client.update_media_metadata(md, mode)
return
md.update(mutagen)
self.api_client.update_media_metadata(md, mode)
elif (mode == self.config.MODE_MOVED): elif (mode == self.config.MODE_DELETE):
md['MDATA_KEY_MD5'] = self.md_manager.get_md5(filepath) self.api_client.update_media_metadata(md, mode)
self.api_client.update_media_metadata(md, mode)
elif (mode == self.config.MODE_DELETE): except Exception, e:
self.api_client.update_media_metadata(md, mode) self.logger.error("failed updating filepath: %s ", d['filepath'])
self.logger.error('Exception: %s', e)
#define which directories the pyinotify WatchManager should watch. #define which directories the pyinotify WatchManager should watch.
def watch_directory(self, directory): def watch_directory(self, directory):

42
python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py
View file

@ -9,7 +9,7 @@ from airtimemetadata import AirtimeMetadata
class MediaMonitorCommon: class MediaMonitorCommon:
timestamp_file = "/var/tmp/airtime/last_index" timestamp_file = "/var/tmp/airtime/last_index"
def __init__(self, airtime_config): def __init__(self, airtime_config):
self.supported_file_formats = ['mp3', 'ogg'] self.supported_file_formats = ['mp3', 'ogg']
self.logger = logging.getLogger() self.logger = logging.getLogger()
@ -38,12 +38,12 @@ class MediaMonitorCommon:
return True return True
else: else:
return False return False
#check if file is readable by "nobody" #check if file is readable by "nobody"
def has_correct_permissions(self, filepath): def has_correct_permissions(self, filepath):
#drop root permissions and become "nobody" #drop root permissions and become "nobody"
os.seteuid(65534) os.seteuid(65534)
try: try:
open(filepath) open(filepath)
readable = True readable = True
@ -56,7 +56,7 @@ class MediaMonitorCommon:
finally: finally:
#reset effective user to root #reset effective user to root
os.seteuid(0) os.seteuid(0)
return readable return readable
def set_needed_file_permissions(self, item, is_dir): def set_needed_file_permissions(self, item, is_dir):
@ -78,7 +78,7 @@ class MediaMonitorCommon:
finally: finally:
os.umask(omask) os.umask(omask)
#checks if path is a directory, and if it doesnt exist, then creates it. #checks if path is a directory, and if it doesnt exist, then creates it.
#Otherwise prints error to log file. #Otherwise prints error to log file.
def ensure_is_dir(self, directory): def ensure_is_dir(self, directory):
@ -92,7 +92,7 @@ class MediaMonitorCommon:
finally: finally:
os.umask(omask) os.umask(omask)
#moves file from source to dest but also recursively removes the #moves file from source to dest but also recursively removes the
#the source file's parent directories if they are now empty. #the source file's parent directories if they are now empty.
def move_file(self, source, dest): def move_file(self, source, dest):
@ -103,10 +103,10 @@ class MediaMonitorCommon:
self.logger.error("failed to move file. %s", e) self.logger.error("failed to move file. %s", e)
finally: finally:
os.umask(omask) os.umask(omask)
dir = os.path.dirname(source) dir = os.path.dirname(source)
self.cleanup_empty_dirs(dir) self.cleanup_empty_dirs(dir)
#keep moving up the file hierarchy and deleting parent #keep moving up the file hierarchy and deleting parent
#directories until we hit a non-empty directory, or we #directories until we hit a non-empty directory, or we
#hit the organize dir. #hit the organize dir.
@ -114,12 +114,12 @@ class MediaMonitorCommon:
if os.path.normpath(dir) != self.config.organize_directory: if os.path.normpath(dir) != self.config.organize_directory:
if len(os.listdir(dir)) == 0: if len(os.listdir(dir)) == 0:
os.rmdir(dir) os.rmdir(dir)
pdir = os.path.dirname(dir) pdir = os.path.dirname(dir)
self.cleanup_empty_dirs(pdir) self.cleanup_empty_dirs(pdir)
#checks if path exists already in stor. If the path exists and the md5s are the
#checks if path exists already in stor. If the path exists and the md5s are the
#same just overwrite. #same just overwrite.
def create_unique_filename(self, filepath, old_filepath): def create_unique_filename(self, filepath, old_filepath):
@ -198,7 +198,7 @@ class MediaMonitorCommon:
elif(md['MDATA_KEY_TRACKNUMBER'] == u'unknown'.encode('utf-8')): elif(md['MDATA_KEY_TRACKNUMBER'] == u'unknown'.encode('utf-8')):
filepath = '%s/%s/%s/%s/%s-%s%s' % (storage_directory, "imported".encode('utf-8'), md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) filepath = '%s/%s/%s/%s/%s-%s%s' % (storage_directory, "imported".encode('utf-8'), md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext)
else: else:
filepath = '%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "imported".encode('utf-8'), md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TRACKNUMBER'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) filepath = '%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "imported".encode('utf-8'), md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TRACKNUMBER'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext)
filepath = self.create_unique_filename(filepath, original_path) filepath = self.create_unique_filename(filepath, original_path)
self.logger.info('Unique filepath: %s', filepath) self.logger.info('Unique filepath: %s', filepath)
@ -208,38 +208,38 @@ class MediaMonitorCommon:
self.logger.error('Exception: %s', e) self.logger.error('Exception: %s', e)
return filepath return filepath
def execCommandAndReturnStdOut(self, command): def execCommandAndReturnStdOut(self, command):
p = Popen(command, shell=True, stdout=PIPE) p = Popen(command, shell=True, stdout=PIPE)
stdout = p.communicate()[0] stdout = p.communicate()[0]
if p.returncode != 0: if p.returncode != 0:
self.logger.warn("command \n%s\n return with a non-zero return value", command) self.logger.warn("command \n%s\n return with a non-zero return value", command)
return stdout return stdout
def scan_dir_for_new_files(self, dir): def scan_dir_for_new_files(self, dir):
command = 'find "%s" -type f -iname "*.ogg" -o -iname "*.mp3" -readable' % dir.replace('"', '\\"') command = 'find "%s" -type f -iname "*.ogg" -o -iname "*.mp3" -readable' % dir.replace('"', '\\"')
self.logger.debug(command) self.logger.debug(command)
stdout = self.execCommandAndReturnStdOut(command) stdout = self.execCommandAndReturnStdOut(command)
stdout = unicode(stdout, "utf_8") stdout = unicode(stdout, "utf_8")
return stdout.splitlines() return stdout.splitlines()
def touch_index_file(self): def touch_index_file(self):
open(self.timestamp_file, "w") open(self.timestamp_file, "w")
def organize_new_file(self, pathname): def organize_new_file(self, pathname):
self.logger.info(u"Organizing new file: %s", pathname) self.logger.info("Organizing new file: %s", pathname)
file_md = self.md_manager.get_md_from_file(pathname) file_md = self.md_manager.get_md_from_file(pathname)
if file_md is not None: if file_md is not None:
#is_recorded_show = 'MDATA_KEY_CREATOR' in file_md and \ #is_recorded_show = 'MDATA_KEY_CREATOR' in file_md and \
# file_md['MDATA_KEY_CREATOR'] == "AIRTIMERECORDERSOURCEFABRIC".encode('utf-8') # file_md['MDATA_KEY_CREATOR'] == "AIRTIMERECORDERSOURCEFABRIC".encode('utf-8')
filepath = self.create_file_path(pathname, file_md) filepath = self.create_file_path(pathname, file_md)
self.logger.debug(u"Moving from %s to %s", pathname, filepath) self.logger.debug("Moving from %s to %s", pathname, filepath)
self.move_file(pathname, filepath) self.move_file(pathname, filepath)
else: else:
filepath = None filepath = None
self.logger.warn("File %s, has invalid metadata", pathname) self.logger.warn("File %s, has invalid metadata", pathname)
return filepath return filepath