From 29869c175a90cc5ed226ce14699c6443dee9c59b Mon Sep 17 00:00:00 2001 From: martin Date: Wed, 13 Jul 2011 17:38:11 -0400 Subject: [PATCH] CC-2279: Upgrade script for converting stor directory to new format -refactor media-monitor --- .../airtime-1.9.0/airtime-upgrade.php | 2 +- python_apps/media-monitor/MediaMonitor.py | 14 +- .../airtimemediamonitorbootstrap.py | 13 +- .../airtimefilemonitor/airtimenotifier.py | 20 +- .../airtimefilemonitor/airtimeprocessevent.py | 304 ++---------------- .../airtimefilemonitor/mediamonitorcommon.py | 255 +++++++++++++++ 6 files changed, 307 insertions(+), 301 deletions(-) create mode 100644 python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py diff --git a/install/upgrades/airtime-1.9.0/airtime-upgrade.php b/install/upgrades/airtime-1.9.0/airtime-upgrade.php index 14975d1c3..1e0af3211 100644 --- a/install/upgrades/airtime-1.9.0/airtime-upgrade.php +++ b/install/upgrades/airtime-1.9.0/airtime-upgrade.php @@ -416,4 +416,4 @@ AirtimeInstall::CreateCronFile(); //old database had a "fullpath" column that stored the absolute path of each track. We have to -//change it so that the "fullpath" column has +//change it so that the "fullpath" column has path relative to the "directory" column. diff --git a/python_apps/media-monitor/MediaMonitor.py b/python_apps/media-monitor/MediaMonitor.py index 773c99fe6..1408bd009 100644 --- a/python_apps/media-monitor/MediaMonitor.py +++ b/python_apps/media-monitor/MediaMonitor.py @@ -13,6 +13,7 @@ from multiprocessing import Process, Queue as mpQueue from pyinotify import WatchManager from airtimefilemonitor.airtimenotifier import AirtimeNotifier +from airtimefilemonitor.mediamonitorcommon import MediaMonitorCommon from airtimefilemonitor.airtimeprocessevent import AirtimeProcessEvent from airtimefilemonitor.mediaconfig import AirtimeMediaConfig from airtimefilemonitor.workerprocess import MediaMonitorWorkerProcess @@ -60,13 +61,16 @@ except Exception, e: logger.error('Exception: %s', e) try: - wm = WatchManager() - pe = AirtimeProcessEvent(queue=multi_queue, airtime_config=config, wm=wm) - bootstrap = AirtimeMediaMonitorBootstrap(logger, pe, api_client) + + wm = WatchManager() + mmc = MediaMonitorCommon(config, wm) + pe = AirtimeProcessEvent(queue=multi_queue, airtime_config=config, wm=wm, mmc=mmc) + + bootstrap = AirtimeMediaMonitorBootstrap(logger, pe, api_client, mmc) bootstrap.scan() - notifier = AirtimeNotifier(wm, pe, read_freq=1, timeout=0, airtime_config=config, api_client=api_client, bootstrap=bootstrap) + notifier = AirtimeNotifier(wm, pe, read_freq=1, timeout=0, airtime_config=config, api_client=api_client, bootstrap=bootstrap, mmc=mmc) notifier.coalesce_events() #create 5 worker processes @@ -76,7 +80,7 @@ try: processes.append(p) p.start() - wdd = pe.watch_directory(storage_directory) + wdd = mmc.watch_directory(storage_directory) logger.info("Added watch to %s", storage_directory) logger.info("wdd result %s", wdd[storage_directory]) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py b/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py index ef8fc0731..012a78aba 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimemediamonitorbootstrap.py @@ -12,10 +12,11 @@ class AirtimeMediaMonitorBootstrap(): pe -- reference to an instance of ProcessEvent api_clients -- reference of api_clients to communicate with airtime-server """ - def __init__(self, logger, pe, api_client): + def __init__(self, logger, pe, api_client, mmc): self.logger = logger self.pe = pe self.api_client = api_client + self.mmc = mmc """On bootup we want to scan all directories and look for files that weren't there or files that changed before media-monitor process @@ -70,20 +71,20 @@ class AirtimeMediaMonitorBootstrap(): for file in files['files']: db_known_files_set.add(file) - new_files = self.pe.scan_dir_for_new_files(dir) + new_files = self.mmc.scan_dir_for_new_files(dir) all_files_set = set() for file_path in new_files: if len(file_path.strip(" \n")) > 0: all_files_set.add(file_path[len(dir):]) - if os.path.exists(self.pe.timestamp_file): + if os.path.exists(self.mmc.timestamp_file): """find files that have been modified since the last time media-monitor process started.""" - time_diff_sec = time.time() - os.path.getmtime(self.pe.timestamp_file) + time_diff_sec = time.time() - os.path.getmtime(self.mmc.timestamp_file) command = "find %s -type f -iname '*.ogg' -o -iname '*.mp3' -readable -mmin -%d" % (dir, time_diff_sec/60+1) else: command = "find %s -type f -iname '*.ogg' -o -iname '*.mp3' -readable" % dir - stdout = self.pe.execCommandAndReturnStdOut(command) + stdout = self.mmc.execCommandAndReturnStdOut(command) stdout = unicode(stdout, "utf_8") new_files = stdout.splitlines() @@ -110,7 +111,7 @@ class AirtimeMediaMonitorBootstrap(): self.logger.info("Modified files: \n%s\n\n"%modified_files_set) #"touch" file timestamp - self.pe.touch_index_file() + self.mmc.touch_index_file() for file_path in deleted_files_set: self.pe.handle_removed_file(False, "%s/%s" % (dir, file_path)) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py index b799c55ce..7f10be94b 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py @@ -15,7 +15,7 @@ from airtimemetadata import AirtimeMetadata class AirtimeNotifier(Notifier): - def __init__(self, watch_manager, default_proc_fun=None, read_freq=0, threshold=0, timeout=None, airtime_config=None, api_client=None, bootstrap=None): + def __init__(self, watch_manager, default_proc_fun=None, read_freq=0, threshold=0, timeout=None, airtime_config=None, api_client=None, bootstrap=None, mmc=None): Notifier.__init__(self, watch_manager, default_proc_fun, read_freq, threshold, timeout) self.logger = logging.getLogger() @@ -25,6 +25,7 @@ class AirtimeNotifier(Notifier): self.md_manager = AirtimeMetadata() self.import_processes = {} self.watched_folders = [] + self.mmc = mmc while not self.init_rabbit_mq(): @@ -65,11 +66,10 @@ class AirtimeNotifier(Notifier): self.md_manager.save_md_to_file(m) elif m['event_type'] == "new_watch": - mm = self.proc_fun() self.logger.info("AIRTIME NOTIFIER add watched folder event " + m['directory']) self.walk_newly_watched_directory(m['directory']) - mm.watch_directory(m['directory']) + self.mmc.watch_directory(m['directory']) elif m['event_type'] == "remove_watch": watched_directory = m['directory'].encode('utf-8') @@ -90,7 +90,7 @@ class AirtimeNotifier(Notifier): self.logger.info("Removing watch on: %s wd %s", storage_directory, wd) mm.wm.rm_watch(wd, rec=True) - mm.set_needed_file_permissions(new_storage_directory, True) + self.mmc.set_needed_file_permissions(new_storage_directory, True) self.bootstrap.sync_database_to_filesystem(new_storage_directory_id, new_storage_directory) @@ -98,11 +98,11 @@ class AirtimeNotifier(Notifier): self.config.imported_directory = os.path.normpath(new_storage_directory + '/imported') self.config.organize_directory = os.path.normpath(new_storage_directory + '/organize') - mm.ensure_is_dir(self.config.storage_directory) - mm.ensure_is_dir(self.config.imported_directory) - mm.ensure_is_dir(self.config.organize_directory) + self.mmc.ensure_is_dir(self.config.storage_directory) + self.mmc.ensure_is_dir(self.config.imported_directory) + self.mmc.ensure_is_dir(self.config.organize_directory) - mm.watch_directory(new_storage_directory) + self.mmc.watch_directory(new_storage_directory) elif m['event_type'] == "file_delete": self.logger.info("Deleting file: %s ", m['filepath']) mm = self.proc_fun() @@ -167,8 +167,8 @@ class AirtimeNotifier(Notifier): for filename in files: full_filepath = path+"/"+filename - if mm.is_audio_file(full_filepath): - if mm.has_correct_permissions(full_filepath): + if self.mmc.is_audio_file(full_filepath): + if self.mmc.has_correct_permissions(full_filepath): self.logger.info("importing %s", full_filepath) event = {'filepath': full_filepath, 'mode': self.config.MODE_CREATE, 'is_recorded_show': False} mm.multi_queue.put(event) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py index 864df358f..1fb227d96 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py @@ -1,11 +1,7 @@ -import os import socket -import grp -import pwd import logging import time -from subprocess import Popen, PIPE import pyinotify from pyinotify import ProcessEvent @@ -19,9 +15,8 @@ from airtimefilemonitor.mediaconfig import AirtimeMediaConfig class AirtimeProcessEvent(ProcessEvent): - timestamp_file = "/var/tmp/airtime/last_index" - - def my_init(self, queue, airtime_config=None, wm=None): + #TODO + def my_init(self, queue, airtime_config=None, wm=None, mmc=None): """ Method automatically called from ProcessEvent.__init__(). Additional keyworded arguments passed to ProcessEvent.__init__() are then @@ -36,223 +31,16 @@ class AirtimeProcessEvent(ProcessEvent): #doesn't need to contact the server and tell it to delete again. self.ignore_event = set() - self.supported_file_formats = ['mp3', 'ogg'] - - """ - self.temp_files = {} - self.renamed_files = {} - self.moved_files = {} - self.gui_replaced = {} - """ - self.cookies_IN_MOVED_FROM = {} self.file_events = [] self.multi_queue = queue - self.mask = pyinotify.ALL_EVENTS self.wm = wm self.md_manager = AirtimeMetadata() + self.mmc = mmc def add_filepath_to_ignore(self, filepath): self.ignore_event.add(filepath) - - #define which directories the pyinotify WatchManager should watch. - def watch_directory(self, directory): - return self.wm.add_watch(directory, self.mask, rec=True, auto_add=True) - - def is_parent_directory(self, filepath, directory): - filepath = os.path.normpath(filepath) - directory = os.path.normpath(directory) - return (directory == filepath[0:len(directory)]) - - """ - def is_temp_file(self, filename): - info = filename.split(".") - - if(info[-2] in self.supported_file_formats): - return True - else: - return False - """ - - def is_audio_file(self, filename): - info = filename.split(".") - - if(info[-1] in self.supported_file_formats): - return True - else: - return False - #check if file is readable by "nobody" - def has_correct_permissions(self, filepath): - #drop root permissions and become "nobody" - os.seteuid(65534) - - try: - open(filepath) - readable = True - except IOError: - self.logger.warn("File does not have correct permissions: '%s'", filepath) - readable = False - except Exception, e: - self.logger.error("Unexpected exception thrown: %s", e) - readable = False - finally: - #reset effective user to root - os.seteuid(0) - - return readable - - def set_needed_file_permissions(self, item, is_dir): - try: - omask = os.umask(0) - - uid = pwd.getpwnam('www-data')[2] - gid = grp.getgrnam('www-data')[2] - - os.chown(item, uid, gid) - - if is_dir is True: - os.chmod(item, 02777) - else: - os.chmod(item, 0666) - - except Exception, e: - self.logger.error("Failed to change file's owner/group/permissions. %s", e) - finally: - os.umask(omask) - - - #checks if path is a directory, and if it doesnt exist, then creates it. - #Otherwise prints error to log file. - def ensure_is_dir(self, directory): - try: - omask = os.umask(0) - if not os.path.exists(directory): - os.makedirs(directory, 02777) - #self.watch_directory(directory) - elif not os.path.isdir(directory): - #path exists but it is a file not a directory! - self.logger.error("path %s exists, but it is not a directory!!!") - finally: - os.umask(omask) - - #moves file from source to dest but also recursively removes the - #the source file's parent directories if they are now empty. - def move_file(self, source, dest): - - try: - omask = os.umask(0) - os.rename(source, dest) - except Exception, e: - self.logger.error("failed to move file. %s", e) - finally: - os.umask(omask) - - dir = os.path.dirname(source) - self.cleanup_empty_dirs(dir) - - #keep moving up the file hierarchy and deleting parent - #directories until we hit a non-empty directory, or we - #hit the organize dir. - def cleanup_empty_dirs(self, dir): - if os.path.normpath(dir) != self.config.organize_directory: - if len(os.listdir(dir)) == 0: - os.rmdir(dir) - - pdir = os.path.dirname(dir) - self.cleanup_empty_dirs(pdir) - - - #checks if path exists already in stor. If the path exists and the md5s are the - #same just overwrite. - def create_unique_filename(self, filepath, old_filepath): - - try: - if(os.path.exists(filepath)): - self.logger.info("Path %s exists", filepath) - - self.logger.info("Checking if md5s are the same.") - md5_fp = self.md_manager.get_md5(filepath) - md5_ofp = self.md_manager.get_md5(old_filepath) - - if(md5_fp == md5_ofp): - self.logger.info("Md5s are the same, moving to same filepath.") - return filepath - - self.logger.info("Md5s aren't the same, appending to filepath.") - file_dir = os.path.dirname(filepath) - filename = os.path.basename(filepath).split(".")[0] - #will be in the format .ext - file_ext = os.path.splitext(filepath)[1] - i = 1; - while(True): - new_filepath = '%s/%s(%s)%s' % (file_dir, filename, i, file_ext) - self.logger.error("Trying %s", new_filepath) - - if(os.path.exists(new_filepath)): - i = i+1; - else: - filepath = new_filepath - break - - except Exception, e: - self.logger.error("Exception %s", e) - - return filepath - - #create path in /srv/airtime/stor/imported/[song-metadata] - def create_file_path(self, original_path, orig_md): - - storage_directory = self.config.storage_directory - - is_recorded_show = False - - try: - #will be in the format .ext - file_ext = os.path.splitext(original_path)[1] - file_ext = file_ext.encode('utf-8') - - path_md = ['MDATA_KEY_TITLE', 'MDATA_KEY_CREATOR', 'MDATA_KEY_SOURCE', 'MDATA_KEY_TRACKNUMBER', 'MDATA_KEY_BITRATE'] - - md = {} - for m in path_md: - if m not in orig_md: - md[m] = u'unknown'.encode('utf-8') - else: - #get rid of any "/" which will interfere with the filepath. - if isinstance(orig_md[m], basestring): - md[m] = orig_md[m].replace("/", "-") - else: - md[m] = orig_md[m] - - if 'MDATA_KEY_TRACKNUMBER' in orig_md: - #make sure all track numbers are at least 2 digits long in the filepath. - md['MDATA_KEY_TRACKNUMBER'] = "%02d" % (int(md['MDATA_KEY_TRACKNUMBER'])) - - #format bitrate as 128kbps - md['MDATA_KEY_BITRATE'] = str(md['MDATA_KEY_BITRATE']/1000)+"kbps" - - filepath = None - #file is recorded by Airtime - #/srv/airtime/stor/recorded/year/month/year-month-day-time-showname-bitrate.ext - if(md['MDATA_KEY_CREATOR'] == "AIRTIMERECORDERSOURCEFABRIC".encode('utf-8')): - #yyyy-mm-dd-hh-MM-ss - y = orig_md['MDATA_KEY_YEAR'].split("-") - filepath = '%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "recorded".encode('utf-8'), y[0], y[1], orig_md['MDATA_KEY_YEAR'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) - elif(md['MDATA_KEY_TRACKNUMBER'] == u'unknown'.encode('utf-8')): - filepath = '%s/%s/%s/%s/%s-%s%s' % (storage_directory, "imported".encode('utf-8'), md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) - else: - filepath = '%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "imported".encode('utf-8'), md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TRACKNUMBER'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) - - filepath = self.create_unique_filename(filepath, original_path) - self.logger.info('Unique filepath: %s', filepath) - self.ensure_is_dir(os.path.dirname(filepath)) - - except Exception, e: - self.logger.error('Exception: %s', e) - - return filepath - #event.dir: True if the event was raised against a directory. #event.name: filename #event.pathname: pathname (str): Concatenation of 'path' and 'name'. @@ -266,46 +54,28 @@ class AirtimeProcessEvent(ProcessEvent): #if self.is_temp_file(name) : #file created is a tmp file which will be modified and then moved back to the original filename. #self.temp_files[pathname] = None - if self.is_audio_file(pathname): - if self.is_parent_directory(pathname, self.config.organize_directory): + if self.mmc.is_audio_file(pathname): + if self.mmc.is_parent_directory(pathname, self.config.organize_directory): #file was created in /srv/airtime/stor/organize. Need to process and move #to /srv/airtime/stor/imported - self.organize_new_file(pathname) + self.mmc.organize_new_file(pathname) else: - self.set_needed_file_permissions(pathname, dir) + self.mmc.set_needed_file_permissions(pathname, dir) self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': pathname, 'is_recorded_show': False}) else: #event is because of a created directory - if self.is_parent_directory(pathname, self.config.storage_directory): - self.set_needed_file_permissions(pathname, dir) - - - def organize_new_file(self, pathname): - self.logger.info(u"Organizing new file: %s", pathname) - file_md = self.md_manager.get_md_from_file(pathname) - - if file_md is not None: - #is_recorded_show = 'MDATA_KEY_CREATOR' in file_md and \ - # file_md['MDATA_KEY_CREATOR'] == "AIRTIMERECORDERSOURCEFABRIC".encode('utf-8') - filepath = self.create_file_path(pathname, file_md) - - self.logger.debug(u"Moving from %s to %s", pathname, filepath) - self.move_file(pathname, filepath) - else: - filepath = None - self.logger.warn("File %s, has invalid metadata", pathname) - - return filepath + if self.mmc.is_parent_directory(pathname, self.config.storage_directory): + self.mmc.set_needed_file_permissions(pathname, dir) def process_IN_MODIFY(self, event): self.logger.info("process_IN_MODIFY: %s", event) self.handle_modified_file(event.dir, event.pathname, event.name) def handle_modified_file(self, dir, pathname, name): - if not dir and self.is_parent_directory(pathname, self.config.organize_directory): + if not dir and self.mmc.is_parent_directory(pathname, self.config.organize_directory): self.logger.info("Modified: %s", pathname) - if self.is_audio_file(name): + if self.mmc.is_audio_file(name): self.file_events.append({'filepath': pathname, 'mode': self.config.MODE_MODIFY}) #if a file is moved somewhere, this callback is run. With details about @@ -315,9 +85,9 @@ class AirtimeProcessEvent(ProcessEvent): def process_IN_MOVED_FROM(self, event): self.logger.info("process_IN_MOVED_FROM: %s", event) if not event.dir: - if not self.is_parent_directory(event.pathname, self.config.organize_directory): + if not self.mmc.is_parent_directory(event.pathname, self.config.organize_directory): #we don't care about moved_from events from the organize dir. - if self.is_audio_file(event.name): + if self.mmc.is_audio_file(event.name): self.cookies_IN_MOVED_FROM[event.cookie] = (event, time.time()) @@ -328,22 +98,22 @@ class AirtimeProcessEvent(ProcessEvent): def process_IN_MOVED_TO(self, event): self.logger.info("process_IN_MOVED_TO: %s", event) #if stuff dropped in stor via a UI move must change file permissions. - self.set_needed_file_permissions(event.pathname, event.dir) + self.mmc.set_needed_file_permissions(event.pathname, event.dir) if not event.dir: - if self.is_audio_file(event.name): + if self.mmc.is_audio_file(event.name): if event.cookie in self.cookies_IN_MOVED_FROM: #files original location was also in a watched directory del self.cookies_IN_MOVED_FROM[event.cookie] - if self.is_parent_directory(event.pathname, self.config.organize_directory): - filepath = self.organize_new_file(event.pathname) + if self.mmc.is_parent_directory(event.pathname, self.config.organize_directory): + filepath = self.mmc.organize_new_file(event.pathname) else: filepath = event.pathname if (filepath is not None): self.file_events.append({'filepath': filepath, 'mode': self.config.MODE_MOVED}) else: - if self.is_parent_directory(event.pathname, self.config.organize_directory): - self.organize_new_file(event.pathname) + if self.mmc.is_parent_directory(event.pathname, self.config.organize_directory): + self.mmc.organize_new_file(event.pathname) else: #show dragged from unwatched folder into a watched folder. Do not "organize". self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': event.pathname, 'is_recorded_show': False}) @@ -351,10 +121,10 @@ class AirtimeProcessEvent(ProcessEvent): #When we move a directory into a watched_dir, we only get a notification that the dir was created, #and no additional information about files that came along with that directory. #need to scan the entire directory for files. - files = self.scan_dir_for_new_files(event.pathname) - if self.is_parent_directory(event.pathname, self.config.organize_directory): + files = self.mmc.scan_dir_for_new_files(event.pathname) + if self.mmc.is_parent_directory(event.pathname, self.config.organize_directory): for file in files: - self.organize_new_file(file) + self.mmc.organize_new_file(file) else: for file in files: self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': file, 'is_recorded_show': False}) @@ -369,46 +139,22 @@ class AirtimeProcessEvent(ProcessEvent): def handle_removed_file(self, dir, pathname): self.logger.info("Deleting %s", pathname) if not dir: - if self.is_audio_file(pathname): + if self.mmc.is_audio_file(pathname): if pathname in self.ignore_event: self.ignore_event.remove(pathname) - elif not self.is_parent_directory(pathname, self.config.organize_directory): + elif not self.mmc.is_parent_directory(pathname, self.config.organize_directory): #we don't care if a file was deleted from the organize directory. self.file_events.append({'filepath': pathname, 'mode': self.config.MODE_DELETE}) def process_default(self, event): - #self.logger.info("PROCESS_DEFAULT: %s", event) pass - def execCommandAndReturnStdOut(self, command): - p = Popen(command, shell=True, stdout=PIPE) - stdout = p.communicate()[0] - if p.returncode != 0: - self.logger.warn("command \n%s\n return with a non-zero return value", command) - return stdout - - def write_file(self, file, string): - f = open(file, 'w') - f.write(string) - f.close() - - def scan_dir_for_new_files(self, dir): - command = 'find "%s" -type f -iname "*.ogg" -o -iname "*.mp3" -readable' % dir.replace('"', '\\"') - self.logger.debug(command) - stdout = self.execCommandAndReturnStdOut(command) - stdout = unicode(stdout, "utf_8") - - return stdout.splitlines() - - def touch_index_file(self): - open(self.timestamp_file, "w") - def notifier_loop_callback(self, notifier): if len(self.file_events) > 0: for event in self.file_events: self.multi_queue.put(event) - self.touch_index_file() + self.mmc.touch_index_file() self.file_events = [] diff --git a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py new file mode 100644 index 000000000..b6bd61585 --- /dev/null +++ b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py @@ -0,0 +1,255 @@ +import os +import grp +import pwd +import logging + +import pyinotify + +from subprocess import Popen, PIPE +from airtimemetadata import AirtimeMetadata + +class MediaMonitorCommon: + + timestamp_file = "/var/tmp/airtime/last_index" + + def __init__(self, airtime_config, watch_manager): + self.supported_file_formats = ['mp3', 'ogg'] + self.mask = pyinotify.ALL_EVENTS + self.logger = logging.getLogger() + self.config = airtime_config + self.md_manager = AirtimeMetadata() + self.wm = watch_manager + + + #define which directories the pyinotify WatchManager should watch. + def watch_directory(self, directory): + return self.wm.add_watch(directory, self.mask, rec=True, auto_add=True) + + def is_parent_directory(self, filepath, directory): + filepath = os.path.normpath(filepath) + directory = os.path.normpath(directory) + return (directory == filepath[0:len(directory)]) + + """ + def is_temp_file(self, filename): + info = filename.split(".") + + if(info[-2] in self.supported_file_formats): + return True + else: + return False + """ + + def is_audio_file(self, filename): + info = filename.split(".") + + if(info[-1] in self.supported_file_formats): + return True + else: + return False + + #check if file is readable by "nobody" + def has_correct_permissions(self, filepath): + #drop root permissions and become "nobody" + os.seteuid(65534) + + try: + open(filepath) + readable = True + except IOError: + self.logger.warn("File does not have correct permissions: '%s'", filepath) + readable = False + except Exception, e: + self.logger.error("Unexpected exception thrown: %s", e) + readable = False + finally: + #reset effective user to root + os.seteuid(0) + + return readable + + def set_needed_file_permissions(self, item, is_dir): + try: + omask = os.umask(0) + + uid = pwd.getpwnam('www-data')[2] + gid = grp.getgrnam('www-data')[2] + + os.chown(item, uid, gid) + + if is_dir is True: + os.chmod(item, 02777) + else: + os.chmod(item, 0666) + + except Exception, e: + self.logger.error("Failed to change file's owner/group/permissions. %s", e) + finally: + os.umask(omask) + + + #checks if path is a directory, and if it doesnt exist, then creates it. + #Otherwise prints error to log file. + def ensure_is_dir(self, directory): + try: + omask = os.umask(0) + if not os.path.exists(directory): + os.makedirs(directory, 02777) + #self.watch_directory(directory) + elif not os.path.isdir(directory): + #path exists but it is a file not a directory! + self.logger.error("path %s exists, but it is not a directory!!!") + finally: + os.umask(omask) + + #moves file from source to dest but also recursively removes the + #the source file's parent directories if they are now empty. + def move_file(self, source, dest): + + try: + omask = os.umask(0) + os.rename(source, dest) + except Exception, e: + self.logger.error("failed to move file. %s", e) + finally: + os.umask(omask) + + dir = os.path.dirname(source) + self.cleanup_empty_dirs(dir) + + #keep moving up the file hierarchy and deleting parent + #directories until we hit a non-empty directory, or we + #hit the organize dir. + def cleanup_empty_dirs(self, dir): + if os.path.normpath(dir) != self.config.organize_directory: + if len(os.listdir(dir)) == 0: + os.rmdir(dir) + + pdir = os.path.dirname(dir) + self.cleanup_empty_dirs(pdir) + + + #checks if path exists already in stor. If the path exists and the md5s are the + #same just overwrite. + def create_unique_filename(self, filepath, old_filepath): + + try: + if(os.path.exists(filepath)): + self.logger.info("Path %s exists", filepath) + + self.logger.info("Checking if md5s are the same.") + md5_fp = self.md_manager.get_md5(filepath) + md5_ofp = self.md_manager.get_md5(old_filepath) + + if(md5_fp == md5_ofp): + self.logger.info("Md5s are the same, moving to same filepath.") + return filepath + + self.logger.info("Md5s aren't the same, appending to filepath.") + file_dir = os.path.dirname(filepath) + filename = os.path.basename(filepath).split(".")[0] + #will be in the format .ext + file_ext = os.path.splitext(filepath)[1] + i = 1; + while(True): + new_filepath = '%s/%s(%s)%s' % (file_dir, filename, i, file_ext) + self.logger.error("Trying %s", new_filepath) + + if(os.path.exists(new_filepath)): + i = i+1; + else: + filepath = new_filepath + break + + except Exception, e: + self.logger.error("Exception %s", e) + + return filepath + + #create path in /srv/airtime/stor/imported/[song-metadata] + def create_file_path(self, original_path, orig_md): + + storage_directory = self.config.storage_directory + + is_recorded_show = False + + try: + #will be in the format .ext + file_ext = os.path.splitext(original_path)[1] + file_ext = file_ext.encode('utf-8') + + path_md = ['MDATA_KEY_TITLE', 'MDATA_KEY_CREATOR', 'MDATA_KEY_SOURCE', 'MDATA_KEY_TRACKNUMBER', 'MDATA_KEY_BITRATE'] + + md = {} + for m in path_md: + if m not in orig_md: + md[m] = u'unknown'.encode('utf-8') + else: + #get rid of any "/" which will interfere with the filepath. + if isinstance(orig_md[m], basestring): + md[m] = orig_md[m].replace("/", "-") + else: + md[m] = orig_md[m] + + if 'MDATA_KEY_TRACKNUMBER' in orig_md: + #make sure all track numbers are at least 2 digits long in the filepath. + md['MDATA_KEY_TRACKNUMBER'] = "%02d" % (int(md['MDATA_KEY_TRACKNUMBER'])) + + #format bitrate as 128kbps + md['MDATA_KEY_BITRATE'] = str(md['MDATA_KEY_BITRATE']/1000)+"kbps" + + filepath = None + #file is recorded by Airtime + #/srv/airtime/stor/recorded/year/month/year-month-day-time-showname-bitrate.ext + if(md['MDATA_KEY_CREATOR'] == "AIRTIMERECORDERSOURCEFABRIC".encode('utf-8')): + #yyyy-mm-dd-hh-MM-ss + y = orig_md['MDATA_KEY_YEAR'].split("-") + filepath = '%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "recorded".encode('utf-8'), y[0], y[1], orig_md['MDATA_KEY_YEAR'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) + elif(md['MDATA_KEY_TRACKNUMBER'] == u'unknown'.encode('utf-8')): + filepath = '%s/%s/%s/%s/%s-%s%s' % (storage_directory, "imported".encode('utf-8'), md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) + else: + filepath = '%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "imported".encode('utf-8'), md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TRACKNUMBER'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) + + filepath = self.create_unique_filename(filepath, original_path) + self.logger.info('Unique filepath: %s', filepath) + self.ensure_is_dir(os.path.dirname(filepath)) + + except Exception, e: + self.logger.error('Exception: %s', e) + + return filepath + + def execCommandAndReturnStdOut(self, command): + p = Popen(command, shell=True, stdout=PIPE) + stdout = p.communicate()[0] + if p.returncode != 0: + self.logger.warn("command \n%s\n return with a non-zero return value", command) + return stdout + + def scan_dir_for_new_files(self, dir): + command = 'find "%s" -type f -iname "*.ogg" -o -iname "*.mp3" -readable' % dir.replace('"', '\\"') + self.logger.debug(command) + stdout = self.execCommandAndReturnStdOut(command) + stdout = unicode(stdout, "utf_8") + + return stdout.splitlines() + + def touch_index_file(self): + open(self.timestamp_file, "w") + + def organize_new_file(self, pathname): + self.logger.info(u"Organizing new file: %s", pathname) + file_md = self.md_manager.get_md_from_file(pathname) + + if file_md is not None: + #is_recorded_show = 'MDATA_KEY_CREATOR' in file_md and \ + # file_md['MDATA_KEY_CREATOR'] == "AIRTIMERECORDERSOURCEFABRIC".encode('utf-8') + filepath = self.create_file_path(pathname, file_md) + + self.logger.debug(u"Moving from %s to %s", pathname, filepath) + self.move_file(pathname, filepath) + else: + filepath = None + self.logger.warn("File %s, has invalid metadata", pathname) + + return filepath \ No newline at end of file