From bd183125da21d3f481ca3bd93628ca57aa773dd5 Mon Sep 17 00:00:00 2001 From: martin Date: Mon, 18 Jul 2011 17:18:21 -0400 Subject: [PATCH] CC-2279: Upgrade script for converting stor directory to new format -move media-monitor python upgrade script to upgrade dir --- .../airtimefilemonitor/mediaconfig.py | 24 ++ .../airtimefilemonitor/mediamonitorcommon.py | 243 ++++++++++++++++++ install/upgrades/airtime-1.9.0/logging.cfg | 22 ++ .../airtime-1.9.0}/media-monitor-upgrade.py | 0 4 files changed, 289 insertions(+) create mode 100644 install/upgrades/airtime-1.9.0/airtimefilemonitor/mediaconfig.py create mode 100644 install/upgrades/airtime-1.9.0/airtimefilemonitor/mediamonitorcommon.py create mode 100644 install/upgrades/airtime-1.9.0/logging.cfg rename {python_apps/media-monitor => install/upgrades/airtime-1.9.0}/media-monitor-upgrade.py (100%) diff --git a/install/upgrades/airtime-1.9.0/airtimefilemonitor/mediaconfig.py b/install/upgrades/airtime-1.9.0/airtimefilemonitor/mediaconfig.py new file mode 100644 index 000000000..2b1197e24 --- /dev/null +++ b/install/upgrades/airtime-1.9.0/airtimefilemonitor/mediaconfig.py @@ -0,0 +1,24 @@ +import sys + +from configobj import ConfigObj + +class AirtimeMediaConfig: + + MODE_CREATE = "create" + MODE_MODIFY = "modify" + MODE_MOVED = "moved" + MODE_DELETE = "delete" + + def __init__(self, logger): + + # loading config file + try: + config = ConfigObj('/etc/airtime/media-monitor.cfg') + self.cfg = config + except Exception, e: + logger.info('Error loading config: ', e) + sys.exit() + + self.storage_directory = None + + diff --git a/install/upgrades/airtime-1.9.0/airtimefilemonitor/mediamonitorcommon.py b/install/upgrades/airtime-1.9.0/airtimefilemonitor/mediamonitorcommon.py new file mode 100644 index 000000000..d3125c5b5 --- /dev/null +++ b/install/upgrades/airtime-1.9.0/airtimefilemonitor/mediamonitorcommon.py @@ -0,0 +1,243 @@ +import os +import grp +import pwd +import logging + +from subprocess import Popen, PIPE +from airtimemetadata import AirtimeMetadata + +class MediaMonitorCommon: + + timestamp_file = "/var/tmp/airtime/last_index" + + def __init__(self, airtime_config): + self.supported_file_formats = ['mp3', 'ogg'] + self.logger = logging.getLogger() + self.config = airtime_config + self.md_manager = AirtimeMetadata() + + def is_parent_directory(self, filepath, directory): + filepath = os.path.normpath(filepath) + directory = os.path.normpath(directory) + return (directory == filepath[0:len(directory)]) + + def is_temp_file(self, filename): + info = filename.split(".") + + if(info[-2] in self.supported_file_formats): + return True + else: + return False + + def is_audio_file(self, filename): + info = filename.split(".") + + if(info[-1] in self.supported_file_formats): + return True + else: + return False + + #check if file is readable by "nobody" + def has_correct_permissions(self, filepath): + #drop root permissions and become "nobody" + os.seteuid(65534) + + try: + open(filepath) + readable = True + except IOError: + self.logger.warn("File does not have correct permissions: '%s'", filepath) + readable = False + except Exception, e: + self.logger.error("Unexpected exception thrown: %s", e) + readable = False + finally: + #reset effective user to root + os.seteuid(0) + + return readable + + def set_needed_file_permissions(self, item, is_dir): + try: + omask = os.umask(0) + + uid = pwd.getpwnam('www-data')[2] + gid = grp.getgrnam('www-data')[2] + + os.chown(item, uid, gid) + + if is_dir is True: + os.chmod(item, 02777) + else: + os.chmod(item, 0666) + + except Exception, e: + self.logger.error("Failed to change file's owner/group/permissions. %s", e) + finally: + os.umask(omask) + + + #checks if path is a directory, and if it doesnt exist, then creates it. + #Otherwise prints error to log file. + def ensure_is_dir(self, directory): + try: + omask = os.umask(0) + if not os.path.exists(directory): + os.makedirs(directory, 02777) + elif not os.path.isdir(directory): + #path exists but it is a file not a directory! + self.logger.error("path %s exists, but it is not a directory!!!") + finally: + os.umask(omask) + + #moves file from source to dest but also recursively removes the + #the source file's parent directories if they are now empty. + def move_file(self, source, dest): + + try: + omask = os.umask(0) + os.rename(source, dest) + except Exception, e: + self.logger.error("failed to move file. %s", e) + finally: + os.umask(omask) + + dir = os.path.dirname(source) + self.cleanup_empty_dirs(dir) + + #keep moving up the file hierarchy and deleting parent + #directories until we hit a non-empty directory, or we + #hit the organize dir. + def cleanup_empty_dirs(self, dir): + if os.path.normpath(dir) != self.config.organize_directory: + if len(os.listdir(dir)) == 0: + os.rmdir(dir) + + pdir = os.path.dirname(dir) + self.cleanup_empty_dirs(pdir) + + + #checks if path exists already in stor. If the path exists and the md5s are the + #same just overwrite. + def create_unique_filename(self, filepath, old_filepath): + + try: + if(os.path.exists(filepath)): + self.logger.info("Path %s exists", filepath) + + self.logger.info("Checking if md5s are the same.") + md5_fp = self.md_manager.get_md5(filepath) + md5_ofp = self.md_manager.get_md5(old_filepath) + + if(md5_fp == md5_ofp): + self.logger.info("Md5s are the same, moving to same filepath.") + return filepath + + self.logger.info("Md5s aren't the same, appending to filepath.") + file_dir = os.path.dirname(filepath) + filename = os.path.basename(filepath).split(".")[0] + #will be in the format .ext + file_ext = os.path.splitext(filepath)[1] + i = 1; + while(True): + new_filepath = '%s/%s(%s)%s' % (file_dir, filename, i, file_ext) + self.logger.error("Trying %s", new_filepath) + + if(os.path.exists(new_filepath)): + i = i+1; + else: + filepath = new_filepath + break + + except Exception, e: + self.logger.error("Exception %s", e) + + return filepath + + #create path in /srv/airtime/stor/imported/[song-metadata] + def create_file_path(self, original_path, orig_md): + + storage_directory = self.config.storage_directory + + is_recorded_show = False + + try: + #will be in the format .ext + file_ext = os.path.splitext(original_path)[1] + file_ext = file_ext.encode('utf-8') + + path_md = ['MDATA_KEY_TITLE', 'MDATA_KEY_CREATOR', 'MDATA_KEY_SOURCE', 'MDATA_KEY_TRACKNUMBER', 'MDATA_KEY_BITRATE'] + + md = {} + for m in path_md: + if m not in orig_md: + md[m] = u'unknown'.encode('utf-8') + else: + #get rid of any "/" which will interfere with the filepath. + if isinstance(orig_md[m], basestring): + md[m] = orig_md[m].replace("/", "-") + else: + md[m] = orig_md[m] + + if 'MDATA_KEY_TRACKNUMBER' in orig_md: + #make sure all track numbers are at least 2 digits long in the filepath. + md['MDATA_KEY_TRACKNUMBER'] = "%02d" % (int(md['MDATA_KEY_TRACKNUMBER'])) + + #format bitrate as 128kbps + md['MDATA_KEY_BITRATE'] = str(md['MDATA_KEY_BITRATE']/1000)+"kbps" + + filepath = None + #file is recorded by Airtime + #/srv/airtime/stor/recorded/year/month/year-month-day-time-showname-bitrate.ext + if(md['MDATA_KEY_CREATOR'] == "AIRTIMERECORDERSOURCEFABRIC".encode('utf-8')): + #yyyy-mm-dd-hh-MM-ss + y = orig_md['MDATA_KEY_YEAR'].split("-") + filepath = '%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "recorded".encode('utf-8'), y[0], y[1], orig_md['MDATA_KEY_YEAR'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) + elif(md['MDATA_KEY_TRACKNUMBER'] == u'unknown'.encode('utf-8')): + filepath = '%s/%s/%s/%s/%s-%s%s' % (storage_directory, "imported".encode('utf-8'), md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) + else: + filepath = '%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "imported".encode('utf-8'), md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TRACKNUMBER'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) + + filepath = self.create_unique_filename(filepath, original_path) + self.logger.info('Unique filepath: %s', filepath) + self.ensure_is_dir(os.path.dirname(filepath)) + + except Exception, e: + self.logger.error('Exception: %s', e) + + return filepath + + def execCommandAndReturnStdOut(self, command): + p = Popen(command, shell=True, stdout=PIPE) + stdout = p.communicate()[0] + if p.returncode != 0: + self.logger.warn("command \n%s\n return with a non-zero return value", command) + return stdout + + def scan_dir_for_new_files(self, dir): + command = 'find "%s" -type f -iname "*.ogg" -o -iname "*.mp3" -readable' % dir.replace('"', '\\"') + self.logger.debug(command) + stdout = self.execCommandAndReturnStdOut(command) + stdout = unicode(stdout, "utf_8") + + return stdout.splitlines() + + def touch_index_file(self): + open(self.timestamp_file, "w") + + def organize_new_file(self, pathname): + self.logger.info("Organizing new file: %s", pathname) + file_md = self.md_manager.get_md_from_file(pathname) + + if file_md is not None: + #is_recorded_show = 'MDATA_KEY_CREATOR' in file_md and \ + # file_md['MDATA_KEY_CREATOR'] == "AIRTIMERECORDERSOURCEFABRIC".encode('utf-8') + filepath = self.create_file_path(pathname, file_md) + + self.logger.debug("Moving from %s to %s", pathname, filepath) + self.move_file(pathname, filepath) + else: + filepath = None + self.logger.warn("File %s, has invalid metadata", pathname) + + return filepath diff --git a/install/upgrades/airtime-1.9.0/logging.cfg b/install/upgrades/airtime-1.9.0/logging.cfg new file mode 100644 index 000000000..c0a43564c --- /dev/null +++ b/install/upgrades/airtime-1.9.0/logging.cfg @@ -0,0 +1,22 @@ +[loggers] +keys=root + +[handlers] +keys=fileOutHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=DEBUG +handlers=fileOutHandler + +[handler_fileOutHandler] +class=logging.handlers.RotatingFileHandler +level=DEBUG +formatter=simpleFormatter +args=("/var/log/airtime/media-monitor/media-monitor.log", 'a', 1000000, 5,) + +[formatter_simpleFormatter] +format=%(asctime)s %(levelname)s - [%(filename)s : %(funcName)s() : line %(lineno)d] - %(message)s +datefmt= diff --git a/python_apps/media-monitor/media-monitor-upgrade.py b/install/upgrades/airtime-1.9.0/media-monitor-upgrade.py similarity index 100% rename from python_apps/media-monitor/media-monitor-upgrade.py rename to install/upgrades/airtime-1.9.0/media-monitor-upgrade.py