import os import grp import pwd import logging import stat import subprocess import traceback from subprocess import Popen, PIPE from airtimemetadata import AirtimeMetadata from api_clients import api_client import pyinotify class MediaMonitorCommon: timestamp_file = "/var/tmp/airtime/media-monitor/last_index" def __init__(self, airtime_config, wm=None): self.supported_file_formats = ['mp3', 'ogg'] self.logger = logging.getLogger() self.config = airtime_config self.md_manager = AirtimeMetadata() self.wm = wm def is_parent_directory(self, filepath, directory): filepath = os.path.normpath(filepath) directory = os.path.normpath(directory) return (directory == filepath[0:len(directory)]) def is_temp_file(self, filename): info = filename.split(".") # if file doesn't have any extension, info[-2] throws exception # Hence, checking length of info before we do anything if(len(info) >= 2): if(info[-2] in self.supported_file_formats): return True else: return False else: return False def is_audio_file(self, filename): info = filename.split(".") if(info[-1] in self.supported_file_formats): return True else: return False #check if file is readable by "nobody" def is_user_readable(self, filepath, euid='nobody', egid='nogroup'): try: uid = pwd.getpwnam(euid)[2] gid = grp.getgrnam(egid)[2] #drop root permissions and become "nobody" os.setegid(gid) os.seteuid(uid) open(filepath) readable = True except IOError: self.logger.warn("File does not have correct permissions: '%s'", filepath) readable = False except Exception, e: self.logger.error("Unexpected exception thrown: %s", e) readable = False self.logger.error("traceback: %s", traceback.format_exc()) finally: #reset effective user to root os.seteuid(0) os.setegid(0) return readable # the function only changes the permission if its not readable by www-data def is_readable(self, item, is_dir): try: return self.is_user_readable(item, 'www-data', 'www-data') \ and self.is_user_readable(item, 'pypo', 'pypo') except Exception, e: self.logger.warn(u"Failed to check owner/group/permissions for %s", item) return False def make_file_readable(self, pathname, is_dir): if is_dir: #set to 755 os.chmod(pathname, stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR | stat.S_IRGRP|stat.S_IXGRP | stat.S_IROTH|stat.S_IXOTH) else: #set to 644 os.chmod(pathname, stat.S_IRUSR|stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) def make_readable(self, pathname): """ Should only call this function if is_readable() returns False. This function will attempt to make the file world readable by modifying the file's permission's as well as the file's parent directory permissions. We should only call this function on files in Airtime's stor directory, not watched directories! Returns True if we were able to make the file world readable. False otherwise. """ original_file = pathname is_dir = False try: while not self.is_readable(original_file, is_dir): #Not readable. Make appropriate permission changes. self.make_file_readable(pathname, is_dir) dirname = os.path.dirname(pathname) if dirname == pathname: #most likey reason for this is that we've hit '/'. Avoid infinite loop by terminating loop raise Exception() else: pathname = dirname is_dir = True except Exception, e: #something went wrong while we were trying to make world readable. return False return True #checks if path is a directory, and if it doesnt exist, then creates it. #Otherwise prints error to log file. def ensure_is_dir(self, directory): try: omask = os.umask(0) if not os.path.exists(directory): os.makedirs(directory, 02777) self.wm.add_watch(directory, pyinotify.ALL_EVENTS, rec=True, auto_add=True) elif not os.path.isdir(directory): #path exists but it is a file not a directory! self.logger.error(u"path %s exists, but it is not a directory!!!", directory) finally: os.umask(omask) #moves file from source to dest but also recursively removes the #the source file's parent directories if they are now empty. def move_file(self, source, dest): try: omask = os.umask(0) os.rename(source, dest) except Exception, e: self.logger.error("failed to move file. %s", e) self.logger.error("traceback: %s", traceback.format_exc()) finally: os.umask(omask) dir = os.path.dirname(source) self.cleanup_empty_dirs(dir) #keep moving up the file hierarchy and deleting parent #directories until we hit a non-empty directory, or we #hit the organize dir. def cleanup_empty_dirs(self, dir): if os.path.normpath(dir) != self.config.organize_directory: if len(os.listdir(dir)) == 0: try: os.rmdir(dir) self.cleanup_empty_dirs(os.path.dirname(dir)) except Exception, e: #non-critical exception because we probably tried to delete a non-empty dir. #Don't need to log this, let's just "return" pass #checks if path exists already in stor. If the path exists and the md5s are the #same just overwrite. def create_unique_filename(self, filepath, old_filepath): try: if(os.path.exists(filepath)): self.logger.info("Path %s exists", filepath) self.logger.info("Checking if md5s are the same.") md5_fp = self.md_manager.get_md5(filepath) md5_ofp = self.md_manager.get_md5(old_filepath) if(md5_fp == md5_ofp): self.logger.info("Md5s are the same, moving to same filepath.") return filepath self.logger.info("Md5s aren't the same, appending to filepath.") file_dir = os.path.dirname(filepath) filename = os.path.basename(filepath).split(".")[0] #will be in the format .ext file_ext = os.path.splitext(filepath)[1] i = 1; while(True): new_filepath = '%s/%s(%s)%s' % (file_dir, filename, i, file_ext) self.logger.error("Trying %s", new_filepath) if(os.path.exists(new_filepath)): i = i+1; else: filepath = new_filepath break except Exception, e: self.logger.error("Exception %s", e) return filepath #create path in /srv/airtime/stor/imported/[song-metadata] def create_file_path(self, original_path, orig_md): storage_directory = self.config.storage_directory try: #will be in the format .ext file_ext = os.path.splitext(original_path)[1] path_md = ['MDATA_KEY_TITLE', 'MDATA_KEY_CREATOR', 'MDATA_KEY_SOURCE', 'MDATA_KEY_TRACKNUMBER', 'MDATA_KEY_BITRATE'] md = {} for m in path_md: if m not in orig_md: md[m] = u'unknown' else: #get rid of any "/" which will interfere with the filepath. if isinstance(orig_md[m], basestring): md[m] = orig_md[m].replace("/", "-") else: md[m] = orig_md[m] if 'MDATA_KEY_TRACKNUMBER' in orig_md: #make sure all track numbers are at least 2 digits long in the filepath. md['MDATA_KEY_TRACKNUMBER'] = "%02d" % (int(md['MDATA_KEY_TRACKNUMBER'])) #format bitrate as 128kbps md['MDATA_KEY_BITRATE'] = str(md['MDATA_KEY_BITRATE']/1000)+"kbps" filepath = None #file is recorded by Airtime #/srv/airtime/stor/recorded/year/month/year-month-day-time-showname-bitrate.ext if(md['MDATA_KEY_CREATOR'] == u"Airtime Show Recorder"): #yyyy-mm-dd-hh-MM-ss y = orig_md['MDATA_KEY_RECORD_DATE'].split("-") filepath = u'%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "recorded", y[0], y[1], orig_md['MDATA_KEY_RECORD_DATE'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) #"Show-Title-2011-03-28-17:15:00" title = md['MDATA_KEY_TITLE'].split("-") show_hour = title[0] show_min = title[1] show_sec = title[2] show_name = '-'.join(title[3:]) new_md = {} new_md["MDATA_KEY_FILEPATH"] = original_path new_md['MDATA_KEY_TITLE'] = '%s-%s-%s:%s:%s' % (show_name, orig_md['MDATA_KEY_RECORD_DATE'], show_hour, show_min, show_sec) self.md_manager.save_md_to_file(new_md) elif(md['MDATA_KEY_TRACKNUMBER'] == u'unknown'): filepath = u'%s/%s/%s/%s/%s-%s%s' % (storage_directory, "imported", md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) else: filepath = u'%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "imported", md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TRACKNUMBER'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext) filepath = self.create_unique_filename(filepath, original_path) self.logger.info('Unique filepath: %s', filepath) self.ensure_is_dir(os.path.dirname(filepath)) except Exception, e: self.logger.error('Exception: %s', e) self.logger.error("traceback: %s", traceback.format_exc()) return filepath def exec_command(self, command): p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate() if p.returncode != 0: self.logger.warn("command \n%s\n return with a non-zero return value", command) self.logger.error(stderr) try: """ File name charset encoding is UTF-8. """ stdout = stdout.decode("UTF-8") except Exception, e: stdout = None self.logger.error("Could not decode %s using UTF-8" % stdout) return stdout def scan_dir_for_new_files(self, dir): command = 'find "%s" -iname "*.ogg" -o -iname "*.mp3" -type f -readable' % dir.replace('"', '\\"') self.logger.debug(command) stdout = self.exec_command(command) return stdout.splitlines() def touch_index_file(self): dirname = os.path.dirname(self.timestamp_file) try: if not os.path.exists(dirname): os.makedirs(dirname) open(self.timestamp_file, "w") except Exception, e: self.logger.error('Exception: %s', e) self.logger.error("traceback: %s", traceback.format_exc()) def organize_new_file(self, pathname, file_md): self.logger.info("Organizing new file: %s", pathname) filepath = self.create_file_path(pathname, file_md) self.logger.debug(u"Moving from %s to %s", pathname, filepath) self.move_file(pathname, filepath) self.make_readable(filepath) return filepath def test_file_playability(self, pathname): #when there is an single apostrophe inside of a string quoted by apostrophes, we can only escape it by replace that apostrophe #with '\''. This breaks the string into two, and inserts an escaped single quote in between them. #We run the command as pypo because otherwise the target file is opened with write permissions, and this causes an inotify ON_CLOSE_WRITE event #to be fired :/ command = "sudo -u pypo airtime-liquidsoap -c 'output.dummy(audio_to_stereo(single(\"%s\")))' > /dev/null 2>&1" % pathname.replace("'", "'\\''") return_code = subprocess.call(command, shell=True) if return_code != 0: #print pathname for py-interpreter.log print pathname return (return_code == 0) def move_to_problem_dir(self, source): dest = os.path.join(self.config.problem_directory, os.path.basename(source)) try: omask = os.umask(0) os.rename(source, dest) except Exception, e: self.logger.error("failed to move file. %s", e) self.logger.error("traceback: %s", traceback.format_exc()) finally: os.umask(omask)