340 lines
13 KiB
Python
340 lines
13 KiB
Python
import os
|
|
import grp
|
|
import pwd
|
|
import logging
|
|
import stat
|
|
import subprocess
|
|
import traceback
|
|
|
|
from subprocess import Popen, PIPE
|
|
from airtimemetadata import AirtimeMetadata
|
|
from api_clients import api_client
|
|
import pyinotify
|
|
|
|
class MediaMonitorCommon:
|
|
|
|
timestamp_file = "/var/tmp/airtime/media-monitor/last_index"
|
|
|
|
def __init__(self, airtime_config, wm=None):
|
|
self.supported_file_formats = ['mp3', 'ogg']
|
|
self.logger = logging.getLogger()
|
|
self.config = airtime_config
|
|
self.md_manager = AirtimeMetadata()
|
|
self.wm = wm
|
|
|
|
def is_parent_directory(self, filepath, directory):
|
|
filepath = os.path.normpath(filepath)
|
|
directory = os.path.normpath(directory)
|
|
return (directory == filepath[0:len(directory)])
|
|
|
|
def is_temp_file(self, filename):
|
|
info = filename.split(".")
|
|
|
|
# if file doesn't have any extension, info[-2] throws exception
|
|
# Hence, checking length of info before we do anything
|
|
if(len(info) >= 2):
|
|
if(info[-2] in self.supported_file_formats):
|
|
return True
|
|
else:
|
|
return False
|
|
else:
|
|
return False
|
|
|
|
def is_audio_file(self, filename):
|
|
info = filename.split(".")
|
|
|
|
if(info[-1] in self.supported_file_formats):
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
#check if file is readable by "nobody"
|
|
def is_user_readable(self, filepath, euid='nobody', egid='nogroup'):
|
|
|
|
try:
|
|
uid = pwd.getpwnam(euid)[2]
|
|
gid = grp.getgrnam(egid)[2]
|
|
|
|
#drop root permissions and become "nobody"
|
|
os.setegid(gid)
|
|
os.seteuid(uid)
|
|
|
|
open(filepath)
|
|
readable = True
|
|
except IOError:
|
|
self.logger.warn("File does not have correct permissions: '%s'", filepath)
|
|
readable = False
|
|
except Exception, e:
|
|
self.logger.error("Unexpected exception thrown: %s", e)
|
|
readable = False
|
|
self.logger.error("traceback: %s", traceback.format_exc())
|
|
finally:
|
|
#reset effective user to root
|
|
os.seteuid(0)
|
|
os.setegid(0)
|
|
|
|
return readable
|
|
|
|
# the function only changes the permission if its not readable by www-data
|
|
def is_readable(self, item, is_dir):
|
|
try:
|
|
return self.is_user_readable(item, 'www-data', 'www-data') \
|
|
and self.is_user_readable(item, 'pypo', 'pypo')
|
|
except Exception, e:
|
|
self.logger.warn(u"Failed to check owner/group/permissions for %s", item)
|
|
return False
|
|
|
|
def make_file_readable(self, pathname, is_dir):
|
|
if is_dir:
|
|
#set to 755
|
|
os.chmod(pathname, stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR | stat.S_IRGRP|stat.S_IXGRP | stat.S_IROTH|stat.S_IXOTH)
|
|
else:
|
|
#set to 644
|
|
os.chmod(pathname, stat.S_IRUSR|stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
|
|
|
|
def make_readable(self, pathname):
|
|
"""
|
|
Should only call this function if is_readable() returns False. This function
|
|
will attempt to make the file world readable by modifying the file's permission's
|
|
as well as the file's parent directory permissions. We should only call this function
|
|
on files in Airtime's stor directory, not watched directories!
|
|
|
|
Returns True if we were able to make the file world readable. False otherwise.
|
|
"""
|
|
original_file = pathname
|
|
is_dir = False
|
|
try:
|
|
while not self.is_readable(original_file, is_dir):
|
|
#Not readable. Make appropriate permission changes.
|
|
self.make_file_readable(pathname, is_dir)
|
|
|
|
dirname = os.path.dirname(pathname)
|
|
if dirname == pathname:
|
|
#most likey reason for this is that we've hit '/'. Avoid infinite loop by terminating loop
|
|
raise Exception()
|
|
else:
|
|
pathname = dirname
|
|
is_dir = True
|
|
except Exception, e:
|
|
#something went wrong while we were trying to make world readable.
|
|
return False
|
|
|
|
return True
|
|
|
|
#checks if path is a directory, and if it doesnt exist, then creates it.
|
|
#Otherwise prints error to log file.
|
|
def ensure_is_dir(self, directory):
|
|
try:
|
|
omask = os.umask(0)
|
|
if not os.path.exists(directory):
|
|
os.makedirs(directory, 02777)
|
|
self.wm.add_watch(directory, pyinotify.ALL_EVENTS, rec=True, auto_add=True)
|
|
elif not os.path.isdir(directory):
|
|
#path exists but it is a file not a directory!
|
|
self.logger.error(u"path %s exists, but it is not a directory!!!", directory)
|
|
finally:
|
|
os.umask(omask)
|
|
|
|
#moves file from source to dest but also recursively removes the
|
|
#the source file's parent directories if they are now empty.
|
|
def move_file(self, source, dest):
|
|
try:
|
|
omask = os.umask(0)
|
|
os.rename(source, dest)
|
|
except Exception, e:
|
|
self.logger.error("failed to move file. %s", e)
|
|
self.logger.error("traceback: %s", traceback.format_exc())
|
|
finally:
|
|
os.umask(omask)
|
|
|
|
dir = os.path.dirname(source)
|
|
self.cleanup_empty_dirs(dir)
|
|
|
|
#keep moving up the file hierarchy and deleting parent
|
|
#directories until we hit a non-empty directory, or we
|
|
#hit the organize dir.
|
|
def cleanup_empty_dirs(self, dir):
|
|
if os.path.normpath(dir) != self.config.organize_directory:
|
|
if len(os.listdir(dir)) == 0:
|
|
try:
|
|
os.rmdir(dir)
|
|
self.cleanup_empty_dirs(os.path.dirname(dir))
|
|
except Exception, e:
|
|
#non-critical exception because we probably tried to delete a non-empty dir.
|
|
#Don't need to log this, let's just "return"
|
|
pass
|
|
|
|
|
|
|
|
#checks if path exists already in stor. If the path exists and the md5s are the
|
|
#same just overwrite.
|
|
def create_unique_filename(self, filepath, old_filepath):
|
|
|
|
try:
|
|
if(os.path.exists(filepath)):
|
|
self.logger.info("Path %s exists", filepath)
|
|
|
|
self.logger.info("Checking if md5s are the same.")
|
|
md5_fp = self.md_manager.get_md5(filepath)
|
|
md5_ofp = self.md_manager.get_md5(old_filepath)
|
|
|
|
if(md5_fp == md5_ofp):
|
|
self.logger.info("Md5s are the same, moving to same filepath.")
|
|
return filepath
|
|
|
|
self.logger.info("Md5s aren't the same, appending to filepath.")
|
|
file_dir = os.path.dirname(filepath)
|
|
filename = os.path.basename(filepath).split(".")[0]
|
|
#will be in the format .ext
|
|
file_ext = os.path.splitext(filepath)[1]
|
|
i = 1;
|
|
while(True):
|
|
new_filepath = '%s/%s(%s)%s' % (file_dir, filename, i, file_ext)
|
|
self.logger.error("Trying %s", new_filepath)
|
|
|
|
if(os.path.exists(new_filepath)):
|
|
i = i+1;
|
|
else:
|
|
filepath = new_filepath
|
|
break
|
|
|
|
except Exception, e:
|
|
self.logger.error("Exception %s", e)
|
|
|
|
return filepath
|
|
|
|
#create path in /srv/airtime/stor/imported/[song-metadata]
|
|
def create_file_path(self, original_path, orig_md):
|
|
|
|
storage_directory = self.config.storage_directory
|
|
|
|
try:
|
|
#will be in the format .ext
|
|
file_ext = os.path.splitext(original_path)[1]
|
|
path_md = ['MDATA_KEY_TITLE', 'MDATA_KEY_CREATOR', 'MDATA_KEY_SOURCE', 'MDATA_KEY_TRACKNUMBER', 'MDATA_KEY_BITRATE']
|
|
|
|
md = {}
|
|
for m in path_md:
|
|
if m not in orig_md:
|
|
md[m] = u'unknown'
|
|
else:
|
|
#get rid of any "/" which will interfere with the filepath.
|
|
if isinstance(orig_md[m], basestring):
|
|
md[m] = orig_md[m].replace("/", "-")
|
|
else:
|
|
md[m] = orig_md[m]
|
|
|
|
if 'MDATA_KEY_TRACKNUMBER' in orig_md:
|
|
#make sure all track numbers are at least 2 digits long in the filepath.
|
|
md['MDATA_KEY_TRACKNUMBER'] = "%02d" % (int(md['MDATA_KEY_TRACKNUMBER']))
|
|
|
|
#format bitrate as 128kbps
|
|
md['MDATA_KEY_BITRATE'] = str(md['MDATA_KEY_BITRATE']/1000)+"kbps"
|
|
|
|
filepath = None
|
|
#file is recorded by Airtime
|
|
#/srv/airtime/stor/recorded/year/month/year-month-day-time-showname-bitrate.ext
|
|
if(md['MDATA_KEY_CREATOR'] == u"Airtime Show Recorder"):
|
|
#yyyy-mm-dd-hh-MM-ss
|
|
y = orig_md['MDATA_KEY_RECORD_DATE'].split("-")
|
|
filepath = u'%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "recorded", y[0], y[1], orig_md['MDATA_KEY_RECORD_DATE'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext)
|
|
|
|
#"Show-Title-2011-03-28-17:15:00"
|
|
title = md['MDATA_KEY_TITLE'].split("-")
|
|
show_hour = title[0]
|
|
show_min = title[1]
|
|
show_sec = title[2]
|
|
show_name = '-'.join(title[3:])
|
|
|
|
new_md = {}
|
|
new_md["MDATA_KEY_FILEPATH"] = original_path
|
|
new_md['MDATA_KEY_TITLE'] = '%s-%s-%s:%s:%s' % (show_name, orig_md['MDATA_KEY_RECORD_DATE'], show_hour, show_min, show_sec)
|
|
self.md_manager.save_md_to_file(new_md)
|
|
|
|
elif(md['MDATA_KEY_TRACKNUMBER'] == u'unknown'):
|
|
filepath = u'%s/%s/%s/%s/%s-%s%s' % (storage_directory, "imported", md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext)
|
|
else:
|
|
filepath = u'%s/%s/%s/%s/%s-%s-%s%s' % (storage_directory, "imported", md['MDATA_KEY_CREATOR'], md['MDATA_KEY_SOURCE'], md['MDATA_KEY_TRACKNUMBER'], md['MDATA_KEY_TITLE'], md['MDATA_KEY_BITRATE'], file_ext)
|
|
|
|
filepath = self.create_unique_filename(filepath, original_path)
|
|
self.logger.info('Unique filepath: %s', filepath)
|
|
self.ensure_is_dir(os.path.dirname(filepath))
|
|
|
|
except Exception, e:
|
|
self.logger.error('Exception: %s', e)
|
|
self.logger.error("traceback: %s", traceback.format_exc())
|
|
|
|
return filepath
|
|
|
|
def exec_command(self, command):
|
|
p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)
|
|
stdout, stderr = p.communicate()
|
|
if p.returncode != 0:
|
|
self.logger.warn("command \n%s\n return with a non-zero return value", command)
|
|
self.logger.error(stderr)
|
|
|
|
try:
|
|
"""
|
|
File name charset encoding is UTF-8.
|
|
"""
|
|
stdout = stdout.decode("UTF-8")
|
|
except Exception, e:
|
|
stdout = None
|
|
self.logger.error("Could not decode %s using UTF-8" % stdout)
|
|
|
|
return stdout
|
|
|
|
def scan_dir_for_new_files(self, dir):
|
|
command = 'find "%s" -iname "*.ogg" -o -iname "*.mp3" -type f -readable' % dir.replace('"', '\\"')
|
|
self.logger.debug(command)
|
|
stdout = self.exec_command(command)
|
|
|
|
return stdout.splitlines()
|
|
|
|
def touch_index_file(self):
|
|
dirname = os.path.dirname(self.timestamp_file)
|
|
try:
|
|
if not os.path.exists(dirname):
|
|
os.makedirs(dirname)
|
|
open(self.timestamp_file, "w")
|
|
except Exception, e:
|
|
self.logger.error('Exception: %s', e)
|
|
self.logger.error("traceback: %s", traceback.format_exc())
|
|
|
|
def organize_new_file(self, pathname, file_md):
|
|
self.logger.info("Organizing new file: %s", pathname)
|
|
|
|
filepath = self.create_file_path(pathname, file_md)
|
|
|
|
self.logger.debug(u"Moving from %s to %s", pathname, filepath)
|
|
self.move_file(pathname, filepath)
|
|
self.make_readable(filepath)
|
|
return filepath
|
|
|
|
def test_file_playability(self, pathname):
|
|
#when there is an single apostrophe inside of a string quoted by apostrophes, we can only escape it by replace that apostrophe
|
|
#with '\''. This breaks the string into two, and inserts an escaped single quote in between them.
|
|
#We run the command as pypo because otherwise the target file is opened with write permissions, and this causes an inotify ON_CLOSE_WRITE event
|
|
#to be fired :/
|
|
command = "sudo -u pypo airtime-liquidsoap -c 'output.dummy(audio_to_stereo(single(\"%s\")))' > /dev/null 2>&1" % pathname.replace("'", "'\\''")
|
|
return_code = subprocess.call(command, shell=True)
|
|
if return_code != 0:
|
|
#print pathname for py-interpreter.log
|
|
print pathname
|
|
|
|
return (return_code == 0)
|
|
|
|
def move_to_problem_dir(self, source):
|
|
|
|
dest = os.path.join(self.config.problem_directory, os.path.basename(source))
|
|
|
|
try:
|
|
omask = os.umask(0)
|
|
os.rename(source, dest)
|
|
except Exception, e:
|
|
self.logger.error("failed to move file. %s", e)
|
|
self.logger.error("traceback: %s", traceback.format_exc())
|
|
finally:
|
|
os.umask(omask)
|
|
|