Merge branch '2.2.x'
Conflicts: install_minimal/airtime-install python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py
This commit is contained in:
Martin Konecny
commit
6500c3312d
750 changed files with 45683 additions and 140897 deletions
|
|
@ -8,7 +8,7 @@ virtualenv_bin="/usr/lib/airtime/airtime_virtualenv/bin/"
|
|||
media_monitor_path="/usr/lib/airtime/media-monitor/"
|
||||
media_monitor_script="media_monitor.py"
|
||||
|
||||
api_client_path="/usr/lib/airtime/"
|
||||
api_client_path="/usr/lib/airtime/:/usr/lib/airtime/media-monitor/mm2/"
|
||||
|
||||
cd ${media_monitor_path}
|
||||
|
||||
|
|
@ -25,7 +25,6 @@ fi
|
|||
export PYTHONPATH=${api_client_path}
|
||||
export LC_ALL=`cat /etc/default/locale | grep "LANG=" | cut -d= -f2 | tr -d "\n\""`
|
||||
|
||||
# Note the -u when calling python! we need it to get unbuffered binary stdout and stderr
|
||||
exec python -u ${media_monitor_path}${media_monitor_script} > /var/log/airtime/media-monitor/py-interpreter.log 2>&1
|
||||
exec python ${media_monitor_path}${media_monitor_script} > /var/log/airtime/media-monitor/py-interpreter.log 2>&1
|
||||
|
||||
# EOF
|
||||
|
|
|
|||
|
|
@ -5,11 +5,8 @@ import time
|
|||
import pyinotify
|
||||
import shutil
|
||||
|
||||
from subprocess import Popen, PIPE
|
||||
from api_clients import api_client
|
||||
|
||||
class AirtimeMediaMonitorBootstrap():
|
||||
|
||||
|
||||
"""AirtimeMediaMonitorBootstrap constructor
|
||||
|
||||
Keyword Arguments:
|
||||
|
|
@ -29,11 +26,11 @@ class AirtimeMediaMonitorBootstrap():
|
|||
self.curr_mtab_file = "/var/tmp/airtime/media-monitor/currMtab"
|
||||
self.logger.info("Adding %s on watch list...", self.mount_file)
|
||||
self.wm.add_watch(self.mount_file, pyinotify.ALL_EVENTS, rec=False, auto_add=False)
|
||||
|
||||
|
||||
tmp_dir = os.path.dirname(self.curr_mtab_file)
|
||||
if not os.path.exists(tmp_dir):
|
||||
os.makedirs(tmp_dir)
|
||||
|
||||
|
||||
# create currMtab file if it's the first time
|
||||
if not os.path.exists(self.curr_mtab_file):
|
||||
shutil.copy('/etc/mtab', self.curr_mtab_file)
|
||||
|
|
@ -43,8 +40,7 @@ class AirtimeMediaMonitorBootstrap():
|
|||
went offline.
|
||||
"""
|
||||
def scan(self):
|
||||
directories = self.get_list_of_watched_dirs();
|
||||
|
||||
directories = self.get_list_of_watched_dirs()
|
||||
self.logger.info("watched directories found: %s", directories)
|
||||
|
||||
for id, dir in directories.iteritems():
|
||||
|
|
@ -60,12 +56,21 @@ class AirtimeMediaMonitorBootstrap():
|
|||
return self.api_client.list_all_db_files(dir_id)
|
||||
|
||||
"""
|
||||
returns the path and the database row id for this path for all watched directories. Also
|
||||
returns the path and its corresponding database row idfor all watched directories. Also
|
||||
returns the Stor directory, which can be identified by its row id (always has value of "1")
|
||||
|
||||
Return type is a dictionary similar to:
|
||||
{"1":"/srv/airtime/stor/"}
|
||||
"""
|
||||
def get_list_of_watched_dirs(self):
|
||||
json = self.api_client.list_all_watched_dirs()
|
||||
return json["dirs"]
|
||||
|
||||
try:
|
||||
return json["dirs"]
|
||||
except KeyError as e:
|
||||
self.logger.error("Could not find index 'dirs' in dictionary: %s", str(json))
|
||||
self.logger.error(e)
|
||||
return {}
|
||||
|
||||
"""
|
||||
This function takes in a path name provided by the database (and its corresponding row id)
|
||||
|
|
@ -77,10 +82,8 @@ class AirtimeMediaMonitorBootstrap():
|
|||
dir -- pathname of the directory
|
||||
"""
|
||||
def sync_database_to_filesystem(self, dir_id, dir):
|
||||
|
||||
# TODO: is this line even necessary?
|
||||
dir = os.path.normpath(dir)+"/"
|
||||
|
||||
|
||||
"""
|
||||
set to hold new and/or modified files. We use a set to make it ok if files are added
|
||||
twice. This is because some of the tests for new files return result sets that are not
|
||||
|
|
@ -91,41 +94,39 @@ class AirtimeMediaMonitorBootstrap():
|
|||
|
||||
db_known_files_set = set()
|
||||
files = self.list_db_files(dir_id)
|
||||
for file in files['files']:
|
||||
db_known_files_set.add(file)
|
||||
|
||||
for f in files:
|
||||
db_known_files_set.add(f)
|
||||
|
||||
all_files = self.mmc.scan_dir_for_new_files(dir)
|
||||
all_files = self.mmc.clean_dirty_file_paths( self.mmc.scan_dir_for_new_files(dir) )
|
||||
|
||||
all_files_set = set()
|
||||
for file_path in all_files:
|
||||
file_path = file_path.strip(" \n")
|
||||
if len(file_path) > 0 and self.config.problem_directory not in file_path:
|
||||
if self.config.problem_directory not in file_path:
|
||||
all_files_set.add(file_path[len(dir):])
|
||||
|
||||
# if dir doesn't exists, update db
|
||||
if not os.path.exists(dir):
|
||||
self.pe.handle_watched_dir_missing(dir)
|
||||
self.pe.handle_stdout_files(dir)
|
||||
|
||||
if os.path.exists(self.mmc.timestamp_file):
|
||||
"""find files that have been modified since the last time media-monitor process started."""
|
||||
time_diff_sec = time.time() - os.path.getmtime(self.mmc.timestamp_file)
|
||||
command = "find '%s' -iname '*.ogg' -o -iname '*.mp3' -type f -readable -mmin -%d" % (dir, time_diff_sec/60+1)
|
||||
command = self.mmc.find_command(directory=dir, extra_arguments=("-type f -readable -mmin -%d" % (time_diff_sec/60+1)))
|
||||
else:
|
||||
command = "find '%s' -iname '*.ogg' -o -iname '*.mp3' -type f -readable" % dir
|
||||
command = self.mmc.find_command(directory=dir, extra_arguments="-type f -readable")
|
||||
|
||||
self.logger.debug(command)
|
||||
stdout = self.mmc.exec_command(command)
|
||||
|
||||
if stdout is None:
|
||||
self.logger.error("Unrecoverable error when syncing db to filesystem.")
|
||||
return
|
||||
|
||||
new_files = stdout.splitlines()
|
||||
if stdout is None:
|
||||
new_files = []
|
||||
else:
|
||||
new_files = stdout.splitlines()
|
||||
|
||||
new_and_modified_files = set()
|
||||
for file_path in new_files:
|
||||
file_path = file_path.strip(" \n")
|
||||
if len(file_path) > 0 and self.config.problem_directory not in file_path:
|
||||
if self.config.problem_directory not in file_path:
|
||||
new_and_modified_files.add(file_path[len(dir):])
|
||||
|
||||
"""
|
||||
|
|
@ -156,16 +157,12 @@ class AirtimeMediaMonitorBootstrap():
|
|||
self.logger.debug(full_file_path)
|
||||
self.pe.handle_removed_file(False, full_file_path)
|
||||
|
||||
for file_path in new_files_set:
|
||||
self.logger.debug("new file")
|
||||
full_file_path = os.path.join(dir, file_path)
|
||||
self.logger.debug(full_file_path)
|
||||
if os.path.exists(full_file_path):
|
||||
self.pe.handle_created_file(False, full_file_path, os.path.basename(full_file_path))
|
||||
|
||||
for file_path in modified_files_set:
|
||||
self.logger.debug("modified file")
|
||||
full_file_path = "%s%s" % (dir, file_path)
|
||||
self.logger.debug(full_file_path)
|
||||
if os.path.exists(full_file_path):
|
||||
self.pe.handle_modified_file(False, full_file_path, os.path.basename(full_file_path))
|
||||
for file_set, debug_message, handle_attribute in [(new_files_set, "new file", "handle_created_file"),
|
||||
(modified_files_set, "modified file", "handle_modified_file")]:
|
||||
for file_path in file_set:
|
||||
self.logger.debug(debug_message)
|
||||
full_file_path = os.path.join(dir, file_path)
|
||||
self.logger.debug(full_file_path)
|
||||
if os.path.exists(full_file_path):
|
||||
getattr(self.pe,handle_attribute)(False,full_file_path, os.path.basename(full_file_path))
|
||||
|
|
|
|||
|
|
@ -1,65 +1,82 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import replaygain
|
||||
|
||||
import os
|
||||
import hashlib
|
||||
import mutagen
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import traceback
|
||||
from api_clients import api_client
|
||||
|
||||
|
||||
"""
|
||||
list of supported easy tags in mutagen version 1.20
|
||||
['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry', 'date', 'performer', 'musicbrainz_albumartistid', 'composer', 'encodedby', 'tracknumber', 'musicbrainz_albumid', 'album', 'asin', 'musicbrainz_artistid', 'mood', 'copyright', 'author', 'media', 'length', 'version', 'artistsort', 'titlesort', 'discsubtitle', 'website', 'musicip_fingerprint', 'conductor', 'compilation', 'barcode', 'performer:*', 'composersort', 'musicbrainz_discid', 'musicbrainz_albumtype', 'genre', 'isrc', 'discnumber', 'musicbrainz_trmid', 'replaygain_*_gain', 'musicip_puid', 'artist', 'title', 'bpm', 'musicbrainz_trackid', 'arranger', 'albumsort', 'replaygain_*_peak', 'organization']
|
||||
['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry',
|
||||
'date', 'performer', 'musicbrainz_albumartistid', 'composer', 'encodedby',
|
||||
'tracknumber', 'musicbrainz_albumid', 'album', 'asin', 'musicbrainz_artistid',
|
||||
'mood', 'copyright', 'author', 'media', 'length', 'version', 'artistsort',
|
||||
'titlesort', 'discsubtitle', 'website', 'musicip_fingerprint', 'conductor',
|
||||
'compilation', 'barcode', 'performer:*', 'composersort', 'musicbrainz_discid',
|
||||
'musicbrainz_albumtype', 'genre', 'isrc', 'discnumber', 'musicbrainz_trmid',
|
||||
'replaygain_*_gain', 'musicip_puid', 'artist', 'title', 'bpm', 'musicbrainz_trackid',
|
||||
'arranger', 'albumsort', 'replaygain_*_peak', 'organization']
|
||||
"""
|
||||
|
||||
class AirtimeMetadata:
|
||||
|
||||
def __init__(self):
|
||||
|
||||
self.airtime2mutagen = {\
|
||||
"MDATA_KEY_TITLE": "title",\
|
||||
"MDATA_KEY_CREATOR": "artist",\
|
||||
"MDATA_KEY_SOURCE": "album",\
|
||||
"MDATA_KEY_GENRE": "genre",\
|
||||
"MDATA_KEY_MOOD": "mood",\
|
||||
"MDATA_KEY_TRACKNUMBER": "tracknumber",\
|
||||
"MDATA_KEY_BPM": "bpm",\
|
||||
"MDATA_KEY_LABEL": "organization",\
|
||||
"MDATA_KEY_COMPOSER": "composer",\
|
||||
"MDATA_KEY_ENCODER": "encodedby",\
|
||||
"MDATA_KEY_CONDUCTOR": "conductor",\
|
||||
"MDATA_KEY_YEAR": "date",\
|
||||
"MDATA_KEY_URL": "website",\
|
||||
"MDATA_KEY_ISRC": "isrc",\
|
||||
"MDATA_KEY_COPYRIGHT": "copyright",\
|
||||
"MDATA_KEY_TITLE": "title", \
|
||||
"MDATA_KEY_CREATOR": "artist", \
|
||||
"MDATA_KEY_SOURCE": "album", \
|
||||
"MDATA_KEY_GENRE": "genre", \
|
||||
"MDATA_KEY_MOOD": "mood", \
|
||||
"MDATA_KEY_TRACKNUMBER": "tracknumber", \
|
||||
"MDATA_KEY_BPM": "bpm", \
|
||||
"MDATA_KEY_LABEL": "organization", \
|
||||
"MDATA_KEY_COMPOSER": "composer", \
|
||||
"MDATA_KEY_ENCODER": "encodedby", \
|
||||
"MDATA_KEY_CONDUCTOR": "conductor", \
|
||||
"MDATA_KEY_YEAR": "date", \
|
||||
"MDATA_KEY_URL": "website", \
|
||||
"MDATA_KEY_ISRC": "isrc", \
|
||||
"MDATA_KEY_COPYRIGHT": "copyright", \
|
||||
}
|
||||
|
||||
self.mutagen2airtime = {\
|
||||
"title": "MDATA_KEY_TITLE",\
|
||||
"artist": "MDATA_KEY_CREATOR",\
|
||||
"album": "MDATA_KEY_SOURCE",\
|
||||
"genre": "MDATA_KEY_GENRE",\
|
||||
"mood": "MDATA_KEY_MOOD",\
|
||||
"tracknumber": "MDATA_KEY_TRACKNUMBER",\
|
||||
"bpm": "MDATA_KEY_BPM",\
|
||||
"organization": "MDATA_KEY_LABEL",\
|
||||
"composer": "MDATA_KEY_COMPOSER",\
|
||||
"encodedby": "MDATA_KEY_ENCODER",\
|
||||
"conductor": "MDATA_KEY_CONDUCTOR",\
|
||||
"date": "MDATA_KEY_YEAR",\
|
||||
"website": "MDATA_KEY_URL",\
|
||||
"isrc": "MDATA_KEY_ISRC",\
|
||||
"copyright": "MDATA_KEY_COPYRIGHT",\
|
||||
"title": "MDATA_KEY_TITLE", \
|
||||
"artist": "MDATA_KEY_CREATOR", \
|
||||
"album": "MDATA_KEY_SOURCE", \
|
||||
"genre": "MDATA_KEY_GENRE", \
|
||||
"mood": "MDATA_KEY_MOOD", \
|
||||
"tracknumber": "MDATA_KEY_TRACKNUMBER", \
|
||||
"bpm": "MDATA_KEY_BPM", \
|
||||
"organization": "MDATA_KEY_LABEL", \
|
||||
"composer": "MDATA_KEY_COMPOSER", \
|
||||
"encodedby": "MDATA_KEY_ENCODER", \
|
||||
"conductor": "MDATA_KEY_CONDUCTOR", \
|
||||
"date": "MDATA_KEY_YEAR", \
|
||||
"website": "MDATA_KEY_URL", \
|
||||
"isrc": "MDATA_KEY_ISRC", \
|
||||
"copyright": "MDATA_KEY_COPYRIGHT", \
|
||||
}
|
||||
|
||||
self.logger = logging.getLogger()
|
||||
|
||||
def get_md5(self, filepath):
|
||||
f = open(filepath, 'rb')
|
||||
m = hashlib.md5()
|
||||
m.update(f.read())
|
||||
md5 = m.hexdigest()
|
||||
"""
|
||||
Returns an md5 of the file located at filepath. Returns an empty string
|
||||
if there was an error reading the file.
|
||||
"""
|
||||
try:
|
||||
f = open(filepath, 'rb')
|
||||
m = hashlib.md5()
|
||||
m.update(f.read())
|
||||
md5 = m.hexdigest()
|
||||
except Exception, e:
|
||||
return ""
|
||||
|
||||
return md5
|
||||
|
||||
|
|
@ -67,9 +84,9 @@ class AirtimeMetadata:
|
|||
## return format hh:mm:ss.uuu
|
||||
def format_length(self, mutagen_length):
|
||||
t = float(mutagen_length)
|
||||
h = int(math.floor(t/3600))
|
||||
h = int(math.floor(t / 3600))
|
||||
t = t % 3600
|
||||
m = int(math.floor(t/60))
|
||||
m = int(math.floor(t / 60))
|
||||
|
||||
s = t % 60
|
||||
# will be ss.uuu
|
||||
|
|
@ -94,12 +111,12 @@ class AirtimeMetadata:
|
|||
for key in m:
|
||||
if key in self.airtime2mutagen:
|
||||
value = m[key]
|
||||
|
||||
|
||||
if value is not None:
|
||||
value = unicode(value)
|
||||
else:
|
||||
value = unicode('');
|
||||
|
||||
|
||||
#if len(value) > 0:
|
||||
self.logger.debug("Saving key '%s' with value '%s' to file", key, value)
|
||||
airtime_file[self.airtime2mutagen[key]] = value
|
||||
|
|
@ -120,35 +137,44 @@ class AirtimeMetadata:
|
|||
return item
|
||||
|
||||
def get_md_from_file(self, filepath):
|
||||
"""
|
||||
Returns None if error retrieving metadata. Otherwise returns a dictionary
|
||||
representing the file's metadata
|
||||
"""
|
||||
|
||||
self.logger.info("getting info from filepath %s", filepath)
|
||||
|
||||
md = {}
|
||||
|
||||
replay_gain_val = replaygain.calculate_replay_gain(filepath)
|
||||
self.logger.info('ReplayGain calculated as %s for %s' % (replay_gain_val, filepath))
|
||||
md['MDATA_KEY_REPLAYGAIN'] = replay_gain_val
|
||||
|
||||
try:
|
||||
md = {}
|
||||
|
||||
md5 = self.get_md5(filepath)
|
||||
md['MDATA_KEY_MD5'] = md5
|
||||
|
||||
file_info = mutagen.File(filepath, easy=True)
|
||||
|
||||
except Exception, e:
|
||||
self.logger.error("failed getting metadata from %s", filepath)
|
||||
self.logger.error("Exception %s", e)
|
||||
return None
|
||||
|
||||
|
||||
#check if file has any metadata
|
||||
if file_info is None:
|
||||
return None
|
||||
#check if file has any metadata
|
||||
if file_info is not None:
|
||||
for key in file_info.keys() :
|
||||
if key in self.mutagen2airtime:
|
||||
val = file_info[key]
|
||||
try:
|
||||
if val is not None and len(val) > 0 and val[0] is not None and len(val[0]) > 0:
|
||||
md[self.mutagen2airtime[key]] = val[0]
|
||||
except Exception, e:
|
||||
self.logger.error('Exception: %s', e)
|
||||
self.logger.error("traceback: %s", traceback.format_exc())
|
||||
|
||||
for key in file_info.keys() :
|
||||
if key in self.mutagen2airtime:
|
||||
val = file_info[key]
|
||||
try:
|
||||
if val is not None and len(val) > 0 and val[0] is not None and len(val[0]) > 0:
|
||||
md[self.mutagen2airtime[key]] = val[0]
|
||||
except Exception, e:
|
||||
self.logger.error('Exception: %s', e)
|
||||
self.logger.error("traceback: %s", traceback.format_exc())
|
||||
if 'MDATA_KEY_TITLE' not in md:
|
||||
#get rid of file extension from original name, name might have more than 1 '.' in it.
|
||||
original_name = os.path.basename(filepath)
|
||||
|
|
@ -165,8 +191,6 @@ class AirtimeMetadata:
|
|||
pass
|
||||
|
||||
if isinstance(md['MDATA_KEY_TRACKNUMBER'], basestring):
|
||||
match = re.search('^(\d*/\d*)?', md['MDATA_KEY_TRACKNUMBER'])
|
||||
|
||||
try:
|
||||
md['MDATA_KEY_TRACKNUMBER'] = int(md['MDATA_KEY_TRACKNUMBER'].split("/")[0], 10)
|
||||
except Exception, e:
|
||||
|
|
@ -222,28 +246,23 @@ class AirtimeMetadata:
|
|||
#end of db truncation checks.
|
||||
|
||||
try:
|
||||
md['MDATA_KEY_BITRATE'] = getattr(file_info.info, "bitrate", "0")
|
||||
except Exception as e:
|
||||
self.logger.warn("Could not get Bitrate")
|
||||
md['MDATA_KEY_BITRATE'] = "0"
|
||||
|
||||
try:
|
||||
md['MDATA_KEY_SAMPLERATE'] = getattr(file_info.info, "sample_rate", "0")
|
||||
except Exception as e:
|
||||
self.logger.warn("Could not get Samplerate")
|
||||
md['MDATA_KEY_SAMPLERATE'] = "0"
|
||||
|
||||
self.logger.info("Bitrate: %s , Samplerate: %s", md['MDATA_KEY_BITRATE'], md['MDATA_KEY_SAMPLERATE'])
|
||||
|
||||
try: md['MDATA_KEY_DURATION'] = self.format_length(file_info.info.length)
|
||||
except Exception as e: self.logger.warn("File: '%s' raises: %s", filepath, str(e))
|
||||
md['MDATA_KEY_BITRATE'] = getattr(file_info.info, "bitrate", 0)
|
||||
md['MDATA_KEY_SAMPLERATE'] = getattr(file_info.info, "sample_rate", 0)
|
||||
|
||||
try: md['MDATA_KEY_MIME'] = file_info.mime[0]
|
||||
except Exception as e: self.logger.warn("File: '%s' has no mime type", filepath, str(e))
|
||||
md['MDATA_KEY_DURATION'] = self.format_length(getattr(file_info.info, "length", 0.0))
|
||||
|
||||
md['MDATA_KEY_MIME'] = ""
|
||||
if len(file_info.mime) > 0:
|
||||
md['MDATA_KEY_MIME'] = file_info.mime[0]
|
||||
except Exception as e:
|
||||
self.logger.warn(e)
|
||||
|
||||
if "mp3" in md['MDATA_KEY_MIME']:
|
||||
md['MDATA_KEY_FTYPE'] = "audioclip"
|
||||
elif "vorbis" in md['MDATA_KEY_MIME']:
|
||||
md['MDATA_KEY_FTYPE'] = "audioclip"
|
||||
else:
|
||||
self.logger.error("File %s of mime type %s does not appear to be a valid vorbis or mp3 file." % (filepath, md['MDATA_KEY_MIME']))
|
||||
return None
|
||||
|
||||
return md
|
||||
|
|
|
|||
|
|
@ -8,13 +8,11 @@ import traceback
|
|||
|
||||
# For RabbitMQ
|
||||
from kombu.connection import BrokerConnection
|
||||
from kombu.messaging import Exchange, Queue, Consumer, Producer
|
||||
from kombu.messaging import Exchange, Queue, Consumer
|
||||
|
||||
import pyinotify
|
||||
from pyinotify import Notifier
|
||||
|
||||
#from api_clients import api_client
|
||||
from api_clients import api_client
|
||||
from airtimemetadata import AirtimeMetadata
|
||||
|
||||
class AirtimeNotifier(Notifier):
|
||||
|
|
@ -38,6 +36,11 @@ class AirtimeNotifier(Notifier):
|
|||
time.sleep(5)
|
||||
|
||||
def init_rabbit_mq(self):
|
||||
"""
|
||||
This function will attempt to connect to RabbitMQ Server and if successful
|
||||
return 'True'. Returns 'False' otherwise.
|
||||
"""
|
||||
|
||||
self.logger.info("Initializing RabbitMQ stuff")
|
||||
try:
|
||||
schedule_exchange = Exchange("airtime-media-monitor", "direct", durable=True, auto_delete=True)
|
||||
|
|
@ -53,13 +56,13 @@ class AirtimeNotifier(Notifier):
|
|||
|
||||
return True
|
||||
|
||||
"""
|
||||
Messages received from RabbitMQ are handled here. These messages
|
||||
instruct media-monitor of events such as a new directory being watched,
|
||||
file metadata has been changed, or any other changes to the config of
|
||||
media-monitor via the web UI.
|
||||
"""
|
||||
def handle_message(self, body, message):
|
||||
"""
|
||||
Messages received from RabbitMQ are handled here. These messages
|
||||
instruct media-monitor of events such as a new directory being watched,
|
||||
file metadata has been changed, or any other changes to the config of
|
||||
media-monitor via the web UI.
|
||||
"""
|
||||
# ACK the message to take it off the queue
|
||||
message.ack()
|
||||
|
||||
|
|
@ -101,16 +104,12 @@ class AirtimeNotifier(Notifier):
|
|||
self.bootstrap.sync_database_to_filesystem(new_storage_directory_id, new_storage_directory)
|
||||
|
||||
self.config.storage_directory = os.path.normpath(new_storage_directory)
|
||||
self.config.imported_directory = os.path.normpath(new_storage_directory + '/imported')
|
||||
self.config.organize_directory = os.path.normpath(new_storage_directory + '/organize')
|
||||
self.config.imported_directory = os.path.normpath(os.path.join(new_storage_directory, '/imported'))
|
||||
self.config.organize_directory = os.path.normpath(os.path.join(new_storage_directory, '/organize'))
|
||||
|
||||
self.mmc.ensure_is_dir(self.config.storage_directory)
|
||||
self.mmc.ensure_is_dir(self.config.imported_directory)
|
||||
self.mmc.ensure_is_dir(self.config.organize_directory)
|
||||
|
||||
self.mmc.is_readable(self.config.storage_directory, True)
|
||||
self.mmc.is_readable(self.config.imported_directory, True)
|
||||
self.mmc.is_readable(self.config.organize_directory, True)
|
||||
for directory in [self.config.storage_directory, self.config.imported_directory, self.config.organize_directory]:
|
||||
self.mmc.ensure_is_dir(directory)
|
||||
self.mmc.is_readable(directory, True)
|
||||
|
||||
self.watch_directory(new_storage_directory)
|
||||
elif m['event_type'] == "file_delete":
|
||||
|
|
@ -129,31 +128,29 @@ class AirtimeNotifier(Notifier):
|
|||
self.logger.error("traceback: %s", traceback.format_exc())
|
||||
|
||||
|
||||
"""
|
||||
Update airtime with information about files discovered in our
|
||||
watched directories.
|
||||
event: a dict() object with the following attributes:
|
||||
-filepath
|
||||
-mode
|
||||
-data
|
||||
-is_recorded_show
|
||||
"""
|
||||
def update_airtime(self, event):
|
||||
|
||||
"""
|
||||
Update airtime with information about files discovered in our
|
||||
watched directories.
|
||||
event: a dict() object with the following attributes:
|
||||
-filepath
|
||||
-mode
|
||||
-data
|
||||
-is_recorded_show
|
||||
"""
|
||||
try:
|
||||
self.logger.info("updating filepath: %s ", event['filepath'])
|
||||
filepath = event['filepath']
|
||||
mode = event['mode']
|
||||
|
||||
md = {}
|
||||
md['MDATA_KEY_FILEPATH'] = filepath
|
||||
md['MDATA_KEY_FILEPATH'] = os.path.normpath(filepath)
|
||||
|
||||
if 'data' in event:
|
||||
file_md = event['data']
|
||||
md.update(file_md)
|
||||
else:
|
||||
file_md = None
|
||||
data = None
|
||||
|
||||
if (os.path.exists(filepath) and (mode == self.config.MODE_CREATE)):
|
||||
if file_md is None:
|
||||
|
|
@ -184,7 +181,7 @@ class AirtimeNotifier(Notifier):
|
|||
self.api_client.update_media_metadata(md, mode)
|
||||
elif (mode == self.config.MODE_DELETE):
|
||||
self.api_client.update_media_metadata(md, mode)
|
||||
|
||||
|
||||
elif (mode == self.config.MODE_DELETE_DIR):
|
||||
self.api_client.update_media_metadata(md, mode)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import difflib
|
|||
import traceback
|
||||
from subprocess import Popen, PIPE
|
||||
|
||||
import pyinotify
|
||||
from pyinotify import ProcessEvent
|
||||
|
||||
from airtimemetadata import AirtimeMetadata
|
||||
|
|
@ -59,10 +58,10 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
if "-unknown-path" in path:
|
||||
unknown_path = path
|
||||
pos = path.find("-unknown-path")
|
||||
path = path[0:pos]+"/"
|
||||
|
||||
path = path[0:pos] + "/"
|
||||
|
||||
list = self.api_client.list_all_watched_dirs()
|
||||
# case where the dir that is being watched is moved to somewhere
|
||||
# case where the dir that is being watched is moved to somewhere
|
||||
if path in list[u'dirs'].values():
|
||||
self.logger.info("Requesting the airtime server to remove '%s'", path)
|
||||
res = self.api_client.remove_watched_dir(path)
|
||||
|
|
@ -81,14 +80,14 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
self.logger.info("Removing watch on: %s wd %s", unknown_path, wd)
|
||||
self.wm.rm_watch(wd, rec=True)
|
||||
self.file_events.append({'mode': self.config.MODE_DELETE_DIR, 'filepath': path})
|
||||
|
||||
|
||||
|
||||
|
||||
def process_IN_DELETE_SELF(self, event):
|
||||
|
||||
|
||||
#we only care about files that have been moved away from imported/ or organize/ dir
|
||||
if event.path in self.config.problem_directory or event.path in self.config.organize_directory:
|
||||
return
|
||||
|
||||
|
||||
self.logger.info("event: %s", event)
|
||||
path = event.path + '/'
|
||||
if event.dir:
|
||||
|
|
@ -103,7 +102,7 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
self.logger.info("%s removed from watch folder list successfully.", path)
|
||||
else:
|
||||
self.logger.info("Removing the watch folder failed: %s", res['msg']['error'])
|
||||
|
||||
|
||||
def process_IN_CREATE(self, event):
|
||||
if event.path in self.mount_file_dir:
|
||||
return
|
||||
|
|
@ -111,18 +110,18 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
if not event.dir:
|
||||
# record the timestamp of the time on IN_CREATE event
|
||||
self.create_dict[event.pathname] = time.time()
|
||||
|
||||
|
||||
#event.dir: True if the event was raised against a directory.
|
||||
#event.name: filename
|
||||
#event.pathname: pathname (str): Concatenation of 'path' and 'name'.
|
||||
# we used to use IN_CREATE event, but the IN_CREATE event gets fired before the
|
||||
# copy was done. Hence, IN_CLOSE_WRITE is the correct one to handle.
|
||||
# copy was done. Hence, IN_CLOSE_WRITE is the correct one to handle.
|
||||
def process_IN_CLOSE_WRITE(self, event):
|
||||
if event.path in self.mount_file_dir:
|
||||
return
|
||||
self.logger.info("event: %s", event)
|
||||
self.logger.info("create_dict: %s", self.create_dict)
|
||||
|
||||
|
||||
try:
|
||||
del self.create_dict[event.pathname]
|
||||
self.handle_created_file(event.dir, event.pathname, event.name)
|
||||
|
|
@ -130,8 +129,8 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
pass
|
||||
#self.logger.warn("%s does not exist in create_dict", event.pathname)
|
||||
#Uncomment the above warning when we fix CC-3830 for 2.1.1
|
||||
|
||||
|
||||
|
||||
|
||||
def handle_created_file(self, dir, pathname, name):
|
||||
if not dir:
|
||||
self.logger.debug("PROCESS_IN_CLOSE_WRITE: %s, name: %s, pathname: %s ", dir, name, pathname)
|
||||
|
|
@ -145,12 +144,12 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
self.temp_files[pathname] = None
|
||||
elif self.mmc.is_audio_file(name):
|
||||
if self.mmc.is_parent_directory(pathname, self.config.organize_directory):
|
||||
|
||||
|
||||
#file was created in /srv/airtime/stor/organize. Need to process and move
|
||||
#to /srv/airtime/stor/imported
|
||||
file_md = self.md_manager.get_md_from_file(pathname)
|
||||
playable = self.mmc.test_file_playability(pathname)
|
||||
|
||||
|
||||
if file_md and playable:
|
||||
self.mmc.organize_new_file(pathname, file_md)
|
||||
else:
|
||||
|
|
@ -182,7 +181,7 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
if self.mmc.is_audio_file(name):
|
||||
is_recorded = self.mmc.is_parent_directory(pathname, self.config.recorded_directory)
|
||||
self.file_events.append({'filepath': pathname, 'mode': self.config.MODE_MODIFY, 'is_recorded_show': is_recorded})
|
||||
|
||||
|
||||
# if change is detected on /etc/mtab, we check what mount(file system) was added/removed
|
||||
# and act accordingly
|
||||
def handle_mount_change(self):
|
||||
|
|
@ -192,41 +191,41 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
shutil.move(self.curr_mtab_file, self.prev_mtab_file)
|
||||
# create the file
|
||||
shutil.copy(self.mount_file, self.curr_mtab_file)
|
||||
|
||||
|
||||
d = difflib.Differ()
|
||||
curr_fh = open(self.curr_mtab_file, 'r')
|
||||
prev_fh = open(self.prev_mtab_file, 'r')
|
||||
|
||||
|
||||
diff = list(d.compare(prev_fh.readlines(), curr_fh.readlines()))
|
||||
added_mount_points = []
|
||||
removed_mount_points = []
|
||||
|
||||
|
||||
for dir in diff:
|
||||
info = dir.split(' ')
|
||||
if info[0] == '+':
|
||||
added_mount_points.append(info[2])
|
||||
elif info[0] == '-':
|
||||
removed_mount_points.append(info[2])
|
||||
|
||||
|
||||
self.logger.info("added: %s", added_mount_points)
|
||||
self.logger.info("removed: %s", removed_mount_points)
|
||||
|
||||
|
||||
# send current mount information to Airtime
|
||||
self.api_client.update_file_system_mount(added_mount_points, removed_mount_points);
|
||||
|
||||
|
||||
def handle_watched_dir_missing(self, dir):
|
||||
self.api_client.handle_watched_dir_missing(dir);
|
||||
|
||||
|
||||
#if a file is moved somewhere, this callback is run. With details about
|
||||
#where the file is being moved from. The corresponding process_IN_MOVED_TO
|
||||
#callback is only called if the destination of the file is also in a watched
|
||||
#directory.
|
||||
def process_IN_MOVED_FROM(self, event):
|
||||
|
||||
|
||||
#we don't care about files that have been moved from problem_directory
|
||||
if event.path in self.config.problem_directory:
|
||||
return
|
||||
|
||||
|
||||
self.logger.info("process_IN_MOVED_FROM: %s", event)
|
||||
if not event.dir:
|
||||
if event.pathname in self.temp_files:
|
||||
|
|
@ -241,10 +240,10 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
def process_IN_MOVED_TO(self, event):
|
||||
self.logger.info("process_IN_MOVED_TO: %s", event)
|
||||
# if /etc/mtab is modified
|
||||
filename = self.mount_file_dir +"/mtab"
|
||||
filename = self.mount_file_dir + "/mtab"
|
||||
if event.pathname in filename:
|
||||
self.handle_mount_change()
|
||||
|
||||
|
||||
if event.path in self.config.problem_directory:
|
||||
return
|
||||
|
||||
|
|
@ -265,15 +264,15 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
#to /srv/airtime/stor/imported
|
||||
file_md = self.md_manager.get_md_from_file(pathname)
|
||||
playable = self.mmc.test_file_playability(pathname)
|
||||
|
||||
|
||||
if file_md and playable:
|
||||
filepath = self.mmc.organize_new_file(pathname, file_md)
|
||||
else:
|
||||
#move to problem_files
|
||||
self.mmc.move_to_problem_dir(pathname)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
else:
|
||||
filepath = event.pathname
|
||||
|
||||
|
|
@ -283,23 +282,23 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
#file's original location is from outside an inotify watched dir.
|
||||
pathname = event.pathname
|
||||
if self.mmc.is_parent_directory(pathname, self.config.organize_directory):
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#file was created in /srv/airtime/stor/organize. Need to process and move
|
||||
#to /srv/airtime/stor/imported
|
||||
file_md = self.md_manager.get_md_from_file(pathname)
|
||||
playable = self.mmc.test_file_playability(pathname)
|
||||
|
||||
|
||||
if file_md and playable:
|
||||
self.mmc.organize_new_file(pathname, file_md)
|
||||
else:
|
||||
#move to problem_files
|
||||
self.mmc.move_to_problem_dir(pathname)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
else:
|
||||
#show moved from unwatched folder into a watched folder. Do not "organize".
|
||||
|
|
@ -309,33 +308,33 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
#When we move a directory into a watched_dir, we only get a notification that the dir was created,
|
||||
#and no additional information about files that came along with that directory.
|
||||
#need to scan the entire directory for files.
|
||||
|
||||
|
||||
if event.cookie in self.cookies_IN_MOVED_FROM:
|
||||
del self.cookies_IN_MOVED_FROM[event.cookie]
|
||||
mode = self.config.MODE_MOVED
|
||||
else:
|
||||
mode = self.config.MODE_CREATE
|
||||
|
||||
|
||||
files = self.mmc.scan_dir_for_new_files(event.pathname)
|
||||
if self.mmc.is_parent_directory(event.pathname, self.config.organize_directory):
|
||||
for pathname in files:
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#file was created in /srv/airtime/stor/organize. Need to process and move
|
||||
#to /srv/airtime/stor/imported
|
||||
file_md = self.md_manager.get_md_from_file(pathname)
|
||||
playable = self.mmc.test_file_playability(pathname)
|
||||
|
||||
|
||||
if file_md and playable:
|
||||
self.mmc.organize_new_file(pathname, file_md)
|
||||
#self.file_events.append({'mode': mode, 'filepath': filepath, 'is_recorded_show': False})
|
||||
else:
|
||||
#move to problem_files
|
||||
self.mmc.move_to_problem_dir(pathname)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
else:
|
||||
for file in files:
|
||||
self.file_events.append({'mode': mode, 'filepath': file, 'is_recorded_show': False})
|
||||
|
|
@ -368,12 +367,12 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
for event in self.file_events:
|
||||
self.multi_queue.put(event)
|
||||
self.mmc.touch_index_file()
|
||||
|
||||
|
||||
self.file_events = []
|
||||
|
||||
#yield to worker thread
|
||||
time.sleep(0)
|
||||
|
||||
|
||||
#use items() because we are going to be modifying this
|
||||
#dictionary while iterating over it.
|
||||
for k, pair in self.cookies_IN_MOVED_FROM.items():
|
||||
|
|
@ -390,7 +389,7 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
#it from the Airtime directory.
|
||||
del self.cookies_IN_MOVED_FROM[k]
|
||||
self.handle_removed_file(False, event.pathname)
|
||||
|
||||
|
||||
# we don't want create_dict grow infinitely
|
||||
# this part is like a garbage collector
|
||||
for k, t in self.create_dict.items():
|
||||
|
|
@ -402,16 +401,16 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
# handling those cases. We are manully calling handle_created_file
|
||||
# function.
|
||||
if os.path.exists(k):
|
||||
# check if file is open
|
||||
# check if file is open
|
||||
try:
|
||||
command = "lsof "+k
|
||||
command = "lsof " + k
|
||||
#f = os.popen(command)
|
||||
f = Popen(command, shell=True, stdout=PIPE).stdout
|
||||
except Exception, e:
|
||||
self.logger.error('Exception: %s', e)
|
||||
self.logger.error("traceback: %s", traceback.format_exc())
|
||||
continue
|
||||
|
||||
|
||||
if not f.readlines():
|
||||
self.logger.info("Handling file: %s", k)
|
||||
self.handle_created_file(False, k, os.path.basename(k))
|
||||
|
|
|
|||
|
|
@ -10,20 +10,31 @@ import traceback
|
|||
|
||||
from subprocess import Popen, PIPE
|
||||
from airtimemetadata import AirtimeMetadata
|
||||
from api_clients import api_client
|
||||
import pyinotify
|
||||
|
||||
class MediaMonitorCommon:
|
||||
|
||||
timestamp_file = "/var/tmp/airtime/media-monitor/last_index"
|
||||
supported_file_formats = ['mp3', 'ogg']
|
||||
|
||||
def __init__(self, airtime_config, wm=None):
|
||||
self.supported_file_formats = ['mp3', 'ogg']
|
||||
self.logger = logging.getLogger()
|
||||
self.config = airtime_config
|
||||
self.md_manager = AirtimeMetadata()
|
||||
self.wm = wm
|
||||
|
||||
|
||||
def clean_dirty_file_paths(self, dirty_files):
|
||||
""" clean dirty file paths by removing blanks and removing trailing/leading whitespace"""
|
||||
return filter(lambda e: len(e) > 0, [ f.strip(" \n") for f in dirty_files ])
|
||||
|
||||
def find_command(self, directory, extra_arguments=""):
|
||||
""" Builds a find command that respects supported_file_formats list
|
||||
Note: Use single quotes to quote arguments """
|
||||
ext_globs = [ "-iname '*.%s'" % ext for ext in self.supported_file_formats ]
|
||||
find_glob = ' -o '.join(ext_globs)
|
||||
return "find '%s' %s %s" % (directory, find_glob, extra_arguments)
|
||||
|
||||
def is_parent_directory(self, filepath, directory):
|
||||
filepath = os.path.normpath(filepath)
|
||||
directory = os.path.normpath(directory)
|
||||
|
|
@ -31,7 +42,6 @@ class MediaMonitorCommon:
|
|||
|
||||
def is_temp_file(self, filename):
|
||||
info = filename.split(".")
|
||||
|
||||
# if file doesn't have any extension, info[-2] throws exception
|
||||
# Hence, checking length of info before we do anything
|
||||
if(len(info) >= 2):
|
||||
|
|
@ -41,20 +51,19 @@ class MediaMonitorCommon:
|
|||
|
||||
def is_audio_file(self, filename):
|
||||
info = filename.split(".")
|
||||
if len(info) < 2: return False # handle cases like filename="mp3"
|
||||
return info[-1].lower() in self.supported_file_formats
|
||||
|
||||
#check if file is readable by "nobody"
|
||||
def is_user_readable(self, filepath, euid='nobody', egid='nogroup'):
|
||||
|
||||
f = None
|
||||
try:
|
||||
uid = pwd.getpwnam(euid)[2]
|
||||
gid = grp.getgrnam(egid)[2]
|
||||
|
||||
#drop root permissions and become "nobody"
|
||||
os.setegid(gid)
|
||||
os.seteuid(uid)
|
||||
|
||||
open(filepath)
|
||||
f = open(filepath)
|
||||
readable = True
|
||||
except IOError:
|
||||
self.logger.warn("File does not have correct permissions: '%s'", filepath)
|
||||
|
|
@ -65,17 +74,16 @@ class MediaMonitorCommon:
|
|||
self.logger.error("traceback: %s", traceback.format_exc())
|
||||
finally:
|
||||
#reset effective user to root
|
||||
if f: f.close()
|
||||
os.seteuid(0)
|
||||
os.setegid(0)
|
||||
|
||||
return readable
|
||||
|
||||
# the function only changes the permission if its not readable by www-data
|
||||
def is_readable(self, item, is_dir):
|
||||
try:
|
||||
return self.is_user_readable(item, 'www-data', 'www-data') \
|
||||
and self.is_user_readable(item, 'pypo', 'pypo')
|
||||
except Exception, e:
|
||||
return self.is_user_readable(item, 'www-data', 'www-data')
|
||||
except Exception:
|
||||
self.logger.warn(u"Failed to check owner/group/permissions for %s", item)
|
||||
return False
|
||||
|
||||
|
|
@ -93,7 +101,7 @@ class MediaMonitorCommon:
|
|||
will attempt to make the file world readable by modifying the file's permission's
|
||||
as well as the file's parent directory permissions. We should only call this function
|
||||
on files in Airtime's stor directory, not watched directories!
|
||||
|
||||
|
||||
Returns True if we were able to make the file world readable. False otherwise.
|
||||
"""
|
||||
original_file = pathname
|
||||
|
|
@ -110,7 +118,7 @@ class MediaMonitorCommon:
|
|||
else:
|
||||
pathname = dirname
|
||||
is_dir = True
|
||||
except Exception, e:
|
||||
except Exception:
|
||||
#something went wrong while we were trying to make world readable.
|
||||
return False
|
||||
|
||||
|
|
@ -154,7 +162,7 @@ class MediaMonitorCommon:
|
|||
try:
|
||||
os.rmdir(dir)
|
||||
self.cleanup_empty_dirs(os.path.dirname(dir))
|
||||
except Exception, e:
|
||||
except Exception:
|
||||
#non-critical exception because we probably tried to delete a non-empty dir.
|
||||
#Don't need to log this, let's just "return"
|
||||
pass
|
||||
|
|
@ -194,7 +202,7 @@ class MediaMonitorCommon:
|
|||
break
|
||||
|
||||
except Exception, e:
|
||||
self.logger.error("Exception %s", e)
|
||||
self.logger.error("Exception %s", e)
|
||||
|
||||
return filepath
|
||||
|
||||
|
|
@ -202,7 +210,6 @@ class MediaMonitorCommon:
|
|||
def create_file_path(self, original_path, orig_md):
|
||||
|
||||
storage_directory = self.config.storage_directory
|
||||
|
||||
try:
|
||||
#will be in the format .ext
|
||||
file_ext = os.path.splitext(original_path)[1].lower()
|
||||
|
|
@ -242,7 +249,7 @@ class MediaMonitorCommon:
|
|||
show_name = '-'.join(title[3:])
|
||||
|
||||
new_md = {}
|
||||
new_md["MDATA_KEY_FILEPATH"] = original_path
|
||||
new_md['MDATA_KEY_FILEPATH'] = os.path.normpath(original_path)
|
||||
new_md['MDATA_KEY_TITLE'] = '%s-%s-%s:%s:%s' % (show_name, orig_md['MDATA_KEY_YEAR'], show_hour, show_min, show_sec)
|
||||
self.md_manager.save_md_to_file(new_md)
|
||||
|
||||
|
|
@ -270,21 +277,24 @@ class MediaMonitorCommon:
|
|||
|
||||
try:
|
||||
"""
|
||||
File name charset encoding is UTF-8.
|
||||
File name charset encoding is UTF-8.
|
||||
"""
|
||||
stdout = stdout.decode("UTF-8")
|
||||
except Exception, e:
|
||||
except Exception:
|
||||
stdout = None
|
||||
self.logger.error("Could not decode %s using UTF-8" % stdout)
|
||||
|
||||
return stdout
|
||||
|
||||
def scan_dir_for_new_files(self, dir):
|
||||
command = 'find "%s" -iname "*.ogg" -o -iname "*.mp3" -type f -readable' % dir.replace('"', '\\"')
|
||||
command = self.find_command(directory=dir, extra_arguments="-type f -readable")
|
||||
self.logger.debug(command)
|
||||
stdout = self.exec_command(command)
|
||||
|
||||
return stdout.splitlines()
|
||||
if stdout is None:
|
||||
return []
|
||||
else:
|
||||
return stdout.splitlines()
|
||||
|
||||
def touch_index_file(self):
|
||||
dirname = os.path.dirname(self.timestamp_file)
|
||||
|
|
@ -316,13 +326,10 @@ class MediaMonitorCommon:
|
|||
if return_code != 0:
|
||||
#print pathname for py-interpreter.log
|
||||
print pathname
|
||||
|
||||
return (return_code == 0)
|
||||
|
||||
def move_to_problem_dir(self, source):
|
||||
|
||||
dest = os.path.join(self.config.problem_directory, os.path.basename(source))
|
||||
|
||||
try:
|
||||
omask = os.umask(0)
|
||||
os.rename(source, dest)
|
||||
|
|
|
|||
132
python_apps/media-monitor/airtimefilemonitor/replaygain.py
Normal file
132
python_apps/media-monitor/airtimefilemonitor/replaygain.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
from subprocess import Popen, PIPE
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import tempfile
|
||||
import logging
|
||||
|
||||
|
||||
logger = logging.getLogger()
|
||||
|
||||
def get_process_output(command):
|
||||
"""
|
||||
Run subprocess and return stdout
|
||||
"""
|
||||
#logger.debug(command)
|
||||
p = Popen(command, shell=True, stdout=PIPE)
|
||||
return p.communicate()[0].strip()
|
||||
|
||||
def run_process(command):
|
||||
"""
|
||||
Run subprocess and return "return code"
|
||||
"""
|
||||
p = Popen(command, shell=True)
|
||||
return os.waitpid(p.pid, 0)[1]
|
||||
|
||||
def get_mime_type(file_path):
|
||||
"""
|
||||
Attempts to get the mime type but will return prematurely if the process
|
||||
takes longer than 5 seconds. Note that this function should only be called
|
||||
for files which do not have a mp3/ogg/flac extension.
|
||||
"""
|
||||
|
||||
return get_process_output("timeout 5 file -b --mime-type %s" % file_path)
|
||||
|
||||
def duplicate_file(file_path):
|
||||
"""
|
||||
Makes a duplicate of the file and returns the path of this duplicate file.
|
||||
"""
|
||||
fsrc = open(file_path, 'r')
|
||||
fdst = tempfile.NamedTemporaryFile(delete=False)
|
||||
|
||||
#logger.info("Copying %s to %s" % (file_path, fdst.name))
|
||||
|
||||
shutil.copyfileobj(fsrc, fdst)
|
||||
|
||||
fsrc.close()
|
||||
fdst.close()
|
||||
|
||||
return fdst.name
|
||||
|
||||
def get_file_type(file_path):
|
||||
file_type = None
|
||||
if re.search(r'mp3$', file_path, re.IGNORECASE):
|
||||
file_type = 'mp3'
|
||||
elif re.search(r'og(g|a)$', file_path, re.IGNORECASE):
|
||||
file_type = 'vorbis'
|
||||
elif re.search(r'flac$', file_path, re.IGNORECASE):
|
||||
file_type = 'flac'
|
||||
else:
|
||||
mime_type = get_mime_type(file_path) == "audio/mpeg"
|
||||
if 'mpeg' in mime_type:
|
||||
file_type = 'mp3'
|
||||
elif 'ogg' in mime_type:
|
||||
file_type = 'vorbis'
|
||||
elif 'flac' in mime_type:
|
||||
file_type = 'flac'
|
||||
|
||||
return file_type
|
||||
|
||||
|
||||
def calculate_replay_gain(file_path):
|
||||
"""
|
||||
This function accepts files of type mp3/ogg/flac and returns a calculated
|
||||
ReplayGain value in dB. If the value cannot be calculated for some reason,
|
||||
then we default to 0 (Unity Gain).
|
||||
|
||||
http://wiki.hydrogenaudio.org/index.php?title=ReplayGain_1.0_specification
|
||||
"""
|
||||
|
||||
try:
|
||||
"""
|
||||
Making a duplicate is required because the ReplayGain extraction
|
||||
utilities we use make unwanted modifications to the file.
|
||||
"""
|
||||
|
||||
search = None
|
||||
temp_file_path = duplicate_file(file_path)
|
||||
|
||||
file_type = get_file_type(file_path)
|
||||
|
||||
if file_type:
|
||||
if file_type == 'mp3':
|
||||
if run_process("which mp3gain > /dev/null") == 0:
|
||||
out = get_process_output('mp3gain -q "%s" 2> /dev/null' % temp_file_path)
|
||||
search = re.search(r'Recommended "Track" dB change: (.*)', out)
|
||||
else:
|
||||
logger.warn("mp3gain not found")
|
||||
elif file_type == 'vorbis':
|
||||
if run_process("which vorbisgain > /dev/null && which ogginfo > /dev/null") == 0:
|
||||
run_process('vorbisgain -q -f "%s" 2>/dev/null >/dev/null' % temp_file_path)
|
||||
out = get_process_output('ogginfo "%s"' % temp_file_path)
|
||||
search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out)
|
||||
else:
|
||||
logger.warn("vorbisgain/ogginfo not found")
|
||||
elif file_type == 'flac':
|
||||
if run_process("which metaflac > /dev/null") == 0:
|
||||
out = get_process_output('metaflac --show-tag=REPLAYGAIN_TRACK_GAIN "%s"' % temp_file_path)
|
||||
search = re.search(r'REPLAYGAIN_TRACK_GAIN=(.*) dB', out)
|
||||
else:
|
||||
logger.warn("metaflac not found")
|
||||
else:
|
||||
pass
|
||||
|
||||
#no longer need the temp, file simply remove it.
|
||||
os.remove(temp_file_path)
|
||||
except Exception, e:
|
||||
logger.error(str(e))
|
||||
|
||||
replay_gain = 0
|
||||
if search:
|
||||
matches = search.groups()
|
||||
if len(matches) == 1:
|
||||
replay_gain = matches[0]
|
||||
|
||||
return replay_gain
|
||||
|
||||
|
||||
# Example of running from command line:
|
||||
# python replay_gain.py /path/to/filename.mp3
|
||||
if __name__ == "__main__":
|
||||
print calculate_replay_gain(sys.argv[1])
|
||||
|
|
@ -1,11 +1,10 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
from mediaconfig import AirtimeMediaConfig
|
||||
import traceback
|
||||
import os
|
||||
|
||||
class MediaMonitorWorkerProcess:
|
||||
|
||||
|
||||
def __init__(self, config, mmc):
|
||||
self.config = config
|
||||
self.mmc = mmc
|
||||
|
|
|
|||
|
|
@ -9,34 +9,35 @@ from configobj import ConfigObj
|
|||
if os.geteuid() != 0:
|
||||
print "Please run this as root."
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def get_current_script_dir():
|
||||
current_script_dir = os.path.realpath(__file__)
|
||||
index = current_script_dir.rindex('/')
|
||||
return current_script_dir[0:index]
|
||||
|
||||
|
||||
def copy_dir(src_dir, dest_dir):
|
||||
if (os.path.exists(dest_dir)) and (dest_dir != "/"):
|
||||
shutil.rmtree(dest_dir)
|
||||
if not (os.path.exists(dest_dir)):
|
||||
#print "Copying directory "+os.path.realpath(src_dir)+" to "+os.path.realpath(dest_dir)
|
||||
shutil.copytree(src_dir, dest_dir)
|
||||
|
||||
|
||||
def create_dir(path):
|
||||
try:
|
||||
os.makedirs(path)
|
||||
# TODO : fix this, at least print the error
|
||||
except Exception, e:
|
||||
pass
|
||||
|
||||
|
||||
def get_rand_string(length=10):
|
||||
return ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(length))
|
||||
|
||||
|
||||
PATH_INI_FILE = '/etc/airtime/media-monitor.cfg'
|
||||
|
||||
try:
|
||||
# Absolute path this script is in
|
||||
current_script_dir = get_current_script_dir()
|
||||
|
||||
|
||||
if not os.path.exists(PATH_INI_FILE):
|
||||
shutil.copy('%s/../media-monitor.cfg'%current_script_dir, PATH_INI_FILE)
|
||||
|
||||
|
|
@ -46,26 +47,24 @@ try:
|
|||
except Exception, e:
|
||||
print 'Error loading config file: ', e
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
#copy monit files
|
||||
shutil.copy('%s/../../monit/monit-airtime-generic.cfg'%current_script_dir, '/etc/monit/conf.d/')
|
||||
subprocess.call('sed -i "s/\$admin_pass/%s/g" /etc/monit/conf.d/monit-airtime-generic.cfg' % get_rand_string(), shell=True)
|
||||
shutil.copy('%s/../monit-airtime-media-monitor.cfg'%current_script_dir, '/etc/monit/conf.d/')
|
||||
|
||||
|
||||
#create log dir
|
||||
create_dir(config['log_dir'])
|
||||
os.system("chown -R pypo:pypo "+config["log_dir"])
|
||||
|
||||
#copy python files
|
||||
copy_dir("%s/.."%current_script_dir, config["bin_dir"])
|
||||
|
||||
#set executable permissions on python files
|
||||
os.system("chown -R pypo:pypo "+config["bin_dir"])
|
||||
# mm2
|
||||
mm2_source = os.path.realpath(os.path.join(current_script_dir,
|
||||
"../../media-monitor2"))
|
||||
copy_dir(mm2_source, os.path.join( config["bin_dir"], "mm2" ))
|
||||
|
||||
#copy init.d script
|
||||
shutil.copy(config["bin_dir"]+"/airtime-media-monitor-init-d", "/etc/init.d/airtime-media-monitor")
|
||||
|
||||
|
||||
except Exception, e:
|
||||
print e
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,16 @@ rabbitmq_password = 'guest'
|
|||
rabbitmq_vhost = '/'
|
||||
|
||||
############################################
|
||||
# Media-Monitor preferences #
|
||||
# Media-Monitor preferences #
|
||||
############################################
|
||||
check_filesystem_events = 5 #how long to queue up events performed on the files themselves.
|
||||
check_airtime_events = 30 #how long to queue metadata input from airtime.
|
||||
|
||||
# MM2 only:
|
||||
touch_interval = 5
|
||||
chunking_number = 450
|
||||
request_max_wait = 3.0
|
||||
rmq_event_wait = 0.1
|
||||
logpath = '/var/log/airtime/media-monitor/media-monitor.log'
|
||||
index_path = '/var/tmp/airtime/media-monitor/last_index'
|
||||
|
||||
|
|
|
|||
|
|
@ -1,144 +1,11 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import time
|
||||
import logging
|
||||
import logging.config
|
||||
import time
|
||||
import sys
|
||||
import os
|
||||
import signal
|
||||
import traceback
|
||||
import locale
|
||||
|
||||
from configobj import ConfigObj
|
||||
|
||||
from api_clients import api_client as apc
|
||||
import mm2.mm2 as mm2
|
||||
from std_err_override import LogWriter
|
||||
|
||||
from multiprocessing import Process, Queue as mpQueue
|
||||
global_cfg = '/etc/airtime/media-monitor.cfg'
|
||||
api_client_cfg = '/etc/airtime/api_client.cfg'
|
||||
logging_cfg = '/usr/lib/airtime/media-monitor/logging.cfg'
|
||||
|
||||
from threading import Thread
|
||||
|
||||
from pyinotify import WatchManager
|
||||
|
||||
from airtimefilemonitor.airtimenotifier import AirtimeNotifier
|
||||
from airtimefilemonitor.mediamonitorcommon import MediaMonitorCommon
|
||||
from airtimefilemonitor.airtimeprocessevent import AirtimeProcessEvent
|
||||
from airtimefilemonitor.mediaconfig import AirtimeMediaConfig
|
||||
from airtimefilemonitor.workerprocess import MediaMonitorWorkerProcess
|
||||
from airtimefilemonitor.airtimemediamonitorbootstrap import AirtimeMediaMonitorBootstrap
|
||||
|
||||
def configure_locale():
|
||||
logger.debug("Before %s", locale.nl_langinfo(locale.CODESET))
|
||||
current_locale = locale.getlocale()
|
||||
|
||||
if current_locale[1] is None:
|
||||
logger.debug("No locale currently set. Attempting to get default locale.")
|
||||
default_locale = locale.getdefaultlocale()
|
||||
|
||||
if default_locale[1] is None:
|
||||
logger.debug("No default locale exists. Let's try loading from /etc/default/locale")
|
||||
if os.path.exists("/etc/default/locale"):
|
||||
config = ConfigObj('/etc/default/locale')
|
||||
lang = config.get('LANG')
|
||||
new_locale = lang
|
||||
else:
|
||||
logger.error("/etc/default/locale could not be found! Please run 'sudo update-locale' from command-line.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
new_locale = default_locale
|
||||
|
||||
logger.info("New locale set to: %s", locale.setlocale(locale.LC_ALL, new_locale))
|
||||
|
||||
|
||||
|
||||
reload(sys)
|
||||
sys.setdefaultencoding("UTF-8")
|
||||
current_locale_encoding = locale.getlocale()[1].lower()
|
||||
logger.debug("sys default encoding %s", sys.getdefaultencoding())
|
||||
logger.debug("After %s", locale.nl_langinfo(locale.CODESET))
|
||||
|
||||
if current_locale_encoding not in ['utf-8', 'utf8']:
|
||||
logger.error("Need a UTF-8 locale. Currently '%s'. Exiting..." % current_locale_encoding)
|
||||
sys.exit(1)
|
||||
|
||||
# configure logging
|
||||
try:
|
||||
logging.config.fileConfig("logging.cfg")
|
||||
|
||||
#need to wait for Python 2.7 for this..
|
||||
#logging.captureWarnings(True)
|
||||
|
||||
logger = logging.getLogger()
|
||||
LogWriter.override_std_err(logger)
|
||||
|
||||
except Exception, e:
|
||||
print 'Error configuring logging: ', e
|
||||
sys.exit(1)
|
||||
|
||||
logger.info("\n\n*** Media Monitor bootup ***\n\n")
|
||||
|
||||
|
||||
try:
|
||||
configure_locale()
|
||||
|
||||
config = AirtimeMediaConfig(logger)
|
||||
api_client = apc.api_client_factory(config.cfg)
|
||||
api_client.register_component("media-monitor")
|
||||
|
||||
logger.info("Setting up monitor")
|
||||
response = None
|
||||
while response is None:
|
||||
response = api_client.setup_media_monitor()
|
||||
time.sleep(5)
|
||||
|
||||
storage_directory = response["stor"]
|
||||
watched_dirs = response["watched_dirs"]
|
||||
logger.info("Storage Directory is: %s", storage_directory)
|
||||
config.storage_directory = os.path.normpath(storage_directory)
|
||||
config.imported_directory = os.path.normpath(os.path.join(storage_directory, 'imported'))
|
||||
config.organize_directory = os.path.normpath(os.path.join(storage_directory, 'organize'))
|
||||
config.recorded_directory = os.path.normpath(os.path.join(storage_directory, 'recorded'))
|
||||
config.problem_directory = os.path.normpath(os.path.join(storage_directory, 'problem_files'))
|
||||
|
||||
dirs = [config.imported_directory, config.organize_directory, config.recorded_directory, config.problem_directory]
|
||||
for d in dirs:
|
||||
if not os.path.exists(d):
|
||||
os.makedirs(d, 02775)
|
||||
|
||||
multi_queue = mpQueue()
|
||||
logger.info("Initializing event processor")
|
||||
|
||||
wm = WatchManager()
|
||||
mmc = MediaMonitorCommon(config, wm=wm)
|
||||
pe = AirtimeProcessEvent(queue=multi_queue, airtime_config=config, wm=wm, mmc=mmc, api_client=api_client)
|
||||
|
||||
bootstrap = AirtimeMediaMonitorBootstrap(logger, pe, api_client, mmc, wm, config)
|
||||
bootstrap.scan()
|
||||
|
||||
notifier = AirtimeNotifier(wm, pe, read_freq=0, timeout=0, airtime_config=config, api_client=api_client, bootstrap=bootstrap, mmc=mmc)
|
||||
notifier.coalesce_events()
|
||||
|
||||
#create 5 worker threads
|
||||
wp = MediaMonitorWorkerProcess(config, mmc)
|
||||
for i in range(5):
|
||||
threadName = "Thread #%d" % i
|
||||
t = Thread(target=wp.process_file_events, name=threadName, args=(multi_queue, notifier))
|
||||
t.start()
|
||||
|
||||
wdd = notifier.watch_directory(storage_directory)
|
||||
logger.info("Added watch to %s", storage_directory)
|
||||
logger.info("wdd result %s", wdd[storage_directory])
|
||||
|
||||
for dir in watched_dirs:
|
||||
wdd = notifier.watch_directory(dir)
|
||||
logger.info("Added watch to %s", dir)
|
||||
logger.info("wdd result %s", wdd[dir])
|
||||
|
||||
notifier.loop(callback=pe.notifier_loop_callback)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
notifier.stop()
|
||||
logger.info("Keyboard Interrupt")
|
||||
except Exception, e:
|
||||
logger.error('Exception: %s', e)
|
||||
logger.error("traceback: %s", traceback.format_exc())
|
||||
mm2.main( global_cfg, api_client_cfg, logging_cfg )
|
||||
|
|
|
|||
142
python_apps/media-monitor/mm1.py
Normal file
142
python_apps/media-monitor/mm1.py
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import time
|
||||
import logging.config
|
||||
import sys
|
||||
import os
|
||||
import traceback
|
||||
import locale
|
||||
|
||||
from configobj import ConfigObj
|
||||
|
||||
from api_clients import api_client as apc
|
||||
from std_err_override import LogWriter
|
||||
|
||||
from multiprocessing import Queue as mpQueue
|
||||
|
||||
from threading import Thread
|
||||
|
||||
from pyinotify import WatchManager
|
||||
|
||||
from airtimefilemonitor.airtimenotifier import AirtimeNotifier
|
||||
from airtimefilemonitor.mediamonitorcommon import MediaMonitorCommon
|
||||
from airtimefilemonitor.airtimeprocessevent import AirtimeProcessEvent
|
||||
from airtimefilemonitor.mediaconfig import AirtimeMediaConfig
|
||||
from airtimefilemonitor.workerprocess import MediaMonitorWorkerProcess
|
||||
from airtimefilemonitor.airtimemediamonitorbootstrap import AirtimeMediaMonitorBootstrap
|
||||
|
||||
def configure_locale():
|
||||
logger.debug("Before %s", locale.nl_langinfo(locale.CODESET))
|
||||
current_locale = locale.getlocale()
|
||||
|
||||
if current_locale[1] is None:
|
||||
logger.debug("No locale currently set. Attempting to get default locale.")
|
||||
default_locale = locale.getdefaultlocale()
|
||||
|
||||
if default_locale[1] is None:
|
||||
logger.debug("No default locale exists. Let's try loading from /etc/default/locale")
|
||||
if os.path.exists("/etc/default/locale"):
|
||||
config = ConfigObj('/etc/default/locale')
|
||||
lang = config.get('LANG')
|
||||
new_locale = lang
|
||||
else:
|
||||
logger.error("/etc/default/locale could not be found! Please run 'sudo update-locale' from command-line.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
new_locale = default_locale
|
||||
|
||||
logger.info("New locale set to: %s", locale.setlocale(locale.LC_ALL, new_locale))
|
||||
|
||||
|
||||
|
||||
reload(sys)
|
||||
sys.setdefaultencoding("UTF-8")
|
||||
current_locale_encoding = locale.getlocale()[1].lower()
|
||||
logger.debug("sys default encoding %s", sys.getdefaultencoding())
|
||||
logger.debug("After %s", locale.nl_langinfo(locale.CODESET))
|
||||
|
||||
if current_locale_encoding not in ['utf-8', 'utf8']:
|
||||
logger.error("Need a UTF-8 locale. Currently '%s'. Exiting..." % current_locale_encoding)
|
||||
sys.exit(1)
|
||||
|
||||
# configure logging
|
||||
try:
|
||||
logging.config.fileConfig("logging.cfg")
|
||||
|
||||
#need to wait for Python 2.7 for this..
|
||||
#logging.captureWarnings(True)
|
||||
|
||||
logger = logging.getLogger()
|
||||
LogWriter.override_std_err(logger)
|
||||
|
||||
except Exception, e:
|
||||
print 'Error configuring logging: ', e
|
||||
sys.exit(1)
|
||||
|
||||
logger.info("\n\n*** Media Monitor bootup ***\n\n")
|
||||
|
||||
|
||||
try:
|
||||
configure_locale()
|
||||
|
||||
config = AirtimeMediaConfig(logger)
|
||||
api_client = apc.AirtimeApiClient()
|
||||
api_client.register_component("media-monitor")
|
||||
|
||||
logger.info("Setting up monitor")
|
||||
response = None
|
||||
while response is None:
|
||||
response = api_client.setup_media_monitor()
|
||||
time.sleep(5)
|
||||
|
||||
storage_directory = response["stor"]
|
||||
watched_dirs = response["watched_dirs"]
|
||||
logger.info("Storage Directory is: %s", storage_directory)
|
||||
config.storage_directory = os.path.normpath(storage_directory)
|
||||
config.imported_directory = os.path.normpath(os.path.join(storage_directory, 'imported'))
|
||||
config.organize_directory = os.path.normpath(os.path.join(storage_directory, 'organize'))
|
||||
config.recorded_directory = os.path.normpath(os.path.join(storage_directory, 'recorded'))
|
||||
config.problem_directory = os.path.normpath(os.path.join(storage_directory, 'problem_files'))
|
||||
|
||||
dirs = [config.imported_directory, config.organize_directory, config.recorded_directory, config.problem_directory]
|
||||
for d in dirs:
|
||||
if not os.path.exists(d):
|
||||
os.makedirs(d, 02775)
|
||||
|
||||
multi_queue = mpQueue()
|
||||
logger.info("Initializing event processor")
|
||||
|
||||
wm = WatchManager()
|
||||
mmc = MediaMonitorCommon(config, wm=wm)
|
||||
pe = AirtimeProcessEvent(queue=multi_queue, airtime_config=config, wm=wm, mmc=mmc, api_client=api_client)
|
||||
|
||||
bootstrap = AirtimeMediaMonitorBootstrap(logger, pe, api_client, mmc, wm, config)
|
||||
bootstrap.scan()
|
||||
|
||||
notifier = AirtimeNotifier(wm, pe, read_freq=0, timeout=0, airtime_config=config, api_client=api_client, bootstrap=bootstrap, mmc=mmc)
|
||||
notifier.coalesce_events()
|
||||
|
||||
#create 5 worker threads
|
||||
wp = MediaMonitorWorkerProcess(config, mmc)
|
||||
for i in range(5):
|
||||
threadName = "Thread #%d" % i
|
||||
t = Thread(target=wp.process_file_events, name=threadName, args=(multi_queue, notifier))
|
||||
t.start()
|
||||
|
||||
wdd = notifier.watch_directory(storage_directory)
|
||||
logger.info("Added watch to %s", storage_directory)
|
||||
logger.info("wdd result %s", wdd[storage_directory])
|
||||
|
||||
for dir in watched_dirs:
|
||||
wdd = notifier.watch_directory(dir)
|
||||
logger.info("Added watch to %s", dir)
|
||||
logger.info("wdd result %s", wdd[dir])
|
||||
|
||||
notifier.loop(callback=pe.notifier_loop_callback)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
notifier.stop()
|
||||
logger.info("Keyboard Interrupt")
|
||||
except Exception, e:
|
||||
logger.error('Exception: %s', e)
|
||||
logger.error("traceback: %s", traceback.format_exc())
|
||||
Loading…
Add table
Add a link
Reference in a new issue