CC-2977: Never delete files from the database
- Files are marked deleted(file_exists to false) on deletion. - Dirs are marked removed(removed flag to true) on removal of watched folder in a usual way. If dir is unmounted, without being removed from watched list first, it will be marked as not exists(exist flag to false) - Manage Media Folders will show if dirs exist or not( mounted or not) - Playlist builder will show if files exists or not
This commit is contained in:
James
parent
61c5355e8a
commit
04b48d47cc
16 changed files with 374 additions and 69 deletions
|
|
@ -48,6 +48,12 @@ remove_watched_dir = 'remove-watched-dir/format/json/api_key/%%api_key%%/path/%%
|
|||
# URL to tell Airtime we want to add watched directory
|
||||
set_storage_dir = 'set-storage-dir/format/json/api_key/%%api_key%%/path/%%path%%'
|
||||
|
||||
# URL to tell Airtime about file system mount change
|
||||
update_fs_mount = 'update-file-system-mount/format/json/api_key/%%api_key%%'
|
||||
|
||||
# URL to tell Airtime about file system mount change
|
||||
handle_watched_dir_missing = 'handle-watched-dir-missing/format/json/api_key/%%api_key%%/dir/%%dir%%'
|
||||
|
||||
#############################
|
||||
## Config for Recorder
|
||||
#############################
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import os
|
|||
from urlparse import urlparse
|
||||
import base64
|
||||
from configobj import ConfigObj
|
||||
import string
|
||||
|
||||
AIRTIME_VERSION = "2.0.0"
|
||||
|
||||
|
|
@ -399,7 +400,7 @@ class AirTimeApiClient(ApiClientInterface):
|
|||
try:
|
||||
url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["media_setup_url"])
|
||||
url = url.replace("%%api_key%%", self.config["api_key"])
|
||||
|
||||
|
||||
response = self.get_response_from_server(url)
|
||||
response = json.loads(response)
|
||||
logger.info("Connected to Airtime Server. Json Media Storage Dir: %s", response)
|
||||
|
|
@ -582,11 +583,55 @@ class AirTimeApiClient(ApiClientInterface):
|
|||
url = url.replace("%%msg%%", encoded_msg)
|
||||
url = url.replace("%%stream_id%%", stream_id)
|
||||
url = url.replace("%%boot_time%%", time)
|
||||
logger.debug(url)
|
||||
|
||||
req = urllib2.Request(url)
|
||||
response = urllib2.urlopen(req).read()
|
||||
except Exception, e:
|
||||
logger.error("Exception: %s", e)
|
||||
|
||||
"""
|
||||
This function updates status of mounted file system information on airtime
|
||||
"""
|
||||
def update_file_system_mount(self, mount_list):
|
||||
logger = logging.getLogger()
|
||||
try:
|
||||
url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["update_fs_mount"])
|
||||
|
||||
url = url.replace("%%api_key%%", self.config["api_key"])
|
||||
|
||||
data_string = string.join(mount_list, ',')
|
||||
map = [("mount_list", data_string)]
|
||||
data = urllib.urlencode(map)
|
||||
|
||||
req = urllib2.Request(url, data)
|
||||
response = urllib2.urlopen(req).read()
|
||||
logger.info("update file system mount: %s", response)
|
||||
except Exception, e:
|
||||
import traceback
|
||||
top = traceback.format_exc()
|
||||
logger.error('Exception: %s', e)
|
||||
logger.error("traceback: %s", top)
|
||||
|
||||
"""
|
||||
When watched dir is missing(unplugged or something) on boot up, this function will get called
|
||||
and will call approperiate function on Airtime.
|
||||
"""
|
||||
def handle_watched_dir_missing(self, dir):
|
||||
logger = logging.getLogger()
|
||||
try:
|
||||
url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["handle_watched_dir_missing"])
|
||||
|
||||
url = url.replace("%%api_key%%", self.config["api_key"])
|
||||
url = url.replace("%%dir%%", base64.b64encode(dir))
|
||||
|
||||
req = urllib2.Request(url)
|
||||
response = urllib2.urlopen(req).read()
|
||||
logger.info("update file system mount: %s", response)
|
||||
except Exception, e:
|
||||
import traceback
|
||||
top = traceback.format_exc()
|
||||
logger.error('Exception: %s', e)
|
||||
logger.error("traceback: %s", top)
|
||||
|
||||
################################################################################
|
||||
# OpenBroadcast API Client
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ try:
|
|||
mmc = MediaMonitorCommon(config, wm=wm)
|
||||
pe = AirtimeProcessEvent(queue=multi_queue, airtime_config=config, wm=wm, mmc=mmc, api_client=api_client)
|
||||
|
||||
bootstrap = AirtimeMediaMonitorBootstrap(logger, pe, api_client, mmc)
|
||||
bootstrap = AirtimeMediaMonitorBootstrap(logger, pe, api_client, mmc, wm)
|
||||
bootstrap.scan()
|
||||
|
||||
notifier = AirtimeNotifier(wm, pe, read_freq=0, timeout=0, airtime_config=config, api_client=api_client, bootstrap=bootstrap, mmc=mmc)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import os
|
||||
import time
|
||||
import pyinotify
|
||||
|
||||
from subprocess import Popen, PIPE
|
||||
from api_clients import api_client
|
||||
|
||||
class AirtimeMediaMonitorBootstrap():
|
||||
|
||||
|
||||
"""AirtimeMediaMonitorBootstrap constructor
|
||||
|
||||
Keyword Arguments:
|
||||
|
|
@ -13,11 +14,16 @@ class AirtimeMediaMonitorBootstrap():
|
|||
pe -- reference to an instance of ProcessEvent
|
||||
api_clients -- reference of api_clients to communicate with airtime-server
|
||||
"""
|
||||
def __init__(self, logger, pe, api_client, mmc):
|
||||
def __init__(self, logger, pe, api_client, mmc, wm):
|
||||
self.logger = logger
|
||||
self.pe = pe
|
||||
self.api_client = api_client
|
||||
self.mmc = mmc
|
||||
self.wm = wm
|
||||
# add /etc on watch list so we can detect mount
|
||||
self.mount_file = "/etc"
|
||||
self.logger.info("Adding %s on watch list...", self.mount_file)
|
||||
self.wm.add_watch(self.mount_file, pyinotify.ALL_EVENTS, rec=False, auto_add=False)
|
||||
|
||||
"""On bootup we want to scan all directories and look for files that
|
||||
weren't there or files that changed before media-monitor process
|
||||
|
|
@ -79,6 +85,10 @@ class AirtimeMediaMonitorBootstrap():
|
|||
if len(file_path.strip(" \n")) > 0:
|
||||
all_files_set.add(file_path[len(dir):])
|
||||
|
||||
# if dir doesn't exists, update db
|
||||
if not os.path.exists(dir):
|
||||
self.pe.handle_watched_dir_missing(dir)
|
||||
|
||||
if os.path.exists(self.mmc.timestamp_file):
|
||||
"""find files that have been modified since the last time media-monitor process started."""
|
||||
time_diff_sec = time.time() - os.path.getmtime(self.mmc.timestamp_file)
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
self.mmc = mmc
|
||||
self.api_client = api_client
|
||||
self.create_dict = {}
|
||||
self.mount_file_dir = "/etc";
|
||||
self.mount_file = "/etc/mtab";
|
||||
|
||||
def add_filepath_to_ignore(self, filepath):
|
||||
self.ignore_event.add(filepath)
|
||||
|
|
@ -74,6 +76,8 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
|
||||
|
||||
def process_IN_DELETE_SELF(self, event):
|
||||
if event.path in self.mount_file_dir:
|
||||
return
|
||||
self.logger.info("event: %s", event)
|
||||
path = event.path + '/'
|
||||
if event.dir:
|
||||
|
|
@ -90,6 +94,8 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
self.logger.info("Removing the watch folder failed: %s", res['msg']['error'])
|
||||
|
||||
def process_IN_CREATE(self, event):
|
||||
if event.path in self.mount_file_dir:
|
||||
return
|
||||
self.logger.info("event: %s", event)
|
||||
if not event.dir:
|
||||
# record the timestamp of the time on IN_CREATE event
|
||||
|
|
@ -101,6 +107,8 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
# we used to use IN_CREATE event, but the IN_CREATE event gets fired before the
|
||||
# copy was done. Hence, IN_CLOSE_WRITE is the correct one to handle.
|
||||
def process_IN_CLOSE_WRITE(self, event):
|
||||
if event.path in self.mount_file_dir:
|
||||
return
|
||||
self.logger.info("event: %s", event)
|
||||
self.logger.info("create_dict: %s", self.create_dict)
|
||||
|
||||
|
|
@ -145,6 +153,9 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
self.handle_modified_file(event.dir, event.pathname, event.name)
|
||||
|
||||
def handle_modified_file(self, dir, pathname, name):
|
||||
# if /etc/mtab is modified
|
||||
if pathname in self.mount_file:
|
||||
self.handle_mount_change()
|
||||
# update timestamp on create_dict for the entry with pathname as the key
|
||||
if pathname in self.create_dict:
|
||||
self.create_dict[pathname] = time.time()
|
||||
|
|
@ -152,12 +163,38 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
self.logger.info("Modified: %s", pathname)
|
||||
if self.mmc.is_audio_file(name):
|
||||
self.file_events.append({'filepath': pathname, 'mode': self.config.MODE_MODIFY})
|
||||
|
||||
|
||||
# if change is detected on /etc/mtab, we check what mount(file system) was added/removed
|
||||
# and act accordingly
|
||||
def handle_mount_change(self):
|
||||
mount_list = [];
|
||||
# parse /etc/mtab
|
||||
fh = open(self.mount_file, 'r')
|
||||
while 1:
|
||||
line = fh.readline()
|
||||
if not line:
|
||||
break
|
||||
|
||||
line_info = line.split(' ')
|
||||
# the line format is like following:
|
||||
# /dev/sdg1 /media/809D-D2A1 vfat rw,nosuid,nodev,uhelper=udisks..........
|
||||
# so we always get [1] after split to get the mount point
|
||||
mount_list.append(line_info[1])
|
||||
fh.close()
|
||||
self.logger.info("Mount List: %s", mount_list)
|
||||
# send current mount information to Airtime
|
||||
self.api_client.update_file_system_mount(mount_list);
|
||||
|
||||
def handle_watched_dir_missing(self, dir):
|
||||
self.api_client.handle_watched_dir_missing(dir);
|
||||
|
||||
#if a file is moved somewhere, this callback is run. With details about
|
||||
#where the file is being moved from. The corresponding process_IN_MOVED_TO
|
||||
#callback is only called if the destination of the file is also in a watched
|
||||
#directory.
|
||||
def process_IN_MOVED_FROM(self, event):
|
||||
if event.path in self.mount_file:
|
||||
return
|
||||
self.logger.info("process_IN_MOVED_FROM: %s", event)
|
||||
if not event.dir:
|
||||
if event.pathname in self.temp_files:
|
||||
|
|
@ -171,6 +208,10 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
|
||||
def process_IN_MOVED_TO(self, event):
|
||||
self.logger.info("process_IN_MOVED_TO: %s", event)
|
||||
# if /etc/mtab is modified
|
||||
filename = self.mount_file_dir +"/mtab"
|
||||
if event.pathname in filename:
|
||||
self.handle_mount_change()
|
||||
#if stuff dropped in stor via a UI move must change file permissions.
|
||||
self.mmc.set_needed_file_permissions(event.pathname, event.dir)
|
||||
if not event.dir:
|
||||
|
|
@ -221,6 +262,8 @@ class AirtimeProcessEvent(ProcessEvent):
|
|||
|
||||
|
||||
def process_IN_DELETE(self, event):
|
||||
if event.path in self.mount_file_dir:
|
||||
return
|
||||
self.logger.info("process_IN_DELETE: %s", event)
|
||||
self.handle_removed_file(event.dir, event.pathname)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue