From 3e251997f88b025e707a31583d3df6feccc705db Mon Sep 17 00:00:00 2001 From: root Date: Fri, 20 Jul 2012 15:37:13 -0400 Subject: [PATCH] cc-4105. Refactored syncdb and fixed a few typos --- python_apps/api_clients/api_client.py | 11 ++---- .../media-monitor2/media/monitor/bootstrap.py | 13 +++---- .../media-monitor2/media/monitor/events.py | 2 +- .../media-monitor2/media/monitor/log.py | 3 ++ .../media-monitor2/media/monitor/syncdb.py | 38 +++++++++++++------ python_apps/media-monitor2/mm2.py | 14 ++++++- 6 files changed, 53 insertions(+), 28 deletions(-) diff --git a/python_apps/api_clients/api_client.py b/python_apps/api_clients/api_client.py index 9f1c100bd..a0c35a2b0 100644 --- a/python_apps/api_clients/api_client.py +++ b/python_apps/api_clients/api_client.py @@ -605,16 +605,13 @@ class AirtimeApiClient(): logger.error('Exception: %s', e) logger.error("traceback: %s", traceback.format_exc()) - """ - Retrive infomations needed on bootstrap time - """ def get_bootstrap_info(self): + """ + Retrive infomations needed on bootstrap time + """ logger = self.logger try: - url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["get_bootstrap_info"]) - - url = url.replace("%%api_key%%", self.config["api_key"]) - + url = self.construct_url("get_bootstrap_info") response = self.get_response_from_server(url) response = json.loads(response) logger.info("Bootstrap info retrieved %s", response) diff --git a/python_apps/media-monitor2/media/monitor/bootstrap.py b/python_apps/media-monitor2/media/monitor/bootstrap.py index 8cffa5b5f..69403f887 100644 --- a/python_apps/media-monitor2/media/monitor/bootstrap.py +++ b/python_apps/media-monitor2/media/monitor/bootstrap.py @@ -9,11 +9,11 @@ class Bootstrapper(Loggable): Bootstrapper reads all the info in the filesystem flushes organize events and watch events """ - def __init__(self,db,last_ran,org_channels,watch_channels): + def __init__(self,db,last_run,org_channels,watch_channels): self.db = db self.org_channels = org_channels self.watch_channels = watch_channels - self.last_ran = last_ran + self.last_run = last_run def flush_organize(self): """ @@ -44,12 +44,12 @@ class Bootstrapper(Loggable): # wasn't aware when this changes occured in the filesystem # hence it will send the correct events to sync the database # with the filesystem - if os.path.getmtime(f) > self.last_ran: + if os.path.getmtime(f) > self.last_run: modded += 1 dispatcher.send(signal=pc.signal, sender=self, event=DeleteFile(f)) dispatcher.send(signal=pc.signal, sender=self, event=NewFile(f)) # Want all files in the database that are not in the filesystem - for to_delete in self.db.exclude(songs): + for to_delete in self.db.difference(songs): for pc in self.watch_channels: if os.path.commonprefix([pc.path, to_delete]) == pc.path: dispatcher.send(signal=pc.signal, sender=self, event=DeleteFile(f)) @@ -57,9 +57,8 @@ class Bootstrapper(Loggable): deleted += 1 break else: - self.logger.info("Error, could not find watch directory of would be deleted \ - file '%s'" % to_delete) - self.logger.info("Flushed watch directories. (modified, deleted) = (%d, %d)" + self.logger.info("Error, could not find watch directory of would be deleted file '%s'" % to_delete) + self.logger.info( "Flushed watch directories. (modified, deleted) = (%d, %d)" % (modded, deleted) ) diff --git a/python_apps/media-monitor2/media/monitor/events.py b/python_apps/media-monitor2/media/monitor/events.py index 67fb4d1e5..4e7490c9f 100644 --- a/python_apps/media-monitor2/media/monitor/events.py +++ b/python_apps/media-monitor2/media/monitor/events.py @@ -2,7 +2,7 @@ import os import abc from media.monitor.pure import LazyProperty -from media.monitor.monitor import Metadata +from media.monitor.metadata import Metadata class PathChannel(object): """a dumb struct; python has no record types""" diff --git a/python_apps/media-monitor2/media/monitor/log.py b/python_apps/media-monitor2/media/monitor/log.py index 186adacbd..3330f809b 100644 --- a/python_apps/media-monitor2/media/monitor/log.py +++ b/python_apps/media-monitor2/media/monitor/log.py @@ -13,3 +13,6 @@ class Loggable(object): # TODO : Clean this up if not hasattr(self,"_logger"): self._logger = logging.getLogger('mediamonitor2') return self._logger + +def get_logger(): + return logging.getLogger('mediamonitor2') diff --git a/python_apps/media-monitor2/media/monitor/syncdb.py b/python_apps/media-monitor2/media/monitor/syncdb.py index 09f00aede..0fca29589 100644 --- a/python_apps/media-monitor2/media/monitor/syncdb.py +++ b/python_apps/media-monitor2/media/monitor/syncdb.py @@ -1,13 +1,27 @@ # -*- coding: utf-8 -*- -class SyncDB(object): - """ - Represents the database returned by airtime_mvc. We do not use a list or some other - fixed data structure because we might want to change the internal representation for - performance reasons later on. - """ - def __init__(self, source): - self.source = source - def has_file(self, path): - return True - def file_mdata(self, path): - return None +from media.monitor.log import Loggable + +class SyncDB(Loggable): + + def __init__(self, apc): + self.apc = apc + dirs = self.apc.list_all_watched_dirs() + directories = None + try: + directories = dirs['dirs'] + except KeyError as e: + self.logger.error("Could not find index 'dirs' in dictionary: %s", str(dirs)) + self.logger.error(e) + raise + # self.directories is a dictionary where a key is the directory and the + # value is the directory's id in the db + self.directories = dict( (v,k) for k,v in directories.iteritems() ) + + def list_directories(self): + return self.directories.keys() + + def directory_get_files(self, directory): + print("trying to access dir id: %s" % self.directories[directory]) + self.apc.list_all_db_files(self.directories[directory]) + + diff --git a/python_apps/media-monitor2/mm2.py b/python_apps/media-monitor2/mm2.py index 9b7107cdf..4a849fb06 100644 --- a/python_apps/media-monitor2/mm2.py +++ b/python_apps/media-monitor2/mm2.py @@ -1,12 +1,16 @@ # -*- coding: utf-8 -*- # testing ground for the script import pyinotify +import time from media.monitor.listeners import OrganizeListener, StoreWatchListener from media.monitor.organizer import Organizer from media.monitor.events import PathChannel from media.monitor.watchersyncer import WatchSyncer from media.monitor.handler import ProblemFileHandler -#from media.monitor.bootstrap import Bootstrapper +from media.monitor.bootstrap import Bootstrapper +from media.monitor.log import get_logger +from media.monitor.syncdb import SyncDB +from api_clients import api_client as apc channels = { # note that org channel still has a 'watch' path because that is the path @@ -21,6 +25,14 @@ org = Organizer(channel=channels['org'],target_path=channels['watch'].path) watch = WatchSyncer(channel=channels['watch']) problem_files = ProblemFileHandler(channel=channels['badfile']) +apiclient = apc.AirtimeApiClient(get_logger()) +raw_bootstrap = apiclient.get_bootstrap_info() +print(raw_bootstrap) +bs = Bootstrapper(db=bootstrap_db, last_run=int(time.time()), org_channels=[channels['org']], watch_channels=[channels['watch']]) + +bs.flush_organize() +bs.flush_watch() + # do the bootstrapping before any listening is going one #conn = Connection('localhost', 'more', 'shit', 'here') #db = DBDumper(conn).dump_block()