cc-4105: Extended api of SyncDB and improved the launcher script

python_apps/api_clients/api_client.py

@@ -426,11 +426,8 @@ class AirtimeApiClient():
     def list_all_db_files(self, dir_id):
         logger = self.logger
         try:
-            url = "http://%s:%s/%s/%s" % (self.config["base_url"], str(self.config["base_port"]), self.config["api_base"], self.config["list_all_db_files"])
+            url = self.construct_url("list_all_db_files")
-
-            url = url.replace("%%api_key%%", self.config["api_key"])
             url = url.replace("%%dir_id%%", dir_id)
-
             response = self.get_response_from_server(url)
             response = json.loads(response)
         except Exception, e:

python_apps/media-monitor2/media/monitor/pure.py

@@ -44,6 +44,11 @@ class IncludeOnly(object):
             if ext in self.exts: func(moi, event, *args, **kwargs)
         return _wrap
 
+
+def partition(f, alist):
+    # TODO : document this function and add doctests
+    return (filter(f, alist), filter(lambda x: not f(x), alist))
+
 def is_file_supported(path):
     # TODO : test and document this function
     return extension(path) in supported_extensions

python_apps/media-monitor2/media/monitor/syncdb.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import os
 from media.monitor.log import Loggable
 
 class SyncDB(Loggable):
@@ -15,12 +16,27 @@ class SyncDB(Loggable):
         # self.directories is a dictionary where a key is the directory and the
         # value is the directory's id in the db
         self.directories = dict( (v,k) for k,v in directories.iteritems() )
+        # Just in case anybody wants to lookup a directory by its id we haev
+        self.id_lookup = directories
 
     def list_directories(self):
+        """
+        returns a list of all the watched directories in the datatabase.
+        """
         return self.directories.keys()
 
     def directory_get_files(self, directory):
-        print("trying to access dir id: %s" % self.directories[directory])
+        """
-        self.apc.list_all_db_files(self.directories[directory])
+        returns all the files(recursively) in a directory. a directory is an "actual" directory
+        path instead of it's id.
+        """
+        return [ os.path.normpath(os.path.join(directory,f)) \
+                for f in self.apc.list_all_db_files(self.directories[directory]) ]
+
+    def id_get_files(self, dir_id):
+        """
+        returns all the files given some dir_id. this method is here for "symmetry". it's not actually used anywhere
+        """
+        return self.directory_get_files(self.id_get_files[dir_id])
 
 

python_apps/media-monitor2/mm2.py

@@ -2,6 +2,7 @@
 # testing ground for the script
 import pyinotify
 import time
+import os
 from media.monitor.listeners import OrganizeListener, StoreWatchListener
 from media.monitor.organizer import Organizer
 from media.monitor.events import PathChannel
@@ -12,23 +13,45 @@ from media.monitor.log import get_logger
 from media.monitor.syncdb import SyncDB
 from api_clients import api_client as apc
 
+# TODO : we should configure locale before doing anything here
+
 channels = {
     # note that org channel still has a 'watch' path because that is the path
     # it supposed to be moving the organized files to. it doesn't matter where
     # are all the "to organize" files are coming from
     'org' : PathChannel('org', '/home/rudi/throwaway/fucking_around/organize'),
-    'watch' : PathChannel('watch', '/home/rudi/throwaway/fucking_around/watch'),
+    'watch' : [],
     'badfile' : PathChannel('badfile', '/home/rudi/throwaway/fucking_around/problem_dir'),
 }
 
+log = get_logger()
+apiclient = apc.AirtimeApiClient(log)
+# We initialize sdb before anything because we must know what our watched
+# directories are.
+sdb = SyncDB(apc)
+for watch_dir in sdb.list_directories():
+    if not os.path.exists(watch_dir):
+        # Create the watch_directory here
+        try: os.makedirs(watch_dir)
+        except Exception as e:
+            log.error("Could not create watch directory: '%s' (given from the database)." % watch_dir)
+    # We must do another existence check for the watched directory because we
+    # the creation of it could have failed above
+    if os.path.exists(watch_dir):
+        channels['watch'].append(PathChannel('watch', watch_dir))
+
 org = Organizer(channel=channels['org'],target_path=channels['watch'].path)
-watch = WatchSyncer(channel=channels['watch'])
+watches = [ WatchSyncer(channel=pc) for pc in channels['watch'] ]
 problem_files = ProblemFileHandler(channel=channels['badfile'])
 
-apiclient = apc.AirtimeApiClient(get_logger())
 raw_bootstrap = apiclient.get_bootstrap_info()
 print(raw_bootstrap)
-bs = Bootstrapper(db=bootstrap_db, last_run=int(time.time()), org_channels=[channels['org']], watch_channels=[channels['watch']])
+# A slight incosistency here, channels['watch'] is already a list while the
+# other items are single elements. For consistency we should make all the
+# values in channels lists later on
+# TODO : get the actual last running time instead of using the current time
+# like now
+bs = Bootstrapper(db=sdb, last_run=int(time.time()), org_channels=[channels['org']], watch_channels=channels['watch'])
 
 bs.flush_organize()
 bs.flush_watch()
@@ -49,7 +72,8 @@ o2 = StoreWatchListener(signal=channels['watch'].signal)
 
 notifier = pyinotify.Notifier(wm)
 wdd1 = wm.add_watch(channels['org'].path, pyinotify.ALL_EVENTS, rec=True, auto_add=True, proc_fun=o1)
-wdd2 = wm.add_watch(channels['watch'].path, pyinotify.ALL_EVENTS, rec=True, auto_add=True, proc_fun=o2)
+for pc in channels['watch']:
+    wdd2 = wm.add_watch(pc.path, pyinotify.ALL_EVENTS, rec=True, auto_add=True, proc_fun=o2)
 
 notifier.loop()