cc-4105: refactored how watched, stored directories are handled

python_apps/media-monitor2/media/monitor/airtime.py

@@ -9,7 +9,7 @@ import copy
 from media.monitor.exceptions import BadSongFile
 from media.monitor.metadata import Metadata
 from media.monitor.log import Loggable
-from media.monitor.syncdb import SyncDB
+from media.monitor.syncdb import AirtimeDB
 from media.monitor.exceptions import DirectoryIsNotListed
 from media.monitor.bootstrap import Bootstrapper
 from media.monitor.listeners import FileMediator
@@ -92,7 +92,7 @@ class AirtimeMessageReceiver(Loggable):
 
 
     def __request_now_bootstrap(self, directory_id=None, directory=None):
-        sdb = SyncDB(apc.AirtimeApiClient.create_right_config())
+        sdb = AirtimeDB(apc.AirtimeApiClient.create_right_config())
         if directory_id == None: directory_id = sdb.directories[directory]
         if directory_id in sdb.id_lookup:
             d = sdb.id_lookup[directory_id]

python_apps/media-monitor2/media/monitor/bootstrap.py

@@ -11,7 +11,7 @@ class Bootstrapper(Loggable):
     """
     def __init__(self,db,watch_signal):
         """
-        db - SyncDB object; small layer over api client
+        db - AirtimeDB object; small layer over api client
         last_ran - last time the program was ran.
         watch_signal - the signals should send events for every file on.
         """
@@ -24,7 +24,7 @@ class Bootstrapper(Loggable):
         note that because of the way list_directories works we also flush
         the import directory as well I think
         """
-        for d in self.db.list_directories():
+        for d in self.db.list_storable_paths():
             self.flush_watch(d, last_ran)
 
     def flush_watch(self, directory, last_ran):

python_apps/media-monitor2/media/monitor/exceptions.py

@@ -38,3 +38,10 @@ class FailedToCreateDir(Exception):
         self.path = path
         self.parent = parent
     def __str__(self): return "Failed to create path '%s'" % self.path
+
+class NoDirectoryInAirtime(Exception):
+    def __init__(self,path, does_exist):
+        self.path = path
+        self.does_exist = does_exist
+    def __str__(self):
+        return "Directory '%s' does not exist in Airtime.\nHowever: %s do exist." % (self.path, self.does_exist)

python_apps/media-monitor2/media/monitor/manager.py

@@ -144,6 +144,8 @@ class Manager(Loggable):
         block until we receive pyinotify events
         """
         pyinotify.Notifier(self.wm).loop()
+        # Experiments with running notifier in different modes
+        # There are 3 options: normal, async, threaded.
         #import asyncore
         #pyinotify.AsyncNotifier(self.wm).loop()
         #asyncore.loop()

python_apps/media-monitor2/media/monitor/syncdb.py

@@ -1,57 +1,86 @@
 # -*- coding: utf-8 -*-
 import os
 from media.monitor.log import Loggable
+from media.monitor.exceptions import NoDirectoryInAirtime
+from os.path import normpath
 
-class SyncDB(Loggable):
+class AirtimeDB(Loggable):
-    def __init__(self, apc):
+    def __init__(self, apc, reload_now=True):
         self.apc = apc
-        dirs = self.apc.list_all_watched_dirs()
+        if reload_now: self.reload_directories()
-
-        directories = None
-        try:
-            directories = dirs['dirs']
-        except KeyError as e:
-            self.logger.error("Could not find index 'dirs' in dictionary: %s", str(dirs))
-            self.logger.error(e)
-            raise
-        # self.directories is a dictionary where a key is the directory and the
-        # value is the directory's id in the db
-        self.directories = dict( (v,k) for k,v in directories.iteritems() )
-        # Just in case anybody wants to lookup a directory by its id we haev
-        self.id_lookup = directories
 
     def reload_directories(self):
+        """
+        this is the 'real' constructor, should be called if you ever want the class reinitialized.
+        there's not much point to doing it yourself however, you should just create a new AirtimeDB
+        instance.
+        """
         # dirs_setup is a dict with keys:
         # u'watched_dirs' and u'stor' which point to lists of corresponding
         # dirs
         dirs_setup = self.apc.setup_media_monitor()
-        self.base_storage = dirs_setup[u'stor']
+        dirs_setup[u'stor'] = normpath( dirs_setup[u'stor'] )
-        self.watched_directories = set(dirs_setup[u'watched_dirs'])
+        dirs_setup[u'watched_dirs'] = map(normpath, dirs_setup[u'watched_dirs'] )
+        dirs_with_id = dict([ (k,normpath(v)) for k,v in self.apc.list_all_watched_dirs()['dirs'].iteritems() ])
 
+        self.id_to_dir = dirs_with_id
+        self.dir_to_id = dict([ (v,k) for k,v in dirs_with_id.iteritems() ])
+
+        self.base_storage = dirs_setup[u'stor']
+        self.base_id = self.dir_to_id[self.base_storage]
+
+        # hack to get around annoying schema of airtime db
+        self.dir_to_id[ self.recorded_path() ] = self.base_id
+        self.dir_to_id[ self.import_path() ] = self.base_id
+
+        # We don't know from the x_to_y dict which directory is watched or
+        # store...
+        self.watched_directories = set([ os.path.normpath(p) for p in dirs_setup[u'watched_dirs'] ])
+
+    def storage_path(self): return self.base_storage
     def organize_path(self): return os.path.join(self.base_storage, 'organize')
     def problem_path(self): return os.path.join(self.base_storage, 'problem_files')
     def import_path(self): return os.path.join(self.base_storage, 'imported')
     def recorded_path(self): return os.path.join(self.base_storage, 'recorded')
 
-    def list_directories(self):
+    def list_watched(self):
+        """
+        returns all watched directories as a list
+        """
+        return list(self.watched_directories)
+
+    def list_storable_paths(self):
         """
         returns a list of all the watched directories in the datatabase.
-        (Includes the imported directory)
+        (Includes the imported directory and the recorded directory)
         """
-        return self.directories.keys()
+        l = self.list_watched()
+        l.append(self.import_path())
+        l.append(self.recorded_path())
+        return l
+
+    def dir_id_get_files(self, dir_id):
+        base_dir = self.id_to_dir[ dir_id ]
+        return set(( os.path.join(base_dir,p) for p in self.apc.list_all_db_files( dir_id ) ))
 
     def directory_get_files(self, directory):
         """
         returns all the files(recursively) in a directory. a directory is an "actual" directory
-        path instead of its id.
+        path instead of its id. This is super hacky because you create one request for the
+        recorded directory and one for the imported directory even though they're the same dir
+        in the database so you get files for both dirs in 1 request...
         """
-        return set( [ os.path.normpath(os.path.join(directory,f)) \
+        normal_dir = os.path.normpath(unicode(directory))
-                for f in self.apc.list_all_db_files(self.directories[directory]) ] )
+        if normal_dir not in self.dir_to_id:
-
+            raise NoDirectoryInAirtime( normal_dir, self.dir_to_id )
-    def id_get_files(self, dir_id):
+        all_files = self.dir_id_get_files( self.dir_to_id[normal_dir] )
-        """
+        if normal_dir == self.recorded_path():
-        returns all the files given some dir_id. this method is here for "symmetry". it's not actually used anywhere
+            all_files = [ p for p in all_files if len(os.path.commonprefix([ self.recorded_path(), p ])) > 0 ]
-        """
+        elif normal_dir == self.import_path():
-        return self.directory_get_files(self.id_lookup[dir_id])
+            all_files = [ p for p in all_files if len(os.path.commonprefix([ self.import_path(), p ])) > 0 ]
+        elif normal_dir == self.storage_path():
+            self.logger.info("Warning, you're getting all files in '%s' which includes imported + record"
+                    % normal_dir)
+        return set(all_files)
 
 

python_apps/media-monitor2/mm2.py

@@ -8,7 +8,7 @@ from media.monitor.log import get_logger
 from media.monitor.events import PathChannel
 from media.monitor.config import MMConfig
 from media.monitor.toucher import ToucherThread
-from media.monitor.syncdb import SyncDB
+from media.monitor.syncdb import AirtimeDB
 from media.monitor.exceptions import FailedToObtainLocale, FailedToSetLocale, NoConfigFile, FailedToCreateDir
 from media.monitor.airtime import AirtimeNotifier, AirtimeMessageReceiver
 from media.monitor.watchersyncer import WatchSyncer
@@ -60,7 +60,7 @@ apiclient = apc.AirtimeApiClient.create_right_config(log=log,config_path=global_
 
 # TODO : Need to do setup_media_monitor call somewhere around here to get
 # import/organize dirs
-sdb = SyncDB(apiclient)
+sdb = AirtimeDB(apiclient)
 
 manager = Manager()
 

python_apps/media-monitor2/tests/test_syncdb.py

@@ -1,31 +1,32 @@
 # -*- coding: utf-8 -*-
 import unittest
 import os
-from media.monitor.syncdb import SyncDB
+from media.monitor.syncdb import AirtimeDB
 from media.monitor.log import get_logger
 from media.monitor.pure import partition
 import api_clients.api_client as ac
 import prepare_tests
 
-class TestSyncDB(unittest.TestCase):
+class TestAirtimeDB(unittest.TestCase):
     def setUp(self):
         self.ac = ac.AirtimeApiClient(logger=get_logger(),
                                       config_path=prepare_tests.real_config)
 
     def test_syncdb_init(self):
-        sdb = SyncDB(self.ac)
+        sdb = AirtimeDB(self.ac)
-        self.assertTrue( len(sdb.directories.keys()) > 0 )
+        self.assertTrue( len(sdb.list_storable_paths()) > 0 )
 
     def test_list(self):
-        self.sdb = SyncDB(self.ac)
+        self.sdb = AirtimeDB(self.ac)
-        for watch_dir in self.sdb.list_directories():
+        for watch_dir in self.sdb.list_storable_paths():
             self.assertTrue( os.path.exists(watch_dir) )
 
     def test_directory_get_files(self):
-        sdb = SyncDB(self.ac)
+        sdb = AirtimeDB(self.ac)
-        print(sdb.directories)
+        print(sdb.list_storable_paths())
-        for wdir in sdb.list_directories():
+        for wdir in sdb.list_storable_paths():
             files = sdb.directory_get_files(wdir)
+            print( "total files: %d" % len(files) )
             self.assertTrue( len(files) >= 0 )
             self.assertTrue( isinstance(files, set) )
             exist, deleted = partition(os.path.exists, files)