CC-3372: Improved pypo cache delete

-Done

airtime_mvc/application/models/Schedule.php

@@ -400,29 +400,22 @@ class Application_Model_Schedule {
     }
 
     /**
-     * Returns array indexed by:
+     * Returns an array of schedule items from cc_schedule table. Tries
-     *    "playlistId"/"playlist_id" (aliases to the same thing)
+     * to return at least 3 items (if they are available). The parameters
-     *    "start"/"starts" (aliases to the same thing) as YYYY-MM-DD HH:MM:SS.nnnnnn
+     * $p_startTime and $p_endTime specify the range. Schedule items returned
-     *    "end"/"ends" (aliases to the same thing) as YYYY-MM-DD HH:MM:SS.nnnnnn
+     * do not have to be entirely within this range. It is enough that the end
-     *    "group_id"/"id" (aliases to the same thing)
+     * or beginning of the scheduled item is in the range.
-     *    "clip_length" (for audio clips this is the length of the audio clip,
+     * 
-     *                   for playlists this is the length of the entire playlist)
-     *    "name" (playlist only)
-     *    "creator" (playlist only)
-     *    "file_id" (audioclip only)
-     *    "count" (number of items in the playlist, always 1 for audioclips.
-     *      Note that playlists with one item will also have count = 1.
      *
-     * @param string $p_fromDateTime
+     * @param string $p_startTime
      *    In the format YYYY-MM-DD HH:MM:SS.nnnnnn
-     * @param string $p_toDateTime
+     * @param string $p_endTime
      *    In the format YYYY-MM-DD HH:MM:SS.nnnnnn
-     * @param boolean $p_playlistsOnly
-     *    Retrieve playlists as a single item.
      * @return array
-     *      Returns null if nothing found
+     *    Returns null if nothing found, else an array of associative
+     *    arrays representing each row.
      */
-    public static function GetItems($p_currentDateTime, $p_toDateTime) {
+    public static function GetItems($p_startTime, $p_endTime) {
         global $CC_CONFIG, $CC_DBC;
         
         $baseQuery = "SELECT st.file_id AS file_id,"
@@ -440,8 +433,8 @@ class Application_Model_Schedule {
             ." ON st.instance_id = si.id";
    
 
-        $predicates = " WHERE st.ends > '$p_currentDateTime'"
+        $predicates = " WHERE st.ends > '$p_startTime'"
-        ." AND st.starts < '$p_toDateTime'"
+        ." AND st.starts < '$p_endTime'"
         ." ORDER BY st.starts";
         
         $sql = $baseQuery.$predicates;
@@ -458,7 +451,7 @@ class Application_Model_Schedule {
             $dt->add(new DateInterval("PT30M"));
             $range_end = $dt->format("Y-m-d H:i:s");
                       
-            $predicates = " WHERE st.ends > '$p_currentDateTime'"
+            $predicates = " WHERE st.ends > '$p_startTime'"
             ." AND st.starts < '$range_end'"
             ." ORDER BY st.starts"
             ." LIMIT 3";

python_apps/pypo/pypofetch.py

@@ -315,12 +315,10 @@ class PypoFetch(Thread):
         self.logger.debug("Pushing to pypo-push")
         self.push_queue.put(media)
 
-        """
+
-        TODO
         # cleanup
-        try: self.cleanup()
+        try: self.cleanup(media)
         except Exception, e: self.logger.error("%s", e)
-        """
 
         
 
@@ -446,31 +444,27 @@ class PypoFetch(Thread):
         else:
             self.logger.debug("try to download %s", media_item['uri'])
             self.api_client.get_media(media_item['uri'], dst)
-    
+            
-    """
+    def cleanup(self, media):
-    Cleans up folders in cache_dir. Look for modification date older than "now - CACHE_FOR"
+        """
-    and deletes them.
+        Get list of all files in the cache dir and remove them if they aren't being used anymore.
-    """
+        Input dict() media, lists all files that are scheduled or currently playing. Not being in this
-    def cleanup(self):
+        dict() means the file is safe to remove. 
-        offset = 3600 * int(config["cache_for"])
+        """
-        now = time.time()
+        cached_file_set = set(os.listdir(self.cache_dir))
-
+        scheduled_file_set = set()
-        for r, d, f in os.walk(self.cache_dir):
+        
-            for dir in d:
+        for mkey in media:
-                try:
+            media_item = media[mkey]
-                    timestamp = calendar.timegm(time.strptime(dir, "%Y-%m-%d-%H-%M-%S"))
+            fileExt = os.path.splitext(media_item['uri'])[1]
-                    if (now - timestamp) > offset:
+            scheduled_file_set.add(media_item["id"] + fileExt)
-                        try:
+        
-                            self.logger.debug('trying to remove  %s - timestamp: %s', os.path.join(r, dir), timestamp)
+        unneeded_files = cached_file_set - scheduled_file_set
-                            shutil.rmtree(os.path.join(r, dir))
+        
-                        except Exception, e:
+        self.logger.debug("Files to remove " + str(unneeded_files))
-                            self.logger.error("%s", e)
+        for file in unneeded_files:
-                            pass
+            self.logger.debug("Removing %s" % os.path.join(self.cache_dir, file))
-                        else:
+            os.remove(os.path.join(self.cache_dir, file))
-                            self.logger.info('sucessfully removed %s', os.path.join(r, dir))
-                except Exception, e:
-                    self.logger.error(e)
-
 
     def main(self):
         # Bootstrap: since we are just starting up, we need to grab the

python_apps/pypo/pypopush.py

@@ -70,7 +70,7 @@ class PypoPush(Thread):
             str_tnow_s = "%04d-%02d-%02d-%02d-%02d-%02d" % (tnow[0], tnow[1], tnow[2], tnow[3], tnow[4], tnow[5])
             str_tcoming_s = "%04d-%02d-%02d-%02d-%02d-%02d" % (tcoming[0], tcoming[1], tcoming[2], tcoming[3], tcoming[4], tcoming[5])
                         
-            for key in media:
+            for key in media.keys():
                 media_item = media[key]
                 item_start = media_item['start'][0:19]