SAAS-818: If the network connection is interrupted, pypo sometimes can't recover

python_apps/pypo/pypofetch.py

@@ -477,6 +477,7 @@ class PypoFetch(Thread):
         loops = 1
         while True:
             self.logger.info("Loop #%s", loops)
+            manual_fetch_needed = False
             try:
                 """
                 our simple_queue.get() requires a timeout, in which case we
@@ -492,17 +493,26 @@ class PypoFetch(Thread):
                 Currently we are checking every POLL_INTERVAL seconds
                 """
 
-
                 message = self.fetch_queue.get(block=True, timeout=self.listener_timeout)
+                manual_fetch_needed = False
                 self.handle_message(message)
-            except Empty, e:
+            except Empty as e:
                 self.logger.info("Queue timeout. Fetching schedule manually")
-                self.persistent_manual_schedule_fetch(max_attempts=5)
-            except Exception, e:
+                manual_fetch_needed = True
+            except Exception as e:
                 top = traceback.format_exc()
                 self.logger.error('Exception: %s', e)
                 self.logger.error("traceback: %s", top)
 
+            try:
+                if manual_fetch_needed:
+                    self.persistent_manual_schedule_fetch(max_attempts=5)
+            except Exception as e:
+                top = traceback.format_exc()
+                self.logger.error('Failed to manually fetch the schedule.')
+                self.logger.error('Exception: %s', e)
+                self.logger.error("traceback: %s", top)
+
             loops += 1
 
     def run(self):
@@ -510,3 +520,4 @@ class PypoFetch(Thread):
         Entry point of the thread
         """
         self.main()
+        self.logger.info('PypoFetch thread exiting')

python_apps/pypo/pypofile.py

@@ -35,6 +35,7 @@ class PypoFile(Thread):
         self.media_queue = schedule_queue
         self.media = None
         self.cache_dir = os.path.join(config["cache_dir"], "scheduler")
+        self._config = self.read_config_file(CONFIG_PATH)
 
     def copy_file(self, media_item):
         """
@@ -65,11 +66,9 @@ class PypoFile(Thread):
         if do_copy:
             self.logger.debug("copying from %s to local cache %s" % (src, dst))
             try:
-                config = self.read_config_file(CONFIG_PATH)
                 CONFIG_SECTION = "general"
-                username = config.get(CONFIG_SECTION, 'api_key')
-
-                host = config.get(CONFIG_SECTION, 'base_url')
+                username = self._config.get(CONFIG_SECTION, 'api_key')
+                host = self._config.get(CONFIG_SECTION, 'base_url')
                 url = "http://%s/rest/media/%s/download" % (host, media_item["id"])
                 with open(dst, "wb") as handle:
                     response = requests.get(url, auth=requests.auth.HTTPBasicAuth(username, ''), stream=True, verify=False)
@@ -210,3 +209,4 @@ class PypoFile(Thread):
         Entry point of the thread
         """
         self.main()
+