From 081ca0a6a455621c91f4de43a808d70bd77b5517 Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Fri, 10 Oct 2014 11:28:44 -0400 Subject: [PATCH 1/6] Reworked upstart config for airtime_analyzer --- .../install/upstart/airtime_analyzer.conf | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/python_apps/airtime_analyzer/install/upstart/airtime_analyzer.conf b/python_apps/airtime_analyzer/install/upstart/airtime_analyzer.conf index 696c8b9a9..eeeb45797 100644 --- a/python_apps/airtime_analyzer/install/upstart/airtime_analyzer.conf +++ b/python_apps/airtime_analyzer/install/upstart/airtime_analyzer.conf @@ -9,14 +9,16 @@ respawn setuid www-data setgid www-data -expect fork +#expect fork env LANG='en_US.UTF-8' env LC_ALL='en_US.UTF-8' -script - airtime_analyzer -end script +#script +# airtime_analyzer +#end script + +exec airtime_analyzer From 28be5c6bd3e33aa007a2fbfc8f53ef6bc875e98f Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Tue, 21 Oct 2014 15:34:10 -0400 Subject: [PATCH 2/6] CC-5929: Certain long filenames can result in the extension getting cut off --- .../airtime_analyzer/filemover_analyzer.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py b/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py index de296e092..79c288124 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/filemover_analyzer.py @@ -42,14 +42,16 @@ class FileMoverAnalyzer(Analyzer): # TODO: Also, handle the case where the move fails and write some code # to possibly move the file to problem_files. - max_dir_len = 32 - max_file_len = 32 + max_dir_len = 48 + max_file_len = 48 final_file_path = import_directory + orig_file_basename, orig_file_extension = os.path.splitext(original_filename) if metadata.has_key("artist_name"): final_file_path += "/" + metadata["artist_name"][0:max_dir_len] # truncating with array slicing if metadata.has_key("album_title"): - final_file_path += "/" + metadata["album_title"][0:max_dir_len] - final_file_path += "/" + original_filename[0:max_file_len] + final_file_path += "/" + metadata["album_title"][0:max_dir_len] + # Note that orig_file_extension includes the "." already + final_file_path += "/" + orig_file_basename[0:max_file_len] + orig_file_extension #Ensure any redundant slashes are stripped final_file_path = os.path.normpath(final_file_path) From 82f251f061ed27c99a7b88dddf9c7fd46c63592c Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Tue, 21 Oct 2014 19:23:48 -0400 Subject: [PATCH 3/6] Fix invalid python in StatusReporter --- .../airtime_analyzer/airtime_analyzer/status_reporter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py b/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py index 7dcfa5d44..ade3c5bdb 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py @@ -134,11 +134,11 @@ def is_web_server_broken(url): test_req = requests.get(url) test_req.raise_for_status() except Exception as e: - return true + return True else: # The request worked fine, so the web server and Airtime are still up. - return false - return false + return False + return False def alert_hung_request(): From 54523e264cb194937c71fa4d1121f7f7ed4dd8cc Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Wed, 22 Oct 2014 11:38:22 -0400 Subject: [PATCH 4/6] Fix rare exception during shutdown in airtime_analyzer --- .../airtime_analyzer/message_listener.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py b/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py index 97613e81f..9b890321c 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/message_listener.py @@ -120,8 +120,13 @@ class MessageListener: def disconnect_from_messaging_server(self): '''Stop consuming RabbitMQ messages and disconnect''' - self._channel.stop_consuming() - self._connection.close() + # If you try to close a connection that's already closed, you're going to have a bad time. + # We're breaking EAFP because this can be called multiple times depending on exception + # handling flow here. + if not self._channel.is_closed and not self._channel.is_closing: + self._channel.stop_consuming() + if not self._connection.is_closed and not self._connection.is_closing: + self._connection.close() def graceful_shutdown(self, signum, frame): '''Disconnect and break out of the message listening loop''' From 53dc92b2041b1287735b3b41da250e84348eaf12 Mon Sep 17 00:00:00 2001 From: Albert Santoni Date: Wed, 22 Oct 2014 11:39:22 -0400 Subject: [PATCH 5/6] StatusReporter exception handler to prevent thread from dying under any circumstances --- .../airtime_analyzer/status_reporter.py | 65 +++++++++++-------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py b/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py index ade3c5bdb..ebf9a12d5 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py @@ -38,9 +38,9 @@ def process_http_requests(ipc_queue, http_retry_queue_path): # retried later: retry_queue = collections.deque() shutdown = False - - # Unpickle retry_queue from disk so that we won't have lost any uploads - # if airtime_analyzer is shut down while the web server is down or unreachable, + + # Unpickle retry_queue from disk so that we won't have lost any uploads + # if airtime_analyzer is shut down while the web server is down or unreachable, # and there were failed HTTP requests pending, waiting to be retried. try: with open(http_retry_queue_path, 'rb') as pickle_file: @@ -57,33 +57,42 @@ def process_http_requests(ipc_queue, http_retry_queue_path): logging.error("Failed to unpickle %s. Continuing..." % http_retry_queue_path) pass - - while not shutdown: + while True: try: - request = ipc_queue.get(block=True, timeout=5) - if isinstance(request, str) and request == "shutdown": # Bit of a cheat - shutdown = True - break - if not isinstance(request, PicklableHttpRequest): - raise TypeError("request must be a PicklableHttpRequest. Was of type " + type(request).__name__) - except Queue.Empty: - request = None - - # If there's no new HTTP request we need to execute, let's check our "retry - # queue" and see if there's any failed HTTP requests we can retry: - if request: - send_http_request(request, retry_queue) - else: - # Using a for loop instead of while so we only iterate over all the requests once! - for i in range(len(retry_queue)): - request = retry_queue.popleft() - send_http_request(request, retry_queue) + while not shutdown: + try: + request = ipc_queue.get(block=True, timeout=5) + if isinstance(request, str) and request == "shutdown": # Bit of a cheat + shutdown = True + break + if not isinstance(request, PicklableHttpRequest): + raise TypeError("request must be a PicklableHttpRequest. Was of type " + type(request).__name__) + except Queue.Empty: + request = None + + # If there's no new HTTP request we need to execute, let's check our "retry + # queue" and see if there's any failed HTTP requests we can retry: + if request: + send_http_request(request, retry_queue) + else: + # Using a for loop instead of while so we only iterate over all the requests once! + for i in range(len(retry_queue)): + request = retry_queue.popleft() + send_http_request(request, retry_queue) + + logging.info("Shutting down status_reporter") + # Pickle retry_queue to disk so that we don't lose uploads if we're shut down while + # while the web server is down or unreachable. + with open(http_retry_queue_path, 'wb') as pickle_file: + pickle.dump(retry_queue, pickle_file) + except Exception as e: # Terrible top-level exception handler to prevent the thread from dying, just in case. + if shutdown: + return + logging.exception("Unhandled exception in StatusReporter") + logging.exception(e) + logging.info("Restarting StatusReporter thread") + time.sleep(2) # Throttle it - logging.info("Shutting down status_reporter") - # Pickle retry_queue to disk so that we don't lose uploads if we're shut down while - # while the web server is down or unreachable. - with open(http_retry_queue_path, 'wb') as pickle_file: - pickle.dump(retry_queue, pickle_file) def send_http_request(picklable_request, retry_queue): if not isinstance(picklable_request, PicklableHttpRequest): From c02ed026f4a334c327cd5be59dc83f4f05f22b57 Mon Sep 17 00:00:00 2001 From: Robert Elder Date: Fri, 24 Oct 2014 03:57:35 +0000 Subject: [PATCH 6/6] Support for tokens in multipart file upload using API. --- .../rest/controllers/MediaController.php | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index dd0497745..1483de816 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -129,6 +129,15 @@ class Rest_MediaController extends Zend_Rest_Controller public function postAction() { + /* If the user presents a valid API key, we don't check CSRF tokens. + CSRF tokens are only used for session based authentication. + */ + if(!$this->verifyAPIKey()){ + if(!$this->verifyCSRFToken($this->_getParam('csrf_token'))){ + return; + } + } + if (!$this->verifyAuth(true, true)) { return; @@ -294,6 +303,21 @@ class Rest_MediaController extends Zend_Rest_Controller } return $id; } + + private function verifyCSRFToken($token){ + $current_namespace = new Zend_Session_Namespace('csrf_namespace'); + $observed_csrf_token = $token; + $expected_csrf_token = $current_namespace->authtoken; + + if($observed_csrf_token == $expected_csrf_token){ + return true; + }else{ + $resp = $this->getResponse(); + $resp->setHttpResponseCode(401); + $resp->appendBody("ERROR: Token Missmatch."); + return false; + } + } private function verifyAuth($checkApiKey, $checkSession) {