Merge branch 'saas' into saas-cloud-storage-config-per-dev-env

Conflicts:

python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py

python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py

python_apps/airtime_analyzer/airtime_analyzer/message_listener.py

airtime_mvc/application/configs/constants.php

@@ -86,3 +86,5 @@ define('SOUNDCLOUD_ERROR'            , -3);
 define("WHMCS_API_URL", "https://account.sourcefabric.com/includes/api.php");
 define("SUBDOMAIN_WHMCS_CUSTOM_FIELD_NAME", "Choose your domain");
 
+//Sentry error logging
+define('SENTRY_CONFIG_PATH', '/etc/airtime-saas/sentry.airtime_web.ini');

airtime_mvc/application/controllers/ApiController.php

@@ -115,7 +115,7 @@ class ApiController extends Zend_Controller_Action
                     header('Content-Disposition: inline; filename="'.$filename.'"');
                 }
 
-                $this->smartReadFile($media);
+                $this->readStoredFileObject($media);
                 exit;
             } else {
                 header ("HTTP/1.1 404 Not Found");
@@ -124,29 +124,53 @@ class ApiController extends Zend_Controller_Action
 
         $this->_helper->json->sendJson(array());
     }
+    
+    /**
+     * Read data from StoredFile object and send with XHR response
+     * 
+     * @param Application_Model_StoredFile $storedFile - StoredFile object holding file information
+     */
+    private function readStoredFileObject($storedFile) {
+        $filepath = $storedFile->getFilePath();
+        $size = $storedFile->getFileSize();
+        $mimeType = $storedFile->getPropelOrm()->getDbMime();
+        
+        $this->smartReadFile($filepath, $mimeType, $size);
+    }
+    
 
     /**
     * Reads the requested portion of a file and sends its contents to the client with the appropriate headers.
     *
     * This HTTP_RANGE compatible read file function is necessary for allowing streaming media to be skipped around in.
     *
-    * @param string $location
+    * @param string $location - the full filepath pointing to the location of the file
-    * @param string $mimeType
+    * @param string $mimeType - the file's mime type. Defaults to 'audio/mp3'
+    * @param integer $size - the file size, in bytes
     * @return void
     *
     * @link https://groups.google.com/d/msg/jplayer/nSM2UmnSKKA/Hu76jDZS4xcJ
     * @link http://php.net/manual/en/function.readfile.php#86244
     */
-    public function smartReadFile($media)
+    private function smartReadFile($location, $mimeType = 'audio/mp3', $size = null)
     {
-        $filepath = $media->getFilePath();
+        if (!$location || $location == "") {
-        $size= $media->getFileSize();
+            throw new FileDoesNotExistException("Requested file does not exist!");
-        $mimeType = $media->getPropelOrm()->getDbMime();
+        }
-
+        
-        $fm = @fopen($filepath, 'rb');
+        // If we're passing in a Stored File object, it's faster 
+        // to use getFileSize() and pass in the result
+        if (!$size || $size <= 0) {
+            $size= filesize($location);
+        }
+        
+        if ($size <= 0) {
+            throw new Exception("Invalid file size returned for file at $location");
+        }
+        
+        $fm = @fopen($location, 'rb');
         if (!$fm) {
             header ("HTTP/1.1 505 Internal server error");
-
             return;
         }
 

airtime_mvc/application/controllers/ShowbuilderController.php

@@ -299,7 +299,7 @@ class ShowbuilderController extends Zend_Controller_Action
         $log_vars["params"]["media_items"] = $mediaItems;
         $log_vars["params"]["scheduled_items"] = $scheduledItems;
         Logging::info($log_vars);
-
+        
         try {
             $scheduler = new Application_Model_Scheduler();
             $scheduler->scheduleAfter($scheduledItems, $mediaItems);

airtime_mvc/application/logging/Logging.php

@@ -1,4 +1,5 @@
 <?php
+require_once('SentryLogging.php');
 
 class Logging {
 
@@ -82,6 +83,7 @@ class Logging {
     {
         $logger = self::getLogger();
         $logger->err(self::getLinePrefix(true) .  self::toString($p_msg));
+        SentryLogger::getInstance()->captureError(self::toString($p_msg));
     }
     
     public static function debug($p_msg)
@@ -123,4 +125,49 @@ class Logging {
         Propel::setLogger(null);
     }
 
+    public static function loggingShutdownCallback()
+    {
+        //Catch the types of errors that PHP doesn't normally let us catch and
+        //would otherwise log to the apache log. We route these to our Airtime log to improve the modularity
+        //and reliability of our error logging. (All errors are in one spot!)
+        $err = error_get_last();
+        if (!is_array($err) || !array_key_exists('type', $err)) {
+            return;
+        }
+
+        switch($err['type'])
+        {
+            case E_ERROR:
+            case E_PARSE:
+            case E_CORE_ERROR:
+            case E_CORE_WARNING:
+            case E_COMPILE_ERROR:
+            case E_COMPILE_WARNING:
+                //error_log("Oh noes, a fatal: " . var_export($err, true), 1, 'fatals@example.com');
+                $errorStr = '';
+                if (array_key_exists('message', $err)) {
+                    $errorStr .= $err['message'];
+                }
+                if (array_key_exists('file', $err))
+                {
+                    $errorStr .= ' at ' .$err['file'];
+                }
+                if (array_key_exists('line', $err))
+                {
+                    $errorStr .= ':' . $err['line'];
+                }
+
+                $errorStr .= "\n" . var_export($err, true);
+                Logging::error($errorStr);
+                break;
+        }
+    }
+
+    public static function setupParseErrorLogging()
+    {
+        //Static callback:
+        register_shutdown_function('Logging::loggingShutdownCallback');
+    }
+
 }
+

airtime_mvc/application/logging/SentryLogging.php

@@ -0,0 +1,131 @@
+<?php
+
+class SentryLogger
+{
+    private static $instance = null;
+    private $sentryClient;
+
+    /** Singleton getter */
+    public static function getInstance()
+    {
+        if (!is_null(self::$instance)) {
+            return self::$instance;
+        } else {
+            self::$instance = new SentryLogger();
+            return self::$instance;
+        }
+    }
+
+    private function __construct()
+    {
+        if (!file_exists(SENTRY_CONFIG_PATH)) {
+            $this->sentryClient = null;
+            return;
+        }
+        // Instantiate a new client with a compatible DSN
+        $sentry_config = parse_ini_file(SENTRY_CONFIG_PATH, false);
+        $dsn = $sentry_config['dsn'];
+        $this->sentryClient = new Raven_Client($dsn,
+            array(
+                //FIXME: This doesn't seem to be working...
+                'processorOptions' => array(
+                    'Raven_SanitizeDataProcessor' => array(
+                        'fields_re' => '/(authorization|password|passwd|user_token|secret|WHMCS_|SESSION)/i',
+                        'values_re' => '/^(?:\d[ -]*?){13,16}$/'
+                    )
+                )
+            ));
+        $client = $this->sentryClient;
+
+        /* The Raven docs suggest not enabling these because they're "too noisy".
+        // Install error handlers and shutdown function to catch fatal errors
+        $error_handler = new Raven_ErrorHandler($client);
+        $error_handler->registerExceptionHandler(true);
+        $error_handler->registerErrorHandler(true);
+        $error_handler->registerShutdownFunction(true);
+        */
+        $error_handler = new Raven_ErrorHandler($client);
+        $error_handler->registerExceptionHandler();
+    }
+
+    public function captureMessage($msg)
+    {
+        if (!$this->sentryClient) {
+            return;
+        }
+        $client = $this->sentryClient;
+        self::addUserData($client);
+        self::addTags($client);
+
+        // Capture a message
+        $event_id = $client->getIdent($client->captureMessage($msg));
+        if ($client->getLastError() !== null) {
+            //printf('There was an error sending the event to Sentry: %s', $client->getLastError());
+        }
+    }
+
+    public function captureException($exception)
+    {
+        if (!$this->sentryClient) {
+            return;
+        }
+
+        $client = $this->sentryClient;
+        self::addUserData($client);
+        self::addTags($client);
+
+        $event_id = $client->getIdent($client->captureException($exception, array(
+            'extra' => $this->getExtraData(),
+            'tags' => $this->getTags(),
+        )));
+        $client->context->clear();
+    }
+
+    public function captureError($errorMessage)
+    {
+        if (!$this->sentryClient) {
+            return;
+        }
+
+        $client = $this->sentryClient;
+
+        // Provide some additional data with an exception
+        self::addUserData($client);
+        self::addTags($client);
+        $event_id = $client->getIdent($client->captureMessage($errorMessage, array(
+            'extra' => $this->getExtraData()
+        )));
+        $client->context->clear();
+    }
+
+    private static function getTags()
+    {
+        $tags = array();
+        $config = Config::getConfig();
+        $tags['Development Environment'] = $config["dev_env"];
+        return $tags;
+    }
+
+    private static function addTags($client)
+    {
+        $client->tags_context(self::getTags());
+    }
+
+    private static function addUserData($client)
+    {
+        $userData = array();
+        $userData['client_id'] = Application_Model_Preference::GetClientId();
+        $userData['station_url'] = array_key_exists('SERVER_NAME', $_SERVER) ? $_SERVER['SERVER_NAME'] : "";
+        $client->user_context($userData);
+    }
+
+    /** Extra data to log with Sentry */
+    private function getExtraData()
+    {
+        $extraData = array();
+        $extraData['php_version'] = phpversion();
+        $extraData['client_id'] = Application_Model_Preference::GetClientId();
+        return $extraData;
+    }
+
+}

airtime_mvc/public/index.php

@@ -79,6 +79,7 @@ $application = new Zend_Application(
 
 require_once (APPLICATION_PATH."/logging/Logging.php");
 Logging::setLogPath('/var/log/airtime/zendphp.log');
+Logging::setupParseErrorLogging();
 
 // Create application, bootstrap, and run
 try {

composer.json

@@ -1,6 +1,7 @@
 {
     "require": {
         "propel/propel1": "1.7.0-stable",
-        "aws/aws-sdk-php": "2.7.9"
+        "aws/aws-sdk-php": "2.7.9",
+        "raven/raven": "0.8.x-dev"
     }
 }

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
         "This file is @generated automatically"
     ],
-    "hash": "311983cf9bb84cfacbfcc6c5dfc9e841",
+    "hash": "30ad5215f679ce0ab55c7210b21a3b32",
     "packages": [
         {
             "name": "aws/aws-sdk-php",
@@ -317,19 +317,71 @@
             ],
             "time": "2013-10-21 12:52:56"
         },
+        {
+            "name": "raven/raven",
+            "version": "dev-master",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/getsentry/raven-php.git",
+                "reference": "407d77059c004d4771e8bb1fa50d8b62c850c7e3"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/getsentry/raven-php/zipball/407d77059c004d4771e8bb1fa50d8b62c850c7e3",
+                "reference": "407d77059c004d4771e8bb1fa50d8b62c850c7e3",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=5.2.4"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "3.7.*"
+            },
+            "bin": [
+                "bin/raven"
+            ],
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "0.8.x-dev"
+                }
+            },
+            "autoload": {
+                "psr-0": {
+                    "Raven_": "lib/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "BSD"
+            ],
+            "authors": [
+                {
+                    "name": "David Cramer",
+                    "email": "dcramer@gmail.com"
+                }
+            ],
+            "description": "A PHP client for Sentry (http://getsentry.com)",
+            "homepage": "http://getsentry.com",
+            "keywords": [
+                "log",
+                "logging"
+            ],
+            "time": "2015-01-23 20:50:15"
+        },
         {
             "name": "symfony/event-dispatcher",
-            "version": "v2.6.1",
+            "version": "v2.6.3",
             "target-dir": "Symfony/Component/EventDispatcher",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/EventDispatcher.git",
-                "reference": "720fe9bca893df7ad1b4546649473b5afddf0216"
+                "reference": "40ff70cadea3785d83cac1c8309514b36113064e"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/720fe9bca893df7ad1b4546649473b5afddf0216",
+                "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/40ff70cadea3785d83cac1c8309514b36113064e",
-                "reference": "720fe9bca893df7ad1b4546649473b5afddf0216",
+                "reference": "40ff70cadea3785d83cac1c8309514b36113064e",
                 "shasum": ""
             },
             "require": {
@@ -337,10 +389,10 @@
             },
             "require-dev": {
                 "psr/log": "~1.0",
-                "symfony/config": "~2.0",
+                "symfony/config": "~2.0,>=2.0.5",
                 "symfony/dependency-injection": "~2.6",
                 "symfony/expression-language": "~2.6",
-                "symfony/stopwatch": "~2.2"
+                "symfony/stopwatch": "~2.3"
             },
             "suggest": {
                 "symfony/dependency-injection": "",
@@ -373,14 +425,17 @@
             ],
             "description": "Symfony EventDispatcher Component",
             "homepage": "http://symfony.com",
-            "time": "2014-12-02 20:19:20"
+            "time": "2015-01-05 14:28:40"
         }
     ],
     "packages-dev": [],
     "aliases": [],
     "minimum-stability": "stable",
-    "stability-flags": [],
+    "stability-flags": {
+        "raven/raven": 20
+    },
     "prefer-stable": false,
+    "prefer-lowest": false,
     "platform": [],
     "platform-dev": []
 }

python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py

@@ -51,6 +51,10 @@ class AnalyzerPipeline:
                 raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__ + " instead.")
             if not isinstance(original_filename, unicode):
                 raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__ + " instead.")
+            if not isinstance(file_prefix, unicode):
+                raise TypeError("file_prefix must be unicode. Was of type " + type(file_prefix).__name__ + " instead.")
+            if not isinstance(cloud_storage_enabled, bool):
+                raise TypeError("cloud_storage_enabled must be a boolean. Was of type " + type(cloud_storage_enabled).__name__ + " instead.")
 
 
             # Analyze the audio file we were told to analyze:
@@ -63,7 +67,6 @@ class AnalyzerPipeline:
             metadata = ReplayGainAnalyzer.analyze(audio_file_path, metadata)
             metadata = PlayabilityAnalyzer.analyze(audio_file_path, metadata)
 
-
             csu = CloudStorageUploader(cloud_storage_config)
             if csu.enabled():
                 metadata = csu.upload_obj(audio_file_path, metadata)

python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py

@@ -22,18 +22,25 @@ class CloudStorageUploader:
 
     def __init__(self, config):
 
-        CLOUD_STORAGE_CONFIG_SECTION = config.get("current_backend", "storage_backend")
+        try:
-        self._storage_backend = CLOUD_STORAGE_CONFIG_SECTION
+            cloud_storage_config_section = config.get("current_backend", "storage_backend")
+            self._storage_backend = cloud_storage_config_section
+        except Exception as e:
+            print e
+            print "Defaulting to file storage"
+            self._storage_backend = STORAGE_BACKEND_FILE
+
         if self._storage_backend == STORAGE_BACKEND_FILE:
             self._host = ""
             self._bucket = ""
             self._api_key = ""
             self._api_key_secret = ""
         else:
-            self._host = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'host')
+            self._host = config.get(cloud_storage_config_section, 'host')
-            self._bucket = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'bucket')
+            self._bucket = config.get(cloud_storage_config_section, 'bucket')
-            self._api_key = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'api_key')
+            self._api_key = config.get(cloud_storage_config_section, 'api_key')
-            self._api_key_secret = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'api_key_secret')
+            self._api_key_secret = config.get(cloud_storage_config_section, 'api_key_secret')
+
 
     def enabled(self):
         if self._storage_backend == "file":

python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader_libcloud.py

@@ -1,7 +1,7 @@
 import os
 import logging
 import uuid
-import config_file
+import ConfigParser
 from libcloud.storage.providers import get_driver
 from libcloud.storage.types import Provider, ContainerDoesNotExistError, ObjectDoesNotExistError
 
@@ -27,20 +27,30 @@ class CloudStorageUploader:
 
     def __init__(self):
 
-        config = config_file.read_config_file(CLOUD_CONFIG_PATH)
+        config = ConfigParser.SafeConfigParser()
+        try:
+            config.readfp(open(CLOUD_CONFIG_PATH))
+            cloud_storage_config_section = config.get("current_backend", "storage_backend")
+            self._storage_backend = cloud_storage_config_section
+        except IOError as e:
+            print "Failed to open config file at " + CLOUD_CONFIG_PATH + ": " + e.strerror
+            print "Defaulting to file storage"
+            self._storage_backend = STORAGE_BACKEND_FILE
+        except Exception as e:
+            print e
+            print "Defaulting to file storage"
+            self._storage_backend = STORAGE_BACKEND_FILE
 
-        CLOUD_STORAGE_CONFIG_SECTION = config.get("current_backend", "storage_backend")
-        self._storage_backend = CLOUD_STORAGE_CONFIG_SECTION
         if self._storage_backend == STORAGE_BACKEND_FILE:
             self._provider = ""
             self._bucket = ""
             self._api_key = ""
             self._api_key_secret = ""
         else:
-            self._provider = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'provider')
+            self._provider = config.get(cloud_storage_config_section, 'provider')
-            self._bucket = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'bucket')
+            self._bucket = config.get(cloud_storage_config_section, 'bucket')
-            self._api_key = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'api_key')
+            self._api_key = config.get(cloud_storage_config_section, 'api_key')
-            self._api_key_secret = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'api_key_secret')
+            self._api_key_secret = config.get(cloud_storage_config_section, 'api_key_secret')
 
     def enabled(self):
         if self._storage_backend == "file":

python_apps/airtime_analyzer/airtime_analyzer/message_listener.py

@@ -213,7 +213,7 @@ class MessageListener:
     def spawn_analyzer_process(audio_file_path, import_directory, original_filename, file_prefix, cloud_storage_config):
         ''' Spawn a child process to analyze and import a new audio file. '''
         q = multiprocessing.Queue()
-        p = multiprocessing.Process(target=AnalyzerPipeline.run_analysis, 
+        p = multiprocessing.Process(target=AnalyzerPipeline.run_analysis,
                         args=(q, audio_file_path, import_directory, original_filename, file_prefix, cloud_storage_config))
         p.start()
         p.join()

python_apps/airtime_analyzer/airtime_analyzer/playability_analyzer.py

@@ -19,7 +19,7 @@ class PlayabilityAnalyzer(Analyzer):
         :param metadata: A metadata dictionary where the results will be put
         :return: The metadata dictionary
         '''
-        command = [PlayabilityAnalyzer.LIQUIDSOAP_EXECUTABLE, '-v', '-c', "output.dummy(audio_to_stereo(single('%s')))" % filename]
+        command = [PlayabilityAnalyzer.LIQUIDSOAP_EXECUTABLE, '-v', '-c', "output.dummy(audio_to_stereo(single(argv(1))))", '--', filename]
         try:
             subprocess.check_output(command, stderr=subprocess.STDOUT, close_fds=True)
 

python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py

@@ -20,8 +20,10 @@ def teardown():
 def test_basic():
     filename = os.path.basename(DEFAULT_AUDIO_FILE)
     q = multiprocessing.Queue()
+    cloud_storage_enabled = False
+    file_prefix = u''
     #This actually imports the file into the "./Test Artist" directory.
-    AnalyzerPipeline.run_analysis(q, DEFAULT_AUDIO_FILE, u'.', filename)
+    AnalyzerPipeline.run_analysis(q, DEFAULT_AUDIO_FILE, u'.', filename, file_prefix, cloud_storage_enabled)
     metadata = q.get()
     assert metadata['track_title'] == u'Test Title'
     assert metadata['artist_name'] == u'Test Artist'