Congegni/sintonia
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'saas' into saas-cloud-storage-config-per-dev-env

Browse source 
Conflicts:

python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py

python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py

python_apps/airtime_analyzer/airtime_analyzer/message_listener.py
This commit is contained in:
drigato 2015-02-04 13:36:35 -05:00
commit c6fc184559
14 changed files with 322 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

2
airtime_mvc/application/configs/constants.php
View file

@ -86,3 +86,5 @@ define('SOUNDCLOUD_ERROR' , -3);
define("WHMCS_API_URL", "https://account.sourcefabric.com/includes/api.php");
define("SUBDOMAIN_WHMCS_CUSTOM_FIELD_NAME", "Choose your domain");
//Sentry error logging
define('SENTRY_CONFIG_PATH', '/etc/airtime-saas/sentry.airtime_web.ini');

42
airtime_mvc/application/controllers/ApiController.php
View file

@ -115,7 +115,7 @@ class ApiController extends Zend_Controller_Action
header('Content-Disposition: inline; filename="'.$filename.'"');
}
$this->smartReadFile($media);
$this->readStoredFileObject($media);
exit;
} else {
header ("HTTP/1.1 404 Not Found");
@ -125,28 +125,52 @@ class ApiController extends Zend_Controller_Action
$this->_helper->json->sendJson(array());
}
/**
* Read data from StoredFile object and send with XHR response
*
* @param Application_Model_StoredFile $storedFile - StoredFile object holding file information
*/
private function readStoredFileObject($storedFile) {
$filepath = $storedFile->getFilePath();
$size = $storedFile->getFileSize();
$mimeType = $storedFile->getPropelOrm()->getDbMime();
$this->smartReadFile($filepath, $mimeType, $size);
}
/**
* Reads the requested portion of a file and sends its contents to the client with the appropriate headers.
*
* This HTTP_RANGE compatible read file function is necessary for allowing streaming media to be skipped around in.
*
* @param string $location
* @param string $mimeType
* @param string $location - the full filepath pointing to the location of the file
* @param string $mimeType - the file's mime type. Defaults to 'audio/mp3'
* @param integer $size - the file size, in bytes
* @return void
*
* @link https://groups.google.com/d/msg/jplayer/nSM2UmnSKKA/Hu76jDZS4xcJ
* @link http://php.net/manual/en/function.readfile.php#86244
*/
public function smartReadFile($media)
private function smartReadFile($location, $mimeType = 'audio/mp3', $size = null)
{
$filepath = $media->getFilePath();
$size= $media->getFileSize();
$mimeType = $media->getPropelOrm()->getDbMime();
if (!$location || $location == "") {
throw new FileDoesNotExistException("Requested file does not exist!");
}
$fm = @fopen($filepath, 'rb');
// If we're passing in a Stored File object, it's faster
// to use getFileSize() and pass in the result
if (!$size || $size <= 0) {
$size= filesize($location);
}
if ($size <= 0) {
throw new Exception("Invalid file size returned for file at $location");
}
$fm = @fopen($location, 'rb');
if (!$fm) {
header ("HTTP/1.1 505 Internal server error");
return;
}

47
airtime_mvc/application/logging/Logging.php
View file

@ -1,4 +1,5 @@
<?php
require_once('SentryLogging.php');
class Logging {
@ -82,6 +83,7 @@ class Logging {
{
$logger = self::getLogger();
$logger->err(self::getLinePrefix(true) . self::toString($p_msg));
SentryLogger::getInstance()->captureError(self::toString($p_msg));
}
public static function debug($p_msg)
@ -123,4 +125,49 @@ class Logging {
Propel::setLogger(null);
}
public static function loggingShutdownCallback()
{
//Catch the types of errors that PHP doesn't normally let us catch and
//would otherwise log to the apache log. We route these to our Airtime log to improve the modularity
//and reliability of our error logging. (All errors are in one spot!)
$err = error_get_last();
if (!is_array($err) || !array_key_exists('type', $err)) {
return;
}
switch($err['type'])
{
case E_ERROR:
case E_PARSE:
case E_CORE_ERROR:
case E_CORE_WARNING:
case E_COMPILE_ERROR:
case E_COMPILE_WARNING:
//error_log("Oh noes, a fatal: " . var_export($err, true), 1, 'fatals@example.com');
$errorStr = '';
if (array_key_exists('message', $err)) {
$errorStr .= $err['message'];
}
if (array_key_exists('file', $err))
{
$errorStr .= ' at ' .$err['file'];
}
if (array_key_exists('line', $err))
{
$errorStr .= ':' . $err['line'];
}
$errorStr .= "\n" . var_export($err, true);
Logging::error($errorStr);
break;
}
}
public static function setupParseErrorLogging()
{
//Static callback:
register_shutdown_function('Logging::loggingShutdownCallback');
}
}

131
airtime_mvc/application/logging/SentryLogging.php Normal file
View file

@ -0,0 +1,131 @@
<?php
class SentryLogger
{
private static $instance = null;
private $sentryClient;
/** Singleton getter */
public static function getInstance()
{
if (!is_null(self::$instance)) {
return self::$instance;
} else {
self::$instance = new SentryLogger();
return self::$instance;
}
}
private function __construct()
{
if (!file_exists(SENTRY_CONFIG_PATH)) {
$this->sentryClient = null;
return;
}
// Instantiate a new client with a compatible DSN
$sentry_config = parse_ini_file(SENTRY_CONFIG_PATH, false);
$dsn = $sentry_config['dsn'];
$this->sentryClient = new Raven_Client($dsn,
array(
//FIXME: This doesn't seem to be working...
'processorOptions' => array(
'Raven_SanitizeDataProcessor' => array(
'fields_re' => '/(authorization|password|passwd|user_token|secret|WHMCS_|SESSION)/i',
'values_re' => '/^(?:\d[ -]*?){13,16}$/'
)
)
));
$client = $this->sentryClient;
/* The Raven docs suggest not enabling these because they're "too noisy".
// Install error handlers and shutdown function to catch fatal errors
$error_handler = new Raven_ErrorHandler($client);
$error_handler->registerExceptionHandler(true);
$error_handler->registerErrorHandler(true);
$error_handler->registerShutdownFunction(true);
*/
$error_handler = new Raven_ErrorHandler($client);
$error_handler->registerExceptionHandler();
}
public function captureMessage($msg)
{
if (!$this->sentryClient) {
return;
}
$client = $this->sentryClient;
self::addUserData($client);
self::addTags($client);
// Capture a message
$event_id = $client->getIdent($client->captureMessage($msg));
if ($client->getLastError() !== null) {
//printf('There was an error sending the event to Sentry: %s', $client->getLastError());
}
}
public function captureException($exception)
{
if (!$this->sentryClient) {
return;
}
$client = $this->sentryClient;
self::addUserData($client);
self::addTags($client);
$event_id = $client->getIdent($client->captureException($exception, array(
'extra' => $this->getExtraData(),
'tags' => $this->getTags(),
)));
$client->context->clear();
}
public function captureError($errorMessage)
{
if (!$this->sentryClient) {
return;
}
$client = $this->sentryClient;
// Provide some additional data with an exception
self::addUserData($client);
self::addTags($client);
$event_id = $client->getIdent($client->captureMessage($errorMessage, array(
'extra' => $this->getExtraData()
)));
$client->context->clear();
}
private static function getTags()
{
$tags = array();
$config = Config::getConfig();
$tags['Development Environment'] = $config["dev_env"];
return $tags;
}
private static function addTags($client)
{
$client->tags_context(self::getTags());
}
private static function addUserData($client)
{
$userData = array();
$userData['client_id'] = Application_Model_Preference::GetClientId();
$userData['station_url'] = array_key_exists('SERVER_NAME', $_SERVER) ? $_SERVER['SERVER_NAME'] : "";
$client->user_context($userData);
}
/** Extra data to log with Sentry */
private function getExtraData()
{
$extraData = array();
$extraData['php_version'] = phpversion();
$extraData['client_id'] = Application_Model_Preference::GetClientId();
return $extraData;
}
}

1
airtime_mvc/public/index.php
View file

@ -79,6 +79,7 @@ $application = new Zend_Application(
require_once (APPLICATION_PATH."/logging/Logging.php");
Logging::setLogPath('/var/log/airtime/zendphp.log');
Logging::setupParseErrorLogging();
// Create application, bootstrap, and run
try {

3
composer.json
View file

@ -1,6 +1,7 @@
{
"require": {
"propel/propel1": "1.7.0-stable",
"aws/aws-sdk-php": "2.7.9"
"aws/aws-sdk-php": "2.7.9",
"raven/raven": "0.8.x-dev"
}
}

73
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"hash": "311983cf9bb84cfacbfcc6c5dfc9e841",
"hash": "30ad5215f679ce0ab55c7210b21a3b32",
"packages": [
{
"name": "aws/aws-sdk-php",
@ -317,19 +317,71 @@
],
"time": "2013-10-21 12:52:56"
},
{
"name": "raven/raven",
"version": "dev-master",
"source": {
"type": "git",
"url": "https://github.com/getsentry/raven-php.git",
"reference": "407d77059c004d4771e8bb1fa50d8b62c850c7e3"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/getsentry/raven-php/zipball/407d77059c004d4771e8bb1fa50d8b62c850c7e3",
"reference": "407d77059c004d4771e8bb1fa50d8b62c850c7e3",
"shasum": ""
},
"require": {
"php": ">=5.2.4"
},
"require-dev": {
"phpunit/phpunit": "3.7.*"
},
"bin": [
"bin/raven"
],
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "0.8.x-dev"
}
},
"autoload": {
"psr-0": {
"Raven_": "lib/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"BSD"
],
"authors": [
{
"name": "David Cramer",
"email": "dcramer@gmail.com"
}
],
"description": "A PHP client for Sentry (http://getsentry.com)",
"homepage": "http://getsentry.com",
"keywords": [
"log",
"logging"
],
"time": "2015-01-23 20:50:15"
},
{
"name": "symfony/event-dispatcher",
"version": "v2.6.1",
"version": "v2.6.3",
"target-dir": "Symfony/Component/EventDispatcher",
"source": {
"type": "git",
"url": "https://github.com/symfony/EventDispatcher.git",
"reference": "720fe9bca893df7ad1b4546649473b5afddf0216"
"reference": "40ff70cadea3785d83cac1c8309514b36113064e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/720fe9bca893df7ad1b4546649473b5afddf0216",
"reference": "720fe9bca893df7ad1b4546649473b5afddf0216",
"url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/40ff70cadea3785d83cac1c8309514b36113064e",
"reference": "40ff70cadea3785d83cac1c8309514b36113064e",
"shasum": ""
},
"require": {
@ -337,10 +389,10 @@
},
"require-dev": {
"psr/log": "~1.0",
"symfony/config": "~2.0",
"symfony/config": "~2.0,>=2.0.5",
"symfony/dependency-injection": "~2.6",
"symfony/expression-language": "~2.6",
"symfony/stopwatch": "~2.2"
"symfony/stopwatch": "~2.3"
},
"suggest": {
"symfony/dependency-injection": "",
@ -373,14 +425,17 @@
],
"description": "Symfony EventDispatcher Component",
"homepage": "http://symfony.com",
"time": "2014-12-02 20:19:20"
"time": "2015-01-05 14:28:40"
}
],
"packages-dev": [],
"aliases": [],
"minimum-stability": "stable",
"stability-flags": [],
"stability-flags": {
"raven/raven": 20
},
"prefer-stable": false,
"prefer-lowest": false,
"platform": [],
"platform-dev": []
}

5
python_apps/airtime_analyzer/airtime_analyzer/analyzer_pipeline.py
View file

@ -51,6 +51,10 @@ class AnalyzerPipeline:
raise TypeError("import_directory must be unicode. Was of type " + type(import_directory).__name__ + " instead.")
if not isinstance(original_filename, unicode):
raise TypeError("original_filename must be unicode. Was of type " + type(original_filename).__name__ + " instead.")
if not isinstance(file_prefix, unicode):
raise TypeError("file_prefix must be unicode. Was of type " + type(file_prefix).__name__ + " instead.")
if not isinstance(cloud_storage_enabled, bool):
raise TypeError("cloud_storage_enabled must be a boolean. Was of type " + type(cloud_storage_enabled).__name__ + " instead.")
# Analyze the audio file we were told to analyze:
@ -63,7 +67,6 @@ class AnalyzerPipeline:
metadata = ReplayGainAnalyzer.analyze(audio_file_path, metadata)
metadata = PlayabilityAnalyzer.analyze(audio_file_path, metadata)
csu = CloudStorageUploader(cloud_storage_config)
if csu.enabled():
metadata = csu.upload_obj(audio_file_path, metadata)

19
python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py
View file

@ -22,18 +22,25 @@ class CloudStorageUploader:
def __init__(self, config):
CLOUD_STORAGE_CONFIG_SECTION = config.get("current_backend", "storage_backend")
self._storage_backend = CLOUD_STORAGE_CONFIG_SECTION
try:
cloud_storage_config_section = config.get("current_backend", "storage_backend")
self._storage_backend = cloud_storage_config_section
except Exception as e:
print e
print "Defaulting to file storage"
self._storage_backend = STORAGE_BACKEND_FILE
if self._storage_backend == STORAGE_BACKEND_FILE:
self._host = ""
self._bucket = ""
self._api_key = ""
self._api_key_secret = ""
else:
self._host = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'host')
self._bucket = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'bucket')
self._api_key = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'api_key')
self._api_key_secret = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'api_key_secret')
self._host = config.get(cloud_storage_config_section, 'host')
self._bucket = config.get(cloud_storage_config_section, 'bucket')
self._api_key = config.get(cloud_storage_config_section, 'api_key')
self._api_key_secret = config.get(cloud_storage_config_section, 'api_key_secret')
def enabled(self):
if self._storage_backend == "file":

26
python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader_libcloud.py
View file

@ -1,7 +1,7 @@
import os
import logging
import uuid
import config_file
import ConfigParser
from libcloud.storage.providers import get_driver
from libcloud.storage.types import Provider, ContainerDoesNotExistError, ObjectDoesNotExistError
@ -27,20 +27,30 @@ class CloudStorageUploader:
def __init__(self):
config = config_file.read_config_file(CLOUD_CONFIG_PATH)
config = ConfigParser.SafeConfigParser()
try:
config.readfp(open(CLOUD_CONFIG_PATH))
cloud_storage_config_section = config.get("current_backend", "storage_backend")
self._storage_backend = cloud_storage_config_section
except IOError as e:
print "Failed to open config file at " + CLOUD_CONFIG_PATH + ": " + e.strerror
print "Defaulting to file storage"
self._storage_backend = STORAGE_BACKEND_FILE
except Exception as e:
print e
print "Defaulting to file storage"
self._storage_backend = STORAGE_BACKEND_FILE
CLOUD_STORAGE_CONFIG_SECTION = config.get("current_backend", "storage_backend")
self._storage_backend = CLOUD_STORAGE_CONFIG_SECTION
if self._storage_backend == STORAGE_BACKEND_FILE:
self._provider = ""
self._bucket = ""
self._api_key = ""
self._api_key_secret = ""
else:
self._provider = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'provider')
self._bucket = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'bucket')
self._api_key = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'api_key')
self._api_key_secret = config.get(CLOUD_STORAGE_CONFIG_SECTION, 'api_key_secret')
self._provider = config.get(cloud_storage_config_section, 'provider')
self._bucket = config.get(cloud_storage_config_section, 'bucket')
self._api_key = config.get(cloud_storage_config_section, 'api_key')
self._api_key_secret = config.get(cloud_storage_config_section, 'api_key_secret')
def enabled(self):
if self._storage_backend == "file":

2
python_apps/airtime_analyzer/airtime_analyzer/playability_analyzer.py
View file

@ -19,7 +19,7 @@ class PlayabilityAnalyzer(Analyzer):
:param metadata: A metadata dictionary where the results will be put
:return: The metadata dictionary
'''
command = [PlayabilityAnalyzer.LIQUIDSOAP_EXECUTABLE, '-v', '-c', "output.dummy(audio_to_stereo(single('%s')))" % filename]
command = [PlayabilityAnalyzer.LIQUIDSOAP_EXECUTABLE, '-v', '-c', "output.dummy(audio_to_stereo(single(argv(1))))", '--', filename]
try:
subprocess.check_output(command, stderr=subprocess.STDOUT, close_fds=True)

4
python_apps/airtime_analyzer/tests/analyzer_pipeline_tests.py
View file

@ -20,8 +20,10 @@ def teardown():
def test_basic():
filename = os.path.basename(DEFAULT_AUDIO_FILE)
q = multiprocessing.Queue()
cloud_storage_enabled = False
file_prefix = u''
#This actually imports the file into the "./Test Artist" directory.
AnalyzerPipeline.run_analysis(q, DEFAULT_AUDIO_FILE, u'.', filename)
AnalyzerPipeline.run_analysis(q, DEFAULT_AUDIO_FILE, u'.', filename, file_prefix, cloud_storage_enabled)
metadata = q.get()
assert metadata['track_title'] == u'Test Title'
assert metadata['artist_name'] == u'Test Artist'