From fa2018a2c5115653e3f026a4feb2b43984a0f1c9 Mon Sep 17 00:00:00 2001 From: Lucas Bickel Date: Tue, 18 Jul 2017 22:27:19 +0200 Subject: [PATCH] Simplify configuration file structure This removes most of the legacy upstream config madness by not using weird config files spread all over the place. This isn't the solution to other config reading fragility issues, but it does move the whole config back to the central airtime.conf file. --- .../application/common/CeleryManager.php | 6 +-- airtime_mvc/application/configs/conf.php | 42 +++++++------------ airtime_mvc/application/models/RabbitMq.php | 22 +--------- airtime_mvc/build/airtime.example.conf | 13 ++++++ airtime_mvc/build/cloud_storage.conf | 8 ---- airtime_mvc/build/rabbitmq-analyzer.ini | 7 ---- airtime_mvc/public/setup/media-setup.php | 17 +------- airtime_mvc/public/setup/rabbitmq-setup.php | 15 +------ airtime_mvc/tests/conf/airtime.conf | 9 ++++ .../tests/conf/testing/cloud_storage.conf | 3 -- install | 4 -- .../cloud_storage_uploader.py | 2 +- .../cloud_storage_uploader_libcloud.py | 2 +- .../airtime_analyzer/bin/airtime_analyzer | 2 +- 14 files changed, 46 insertions(+), 106 deletions(-) delete mode 100644 airtime_mvc/build/cloud_storage.conf delete mode 100644 airtime_mvc/build/rabbitmq-analyzer.ini delete mode 100644 airtime_mvc/tests/conf/testing/cloud_storage.conf diff --git a/airtime_mvc/application/common/CeleryManager.php b/airtime_mvc/application/common/CeleryManager.php index 065da9009..dc2c97ba4 100644 --- a/airtime_mvc/application/common/CeleryManager.php +++ b/airtime_mvc/application/common/CeleryManager.php @@ -55,7 +55,7 @@ class CeleryManager { * results asynchronously later */ public static function sendCeleryMessage($task, $exchange, $data) { - $config = parse_ini_file(Application_Model_RabbitMq::getRmqConfigPath(), true); + $config = Config::getConfig(); $queue = $routingKey = $exchange; $c = self::_setupCeleryExchange($config, $exchange, $queue); // Use the exchange name for the queue $result = $c->PostTask($task, $data, true, $routingKey); // and routing key @@ -75,7 +75,7 @@ class CeleryManager { * $_CELERY_MESSAGE_TIMEOUT milliseconds have passed */ private static function getAsyncResultMessage($task) { - $config = parse_ini_file(Application_Model_RabbitMq::getRmqConfigPath(), true); + $config = Config::getConfig(); $queue = self::$_CELERY_RESULTS_EXCHANGE . "." . $task; $c = self::_setupCeleryExchange($config, self::$_CELERY_RESULTS_EXCHANGE, $queue); $message = $c->getAsyncResultMessage($task->getDbName(), $task->getDbTaskId()); @@ -208,4 +208,4 @@ class CeleryManager { return (empty($dispatchTime) || $dispatchTime->add($timeoutInterval) <= $now); } -} \ No newline at end of file +} diff --git a/airtime_mvc/application/configs/conf.php b/airtime_mvc/application/configs/conf.php index f12d04c86..54d89b0ba 100644 --- a/airtime_mvc/application/configs/conf.php +++ b/airtime_mvc/application/configs/conf.php @@ -49,20 +49,13 @@ class Config { $CC_CONFIG['staticBaseDir'] = '/'; } - // Parse separate conf file for cloud storage values - $cloudStorageConfig = LIBRETIME_CONF_DIR . '/' . $CC_CONFIG['dev_env']."/cloud_storage.conf"; - if (!file_exists($cloudStorageConfig)) { - // If the dev env specific cloud_storage.conf doesn't exist default - // to the production cloud_storage.conf - $cloudStorageConfig = LIBRETIME_CONF_DIR . "/production/cloud_storage.conf"; - } - $cloudStorageValues = parse_ini_file($cloudStorageConfig, true); - - $CC_CONFIG["supportedStorageBackends"] = array('amazon_S3'); - foreach ($CC_CONFIG["supportedStorageBackends"] as $backend) { - $CC_CONFIG[$backend] = $cloudStorageValues[$backend]; - } - + $CC_CONFIG['amazon_S3'] = array( + 'provider' => $values['amazon_S3']['provider'], + 'bucket' => $values['amazon_S3']['bucket'], + 'api_key' => $values['amazon_S3']['api_key'], + 'api_key_secret' => $values['amazon_S3']['api_key_secret'] + ); + // Tells us where file uploads will be uploaded to. // It will either be set to a cloud storage backend or local file storage. $CC_CONFIG["current_backend"] = $cloudStorageValues["current_backend"]["storage_backend"]; @@ -81,20 +74,13 @@ class Config { $CC_CONFIG['soundcloud-connection-retries'] = $values['soundcloud']['connection_retries']; $CC_CONFIG['soundcloud-connection-wait'] = $values['soundcloud']['time_between_retries']; - $globalAirtimeConfig = LIBRETIME_CONF_DIR . '/' . $CC_CONFIG['dev_env']."/airtime.conf"; - if (!file_exists($globalAirtimeConfig)) { - // If the dev env specific airtime.conf doesn't exist default - // to the production airtime.conf - $globalAirtimeConfig = LIBRETIME_CONF_DIR . "/production/airtime.conf"; - } - $globalAirtimeConfigValues = parse_ini_file($globalAirtimeConfig, true); - $CC_CONFIG['soundcloud-client-id'] = $globalAirtimeConfigValues['soundcloud']['soundcloud_client_id']; - $CC_CONFIG['soundcloud-client-secret'] = $globalAirtimeConfigValues['soundcloud']['soundcloud_client_secret']; - $CC_CONFIG['soundcloud-redirect-uri'] = $globalAirtimeConfigValues['soundcloud']['soundcloud_redirect_uri']; - if (isset($globalAirtimeConfigValues['facebook']['facebook_app_id'])) { - $CC_CONFIG['facebook-app-id'] = $globalAirtimeConfigValues['facebook']['facebook_app_id']; - $CC_CONFIG['facebook-app-url'] = $globalAirtimeConfigValues['facebook']['facebook_app_url']; - $CC_CONFIG['facebook-app-api-key'] = $globalAirtimeConfigValues['facebook']['facebook_app_api_key']; + $CC_CONFIG['soundcloud-client-id'] = $values['soundcloud']['soundcloud_client_id']; + $CC_CONFIG['soundcloud-client-secret'] = $values['soundcloud']['soundcloud_client_secret']; + $CC_CONFIG['soundcloud-redirect-uri'] = $values['soundcloud']['soundcloud_redirect_uri']; + if (isset($values['facebook']['facebook_app_id'])) { + $CC_CONFIG['facebook-app-id'] = $values['facebook']['facebook_app_id']; + $CC_CONFIG['facebook-app-url'] = $values['facebook']['facebook_app_url']; + $CC_CONFIG['facebook-app-api-key'] = $values['facebook']['facebook_app_api_key']; } // ldap config diff --git a/airtime_mvc/application/models/RabbitMq.php b/airtime_mvc/application/models/RabbitMq.php index 0269971e6..98ed388f1 100644 --- a/airtime_mvc/application/models/RabbitMq.php +++ b/airtime_mvc/application/models/RabbitMq.php @@ -78,29 +78,11 @@ class Application_Model_RabbitMq self::sendMessage($exchange, 'direct', true, $data); } - public static function getRmqConfigPath() { - //Hack for Airtime Pro. The RabbitMQ settings for communicating with airtime_analyzer are global - //and shared between all instances on Airtime Pro. - // - // todo: rewrite me to only use the config class and not access /etc/airtime directly - $CC_CONFIG = Config::getConfig(); - $devEnv = "production"; //Default - if (array_key_exists("dev_env", $CC_CONFIG)) { - $devEnv = $CC_CONFIG["dev_env"]; - } - $rmq_config_path = LIBRETIME_CONF_DIR . '/' . $devEnv."/rabbitmq-analyzer.ini"; - if (!file_exists($rmq_config_path)) { - // If the dev env specific rabbitmq-analyzer.ini doesn't exist default - // to the production rabbitmq-analyzer.ini - $rmq_config_path = LIBRETIME_CONF_PATH . "/production/rabbitmq-analyzer.ini"; - } - return $rmq_config_path; - } - public static function SendMessageToAnalyzer($tmpFilePath, $importedStorageDirectory, $originalFilename, $callbackUrl, $apiKey, $storageBackend, $filePrefix) { - $config = parse_ini_file(self::getRmqConfigPath(), true); + $config = Config::getConfig(); + $conn = new \PhpAmqpLib\Connection\AMQPConnection($config["rabbitmq"]["host"], $config["rabbitmq"]["port"], $config["rabbitmq"]["user"], diff --git a/airtime_mvc/build/airtime.example.conf b/airtime_mvc/build/airtime.example.conf index a4459338b..9000a8767 100644 --- a/airtime_mvc/build/airtime.example.conf +++ b/airtime_mvc/build/airtime.example.conf @@ -120,6 +120,19 @@ vhost = /airtime # ---------------------------------------------------------------------- +# ---------------------------------------------------------------------- +# S T O R A G E +# ---------------------------------------------------------------------- +# +[current_backend] +storage_backend=file + +[amazon_S3] +provider=amazon_S3 +bucket=0 +api_key=0 +api_key_secret=0 + # ---------------------------------------------------------------------- # M O N I T # ---------------------------------------------------------------------- diff --git a/airtime_mvc/build/cloud_storage.conf b/airtime_mvc/build/cloud_storage.conf deleted file mode 100644 index 3e93e0512..000000000 --- a/airtime_mvc/build/cloud_storage.conf +++ /dev/null @@ -1,8 +0,0 @@ -[current_backend] -storage_backend=file - -[amazon_S3] -provider=amazon_S3 -bucket=0 -api_key=0 -api_key_secret=0 diff --git a/airtime_mvc/build/rabbitmq-analyzer.ini b/airtime_mvc/build/rabbitmq-analyzer.ini deleted file mode 100644 index 7f053ef97..000000000 --- a/airtime_mvc/build/rabbitmq-analyzer.ini +++ /dev/null @@ -1,7 +0,0 @@ -[rabbitmq] -host = 127.0.0.1 -port = 5672 -user = airtime -password = airtime -vhost = /airtime - diff --git a/airtime_mvc/public/setup/media-setup.php b/airtime_mvc/public/setup/media-setup.php index 3816330ab..8fce8ba53 100644 --- a/airtime_mvc/public/setup/media-setup.php +++ b/airtime_mvc/public/setup/media-setup.php @@ -26,7 +26,6 @@ class MediaSetup extends Setup { const MEDIA_FOLDER = "mediaFolder"; const LIBRETIME_CONF_FILE_NAME = "airtime.conf"; - const RMQ_INI_FILE_NAME = "rabbitmq-analyzer.ini"; static $path; static $message = null; @@ -66,10 +65,6 @@ class MediaSetup extends Setup { self::$message = "Error moving airtime.conf or deleting /tmp/airtime.conf.temp!"; self::$errors[] = "ERR"; } - if (!$this->moveRmqConfig()) { - self::$message = "Error moving rabbitmq-analyzer.ini or deleting /tmp/rabbitmq.ini.tmp!"; - self::$errors[] = "ERR"; - } /* * If we're upgrading from an old Airtime instance (pre-2.5.2) we rename their old @@ -102,16 +97,6 @@ class MediaSetup extends Setup { && unlink(AIRTIME_CONF_TEMP_PATH); } - /** - * Moves /tmp/airtime.conf.temp to /etc/airtime.conf and then removes it to complete setup - * @return boolean false if either of the copy or removal operations fail - */ - function moveRmqConfig() { - return copy(RMQ_INI_TEMP_PATH, LIBRETIME_CONF_DIR . '/' . self::RMQ_INI_FILE_NAME) - && copy(RMQ_INI_TEMP_PATH, LIBRETIME_CONF_DIR . '/production/' . self::RMQ_INI_FILE_NAME) - && unlink(RMQ_INI_TEMP_PATH); - } - /** * Add the given directory to cc_music_dirs * TODO Should we check for an existing entry in cc_music_dirs? @@ -156,4 +141,4 @@ class MediaSetup extends Setup { return isset($entry) && $entry; } -} \ No newline at end of file +} diff --git a/airtime_mvc/public/setup/rabbitmq-setup.php b/airtime_mvc/public/setup/rabbitmq-setup.php index 25a7246d6..53a09d787 100644 --- a/airtime_mvc/public/setup/rabbitmq-setup.php +++ b/airtime_mvc/public/setup/rabbitmq-setup.php @@ -51,10 +51,6 @@ class RabbitMQSetup extends Setup { $this->identifyRMQConnectionError(); } - if (count(self::$errors) <= 0) { - $this->writeToTemp(); - } - return array( "message" => self::$message, "errors" => self::$errors @@ -81,13 +77,4 @@ class RabbitMQSetup extends Setup { self::$errors[] = self::RMQ_PORT; self::$errors[] = self::RMQ_VHOST; } - - protected function writeToTemp() { - if (!file_exists(RMQ_INI_TEMP_PATH)) { - copy(BUILD_PATH . "rabbitmq-analyzer.ini", RMQ_INI_TEMP_PATH); - } - $this->_write(RMQ_INI_TEMP_PATH); - parent::writeToTemp(); - } - -} \ No newline at end of file +} diff --git a/airtime_mvc/tests/conf/airtime.conf b/airtime_mvc/tests/conf/airtime.conf index 1d48b4888..5ddfd462c 100644 --- a/airtime_mvc/tests/conf/airtime.conf +++ b/airtime_mvc/tests/conf/airtime.conf @@ -23,6 +23,15 @@ base_dir = / cache_ahead_hours = 1 station_id = teststation +[current_backend] +storage_backend=file + +[amazon_S3] +provider=amazon_S3 +bucket=0 +api_key=0 +api_key_secret=0 + [monit] monit_user = guest monit_password = airtime diff --git a/airtime_mvc/tests/conf/testing/cloud_storage.conf b/airtime_mvc/tests/conf/testing/cloud_storage.conf deleted file mode 100644 index 902114206..000000000 --- a/airtime_mvc/tests/conf/testing/cloud_storage.conf +++ /dev/null @@ -1,3 +0,0 @@ -[amazon_S3] -[current_backend] -storage_backend=file diff --git a/install b/install index 7e2cf2dd3..e5d95c915 100755 --- a/install +++ b/install @@ -1032,10 +1032,6 @@ if [ ! -d "/etc/airtime" ]; then verbose "\n * Creating /etc/airtime/ directory..." mkdir /etc/airtime - # workaround for reintegrated airtime-saas dir, will get removed after we refactored config loading - ln -s /etc/airtime/ /etc/airtime/production - # put the default cloud_storage.conf using local file storage into directory - cp ${AIRTIMEROOT}/airtime_mvc/build/cloud_storage.conf /etc/airtime/cloud_storage.conf fi diff --git a/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py b/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py index ed80ee1c5..55e26a9d0 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py @@ -10,7 +10,7 @@ from boto.s3.key import Key # https://github.com/docker/docker-registry/issues/400 u'fix getaddrinfo deadlock'.encode('idna') -CLOUD_CONFIG_PATH = os.path.join(os.getenv('LIBRETIME_CONF_DIR', '/etc/airtime'), 'cloud_storage.conf') +CLOUD_CONFIG_PATH = os.path.join(os.getenv('LIBRETIME_CONF_DIR', '/etc/airtime'), 'airtime.conf') STORAGE_BACKEND_FILE = "file" SOCKET_TIMEOUT = 240 diff --git a/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader_libcloud.py b/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader_libcloud.py index e98dfe0b2..162493dd5 100644 --- a/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader_libcloud.py +++ b/python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader_libcloud.py @@ -6,7 +6,7 @@ from libcloud.storage.providers import get_driver from libcloud.storage.types import Provider, ContainerDoesNotExistError, ObjectDoesNotExistError -CLOUD_CONFIG_PATH = os.path.join(os.getenv('LIBRETIME_CONF_DIR', '/etc/airtime'), 'cloud_storage.conf') +CLOUD_CONFIG_PATH = os.path.join(os.getenv('LIBRETIME_CONF_DIR', '/etc/airtime'), 'airtime.conf') STORAGE_BACKEND_FILE = "file" class CloudStorageUploader: diff --git a/python_apps/airtime_analyzer/bin/airtime_analyzer b/python_apps/airtime_analyzer/bin/airtime_analyzer index 18142a0f4..ed2c296bb 100755 --- a/python_apps/airtime_analyzer/bin/airtime_analyzer +++ b/python_apps/airtime_analyzer/bin/airtime_analyzer @@ -10,7 +10,7 @@ import airtime_analyzer.airtime_analyzer as aa VERSION = "1.0" LIBRETIME_CONF_DIR = os.getenv('LIBRETIME_CONF_DIR', '/etc/airtime') DEFAULT_RMQ_CONFIG_PATH = os.path.join(LIBRETIME_CONF_DIR, 'airtime.conf') -DEFAULT_CLOUD_STORAGE_CONFIG_PATH = os.path.join(LIBRETIME_CONF_DIR, os.getenv('ENVIRONMENT', 'production'), 'airtime.conf') +DEFAULT_CLOUD_STORAGE_CONFIG_PATH = os.path.join(LIBRETIME_CONF_DIR, 'airtime.conf') DEFAULT_HTTP_RETRY_PATH = '/tmp/airtime_analyzer_http_retries' def run():