Merge branch 'devel' of dev.sourcefabric.org:airtime into devel

install_minimal/include/AirtimeIni.php

@@ -237,7 +237,7 @@ class AirtimeIni
         $n=count($lines);
         for ($i=0; $i<$n; $i++) {
             if (strlen($lines[$i]) && !in_array(substr($lines[$i], 0, 1), array('#', PHP_EOL))){
-                 $info = explode("=", $lines[$i]);
+                 $info = explode("=", $lines[$i], 2);
                  $values[trim($info[0])] = trim($info[1]);
              }
         }

install_minimal/include/airtime-upgrade.php

@@ -116,5 +116,8 @@ if (strcmp($version, "2.1.3") < 0){
     passthru("php --php-ini $SCRIPTPATH/../airtime-php.ini $SCRIPTPATH/../upgrades/airtime-2.1.3/airtime-upgrade.php");
     pause();
 }  
-
+if (strcmp($version, "2.2.0") < 0){
+    passthru("php --php-ini $SCRIPTPATH/../airtime-php.ini $SCRIPTPATH/../upgrades/airtime-2.2.0/airtime-upgrade.php");
+    pause();
+}  
 echo "******************************* Upgrade Complete *******************************".PHP_EOL;

install_minimal/upgrades/airtime-2.2.0/ConfFileUpgrade.php

@@ -0,0 +1,16 @@
+<?php
+
+/* This class deals with any modifications to config files in /etc/airtime
+ * as well as backups. All functions other than start() should be marked
+ * as private. */
+class AirtimeConfigFileUpgrade{
+
+    public static function start(){
+        echo "* Updating configFiles".PHP_EOL;
+        self::task0();
+    }
+    
+    private static function task0(){
+        UpgradeCommon::upgradeConfigFiles();
+    }
+}

install_minimal/upgrades/airtime-2.2.0/DbUpgrade.php

@@ -0,0 +1,24 @@
+<?php
+
+/* All functions other than start() should be marked as
+ * private.
+ */
+class AirtimeDatabaseUpgrade{
+
+    public static function start($p_dbValues){
+        echo "* Updating Database".PHP_EOL;
+        self::task0($p_dbValues);
+        echo " * Complete".PHP_EOL;
+    }
+
+    private static function task0($p_dbValues){
+        
+        $username = $p_dbValues['database']['dbuser'];
+        $password = $p_dbValues['database']['dbpass'];
+        $host = $p_dbValues['database']['host'];
+        $database = $p_dbValues['database']['dbname'];
+        $dir = __DIR__;
+        
+        passthru("export PGPASSWORD=$password && psql -h $host -U $username -q -f $dir/data/upgrade.sql $database 2>&1 | grep -v \"will create implicit index\"");
+    }
+}

install_minimal/upgrades/airtime-2.2.0/airtime-upgrade.php

@@ -0,0 +1,11 @@
+<?php
+
+require_once 'DbUpgrade.php';
+require_once 'ConfFileUpgrade.php';
+require_once 'common/UpgradeCommon.php';
+
+$filename = "/etc/airtime/airtime.conf";
+$values = parse_ini_file($filename, true);
+
+AirtimeConfigFileUpgrade::start();
+AirtimeDatabaseUpgrade::start($values);

install_minimal/upgrades/airtime-2.2.0/common/UpgradeCommon.php

@@ -0,0 +1,246 @@
+<?php
+/* These are helper functions that are common to each upgrade such as
+ * creating connections to a database, backing up config files etc.
+ */
+class UpgradeCommon{
+    const CONF_FILE_AIRTIME = "/etc/airtime/airtime.conf";
+    const CONF_FILE_PYPO = "/etc/airtime/pypo.cfg";
+    const CONF_FILE_LIQUIDSOAP = "/etc/airtime/liquidsoap.cfg";
+    const CONF_FILE_MEDIAMONITOR = "/etc/airtime/media-monitor.cfg";
+    const CONF_FILE_API_CLIENT = "/etc/airtime/api_client.cfg";
+
+    const CONF_PYPO_GRP = "pypo";
+    const CONF_WWW_DATA_GRP = "www-data";
+    const CONF_BACKUP_SUFFIX = "220";
+    const VERSION_NUMBER = "2.2.0";
+    
+    public static function SetDefaultTimezone()
+    {       
+        $sql = "SELECT valstr from cc_pref WHERE keystr = 'timezone'";
+
+        $result = self::queryDb($sql);
+        $timezone = $result->fetchColumn();
+                
+        date_default_timezone_set($timezone);
+    }
+    
+    public static function connectToDatabase($p_exitOnError = true)
+    {
+        try {
+            $con = Propel::getConnection();
+        } catch (Exception $e) {
+            echo $e->getMessage().PHP_EOL;
+            echo "Database connection problem.".PHP_EOL;
+            echo "Check if database exists with corresponding permissions.".PHP_EOL;
+            if ($p_exitOnError) {
+                exit(1);
+            }
+            return false;
+        }
+        return true;
+    }
+
+    
+    public static function DbTableExists($p_name)
+    {
+        $con = Propel::getConnection();
+        try {
+            $sql = "SELECT * FROM ".$p_name." LIMIT 1";
+            $con->query($sql);
+        } catch (PDOException $e){
+            return false;
+        }
+        return true;
+    }
+
+    private static function GetAirtimeSrcDir()
+    {
+        return __DIR__."/../../../../airtime_mvc";
+    }
+
+    public static function MigrateTablesToVersion($dir, $version)
+    {
+        echo "Upgrading database, may take several minutes, please wait".PHP_EOL;
+        
+        $appDir = self::GetAirtimeSrcDir();
+        $command = "php --php-ini $dir/../../airtime-php.ini ".
+                    "$appDir/library/doctrine/migrations/doctrine-migrations.phar ".
+                    "--configuration=$dir/common/migrations.xml ".
+                    "--db-configuration=$appDir/library/doctrine/migrations/migrations-db.php ".
+                    "--no-interaction migrations:migrate $version";
+        system($command);
+    }
+
+    public static function BypassMigrations($dir, $version)
+    {
+        $appDir = self::GetAirtimeSrcDir();
+        $command = "php --php-ini $dir/../../airtime-php.ini ".
+                    "$appDir/library/doctrine/migrations/doctrine-migrations.phar ".
+                    "--configuration=$dir/common/migrations.xml ".
+                    "--db-configuration=$appDir/library/doctrine/migrations/migrations-db.php ".
+                    "--no-interaction --add migrations:version $version";
+        system($command);
+    }
+
+    public static function upgradeConfigFiles(){
+
+        $configFiles = array(UpgradeCommon::CONF_FILE_AIRTIME,
+                             UpgradeCommon::CONF_FILE_PYPO,
+                             UpgradeCommon::CONF_FILE_LIQUIDSOAP,
+                             UpgradeCommon::CONF_FILE_MEDIAMONITOR,
+                             UpgradeCommon::CONF_FILE_API_CLIENT);
+
+        // Backup the config files
+        $suffix = date("Ymdhis")."-".UpgradeCommon::VERSION_NUMBER;
+        foreach ($configFiles as $conf) {
+            // do not back up monit cfg
+            if (file_exists($conf)) {
+                echo "Backing up $conf to $conf$suffix.bak".PHP_EOL;
+                //copy($conf, $conf.$suffix.".bak");
+                exec("cp -p $conf $conf$suffix.bak"); //use cli version to preserve file attributes
+            }
+        }
+
+        self::CreateIniFiles(UpgradeCommon::CONF_BACKUP_SUFFIX);
+        self::MergeConfigFiles($configFiles, $suffix);
+    }
+
+    /**
+     * This function creates the /etc/airtime configuration folder
+     * and copies the default config files to it.
+     */
+    public static function CreateIniFiles($suffix)
+    {
+        if (!file_exists("/etc/airtime/")){
+            if (!mkdir("/etc/airtime/", 0755, true)){
+                echo "Could not create /etc/airtime/ directory. Exiting.";
+                exit(1);
+            }
+        }
+
+        if (!copy(__DIR__."/../etc/airtime.conf.$suffix", self::CONF_FILE_AIRTIME)){
+            echo "Could not copy airtime.conf to /etc/airtime/. Exiting.";
+            exit(1);
+        }
+        if (!copy(__DIR__."/../etc/pypo.cfg.$suffix", self::CONF_FILE_PYPO)){
+            echo "Could not copy pypo.cfg to /etc/airtime/. Exiting.";
+            exit(1);
+        }
+        if (!copy(__DIR__."/../etc/media-monitor.cfg.$suffix", self::CONF_FILE_MEDIAMONITOR)){
+            echo "Could not copy meadia-monitor.cfg to /etc/airtime/. Exiting.";
+            exit(1);
+        }
+        if (!copy(__DIR__."/../etc/api_client.cfg.$suffix", self::CONF_FILE_API_CLIENT)){
+            echo "Could not copy api_client.cfg to /etc/monit/conf.d/. Exiting.";
+            exit(1);
+        }
+    }
+
+    private static function MergeConfigFiles($configFiles, $suffix) {
+        foreach ($configFiles as $conf) {
+            // we want to use new liquidsoap.cfg so don't merge
+            // also for monit
+            if( $conf == self::CONF_FILE_LIQUIDSOAP){
+                continue;
+            }
+            if (file_exists("$conf$suffix.bak")) {
+
+                if($conf === self::CONF_FILE_AIRTIME) {
+                    // Parse with sections
+                    $newSettings = parse_ini_file($conf, true);
+                    $oldSettings = parse_ini_file("$conf$suffix.bak", true);
+                }
+                else {
+                    $newSettings = self::ReadPythonConfig($conf);
+                    $oldSettings = self::ReadPythonConfig("$conf$suffix.bak");
+                }
+
+                $settings = array_keys($newSettings);
+
+                foreach($settings as $section) {
+                    if(isset($oldSettings[$section])) {
+                        if(is_array($oldSettings[$section])) {
+                            $sectionKeys = array_keys($newSettings[$section]);
+                            foreach($sectionKeys as $sectionKey) {
+
+                                if(isset($oldSettings[$section][$sectionKey])) {
+                                    self::UpdateIniValue($conf, $sectionKey, $oldSettings[$section][$sectionKey]);
+                                }
+                            }
+                        } else {
+                            self::UpdateIniValue($conf, $section, $oldSettings[$section]);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private static function ReadPythonConfig($p_filename)
+    {
+        $values = array();
+        
+        $fh = fopen($p_filename, 'r');
+        
+        while(!feof($fh)){
+            $line = fgets($fh);
+            if(substr(trim($line), 0, 1) == '#' || trim($line) == ""){
+                continue;
+            }else{
+                $info = explode('=', $line, 2);
+                $values[trim($info[0])] = trim($info[1]);
+            }
+        }
+
+        return $values;
+    }
+
+    /**
+     * This function updates an INI style config file.
+     *
+     * A property and the value the property should be changed to are
+     * supplied. If the property is not found, then no changes are made.
+     *
+     * @param string $p_filename
+     *      The path the to the file.
+     * @param string $p_property
+     *      The property to look for in order to change its value.
+     * @param string $p_value
+     *      The value the property should be changed to.
+     *
+     */
+    private static function UpdateIniValue($p_filename, $p_property, $p_value)
+    {
+        $lines = file($p_filename);
+        $n=count($lines);
+        foreach ($lines as &$line) {
+            if ($line[0] != "#"){
+                $key_value = explode("=", $line);
+                $key = trim($key_value[0]);
+
+                if ($key == $p_property){
+                    $line = "$p_property = $p_value".PHP_EOL;
+                }
+            }
+        }
+
+        $fp=fopen($p_filename, 'w');
+        for($i=0; $i<$n; $i++){
+            fwrite($fp, $lines[$i]);
+        }
+        fclose($fp);
+    }
+    
+    public static function queryDb($p_sql){
+        $con = Propel::getConnection();
+
+        try {
+            $result = $con->query($p_sql);
+        } catch (Exception $e) {
+            echo "Error executing $p_sql. Exiting.";
+            exit(1);
+        }
+
+        return $result;
+    }
+}

python_apps/api_clients/install/api_client_install.py

@@ -14,12 +14,12 @@ def copy_dir(src_dir, dest_dir):
         shutil.copytree(src_dir, dest_dir)
 
 PATH_INI_FILE = '/etc/airtime/api_client.cfg'
-    
+
 current_script_dir = get_current_script_dir()
 
 if not os.path.exists(PATH_INI_FILE):
     shutil.copy('%s/../api_client.cfg'%current_script_dir, PATH_INI_FILE)
-    
+
 """load config file"""
 try:
     config = ConfigObj("%s/../api_client.cfg" % current_script_dir)

python_apps/media-monitor2/media/monitor/pure.py

@@ -273,6 +273,8 @@ def organized_path(old_path, root_path, orig_md):
     def default_f(dictionary, key):
         if key in dictionary: return len(dictionary[key]) == 0
         else: return True
+    # We set some metadata elements to a default "unknown" value because we use
+    # these fields to create a path hence they cannot be empty
     normal_md = default_to_f(orig_md, path_md, unicode_unknown, default_f)
     if is_airtime_recorded(normal_md):
         fname = u'%s-%s-%s.%s' % ( normal_md['MDATA_KEY_YEAR'],

python_apps/media-monitor2/tests/test_upgrade2dot2.py

@@ -0,0 +1,51 @@
+from configobj import ConfigObj
+import unittest
+import os
+import upgrade2dot2
+
+
+def create_cfg(cfg):
+    config = ConfigObj()
+    config.filename = cfg['path']
+    for k,v in cfg['data'].iteritems(): config[k] = v
+    return config
+
+class TestUpgrade(unittest.TestCase):
+
+    def setUp(self):
+        self.source = 'ttt1.cfg'
+        self.dest = 'ttt2.cfg'
+
+    def test_upgrade(self):
+        cf = {
+                'source' : {
+                    'path' : self.source,
+                    'data' : {
+                        'key1' : 'val1',
+                        'key2' : 'val2',
+                        'key3' : 5,
+                        'key4' : 10,},
+                },
+                'dest' : {
+                    'path' : self.dest,
+                    'data' : {
+                        'key1' : 'NEW_VAL',
+                        'key3' : 25, }
+                }
+        }
+        config1, config2 = create_cfg(cf['source']), create_cfg(cf['dest'])
+        for c in [config1,config2]: c.write()
+        self.assertTrue( os.path.exists(cf['source']['path']) )
+        self.assertTrue( os.path.exists(cf['dest']['path']) )
+        # Finished preparing
+        upgrade2dot2.upgrade(cf['source']['path'], cf['dest']['path'] )
+        c1, c2 = ConfigObj(cf['source']['path']), ConfigObj(cf['dest']['path'])
+        self.assertEqual( c2['key2'], 'val2')
+        self.assertEqual( c2['key4'], '10')
+        self.assertEqual( c2['key3'], '25')
+
+    def tearDown(self):
+        for clean in [ self.source, self.dest ]:
+            os.unlink(clean)
+
+if __name__ == '__main__': unittest.main()

python_apps/media-monitor2/upgrade2.2.py

@@ -1,37 +0,0 @@
-import os
-from configobj import ConfigObj
-import traceback
-
-upgrades_config = {
-        '/etc/airtime/media-monitor.cfg' :
-        {
-            'check_filesystem_events' : 5,
-            'check_airtime_events' : 30,
-            'touch_interval' : 5,
-            'chunking_number' : 450,
-            'request_max_wait' : 3.0,
-            'rmq_event_wait' : 0.1,
-            'logpath' : '/var/log/airtime/media-monitor/media-monitor.log',
-            'index_path' : '/var/tmp/airtime/media-monitor/last_index',
-        },
-        '/etc/airtime/api_client.cfg' :
-        {
-            'reload_metadata_group' :
-                'reload-metadata-group/format/json/api_key/%%api_key%%',
-        }
-}
-
-def upgrade(upgrade_data):
-    for f, values in upgrade_data:
-        if not os.path.exists(f):
-            print("Cannot upgrade '%s'. Skipping this file" % f)
-            continue
-        try:
-            cfg = ConfigObj(f)
-            for k,v in values:
-                if k not in cfg: cfg[k] = v
-        except Exception:
-            print("Error upgrading")
-            print( traceback.format_exc() )
-
-if __name__ == "__main__": upgrade(upgrades_config)

python_apps/media-monitor2/upgrade2dot2.py

@@ -0,0 +1,20 @@
+import os
+from configobj import ConfigObj
+import traceback
+
+def upgrade(source, destination):
+    """
+    Must be ran as sudo. will do upgrade of configuration files by filling in
+    missing values according to upgrade_data
+    """
+    if not os.path.exists(source):
+        print("Cannot upgrade '%s'. Skipping this file" % source)
+        return
+    try:
+        cfg_source, cfg_dest = ConfigObj(source), ConfigObj(destination)
+        for key, val in cfg_source.iteritems():
+            if key not in cfg_dest: cfg_dest[key] = val
+        cfg_dest.write()
+    except Exception:
+        print("Error upgrading")
+        print( traceback.format_exc() )