From 5971f51b53b18413bcf226c19c3fb7e00712edc7 Mon Sep 17 00:00:00 2001 From: drigato Date: Thu, 10 Apr 2014 07:12:31 -0400 Subject: [PATCH 1/9] CC-5781: Upgrade script for new storage quota implementation First draft of an upgrade controller --- airtime_mvc/application/configs/ACL.php | 4 +- .../controllers/UpgradeController.php | 69 +++++++++++++++++++ .../upgrade_sql/airtime_2.5.3/upgrade.sql | 6 ++ .../airtime-2.5.3/airtime-upgrade.php | 6 -- 4 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 airtime_mvc/application/controllers/UpgradeController.php create mode 100644 airtime_mvc/application/controllers/upgrade_sql/airtime_2.5.3/upgrade.sql diff --git a/airtime_mvc/application/configs/ACL.php b/airtime_mvc/application/configs/ACL.php index 83cba4b08..842778065 100644 --- a/airtime_mvc/application/configs/ACL.php +++ b/airtime_mvc/application/configs/ACL.php @@ -28,7 +28,8 @@ $ccAcl->add(new Zend_Acl_Resource('library')) ->add(new Zend_Acl_Resource('usersettings')) ->add(new Zend_Acl_Resource('audiopreview')) ->add(new Zend_Acl_Resource('webstream')) - ->add(new Zend_Acl_Resource('locale')); + ->add(new Zend_Acl_Resource('locale')) + ->add(new Zend_Acl_Resource('upgrade')); /** Creating permissions */ $ccAcl->allow('G', 'index') @@ -42,6 +43,7 @@ $ccAcl->allow('G', 'index') ->allow('G', 'audiopreview') ->allow('G', 'webstream') ->allow('G', 'locale') + ->allow('G', 'upgrade') ->allow('H', 'preference', 'is-import-in-progress') ->allow('H', 'usersettings') ->allow('H', 'plupload') diff --git a/airtime_mvc/application/controllers/UpgradeController.php b/airtime_mvc/application/controllers/UpgradeController.php new file mode 100644 index 000000000..e1606b98d --- /dev/null +++ b/airtime_mvc/application/controllers/UpgradeController.php @@ -0,0 +1,69 @@ +view->layout()->disableLayout(); + $this->_helper->viewRenderer->setNoRender(true); + + //TODO: check api key + //The API key is passed in via HTTP "basic authentication": + //http://en.wikipedia.org/wiki/Basic_access_authentication + + $CC_CONFIG = Config::getConfig(); + + //Decode the API key that was passed to us in the HTTP request. + $authHeader = $this->getRequest()->getHeader("Authorization"); + $encodedRequestApiKey = substr($authHeader, strlen("Basic ")); + $encodedStoredApiKey = base64_encode($CC_CONFIG["apiKey"][0] . ":"); + + if (!$encodedRequestApiKey === $encodedStoredApiKey) + { + $this->getResponse() + ->setHttpResponseCode(401) + ->appendBody("Bad Authorization."); + return; + } + + //check current airtime version + $airtime_version = Application_Model_Preference::GetAirtimeVersion(); + if ($airtime_version != '2.5.2') { + $this->getResponse() + ->setHttpResponseCode(400) + ->appendBody("Upgrade to Airtime 2.5.3 FAILED. You must be using Airtime 2.5.2 to upgrade."); + return; + } + + $filename = "/etc/airtime/airtime.conf"; + $values = parse_ini_file($filename, true); + + $username = $values['database']['dbuser']; + $password = $values['database']['dbpass']; + $host = $values['database']['host']; + $database = $values['database']['dbname']; + $dir = __DIR__; + + passthru("export PGPASSWORD=$password && psql -h $host -U $username -q -f $dir/upgrade_sql/airtime_$airtime_version/upgrade.sql $database 2>&1 | grep -v \"will create implicit index\""); + + + $musicDir = CcMusicDirsQuery::create() + ->filterByType('stor') + ->filterByExists(true) + ->findOne(); + $storPath = $musicDir->getDirectory(); + + $freeSpace = disk_free_space($storPath); + $totalSpace = disk_total_space($storPath); + + Application_Model_Preference::setDiskUsage($totalSpace - $freeSpace); + + $this->getResponse() + ->setHttpResponseCode(200) + ->appendBody("Upgrade to Airtime 2.5.3 OK"); + } + + +} \ No newline at end of file diff --git a/airtime_mvc/application/controllers/upgrade_sql/airtime_2.5.3/upgrade.sql b/airtime_mvc/application/controllers/upgrade_sql/airtime_2.5.3/upgrade.sql new file mode 100644 index 000000000..6c7980983 --- /dev/null +++ b/airtime_mvc/application/controllers/upgrade_sql/airtime_2.5.3/upgrade.sql @@ -0,0 +1,6 @@ +DELETE FROM cc_pref WHERE keystr = 'system_version'; +INSERT INTO cc_pref (keystr, valstr) VALUES ('system_version', '2.5.3'); + +ALTER TABLE cc_files DROP COLUMN state; +ALTER TABLE cc_files ADD import_status integer default 1; -- Default is "pending" +UPDATE cc_files SET import_status=0; -- Existing files are already "imported" diff --git a/install_minimal/upgrades/airtime-2.5.3/airtime-upgrade.php b/install_minimal/upgrades/airtime-2.5.3/airtime-upgrade.php index 09ecd7ed2..31792eb7a 100644 --- a/install_minimal/upgrades/airtime-2.5.3/airtime-upgrade.php +++ b/install_minimal/upgrades/airtime-2.5.3/airtime-upgrade.php @@ -1,11 +1,5 @@ Date: Thu, 10 Apr 2014 09:28:23 -0400 Subject: [PATCH 2/9] CC-5781: Upgrade script for new storage quota implementation Returns error if API key is incorrect Set the upgrade controller to skip login authentication --- airtime_mvc/application/controllers/UpgradeController.php | 8 ++++---- .../application/controllers/plugins/Acl_plugin.php | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airtime_mvc/application/controllers/UpgradeController.php b/airtime_mvc/application/controllers/UpgradeController.php index e1606b98d..a01eb5eca 100644 --- a/airtime_mvc/application/controllers/UpgradeController.php +++ b/airtime_mvc/application/controllers/UpgradeController.php @@ -17,10 +17,11 @@ class UpgradeController extends Zend_Controller_Action //Decode the API key that was passed to us in the HTTP request. $authHeader = $this->getRequest()->getHeader("Authorization"); + $encodedRequestApiKey = substr($authHeader, strlen("Basic ")); $encodedStoredApiKey = base64_encode($CC_CONFIG["apiKey"][0] . ":"); - - if (!$encodedRequestApiKey === $encodedStoredApiKey) + + if ($encodedRequestApiKey !== $encodedStoredApiKey) { $this->getResponse() ->setHttpResponseCode(401) @@ -46,8 +47,7 @@ class UpgradeController extends Zend_Controller_Action $database = $values['database']['dbname']; $dir = __DIR__; - passthru("export PGPASSWORD=$password && psql -h $host -U $username -q -f $dir/upgrade_sql/airtime_$airtime_version/upgrade.sql $database 2>&1 | grep -v \"will create implicit index\""); - + passthru("export PGPASSWORD=$password && psql -h $host -U $username -q -f $dir/upgrade_sql/airtime_$airtime_upgrade_version/upgrade.sql $database 2>&1 | grep -v \"will create implicit index\""); $musicDir = CcMusicDirsQuery::create() ->filterByType('stor') diff --git a/airtime_mvc/application/controllers/plugins/Acl_plugin.php b/airtime_mvc/application/controllers/plugins/Acl_plugin.php index 4cf9f97b5..c5dc4b9f4 100644 --- a/airtime_mvc/application/controllers/plugins/Acl_plugin.php +++ b/airtime_mvc/application/controllers/plugins/Acl_plugin.php @@ -117,7 +117,7 @@ class Zend_Controller_Plugin_Acl extends Zend_Controller_Plugin_Abstract return; } - if (in_array($controller, array("api", "auth", "locale"))) { + if (in_array($controller, array("api", "auth", "locale", "upgrade"))) { $this->setRoleName("G"); } elseif (!Zend_Auth::getInstance()->hasIdentity()) { From 819862c37bf456be2f16816c0d35c9e3455dc0c3 Mon Sep 17 00:00:00 2001 From: drigato Date: Thu, 10 Apr 2014 10:55:47 -0400 Subject: [PATCH 3/9] CC-5781: Upgrade script for new storage quota implementation cleaned up upgrade controller --- .../controllers/UpgradeController.php | 69 ++++++++++++------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/airtime_mvc/application/controllers/UpgradeController.php b/airtime_mvc/application/controllers/UpgradeController.php index a01eb5eca..bad83f409 100644 --- a/airtime_mvc/application/controllers/UpgradeController.php +++ b/airtime_mvc/application/controllers/UpgradeController.php @@ -9,35 +9,15 @@ class UpgradeController extends Zend_Controller_Action $this->view->layout()->disableLayout(); $this->_helper->viewRenderer->setNoRender(true); - //TODO: check api key - //The API key is passed in via HTTP "basic authentication": - //http://en.wikipedia.org/wiki/Basic_access_authentication + if (!$this->verifyAuth()) { + retrun; + } - $CC_CONFIG = Config::getConfig(); - - //Decode the API key that was passed to us in the HTTP request. - $authHeader = $this->getRequest()->getHeader("Authorization"); - - $encodedRequestApiKey = substr($authHeader, strlen("Basic ")); - $encodedStoredApiKey = base64_encode($CC_CONFIG["apiKey"][0] . ":"); - - if ($encodedRequestApiKey !== $encodedStoredApiKey) - { - $this->getResponse() - ->setHttpResponseCode(401) - ->appendBody("Bad Authorization."); - return; - } - - //check current airtime version - $airtime_version = Application_Model_Preference::GetAirtimeVersion(); - if ($airtime_version != '2.5.2') { - $this->getResponse() - ->setHttpResponseCode(400) - ->appendBody("Upgrade to Airtime 2.5.3 FAILED. You must be using Airtime 2.5.2 to upgrade."); + if (!$this->verifyAirtimeVersion()) { return; } + //Begin upgrade $filename = "/etc/airtime/airtime.conf"; $values = parse_ini_file($filename, true); @@ -65,5 +45,42 @@ class UpgradeController extends Zend_Controller_Action ->appendBody("Upgrade to Airtime 2.5.3 OK"); } - + private function verifyAuth() + { + //The API key is passed in via HTTP "basic authentication": + //http://en.wikipedia.org/wiki/Basic_access_authentication + + $CC_CONFIG = Config::getConfig(); + + //Decode the API key that was passed to us in the HTTP request. + $authHeader = $this->getRequest()->getHeader("Authorization"); + + $encodedRequestApiKey = substr($authHeader, strlen("Basic ")); + $encodedStoredApiKey = base64_encode($CC_CONFIG["apiKey"][0] . ":"); + + if ($encodedRequestApiKey !== $encodedStoredApiKey) + { + $this->getResponse() + ->setHttpResponseCode(401) + ->appendBody("Error: Incorrect API key."); + return false; + } + return true; + } + + private function verifyAirtimeVersion() + { + $pref = CcPrefQuery::create() + ->filterByKeystr('system_version') + ->findOne(); + $airtime_version = $pref->getValStr(); + + if ($airtime_version != '2.5.2') { + $this->getResponse() + ->setHttpResponseCode(400) + ->appendBody("Upgrade to Airtime 2.5.3 FAILED. You must be using Airtime 2.5.2 to upgrade."); + return false; + } + return true; + } } \ No newline at end of file From 7fbd285dd0523be5870bd7d85f8f9b552984ce71 Mon Sep 17 00:00:00 2001 From: drigato Date: Thu, 10 Apr 2014 15:55:21 -0400 Subject: [PATCH 4/9] Fix typo --- airtime_mvc/application/controllers/UpgradeController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtime_mvc/application/controllers/UpgradeController.php b/airtime_mvc/application/controllers/UpgradeController.php index bad83f409..4a745b657 100644 --- a/airtime_mvc/application/controllers/UpgradeController.php +++ b/airtime_mvc/application/controllers/UpgradeController.php @@ -10,7 +10,7 @@ class UpgradeController extends Zend_Controller_Action $this->_helper->viewRenderer->setNoRender(true); if (!$this->verifyAuth()) { - retrun; + return; } if (!$this->verifyAirtimeVersion()) { From 4add0f0b7bbc95dc5baf1a3944629751a01190f4 Mon Sep 17 00:00:00 2001 From: drigato Date: Mon, 14 Apr 2014 11:24:39 -0400 Subject: [PATCH 5/9] CC-5786: Quota Enforcement in the File Upload API --- .../controllers/PluploadController.php | 5 ++++ .../rest/controllers/MediaController.php | 23 ++++++++++++++++++- .../views/scripts/plupload/index.phtml | 11 +++++++-- airtime_mvc/public/css/styles.css | 2 ++ 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/airtime_mvc/application/controllers/PluploadController.php b/airtime_mvc/application/controllers/PluploadController.php index 0695f0def..a121bd7ec 100644 --- a/airtime_mvc/application/controllers/PluploadController.php +++ b/airtime_mvc/application/controllers/PluploadController.php @@ -25,6 +25,11 @@ class PluploadController extends Zend_Controller_Action $this->view->headLink()->appendStylesheet($baseUrl.'css/plupload.queue.css?'.$CC_CONFIG['airtime_version']); $this->view->headLink()->appendStylesheet($baseUrl.'css/addmedia.css?'.$CC_CONFIG['airtime_version']); + + $this->view->quotaLimitReached = false; + if (Application_Model_Preference::getDiskUsage() > Application_Model_Preference::getDiskQuota()) { + $this->view->quotaLimitReached = true; + } } public function recentUploadsAction() diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index ad8cada08..ffe09ba89 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -110,7 +110,7 @@ class Rest_MediaController extends Zend_Rest_Controller { return; } - + //If we do get an ID on a POST, then that doesn't make any sense //since POST is only for creating. if ($id = $this->_getParam('id', false)) { @@ -120,6 +120,13 @@ class Rest_MediaController extends Zend_Rest_Controller return; } + if (!$this->isEnoughDiskSpace()) { + $this->getResponse() + ->setHttpResponseCode(400) + ->appendBody("ERROR: Disk Quota limit reached."); + return; + } + $file = new CcFiles(); $whiteList = $this->removeBlacklistedFieldsFromRequestData($this->getRequest()->getPost()); @@ -423,5 +430,19 @@ class Rest_MediaController extends Zend_Rest_Controller return $response; } + /** + * + * Checks if there is enough disk space to upload the file in question + * We allow one file to exceed to the disk quota so it is possible for the + * disk usage to be greater than the disk usage value + */ + private function isEnoughDiskSpace() + { + if (Application_Model_Preference::getDiskUsage() < Application_Model_Preference::GetDiskQuota()) { + return true; + } + return false; + } + } diff --git a/airtime_mvc/application/views/scripts/plupload/index.phtml b/airtime_mvc/application/views/scripts/plupload/index.phtml index 4eec76438..bd25abe61 100644 --- a/airtime_mvc/application/views/scripts/plupload/index.phtml +++ b/airtime_mvc/application/views/scripts/plupload/index.phtml @@ -2,8 +2,15 @@ #plupload_files input[type="file"] { font-size: 200px !important; } - -
+ +quotaLimitReached) { ?> +
+ Disk quota exceeded. You cannot upload files until you upgrade your storage. +
+ +quotaLimitReached) { ?> class="hidden" >
diff --git a/airtime_mvc/public/css/styles.css b/airtime_mvc/public/css/styles.css index a63f4975e..8cdf05891 100644 --- a/airtime_mvc/public/css/styles.css +++ b/airtime_mvc/public/css/styles.css @@ -3087,3 +3087,5 @@ dd .stream-status { #popup-share-link { width: 320px; } +.quota-reached { + font-size: 14px !important; From 63e2eda64b082cc7af854f9e10a5cb4cf0c0b7e6 Mon Sep 17 00:00:00 2001 From: drigato Date: Mon, 14 Apr 2014 12:09:15 -0400 Subject: [PATCH 6/9] CC-5786: Quota Enforcement in the File Upload API Tweaked this so it will work on self-hosted instances --- .../application/controllers/PluploadController.php | 2 +- airtime_mvc/application/models/Systemstatus.php | 12 ++++++++++++ .../modules/rest/controllers/MediaController.php | 12 +++--------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/airtime_mvc/application/controllers/PluploadController.php b/airtime_mvc/application/controllers/PluploadController.php index a121bd7ec..abd03afc3 100644 --- a/airtime_mvc/application/controllers/PluploadController.php +++ b/airtime_mvc/application/controllers/PluploadController.php @@ -27,7 +27,7 @@ class PluploadController extends Zend_Controller_Action $this->view->headLink()->appendStylesheet($baseUrl.'css/addmedia.css?'.$CC_CONFIG['airtime_version']); $this->view->quotaLimitReached = false; - if (Application_Model_Preference::getDiskUsage() > Application_Model_Preference::getDiskQuota()) { + if (Application_Model_Systemstatus::isDiskOverQuota()) { $this->view->quotaLimitReached = true; } } diff --git a/airtime_mvc/application/models/Systemstatus.php b/airtime_mvc/application/models/Systemstatus.php index 4a0480a04..1185609f2 100644 --- a/airtime_mvc/application/models/Systemstatus.php +++ b/airtime_mvc/application/models/Systemstatus.php @@ -235,4 +235,16 @@ class Application_Model_Systemstatus return array_values($partitions); } + + public static function isDiskOverQuota() + { + $diskInfo = self::GetDiskInfo(); + $diskInfo = $diskInfo[0]; + $diskUsage = $diskInfo->totalSpace - $diskInfo->totalFreeSpace; + if ($diskUsage > $diskInfo->totalSpace) { + return true; + } + + return false; + } } diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index ffe09ba89..72fc067b1 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -120,7 +120,7 @@ class Rest_MediaController extends Zend_Rest_Controller return; } - if (!$this->isEnoughDiskSpace()) { + if (!$this->isDiskOverQuota()) { $this->getResponse() ->setHttpResponseCode(400) ->appendBody("ERROR: Disk Quota limit reached."); @@ -430,15 +430,9 @@ class Rest_MediaController extends Zend_Rest_Controller return $response; } - /** - * - * Checks if there is enough disk space to upload the file in question - * We allow one file to exceed to the disk quota so it is possible for the - * disk usage to be greater than the disk usage value - */ - private function isEnoughDiskSpace() + private function isDiskOverQuota() { - if (Application_Model_Preference::getDiskUsage() < Application_Model_Preference::GetDiskQuota()) { + if (Application_Model_Systemstatus::isDiskOverQuota()) { return true; } return false; From acf91bc627b8feadb146229032455fa1e13c2ea1 Mon Sep 17 00:00:00 2001 From: drigato Date: Mon, 14 Apr 2014 12:13:48 -0400 Subject: [PATCH 7/9] CC-5786: Quota Enforcement in the File Upload API Small refactor --- .../modules/rest/controllers/MediaController.php | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index 72fc067b1..ecd538a12 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -120,10 +120,10 @@ class Rest_MediaController extends Zend_Rest_Controller return; } - if (!$this->isDiskOverQuota()) { + if (Application_Model_Systemstatus::isDiskOverQuota()) { $this->getResponse() ->setHttpResponseCode(400) - ->appendBody("ERROR: Disk Quota limit reached."); + ->appendBody("ERROR: Disk Quota reached."); return; } @@ -430,13 +430,5 @@ class Rest_MediaController extends Zend_Rest_Controller return $response; } - private function isDiskOverQuota() - { - if (Application_Model_Systemstatus::isDiskOverQuota()) { - return true; - } - return false; - } - } From 53ffa29b6bd4c711f8549e96f38f8fed1fa4022f Mon Sep 17 00:00:00 2001 From: drigato Date: Mon, 14 Apr 2014 12:21:59 -0400 Subject: [PATCH 8/9] CC-5786: Quota Enforcement in the File Upload API small fix --- airtime_mvc/application/models/Systemstatus.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtime_mvc/application/models/Systemstatus.php b/airtime_mvc/application/models/Systemstatus.php index 1185609f2..05e69bc34 100644 --- a/airtime_mvc/application/models/Systemstatus.php +++ b/airtime_mvc/application/models/Systemstatus.php @@ -241,7 +241,7 @@ class Application_Model_Systemstatus $diskInfo = self::GetDiskInfo(); $diskInfo = $diskInfo[0]; $diskUsage = $diskInfo->totalSpace - $diskInfo->totalFreeSpace; - if ($diskUsage > $diskInfo->totalSpace) { + if ($diskUsage >= $diskInfo->totalSpace) { return true; } From 7aba416c3b47b1fbbf899bcd342f8501a3fc149a Mon Sep 17 00:00:00 2001 From: drigato Date: Wed, 16 Apr 2014 10:54:55 -0400 Subject: [PATCH 9/9] Small fix to ftp hook script. Removed space that was getting added to the api key --- python_apps/airtime_analyzer/tools/ftp-upload-hook.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/airtime_analyzer/tools/ftp-upload-hook.sh b/python_apps/airtime_analyzer/tools/ftp-upload-hook.sh index faf677a29..e456d8be9 100755 --- a/python_apps/airtime_analyzer/tools/ftp-upload-hook.sh +++ b/python_apps/airtime_analyzer/tools/ftp-upload-hook.sh @@ -26,7 +26,7 @@ post_file() { #path to specific instance's airtime.conf instance_conf_path=$base_instance_path$instance_path$airtime_conf_path - api_key=$(sudo awk -F "=" '/api_key/ {print $2}' $instance_conf_path) + api_key=$(awk -F "= " '/api_key/ {print $2}' $instance_conf_path) until curl --max-time 30 $url -u $api_key":" -X POST -F "file=@${file_path}" -F "name=${filename}" do