From fae5b15a9f8f00245a0dc7377db5d3e8371051f8 Mon Sep 17 00:00:00 2001 From: drigato Date: Mon, 28 Apr 2014 14:44:52 -0400 Subject: [PATCH 1/8] CC-5821: Airtime Analyzer: Ftp-uploaded folder doesn't get removed from organize $_FILES does not store the original file path so we were losing the folder name. Fixed by explicitly passing the full filepath in via the ftp-upload-hook.sh script --- .../rest/controllers/MediaController.php | 21 +++++++++++++------ .../airtime_analyzer/tools/ftp-upload-hook.sh | 2 +- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index ecd538a12..eec9e1b10 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -136,7 +136,19 @@ class Rest_MediaController extends Zend_Rest_Controller $file->save(); return; } else { - + /* If full_path is set, the post request came from ftp. + * Users are allowed to upload folders via ftp. If this is the case + * we need to include the folder name with the file name, otherwise + * files won't get removed from the organize folder. + */ + if (isset($whiteList["full_path"])) { + $fullPath = $whiteList["full_path"]; + $basePath = isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/organize/" : "/srv/airtime/stor/organize/"; + $relativePath = substr($fullPath, strlen($basePath)); + } else { + $relativePath = $_FILES["file"]["name"]; + } + $file->fromArray($whiteList); $file->setDbOwnerId($this->getOwnerId()); $now = new DateTime("now", new DateTimeZone("UTC")); @@ -146,8 +158,8 @@ class Rest_MediaController extends Zend_Rest_Controller $file->save(); $callbackUrl = $this->getRequest()->getScheme() . '://' . $this->getRequest()->getHttpHost() . $this->getRequest()->getRequestUri() . "/" . $file->getPrimaryKey(); - - $this->processUploadedFile($callbackUrl, $_FILES["file"]["name"], $this->getOwnerId()); + + $this->processUploadedFile($callbackUrl, $relativePath, $this->getOwnerId()); $this->getResponse() ->setHttpResponseCode(201) @@ -365,9 +377,6 @@ class Rest_MediaController extends Zend_Rest_Controller Logging::error($e->getMessage()); return; } - - Logging::info($newTempFilePath); - //Logging::info("Old temp file path: " . $tempFilePath); //Dispatch a message to airtime_analyzer through RabbitMQ, //notifying it that there's a new upload to process! diff --git a/python_apps/airtime_analyzer/tools/ftp-upload-hook.sh b/python_apps/airtime_analyzer/tools/ftp-upload-hook.sh index e456d8be9..f5be38183 100755 --- a/python_apps/airtime_analyzer/tools/ftp-upload-hook.sh +++ b/python_apps/airtime_analyzer/tools/ftp-upload-hook.sh @@ -28,7 +28,7 @@ post_file() { api_key=$(awk -F "= " '/api_key/ {print $2}' $instance_conf_path) - until curl --max-time 30 $url -u $api_key":" -X POST -F "file=@${file_path}" -F "name=${filename}" + until curl --max-time 30 $url -u $api_key":" -X POST -F "file=@${file_path}" -F "full_path=${file_path}" do retry_count=$[$retry_count+1] if [ $retry_count -ge $max_retry ]; then From c586e3e2b8e1d41be6dd302a3f0638b2db178c68 Mon Sep 17 00:00:00 2001 From: drigato Date: Mon, 28 Apr 2014 15:52:08 -0400 Subject: [PATCH 2/8] CC-5821: Airtime Analyzer: Ftp-uploaded folder doesn't get removed from organize Was using wrong position when using substr() --- .../application/modules/rest/controllers/MediaController.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index eec9e1b10..d6881e831 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -144,7 +144,8 @@ class Rest_MediaController extends Zend_Rest_Controller if (isset($whiteList["full_path"])) { $fullPath = $whiteList["full_path"]; $basePath = isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/organize/" : "/srv/airtime/stor/organize/"; - $relativePath = substr($fullPath, strlen($basePath)); + $relativePath = substr($fullPath, strlen($basePath)-1); + } else { $relativePath = $_FILES["file"]["name"]; } From f428a2d9609a37676b7f56133d55d0ecc7d5328f Mon Sep 17 00:00:00 2001 From: drigato Date: Mon, 28 Apr 2014 16:20:26 -0400 Subject: [PATCH 3/8] CC-5821: Airtime Analyzer: Ftp-uploaded folder doesn't get removed from organize Delete the empty folder after files have been copied into imported --- .../rest/controllers/MediaController.php | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index d6881e831..8838d4e4f 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -144,11 +144,24 @@ class Rest_MediaController extends Zend_Rest_Controller if (isset($whiteList["full_path"])) { $fullPath = $whiteList["full_path"]; $basePath = isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/organize/" : "/srv/airtime/stor/organize/"; + //$relativePath is the folder name(if one) + track name, that was uploaded via ftp $relativePath = substr($fullPath, strlen($basePath)-1); + //after the file is moved from organize store it's parent folder so we can delete it after + if (dirname($relativePath) != '/') { + $pathToDelete = Application_Common_OsPath::join( + $basePath, + dirname($relativePath) + ); + } else { + //if the uploaded file was not in a folder, DO NOT delete + $pathToDelete = False; + } } else { $relativePath = $_FILES["file"]["name"]; + $pathToDelete = False; } + $file->fromArray($whiteList); $file->setDbOwnerId($this->getOwnerId()); @@ -161,6 +174,11 @@ class Rest_MediaController extends Zend_Rest_Controller $callbackUrl = $this->getRequest()->getScheme() . '://' . $this->getRequest()->getHttpHost() . $this->getRequest()->getRequestUri() . "/" . $file->getPrimaryKey(); $this->processUploadedFile($callbackUrl, $relativePath, $this->getOwnerId()); + + if (!$pathToDelete) { + //delete the empty folder that was uploaded via ftp (if one) + rmdir($pathToDelete); + } $this->getResponse() ->setHttpResponseCode(201) From 79687499aa3b2812929a5d338ab6fdfe31f4de63 Mon Sep 17 00:00:00 2001 From: drigato Date: Mon, 28 Apr 2014 16:24:59 -0400 Subject: [PATCH 4/8] Logging statement. To remove later. --- .../application/modules/rest/controllers/MediaController.php | 1 + 1 file changed, 1 insertion(+) diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index 8838d4e4f..7c3855824 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -176,6 +176,7 @@ class Rest_MediaController extends Zend_Rest_Controller $this->processUploadedFile($callbackUrl, $relativePath, $this->getOwnerId()); if (!$pathToDelete) { + Logging::info($pathToDelete); //delete the empty folder that was uploaded via ftp (if one) rmdir($pathToDelete); } From 6f1dd7987f1b6df012b9faa3b6bb25cd747dd844 Mon Sep 17 00:00:00 2001 From: drigato Date: Tue, 29 Apr 2014 10:05:17 -0400 Subject: [PATCH 5/8] CC-5821: Airtime Analyzer: Ftp-uploaded folder doesn't get removed from organize --- .../modules/rest/controllers/MediaController.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index 7c3855824..2c7a667ca 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -147,7 +147,7 @@ class Rest_MediaController extends Zend_Rest_Controller //$relativePath is the folder name(if one) + track name, that was uploaded via ftp $relativePath = substr($fullPath, strlen($basePath)-1); - //after the file is moved from organize store it's parent folder so we can delete it after + //after the file is moved from organize, store it's parent folder so we can delete it after if (dirname($relativePath) != '/') { $pathToDelete = Application_Common_OsPath::join( $basePath, @@ -155,11 +155,11 @@ class Rest_MediaController extends Zend_Rest_Controller ); } else { //if the uploaded file was not in a folder, DO NOT delete - $pathToDelete = False; + $pathToDelete = false; } } else { $relativePath = $_FILES["file"]["name"]; - $pathToDelete = False; + $pathToDelete = false; } @@ -175,7 +175,7 @@ class Rest_MediaController extends Zend_Rest_Controller $this->processUploadedFile($callbackUrl, $relativePath, $this->getOwnerId()); - if (!$pathToDelete) { + if ($pathToDelete) { Logging::info($pathToDelete); //delete the empty folder that was uploaded via ftp (if one) rmdir($pathToDelete); From 94375ee441be4acae99ee5aa038f1a004aebc0e4 Mon Sep 17 00:00:00 2001 From: drigato Date: Tue, 29 Apr 2014 10:35:39 -0400 Subject: [PATCH 6/8] CC-5821: Airtime Analyzer: Ftp-uploaded folder doesn't get removed from organize Cleaner way of removing empty sub folders after ftp uploads --- .../rest/controllers/MediaController.php | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index 2c7a667ca..e74f491c4 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -146,20 +146,8 @@ class Rest_MediaController extends Zend_Rest_Controller $basePath = isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/organize/" : "/srv/airtime/stor/organize/"; //$relativePath is the folder name(if one) + track name, that was uploaded via ftp $relativePath = substr($fullPath, strlen($basePath)-1); - - //after the file is moved from organize, store it's parent folder so we can delete it after - if (dirname($relativePath) != '/') { - $pathToDelete = Application_Common_OsPath::join( - $basePath, - dirname($relativePath) - ); - } else { - //if the uploaded file was not in a folder, DO NOT delete - $pathToDelete = false; - } } else { $relativePath = $_FILES["file"]["name"]; - $pathToDelete = false; } @@ -175,12 +163,6 @@ class Rest_MediaController extends Zend_Rest_Controller $this->processUploadedFile($callbackUrl, $relativePath, $this->getOwnerId()); - if ($pathToDelete) { - Logging::info($pathToDelete); - //delete the empty folder that was uploaded via ftp (if one) - rmdir($pathToDelete); - } - $this->getResponse() ->setHttpResponseCode(201) ->appendBody(json_encode($this->sanitizeResponse($file))); @@ -233,6 +215,10 @@ class Rest_MediaController extends Zend_Rest_Controller $now = new DateTime("now", new DateTimeZone("UTC")); $file->setDbMtime($now); $file->save(); + + $this->removeEmptySubFolders( + isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/organize/" : "/srv/airtime/stor/organize/"); + $this->getResponse() ->setHttpResponseCode(200) ->appendBody(json_encode($this->sanitizeResponse($file))); @@ -459,5 +445,10 @@ class Rest_MediaController extends Zend_Rest_Controller return $response; } + private function removeEmptySubFolders($path) + { + exec("find $path -empty -type d -delete"); + } + } From 04da9b3d61100d62b052d1bea8bed43593a31166 Mon Sep 17 00:00:00 2001 From: drigato Date: Tue, 29 Apr 2014 11:09:31 -0400 Subject: [PATCH 7/8] CC-5821: Airtime Analyzer: Ftp-uploaded folder doesn't get removed from organize Imported file path had album name twice - fixed --- .../application/modules/rest/controllers/MediaController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index e74f491c4..ba116af0f 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -387,7 +387,7 @@ class Rest_MediaController extends Zend_Rest_Controller //Dispatch a message to airtime_analyzer through RabbitMQ, //notifying it that there's a new upload to process! Application_Model_RabbitMq::SendMessageToAnalyzer($newTempFilePath, - $importedStorageDirectory, $originalFilename, + $importedStorageDirectory, basename($originalFilename), $callbackUrl, $apiKey); } From a5eb5e9901c46a112d23f8d6d2f26737ba20cc3a Mon Sep 17 00:00:00 2001 From: drigato Date: Tue, 29 Apr 2014 16:06:25 -0400 Subject: [PATCH 8/8] CC-5806: Airtime Analyzer: Please implement "remove all files" --- .../application/modules/rest/Bootstrap.php | 12 ++++++- .../rest/controllers/MediaController.php | 36 +++++++++++++++++++ .../rest/views/scripts/media/clear.phtml | 0 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 airtime_mvc/application/modules/rest/views/scripts/media/clear.phtml diff --git a/airtime_mvc/application/modules/rest/Bootstrap.php b/airtime_mvc/application/modules/rest/Bootstrap.php index e7017ba16..31691ca96 100644 --- a/airtime_mvc/application/modules/rest/Bootstrap.php +++ b/airtime_mvc/application/modules/rest/Bootstrap.php @@ -23,5 +23,15 @@ class Rest_Bootstrap extends Zend_Application_Module_Bootstrap ) ); $router->addRoute('download', $downloadRoute); + + $clearLibraryRoute = new Zend_Controller_Router_Route( + 'rest/media/clear', + array( + 'controller' => 'media', + 'action' => 'clear', + 'module' => 'rest' + ) + ); + $router->addRoute('clear', $clearLibraryRoute); } -} \ No newline at end of file +} diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index ba116af0f..65f30b7e9 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -80,6 +80,42 @@ class Rest_MediaController extends Zend_Rest_Controller $this->fileNotFoundResponse(); } } + + public function clearAction() + { + //TODO:: make this not accessible via public api?? + if (!$this->verifyAuth(true, true)) + { + return; + } + + //set file_exists flag to false for every file + $con = Propel::getConnection(CcFilesPeer::DATABASE_NAME); + $selectCriteria = new Criteria(); + $selectCriteria->add(CcFilesPeer::FILE_EXISTS, true); + $updateCriteria = new Criteria(); + $updateCriteria->add(CcFilesPeer::FILE_EXISTS, false); + BasePeer::doUpdate($selectCriteria, $updateCriteria, $con); + + $path = isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/imported/*" : "/srv/airtime/stor/imported/*"; + exec("rm -rf $path"); + + //update disk_usage value in cc_pref + $musicDir = CcMusicDirsQuery::create() + ->filterByType('stor') + ->filterByExists(true) + ->findOne(); + $storPath = $musicDir->getDirectory(); + + $freeSpace = disk_free_space($storPath); + $totalSpace = disk_total_space($storPath); + + Application_Model_Preference::setDiskUsage($totalSpace - $freeSpace); + + $this->getResponse() + ->setHttpResponseCode(200) + ->appendBody("Library has been cleared"); + } public function getAction() { diff --git a/airtime_mvc/application/modules/rest/views/scripts/media/clear.phtml b/airtime_mvc/application/modules/rest/views/scripts/media/clear.phtml new file mode 100644 index 000000000..e69de29bb