CC-5888: Handle file deletion if the file is stored in the cloud
This commit is contained in:
drigato
parent
8e714bcb64
commit
aaee522ec6
7 changed files with 134 additions and 39 deletions
|
|
@ -79,10 +79,11 @@ class Application_Model_RabbitMq
|
|||
self::sendMessage($exchange, 'direct', true, $data);
|
||||
}
|
||||
|
||||
public static function SendMessageToAnalyzer($tmpFilePath, $importedStorageDirectory, $originalFilename,
|
||||
public static function SendUploadMessageToAnalyzer($tmpFilePath, $importedStorageDirectory, $originalFilename,
|
||||
$callbackUrl, $apiKey, $messageType)
|
||||
{
|
||||
$exchange = 'airtime-uploads';
|
||||
|
||||
$data['message_type'] = $messageType;
|
||||
$data['tmp_file_path'] = $tmpFilePath;
|
||||
$data['import_directory'] = $importedStorageDirectory;
|
||||
|
|
@ -93,4 +94,17 @@ class Application_Model_RabbitMq
|
|||
$jsonData = json_encode($data);
|
||||
self::sendMessage($exchange, 'topic', false, $jsonData, 'airtime-uploads');
|
||||
}
|
||||
|
||||
public static function SendDeleteMessageToAnalyzer($callbackUrl, $objectName, $apiKey, $messageType)
|
||||
{
|
||||
$exchange = 'airtime-uploads';
|
||||
|
||||
$data['message_type'] = $messageType;
|
||||
$data['api_key'] = $apiKey;
|
||||
$data['object_name'] = $objectName;
|
||||
$data['callback_url'] = $callbackUrl;
|
||||
|
||||
$jsonData = json_encode($data);
|
||||
self::sendMessage($exchange, 'topic', false, $jsonData, 'airtime-uploads');
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -363,9 +363,7 @@ SQL;
|
|||
}
|
||||
|
||||
/**
|
||||
* Delete stored virtual file
|
||||
*
|
||||
* @param boolean $p_deleteFile
|
||||
* Deletes the physical file from the local file system or from the cloud
|
||||
*
|
||||
*/
|
||||
public function delete()
|
||||
|
|
@ -383,10 +381,11 @@ SQL;
|
|||
if (!$isAdminOrPM && $this->getFileOwnerId() != $user->getId()) {
|
||||
throw new FileNoPermissionException();
|
||||
}
|
||||
$file_id = $this->_file->getDbId();
|
||||
Logging::info("User ".$user->getLogin()." is deleting file: ".$this->_file->getDbTrackTitle()." - file id: ".$file_id);
|
||||
|
||||
$isInCloud = $this->isInCloud();
|
||||
|
||||
$filesize = $this->getFileSize();
|
||||
if (file_exists($filepath) && !$isInCloud) {
|
||||
try {
|
||||
unlink($filepath);
|
||||
|
|
@ -394,23 +393,46 @@ SQL;
|
|||
Logging::error($e->getMessage());
|
||||
return;
|
||||
}
|
||||
|
||||
$this->doFileDeletionCleanup($this->getFileSize());
|
||||
|
||||
} elseif ($isInCloud) {
|
||||
//delete file from cloud
|
||||
//Dispatch a message to airtime_analyzer through RabbitMQ,
|
||||
//notifying it that we need to delete a file from the cloud
|
||||
$CC_CONFIG = Config::getConfig();
|
||||
$apiKey = $CC_CONFIG["apiKey"][0];
|
||||
|
||||
$callbackUrl = 'http://'.$_SERVER['HTTP_HOST'].'/rest/media/'.$file_id.'/delete-success';
|
||||
|
||||
Application_Model_RabbitMq::SendDeleteMessageToAnalyzer(
|
||||
$callbackUrl, $this->_file->getDbResourceId(), $apiKey, 'delete');
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* This function handles all the actions required when a file is deleted
|
||||
*/
|
||||
public function doFileDeletionCleanup($filesize)
|
||||
{
|
||||
//Update the user's disk usage
|
||||
Application_Model_Preference::updateDiskUsage(-1 * $filesize);
|
||||
|
||||
//Explicitly update any playlist's and block's length that contains
|
||||
//the file getting deleted
|
||||
self::updateBlockAndPlaylistLength($this->_file->getDbId());
|
||||
|
||||
//delete the file record from cc_files
|
||||
$this->_file->delete();
|
||||
}
|
||||
|
||||
Logging::info("User ".$user->getLogin()." is deleting file: ".$this->_file->getDbTrackTitle()." - file id: ".$this->_file->getDbId());
|
||||
// set hidden flag to true
|
||||
//$this->_file->setDbHidden(true);
|
||||
//$this->_file->setDbFileExists(false);
|
||||
//$this->_file->save();
|
||||
|
||||
// need to explicitly update any playlist's and block's length
|
||||
// that contains the file getting deleted
|
||||
$fileId = $this->_file->getDbId();
|
||||
$plRows = CcPlaylistcontentsQuery::create()->filterByDbFileId()->find();
|
||||
/*
|
||||
* This function is meant to be called when a file is getting
|
||||
* deleted from the library. It re-calculates the length of
|
||||
* all blocks and playlists that contained the deleted file.
|
||||
*/
|
||||
public static function updateBlockAndPlaylistLength($fileId)
|
||||
{
|
||||
$plRows = CcPlaylistcontentsQuery::create()->filterByDbFileId($fileId)->find();
|
||||
foreach ($plRows as $row) {
|
||||
$pl = CcPlaylistQuery::create()->filterByDbId($row->getDbPlaylistId($fileId))->findOne();
|
||||
$pl->setDbLength($pl->computeDbLength(Propel::getConnection(CcPlaylistPeer::DATABASE_NAME)));
|
||||
|
|
@ -423,8 +445,6 @@ SQL;
|
|||
$bl->setDbLength($bl->computeDbLength(Propel::getConnection(CcBlockPeer::DATABASE_NAME)));
|
||||
$bl->save();
|
||||
}
|
||||
|
||||
$this->_file->delete();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -33,5 +33,18 @@ class Rest_Bootstrap extends Zend_Application_Module_Bootstrap
|
|||
)
|
||||
);
|
||||
$router->addRoute('clear', $clearLibraryRoute);
|
||||
|
||||
$deleteSuccessRoute = new Zend_Controller_Router_Route(
|
||||
'rest/media/:id/delete-success',
|
||||
array(
|
||||
'controller' => 'media',
|
||||
'action' => 'delete-success',
|
||||
'module' => 'rest'
|
||||
),
|
||||
array(
|
||||
'id' => '\d+'
|
||||
)
|
||||
);
|
||||
$router->addRoute('delete-success', $deleteSuccessRoute);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,9 @@ class Rest_MediaController extends Zend_Rest_Controller
|
|||
public function init()
|
||||
{
|
||||
$this->view->layout()->disableLayout();
|
||||
|
||||
$ajaxContext = $this->_helper->getHelper('AjaxContext');
|
||||
$ajaxContext->addActionContext('delete-success', 'json');
|
||||
}
|
||||
|
||||
public function indexAction()
|
||||
|
|
@ -278,7 +281,7 @@ class Rest_MediaController extends Zend_Rest_Controller
|
|||
|
||||
//get the filesize and update disk_usage
|
||||
$storedFile = Application_Model_StoredFile::RecallById($file->getDbId());
|
||||
Application_Model_Preference::updateDiskUsage($storedFile->getFileSize());
|
||||
Application_Model_Preference::updateDiskUsage($requestData["filesize"]);
|
||||
|
||||
$this->getResponse()
|
||||
->setHttpResponseCode(200)
|
||||
|
|
@ -304,9 +307,8 @@ class Rest_MediaController extends Zend_Rest_Controller
|
|||
if ($file) {
|
||||
$con = Propel::getConnection();
|
||||
$storedFile = new Application_Model_StoredFile($file, $con);
|
||||
if ($storedFile->existsOnDisk()) {
|
||||
$storedFile->delete(); //TODO: This checks your session permissions... Make it work without a session?
|
||||
}
|
||||
$storedFile->delete(); //TODO: This checks your session permissions... Make it work without a session?
|
||||
|
||||
$this->getResponse()
|
||||
->setHttpResponseCode(204);
|
||||
} else {
|
||||
|
|
@ -314,6 +316,26 @@ class Rest_MediaController extends Zend_Rest_Controller
|
|||
}
|
||||
}
|
||||
|
||||
public function deleteSuccessAction()
|
||||
{
|
||||
if (!$this->verifyAuth(true, true))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
$id = $this->getId();
|
||||
if (!$id) {
|
||||
return;
|
||||
}
|
||||
|
||||
$requestData = json_decode($this->getRequest()->getRawBody(), true);
|
||||
|
||||
$con = Propel::getConnection();
|
||||
$storedFile = new Application_Model_StoredFile(CcFilesQuery::create()->findPk($id), $con);
|
||||
|
||||
$storedFile->doFileDeletionCleanup($requestData["filesize"]);
|
||||
}
|
||||
|
||||
private function getId()
|
||||
{
|
||||
if (!$id = $this->_getParam('id', false)) {
|
||||
|
|
@ -479,9 +501,9 @@ class Rest_MediaController extends Zend_Rest_Controller
|
|||
|
||||
//Dispatch a message to airtime_analyzer through RabbitMQ,
|
||||
//notifying it that there's a new upload to process!
|
||||
Application_Model_RabbitMq::SendMessageToAnalyzer($newTempFilePath,
|
||||
Application_Model_RabbitMq::SendUploadMessageToAnalyzer($newTempFilePath,
|
||||
$importedStorageDirectory, basename($originalFilename),
|
||||
$callbackUrl, $apiKey);
|
||||
$callbackUrl, $apiKey, 'upload');
|
||||
}
|
||||
|
||||
private function getOwnerId()
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import logging
|
|||
import uuid
|
||||
|
||||
from libcloud.storage.providers import get_driver
|
||||
from libcloud.storage.types import Provider, ContainerDoesNotExistError
|
||||
from libcloud.storage.types import Provider, ContainerDoesNotExistError, ObjectDoesNotExistError
|
||||
|
||||
class CloudStorageUploader:
|
||||
def __init__(self, provider, bucket, api_key, api_key_secret):
|
||||
|
|
@ -17,15 +17,16 @@ class CloudStorageUploader:
|
|||
file_name, extension = os.path.splitext(file_base_name)
|
||||
object_name = "%s_%s%s" % (file_name, str(uuid.uuid4()), extension)
|
||||
|
||||
cls = get_driver(getattr(Provider, self._provider))
|
||||
driver = cls(self._api_key, self._api_key_secret)
|
||||
#cls = get_driver(getattr(Provider, self._provider))
|
||||
driver = self.get_cloud_driver()
|
||||
|
||||
try:
|
||||
container = driver.get_container(self._bucket)
|
||||
except ContainerDoesNotExistError:
|
||||
container = driver.create_container(self._bucket)
|
||||
|
||||
extra = {'meta_data': {'filename': file_base_name}}
|
||||
extra = {'meta_data': {'filename': file_base_name},
|
||||
'acl': 'public-read-write'}
|
||||
|
||||
with open(audio_file_path, 'rb') as iterator:
|
||||
obj = driver.upload_object_via_stream(iterator=iterator,
|
||||
|
|
@ -33,14 +34,32 @@ class CloudStorageUploader:
|
|||
object_name=object_name,
|
||||
extra=extra)
|
||||
|
||||
|
||||
metadata["filesize"] = os.path.getsize(audio_file_path)
|
||||
|
||||
'''remove file from organize directory'''
|
||||
try:
|
||||
os.remove(audio_file_path)
|
||||
except OSError:
|
||||
logging.info("Could not remove %s" % audio_file_path)
|
||||
logging.info("Could not remove %s from organize directory" % audio_file_path)
|
||||
|
||||
metadata["s3_object_name"] = object_name
|
||||
return metadata
|
||||
|
||||
def delete_obj(self, object_name):
|
||||
pass
|
||||
def delete_obj(self, obj_name):
|
||||
driver = self.get_cloud_driver()
|
||||
|
||||
return_msg = dict()
|
||||
return_msg["success"] = False
|
||||
try:
|
||||
cloud_obj = driver.get_object(container_name=self._bucket,
|
||||
object_name=obj_name)
|
||||
return_msg["filesize"] = getattr(cloud_obj, 'size')
|
||||
return_msg["success"] = driver.delete_object(obj=cloud_obj)
|
||||
return return_msg
|
||||
except ObjectDoesNotExistError:
|
||||
logging.info("Could not find object on %s" % self._provider)
|
||||
|
||||
def get_cloud_driver(self):
|
||||
cls = get_driver(getattr(Provider, self._provider))
|
||||
return cls(self._api_key, self._api_key_secret)
|
||||
|
|
|
|||
|
|
@ -162,19 +162,26 @@ class MessageListener:
|
|||
'''
|
||||
try:
|
||||
msg_dict = json.loads(body)
|
||||
audio_file_path = msg_dict["tmp_file_path"]
|
||||
#final_file_path = msg_dict["final_file_path"]
|
||||
import_directory = msg_dict["import_directory"]
|
||||
original_filename = msg_dict["original_filename"]
|
||||
callback_url = msg_dict["callback_url"]
|
||||
api_key = msg_dict["api_key"]
|
||||
message_type = msg_dict["message_type"]
|
||||
callback_url = msg_dict["callback_url"]
|
||||
|
||||
if event_type == "upload":
|
||||
if message_type == "upload":
|
||||
audio_file_path = msg_dict["tmp_file_path"]
|
||||
import_directory = msg_dict["import_directory"]
|
||||
original_filename = msg_dict["original_filename"]
|
||||
|
||||
audio_metadata = self.spawn_analyzer_process(audio_file_path, import_directory, original_filename)
|
||||
StatusReporter.report_success_to_callback_url(callback_url, api_key, audio_metadata)
|
||||
elif event_type == "delete":
|
||||
pass
|
||||
elif message_type == "delete":
|
||||
object_name = msg_dict["object_name"]
|
||||
csu = CloudStorageUploader(self._provider, self._bucket, self._api_key, self._api_key_secret)
|
||||
response = csu.delete_obj(object_name)
|
||||
if response["success"]:
|
||||
audio_metadata = dict()
|
||||
audio_metadata["delete_success"] = True
|
||||
audio_metadata["filesize"] = response["filesize"]
|
||||
StatusReporter.report_success_to_callback_url(callback_url, api_key, audio_metadata)
|
||||
|
||||
except KeyError as e:
|
||||
# A field in msg_dict that we needed was missing (eg. audio_file_path)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue