Congegni
/
sintonia
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CC-5896: Store cloud files in separate table, inherited from cc_files

This commit is contained in:
drigato 2014-07-24 16:56:15 -04:00
parent b38f3d7e03
commit f1ea100411
11 changed files with 165 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
airtime_mvc/application/controllers/ApiController.php
View File

@ -81,34 +81,10 @@ class ApiController extends Zend_Controller_Action
$media = Application_Model_StoredFile::RecallById($fileId);
if ($media != null) {
if ($media->isInCloud()) {
if ("true" == $this->_getParam('download')) {
header('Content-type:'.$media->getPropelOrm()->getDbMime());
header('Content-Disposition: attachment; filename="'.$media->getResourceId().'"');
header('Content-length:'.$media->getFileSize());
echo $media->getCloudUrl();
exit;
} else {
$this->_redirect($media->getCloudUrl());
}
}
$filepath = $media->getFilePath();
// Make sure we don't have some wrong result beecause of caching
clearstatcache();
if (is_file($filepath)) {
$full_path = $media->getPropelOrm()->getDbFilepath();
$file_base_name = strrchr($full_path, '/');
/* If $full_path does not contain a '/', strrchr will return false,
* in which case we can use $full_path as the base name.
*/
if (!$file_base_name) {
$file_base_name = $full_path;
} else {
$file_base_name = substr($file_base_name, 1);
}
if ($media->getPropelOrm()->isValidFile()) {
$filename = $media->getPropelOrm()->getFilename();
//Download user left clicks a track and selects Download.
if ("true" == $this->_getParam('download')) {
@ -116,14 +92,13 @@ class ApiController extends Zend_Controller_Action
//We just want the basename which is the file name with the path
//information stripped away. We are using Content-Disposition to specify
//to the browser what name the file should be saved as.
header('Content-Disposition: attachment; filename="'.$file_base_name.'"');
header('Content-Disposition: attachment; filename="'.$filename.'"');
} else {
//user clicks play button for track and downloads it.
header('Content-Disposition: inline; filename="'.$file_base_name.'"');
header('Content-Disposition: inline; filename="'.$filename.'"');
}
$this->smartReadFile($filepath, $media->getPropelOrm()->getDbMime());
exit;
$this->_redirect($media->getFilePath());
} else {
header ("HTTP/1.1 404 Not Found");
}

77
airtime_mvc/application/models/StoredFile.php
View File

@ -97,7 +97,7 @@ class Application_Model_StoredFile
}
public static function createWithFile($f, $con) {
$storedFile = new Application_Model_StoredFile($f, $con);
$storedFile = new Application_Model_StoredFile($f, $con);
return $storedFile;
}
@ -384,19 +384,16 @@ SQL;
$file_id = $this->_file->getDbId();
Logging::info("User ".$user->getLogin()." is deleting file: ".$this->_file->getDbTrackTitle()." - file id: ".$file_id);
$isInCloud = $this->isInCloud();
if (file_exists($filepath) && !$isInCloud) {
$music_dir = Application_Model_MusicDir::getDirByPK($this->_file->getDbDirectory());
if (!is_null($music_dir) && $music_dir->getType() == "stor" && file_exists($filepath)) {
try {
$this->doFileDeletionCleanup($this->getFileSize());
unlink($filepath);
} catch (Exception $e) {
Logging::error($e->getMessage());
return;
}
$this->doFileDeletionCleanup($this->getFileSize());
} elseif ($isInCloud) {
} /*elseif ($isInCloud) {
//Dispatch a message to airtime_analyzer through RabbitMQ,
//notifying it that we need to delete a file from the cloud
$CC_CONFIG = Config::getConfig();
@ -408,7 +405,7 @@ SQL;
Application_Model_RabbitMq::SendDeleteMessageToAnalyzer(
$callbackUrl, $this->_file->getDbResourceId(), $apiKey, 'delete');
}
}*/
}
/*
@ -416,6 +413,9 @@ SQL;
*/
public function doFileDeletionCleanup($filesize)
{
if ($filesize <= 0) {
throw new Exception ("Could not delete file with ".$filesize." filesize");
}
//Update the user's disk usage
Application_Model_Preference::updateDiskUsage(-1 * $filesize);
@ -432,7 +432,7 @@ SQL;
* deleted from the library. It re-calculates the length of
* all blocks and playlists that contained the deleted file.
*/
public static function updateBlockAndPlaylistLength($fileId)
private static function updateBlockAndPlaylistLength($fileId)
{
$plRows = CcPlaylistcontentsQuery::create()->filterByDbFileId($fileId)->find();
foreach ($plRows as $row) {
@ -511,7 +511,7 @@ SQL;
}
/**
* Get real filename of raw media data
* Get the absolute filepath
*
* @return string
*/
@ -519,20 +519,7 @@ SQL;
{
assert($this->_file);
if ($this->isInCloud()) {
return $this->getCloudUrl();
} else {
$music_dir = Application_Model_MusicDir::getDirByPK($this->
_file->getDbDirectory());
if (!$music_dir) {
throw new Exception("Invalid music_dir for file in database.");
}
$directory = $music_dir->getDirectory();
$filepath = $this->_file->getDbFilepath();
return Application_Common_OsPath::join($directory, $filepath);
}
return $this->_file->getAbsoluteFilePath();
}
/**
@ -585,21 +572,6 @@ SQL;
return $baseUrl."api/get-media/file/".$this->getId().".".$this->getFileExtension();
}
public function isInCloud()
{
$location = CcMusicDirsQuery::create()->findPk($this->_file->getDbDirectory());
if ($location->getType() == "cloud") {
return true;
}
return false;
}
public function getCloudUrl()
{
$CC_CONFIG = Config::getConfig();
return $CC_CONFIG["cloud_storage"]["host"]."/".$CC_CONFIG["cloud_storage"]["bucket"]."/" . urlencode($this->getResourceId());
}
public function getResourceId()
{
return $this->_file->getDbResourceId();
@ -607,12 +579,7 @@ SQL;
public function getFileSize()
{
if ($this->isInCloud()) {
//TODO: error checking - 403 forbidden>
return strlen(file_get_contents($this->getCloudUrl()));
} else {
return filesize($this->getFilePath());
}
return $this->_file->getFileSize();
}
public static function Insert($md, $con)
@ -653,8 +620,19 @@ SQL;
}
if (isset($p_id)) {
$f = CcFilesQuery::create()->findPK(intval($p_id), $con);
return is_null($f) ? null : self::createWithFile($f, $con);
$storedFile = CcFilesQuery::create()->findPK(intval($p_id), $con);
if (is_null($storedFile)) {
throw new Exception("Could not recall file with id: ".$p_id);
}
//Attempt to get the cloud file object and return it. If no cloud
//file object is found then we are dealing with a regular stored
//object so return that
$cloudFile = $storedFile->getCloudFiles()->getFirst();
if (is_null($cloudFile)) {
return self::createWithFile($storedFile, $con);
} else {
return self::createWithFile($cloudFile, $con);
}
} else {
throw new Exception("No arguments passed to RecallById");
}
@ -662,8 +640,7 @@ SQL;
public function getName()
{
$info = pathinfo($this->getFilePath());
return $info['filename'];
return $this->_file->getFilename();
}
/**

30
airtime_mvc/application/models/airtime/CcFiles.php
View File

@ -70,4 +70,34 @@ class CcFiles extends BaseCcFiles {
return $response;
}
public function getFileSize()
{
return filesize($this->getAbsoluteFilePath());
}
public function getFilename()
{
$info = pathinfo($this->getAbsoluteFilePath());
return $info['filename'];
}
public function getAbsoluteFilePath()
{
$music_dir = Application_Model_MusicDir::getDirByPK($this->
_file->getDbDirectory());
if (!$music_dir) {
throw new Exception("Invalid music_dir for file in database.");
}
$directory = $music_dir->getDirectory();
$filepath = $this->_file->getDbFilepath();
return Application_Common_OsPath::join($directory, $filepath);
}
public function isValidFile()
{
return is_file($this->getAbsoluteFilePath());
}
} // CcFiles

35
airtime_mvc/application/models/airtime/CloudFile.php
View File

@ -15,4 +15,39 @@
*/
class CloudFile extends BaseCloudFile
{
public function getAbsoluteFilePath()
{
$CC_CONFIG = Config::getConfig();
return $CC_CONFIG["cloud_storage"]["host"]."/".$CC_CONFIG["cloud_storage"]["bucket"]."/" . urlencode($this->getResourceId());
}
public function getFileSize()
{
return strlen(file_get_contents($this->getAbsoluteFilePath()));
}
public function getFilename()
{
return $this->getResourceId();
}
public function isValidFile()
{
$ch = curl_init();
curl_setopt_array($ch, array(
CURLOPT_URL => $this->getAbsoluteFilePath(),
CURLOPT_RETURNTRANSFER => true,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_VERBOSE => false
));
curl_exec($ch);
$http_status = curl_getinfo($ch);
if ($http_status["http_code"] === 200)
{
return true;
} else {
return false;
}
}
}

22
airtime_mvc/application/models/airtime/om/BaseCloudFile.php
View File

@ -77,7 +77,7 @@ abstract class BaseCloudFile extends BaseObject implements Persistent
*
* @return int
*/
public function getId()
public function getDbId()
{
return $this->id;
@ -111,7 +111,7 @@ abstract class BaseCloudFile extends BaseObject implements Persistent
* @param int $v new value
* @return CloudFile The current object (for fluent API support)
*/
public function setId($v)
public function setDbId($v)
{
if ($v !== null && is_numeric($v)) {
$v = (int) $v;
@ -124,7 +124,7 @@ abstract class BaseCloudFile extends BaseObject implements Persistent
return $this;
} // setId()
} // setDbId()
/**
* Set the value of [resource_id] column.
@ -623,7 +623,7 @@ abstract class BaseCloudFile extends BaseObject implements Persistent
{
switch ($pos) {
case 0:
return $this->getId();
return $this->getDbId();
break;
case 1:
return $this->getResourceId();
@ -660,7 +660,7 @@ abstract class BaseCloudFile extends BaseObject implements Persistent
$alreadyDumpedObjects['CloudFile'][$this->getPrimaryKey()] = true;
$keys = CloudFilePeer::getFieldNames($keyType);
$result = array(
$keys[0] => $this->getId(),
$keys[0] => $this->getDbId(),
$keys[1] => $this->getResourceId(),
$keys[2] => $this->getCcFileId(),
);
@ -708,7 +708,7 @@ abstract class BaseCloudFile extends BaseObject implements Persistent
{
switch ($pos) {
case 0:
$this->setId($value);
$this->setDbId($value);
break;
case 1:
$this->setResourceId($value);
@ -740,7 +740,7 @@ abstract class BaseCloudFile extends BaseObject implements Persistent
{
$keys = CloudFilePeer::getFieldNames($keyType);
if (array_key_exists($keys[0], $arr)) $this->setId($arr[$keys[0]]);
if (array_key_exists($keys[0], $arr)) $this->setDbId($arr[$keys[0]]);
if (array_key_exists($keys[1], $arr)) $this->setResourceId($arr[$keys[1]]);
if (array_key_exists($keys[2], $arr)) $this->setCcFileId($arr[$keys[2]]);
}
@ -783,7 +783,7 @@ abstract class BaseCloudFile extends BaseObject implements Persistent
*/
public function getPrimaryKey()
{
return $this->getId();
return $this->getDbId();
}
/**
@ -794,7 +794,7 @@ abstract class BaseCloudFile extends BaseObject implements Persistent
*/
public function setPrimaryKey($key)
{
$this->setId($key);
$this->setDbId($key);
}
/**
@ -804,7 +804,7 @@ abstract class BaseCloudFile extends BaseObject implements Persistent
public function isPrimaryKeyNull()
{
return null === $this->getId();
return null === $this->getDbId();
}
/**
@ -836,7 +836,7 @@ abstract class BaseCloudFile extends BaseObject implements Persistent
if ($makeNew) {
$copyObj->setNew(true);
$copyObj->setId(NULL); // this is a auto-increment column, so set to default value
$copyObj->setDbId(NULL); // this is a auto-increment column, so set to default value
}
}

12
airtime_mvc/application/models/airtime/om/BaseCloudFilePeer.php
View File

@ -60,8 +60,8 @@ abstract class BaseCloudFilePeer
* e.g. CloudFilePeer::$fieldNames[CloudFilePeer::TYPE_PHPNAME][0] = 'Id'
*/
protected static $fieldNames = array (
BasePeer::TYPE_PHPNAME => array ('Id', 'ResourceId', 'CcFileId', ),
BasePeer::TYPE_STUDLYPHPNAME => array ('id', 'resourceId', 'ccFileId', ),
BasePeer::TYPE_PHPNAME => array ('DbId', 'ResourceId', 'CcFileId', ),
BasePeer::TYPE_STUDLYPHPNAME => array ('dbId', 'resourceId', 'ccFileId', ),
BasePeer::TYPE_COLNAME => array (CloudFilePeer::ID, CloudFilePeer::RESOURCE_ID, CloudFilePeer::CC_FILE_ID, ),
BasePeer::TYPE_RAW_COLNAME => array ('ID', 'RESOURCE_ID', 'CC_FILE_ID', ),
BasePeer::TYPE_FIELDNAME => array ('id', 'resource_id', 'cc_file_id', ),
@ -75,8 +75,8 @@ abstract class BaseCloudFilePeer
* e.g. CloudFilePeer::$fieldNames[BasePeer::TYPE_PHPNAME]['Id'] = 0
*/
protected static $fieldKeys = array (
BasePeer::TYPE_PHPNAME => array ('Id' => 0, 'ResourceId' => 1, 'CcFileId' => 2, ),
BasePeer::TYPE_STUDLYPHPNAME => array ('id' => 0, 'resourceId' => 1, 'ccFileId' => 2, ),
BasePeer::TYPE_PHPNAME => array ('DbId' => 0, 'ResourceId' => 1, 'CcFileId' => 2, ),
BasePeer::TYPE_STUDLYPHPNAME => array ('dbId' => 0, 'resourceId' => 1, 'ccFileId' => 2, ),
BasePeer::TYPE_COLNAME => array (CloudFilePeer::ID => 0, CloudFilePeer::RESOURCE_ID => 1, CloudFilePeer::CC_FILE_ID => 2, ),
BasePeer::TYPE_RAW_COLNAME => array ('ID' => 0, 'RESOURCE_ID' => 1, 'CC_FILE_ID' => 2, ),
BasePeer::TYPE_FIELDNAME => array ('id' => 0, 'resource_id' => 1, 'cc_file_id' => 2, ),
@ -287,7 +287,7 @@ abstract class BaseCloudFilePeer
{
if (Propel::isInstancePoolingEnabled()) {
if ($key === null) {
$key = (string) $obj->getId();
$key = (string) $obj->getDbId();
} // if key === null
CloudFilePeer::$instances[$key] = $obj;
}
@ -310,7 +310,7 @@ abstract class BaseCloudFilePeer
{
if (Propel::isInstancePoolingEnabled() && $value !== null) {
if (is_object($value) && $value instanceof CloudFile) {
$key = (string) $value->getId();
$key = (string) $value->getDbId();
} elseif (is_scalar($value)) {
// assume we've been passed a primary key
$key = (string) $value;

34
airtime_mvc/application/models/airtime/om/BaseCloudFileQuery.php
View File

@ -6,11 +6,11 @@
*
*
*
* @method CloudFileQuery orderById($order = Criteria::ASC) Order by the id column
* @method CloudFileQuery orderByDbId($order = Criteria::ASC) Order by the id column
* @method CloudFileQuery orderByResourceId($order = Criteria::ASC) Order by the resource_id column
* @method CloudFileQuery orderByCcFileId($order = Criteria::ASC) Order by the cc_file_id column
*
* @method CloudFileQuery groupById() Group by the id column
* @method CloudFileQuery groupByDbId() Group by the id column
* @method CloudFileQuery groupByResourceId() Group by the resource_id column
* @method CloudFileQuery groupByCcFileId() Group by the cc_file_id column
*
@ -28,7 +28,7 @@
* @method CloudFile findOneByResourceId(string $resource_id) Return the first CloudFile filtered by the resource_id column
* @method CloudFile findOneByCcFileId(int $cc_file_id) Return the first CloudFile filtered by the cc_file_id column
*
* @method array findById(int $id) Return CloudFile objects filtered by the id column
* @method array findByDbId(int $id) Return CloudFile objects filtered by the id column
* @method array findByResourceId(string $resource_id) Return CloudFile objects filtered by the resource_id column
* @method array findByCcFileId(int $cc_file_id) Return CloudFile objects filtered by the cc_file_id column
*
@ -121,7 +121,7 @@ abstract class BaseCloudFileQuery extends ModelCriteria
* @return CloudFile A model object, or null if the key is not found
* @throws PropelException
*/
public function findOneById($key, $con = null)
public function findOneByDbId($key, $con = null)
{
return $this->findPk($key, $con);
}
@ -232,13 +232,13 @@ abstract class BaseCloudFileQuery extends ModelCriteria
*
* Example usage:
* <code>
* $query->filterById(1234); // WHERE id = 1234
* $query->filterById(array(12, 34)); // WHERE id IN (12, 34)
* $query->filterById(array('min' => 12)); // WHERE id >= 12
* $query->filterById(array('max' => 12)); // WHERE id <= 12
* $query->filterByDbId(1234); // WHERE id = 1234
* $query->filterByDbId(array(12, 34)); // WHERE id IN (12, 34)
* $query->filterByDbId(array('min' => 12)); // WHERE id >= 12
* $query->filterByDbId(array('max' => 12)); // WHERE id <= 12
* </code>
*
* @param mixed $id The value to use as filter.
* @param mixed $dbId The value to use as filter.
* Use scalar values for equality.
* Use array values for in_array() equivalent.
* Use associative array('min' => $minValue, 'max' => $maxValue) for intervals.
@ -246,16 +246,16 @@ abstract class BaseCloudFileQuery extends ModelCriteria
*
* @return CloudFileQuery The current query, for fluid interface
*/
public function filterById($id = null, $comparison = null)
public function filterByDbId($dbId = null, $comparison = null)
{
if (is_array($id)) {
if (is_array($dbId)) {
$useMinMax = false;
if (isset($id['min'])) {
$this->addUsingAlias(CloudFilePeer::ID, $id['min'], Criteria::GREATER_EQUAL);
if (isset($dbId['min'])) {
$this->addUsingAlias(CloudFilePeer::ID, $dbId['min'], Criteria::GREATER_EQUAL);
$useMinMax = true;
}
if (isset($id['max'])) {
$this->addUsingAlias(CloudFilePeer::ID, $id['max'], Criteria::LESS_EQUAL);
if (isset($dbId['max'])) {
$this->addUsingAlias(CloudFilePeer::ID, $dbId['max'], Criteria::LESS_EQUAL);
$useMinMax = true;
}
if ($useMinMax) {
@ -266,7 +266,7 @@ abstract class BaseCloudFileQuery extends ModelCriteria
}
}
return $this->addUsingAlias(CloudFilePeer::ID, $id, $comparison);
return $this->addUsingAlias(CloudFilePeer::ID, $dbId, $comparison);
}
/**
@ -428,7 +428,7 @@ abstract class BaseCloudFileQuery extends ModelCriteria
public function prune($cloudFile = null)
{
if ($cloudFile) {
$this->addUsingAlias(CloudFilePeer::ID, $cloudFile->getId(), Criteria::NOT_EQUAL);
$this->addUsingAlias(CloudFilePeer::ID, $cloudFile->getDbId(), Criteria::NOT_EQUAL);
}
return $this;

44
airtime_mvc/application/modules/rest/controllers/MediaController.php
View File

@ -217,12 +217,6 @@ class Rest_MediaController extends Zend_Rest_Controller
$requestData = json_decode($this->getRequest()->getRawBody(), true);
$whiteList = $this->removeBlacklistedFieldsFromRequestData($requestData);
$whiteList = $this->stripTimeStampFromYearTag($whiteList);
if ($requestData["import_status"] == 2) {
$file->setDbImportStatus(2)->save();
$this->importFailedResponse();
return;
}
if (!$this->validateRequestData($file, $whiteList)) {
$file->save();
@ -230,16 +224,40 @@ class Rest_MediaController extends Zend_Rest_Controller
} else if ($file) {
$file->fromArray($whiteList, BasePeer::TYPE_FIELDNAME);
if (isset($requestData["s3_object_name"])) {
$cloud_cc_music_dir = CcMusicDirsQuery::create()
->filterByType("cloud")
->findOne();
$file->setDbDirectory($cloud_cc_music_dir->getId());
$file->setDbResourceId($requestData["s3_object_name"]);
//file is stored in the cloud
if (isset($requestData["resource_id"])) {
$fileSizeBytes = $requestData["filesize"];
$cloudFile = new CloudFile();
$cloudFile->setResourceId($requestData["resource_id"]);
$cloudFile->setCcFiles($file);
$cloudFile->save();
Application_Model_Preference::updateDiskUsage($requestData["filesize"]);
//file is stored locally
} else if (isset($requestData["full_path"])) {
$fileSizeBytes = filesize($requestData["full_path"]);
if ($fileSizeBytes === false)
{
$file->setDbImportStatus(2)->save();
$this->fileNotFoundResponse();
return;
}
$fullPath = $requestData["full_path"];
$storDir = Application_Model_MusicDir::getStorDir()->getDirectory();
$pos = strpos($fullPath, $storDir);
if ($pos !== FALSE)
{
assert($pos == 0); //Path must start with the stor directory path
$filePathRelativeToStor = substr($fullPath, strlen($storDir));
$file->setDbFilepath($filePathRelativeToStor);
$file->setDbDirectory(1); //1 corresponds to the default stor/imported directory.
}
}
Application_Model_Preference::updateDiskUsage($fileSizeBytes);
$now = new DateTime("now", new DateTimeZone("UTC"));
$file->setDbMtime($now);
$file->save();

2
airtime_mvc/build/schema.xml
View File

@ -100,7 +100,7 @@
</table>
<!-- Class Table Inheritance -->
<table name="cloud_file" phpName="CloudFile">
<column name="id" required="true" primaryKey="true" autoIncrement="true" type="INTEGER"/>
<column name="id" phpName="DbId" required="true" primaryKey="true" autoIncrement="true" type="INTEGER"/>
<column name="resource_id" phpName="ResourceId" type="LONGVARCHAR" required="true"/>
<column name="cc_file_id" type="INTEGER"/>
<foreign-key foreignTable="cc_files">

2
dev_tools/propel_regenerate.sh
View File

@ -8,4 +8,4 @@ cd $SCRIPTPATH/../airtime_mvc/
path=`pwd`
cd build
sed -i s#"project\.home =.*$"#"project.home = $path"#g build.properties
../library/propel/vendor/propel/propel1/generator/bin/propel-gen
../../vendor/propel/propel1/generator/bin/propel-gen

2
python_apps/airtime_analyzer/airtime_analyzer/cloud_storage_uploader.py
View File

@ -41,7 +41,7 @@ class CloudStorageUploader:
except OSError:
logging.info("Could not remove %s from organize directory" % audio_file_path)
metadata["s3_object_name"] = object_name
metadata["resource_id"] = object_name
return metadata
def delete_obj(self, obj_name):