From 35dc3fd01fcb210365df5ec57bd243920189fe9c Mon Sep 17 00:00:00 2001 From: "paul.baranowski" Date: Thu, 11 Nov 2010 16:50:30 -0500 Subject: [PATCH] CC-1713 - New storage directory structure Changed the GUNID to a 32 hex string (MD5 value), there is no more conversion between hex strings and bigints in the database anymore. Added added the file extension to the file name. Stored the complete path to the file in the database. This means that linking doesnt need to create any files at all. (It used to create a symlink to the file you were importing) The structure and file naming should be fine - there are 4096 possibilities for the first directory level, and even will a million files this is only 244 files per sub-directory. The GUID is fine for the file name --- api/api_version.php | 13 ++ api/get_media.php | 38 ++++++ api/schedule.php | 25 ++++ backend/BasicStor.php | 28 +++- backend/StoredFile.php | 173 +++++++++++++++---------- backend/propel-db/build/sql/schema.sql | 7 +- backend/propel-db/schema.xml | 7 +- utils/campcaster-import.php | 2 +- 8 files changed, 215 insertions(+), 78 deletions(-) create mode 100644 api/api_version.php create mode 100644 api/get_media.php create mode 100644 api/schedule.php diff --git a/api/api_version.php b/api/api_version.php new file mode 100644 index 000000000..350f477d9 --- /dev/null +++ b/api/api_version.php @@ -0,0 +1,13 @@ +CAMPCASTER_VERSION)); +?> \ No newline at end of file diff --git a/api/get_media.php b/api/get_media.php new file mode 100644 index 000000000..824d7b453 --- /dev/null +++ b/api/get_media.php @@ -0,0 +1,38 @@ +getMessage()." ".$CC_DBC->getUserInfo()."\n"; + exit(1); +} +$CC_DBC->setFetchMode(DB_FETCHMODE_ASSOC); + +$file_id = $_GET[""] +if(!is_file($src)) +{ + header($_SERVER["SERVER_PROTOCOL"]." 404 Not Found"); + //print 'Ressource in database, but not in storage. Sorry.'; + exit; +} + +// !! binary mode !! +$fp = fopen($src, 'rb'); + +header("Content-Type: audio/mpeg"); +header("Content-Length: " . filesize($src)); + +fpassthru($fp); +exit; +?> \ No newline at end of file diff --git a/api/schedule.php b/api/schedule.php new file mode 100644 index 000000000..14f9247bf --- /dev/null +++ b/api/schedule.php @@ -0,0 +1,25 @@ +getMessage()." ".$CC_DBC->getUserInfo()."\n"; + exit(1); +} +$CC_DBC->setFetchMode(DB_FETCHMODE_ASSOC); +$from = $_GET["from"]; +$to = $_GET["to"]; +echo Schedule::ExportRangeAsJson($from, $to); +?> \ No newline at end of file diff --git a/backend/BasicStor.php b/backend/BasicStor.php index 4d4e2734a..ef9ffd6c6 100644 --- a/backend/BasicStor.php +++ b/backend/BasicStor.php @@ -935,6 +935,9 @@ class BasicStor { if (!is_array($values)) { $values = array($values); } + if (count($values) == 0) { + return true; + } if (is_a($id, "StoredFile")) { $storedFile =& $id; } else { @@ -949,10 +952,31 @@ class BasicStor { if ($category == 'dcterms:extent') { $oneValue = BasicStor::NormalizeExtent($oneValue); } - $escapedValue = pg_escape_string($oneValue); - $sqlValues[] = "$columnName = '$escapedValue'"; + // Since track_number is an integer, you cannot set + // it to be the empty string, so we NULL it instead. + if ($columnName == 'track_number' && empty($oneValue)) { + $sqlPart = "$columnName = NULL"; + } elseif (($columnName == 'length') && (strlen($oneValue) > 8)) { + // Postgres doesnt like it if you try to store really large hour + // values. TODO: We need to fix the underlying problem of getting the + // right values. + $parts = explode(':', $oneValue); + $hour = intval($parts[0]); + if ($hour > 24) { + continue; + } else { + $sqlPart = "$columnName = '$oneValue'"; + } + } else { + $escapedValue = pg_escape_string($oneValue); + $sqlPart = "$columnName = '$escapedValue'"; + } + $sqlValues[] = $sqlPart; } } + if (count($sqlValues)==0) { + return TRUE; + } $sql = "UPDATE ".$CC_CONFIG["filesTable"] ." SET ".join(",", $sqlValues) ." WHERE id=$id"; diff --git a/backend/StoredFile.php b/backend/StoredFile.php index 28de8b0dd..d0fd7d78b 100644 --- a/backend/StoredFile.php +++ b/backend/StoredFile.php @@ -1,6 +1,7 @@ gunid = $p_gunid; if (empty($this->gunid)) { - $this->gunid = StoredFile::CreateGunid(); + $this->gunid = StoredFile::generateGunid(); } - $this->resDir = $this->_getResDir($this->gunid); - $this->filepath = "{$this->resDir}/{$this->gunid}"; + //$this->resDir = $this->_getResDir($this->gunid); + //$this->filepath = "{$this->resDir}/{$this->gunid}"; $this->exists = is_file($this->filepath) && is_readable($this->filepath); $this->md = $this->loadMetadata(); } @@ -401,7 +402,7 @@ class StoredFile { global $CC_CONFIG, $CC_DBC; $escapedValue = pg_escape_string($this->gunid); $sql = "SELECT * FROM ".$CC_CONFIG["filesTable"] - ." WHERE gunid=x'$escapedValue'::bigint"; + ." WHERE gunid='$escapedValue'"; //var_dump($sql); $this->md = $CC_DBC->getRow($sql); if (PEAR::isError($this->md)) { @@ -409,6 +410,7 @@ class StoredFile { $this->md = null; return $error; } + $this->filepath = $this->md["filepath"]; if (is_null($this->md)) { $this->md = array(); return; @@ -465,7 +467,7 @@ class StoredFile { * Create instance of StoredFile object and insert new file * * @param array $p_values - * "filepath" - required, local path to media file + * "filepath" - required, local path to media file (where it is before import) * "id" - optional, local object id, will be generated if not given * "gunid" - optional, unique id, for insert file with gunid, will be generated if not given * "filename" - optional, will use "filepath" if not given @@ -503,9 +505,9 @@ class StoredFile { } $storedFile->name = isset($p_values['filename']) ? $p_values['filename'] : $p_values["filepath"]; - // NOTE: POSTGRES-SPECIFIC KEYWORD "DEFAULT" BEING USED, WOULD BE "NULL" IN MYSQL $storedFile->id = isset($p_values['id']) && is_integer($p_values['id'])?(int)$p_values['id']:null; - $sqlId = !is_null($storedFile->id)?"'".$storedFile->id."'":'DEFAULT'; + // NOTE: POSTGRES-SPECIFIC KEYWORD "DEFAULT" BEING USED, WOULD BE "NULL" IN MYSQL + $sqlId = !is_null($storedFile->id)?"'".$storedFile->id."'":'DEFAULT'; $storedFile->ftype = isset($p_values['filetype']) ? strtolower($p_values['filetype']) : "audioclip"; $storedFile->mime = (isset($p_values["mime"]) ? $p_values["mime"] : NULL ); // $storedFile->filepath = $p_values['filepath']; @@ -531,7 +533,7 @@ class StoredFile { $sql = "INSERT INTO ".$CC_CONFIG['filesTable'] ."(id, name, gunid, mime, state, ftype, mtime, md5)" ."VALUES ({$sqlId}, '{$escapedName}', " - ." x'{$storedFile->gunid}'::bigint," + ." '{$storedFile->gunid}'," ." '{$storedFile->mime}', 'incomplete', '$escapedFtype'," ." now(), '{$storedFile->md5}')"; //$_SESSION["debug"] .= "sql: ".$sql."
"; @@ -550,6 +552,7 @@ class StoredFile { // Save media file $res = $storedFile->addFile($p_values['filepath'], $p_copyMedia); + if (PEAR::isError($res)) { echo "StoredFile::Insert: ERROR adding file: '".$res->getMessage()."'\n"; $CC_DBC->query("ROLLBACK"); @@ -596,17 +599,18 @@ class StoredFile { if (!is_null($p_oid)) { $cond = "id='".intval($p_oid)."'"; } elseif (!is_null($p_gunid)) { - $cond = "gunid=x'$p_gunid'::bigint"; + $cond = "gunid='$p_gunid'"; } elseif (!is_null($p_md5sum)) { $cond = "md5='$p_md5sum'"; } else { return null; } - $sql = "SELECT id, to_hex(gunid)as gunid, gunid as gunid_bigint," + $sql = "SELECT id, gunid," ." name, mime, ftype, state, currentlyaccessing, editedby, " ." mtime, md5" ." FROM ".$CC_CONFIG['filesTable'] ." WHERE $cond"; + //echo $sql; $row = $CC_DBC->getRow($sql); if (PEAR::isError($row)) { return $row; @@ -614,7 +618,7 @@ class StoredFile { if (is_null($row)) { return null; } - $gunid = StoredFile::NormalizeGunid($row['gunid']); + $gunid = $row['gunid']; if ($row['ftype'] == 'audioclip') { $storedFile = new StoredFile($gunid); } elseif ($row['ftype'] == 'playlist') { @@ -623,9 +627,9 @@ class StoredFile { $storedFile = new StoredFile($gunid); } $storedFile->loadMetadata(); - $storedFile->gunidBigint = $row['gunid_bigint']; + //$storedFile->gunidBigint = $row['gunid_bigint']; //$storedFile->md->gunidBigint = $row['gunid_bigint']; - $storedFile->md["gunid"] = $row['gunid_bigint']; + $storedFile->md["gunid"] = $row['gunid']; $storedFile->id = $row['id']; $storedFile->name = $row['name']; $storedFile->mime = $row['mime']; @@ -678,7 +682,7 @@ class StoredFile { public static function RecallByToken($p_token) { global $CC_CONFIG, $CC_DBC; - $sql = "SELECT to_hex(gunid) as gunid" + $sql = "SELECT gunid" ." FROM ".$CC_CONFIG['accessTable'] ." WHERE token=x'$p_token'::bigint"; $gunid = $CC_DBC->getOne($sql); @@ -689,11 +693,28 @@ class StoredFile { return PEAR::raiseError( "StoredFile::RecallByToken: invalid token ($p_token)", GBERR_AOBJNEX); } - $gunid = StoredFile::NormalizeGunid($gunid); return StoredFile::Recall(null, $gunid); } + /** + * Generate the location to store the file. + * It creates the subdirectory if needed. + */ + private function generateFilePath() + { + global $CC_CONFIG, $CC_DBC; + $resDir = $CC_CONFIG['storageDir']."/".substr($this->gunid, 0, 3); + // see Transport::_getResDir too for resDir name create code + if (!is_dir($resDir)) { + mkdir($resDir, 02775); + chmod($resDir, 02775); + } + $info = pathinfo($this->name); + $fileExt = strtolower($info["extension"]); + return "{$resDir}/{$this->gunid}.{$fileExt}"; + } + /** * Insert media file to filesystem * @@ -705,30 +726,43 @@ class StoredFile { */ public function addFile($p_localFilePath, $p_copyMedia=TRUE) { + global $CC_CONFIG, $CC_DBC; if ($this->exists) { return FALSE; } - // for files downloaded from archive: + // for files downloaded from remote instance: if ($p_localFilePath == $this->filepath) { $this->exists = TRUE; return TRUE; } umask(0002); + $dstFile = ''; if ($p_copyMedia) { - $r = @copy($p_localFilePath, $this->filepath); + $dstFile = $this->generateFilePath(); + $r = @copy($p_localFilePath, $dstFile); + if (!$r) { + $this->exists = FALSE; + return PEAR::raiseError( + "StoredFile::addFile: file save failed". + " ($p_localFilePath, {$this->filepath})",GBERR_FILEIO + ); + } } else { - $r = @symlink($p_localFilePath, $this->filepath); + $dstFile = $p_localFilePath; + $r = TRUE; + //$r = @symlink($p_localFilePath, $dstFile); } - if ($r) { - $this->exists = TRUE; - return TRUE; - } else { - $this->exists = FALSE; - return PEAR::raiseError( - "StoredFile::addFile: file save failed". - " ($p_localFilePath, {$this->filepath})",GBERR_FILEIO - ); + $this->filepath = $dstFile; + $sqlPath = pg_escape_string($this->filepath); + $sql = "UPDATE ".$CC_CONFIG["filesTable"] + ." SET filepath='{$sqlPath}'" + ." WHERE id={$this->id}"; + $res = $CC_DBC->query($sql); + if (PEAR::isError($res)) { + return $res; } + $this->exists = TRUE; + return TRUE; } @@ -1013,7 +1047,7 @@ class StoredFile { $escapedName = pg_escape_string($p_newname); $sql = "UPDATE ".$CC_CONFIG['filesTable'] ." SET name='$escapedName', mtime=now()" - ." WHERE gunid=x'{$this->gunid}'::bigint"; + ." WHERE gunid='{$this->gunid}'"; $res = $CC_DBC->query($sql); if (PEAR::isError($res)) { return $res; @@ -1039,7 +1073,7 @@ class StoredFile { $eb = (!is_null($p_editedby) ? ", editedBy=$p_editedby" : ''); $sql = "UPDATE ".$CC_CONFIG['filesTable'] ." SET state='$escapedState'$eb, mtime=now()" - ." WHERE gunid=x'{$this->gunid}'::bigint"; + ." WHERE gunid='{$this->gunid}'"; $res = $CC_DBC->query($sql); if (PEAR::isError($res)) { return $res; @@ -1065,7 +1099,7 @@ class StoredFile { $escapedMime = pg_escape_string($p_mime); $sql = "UPDATE ".$CC_CONFIG['filesTable'] ." SET mime='$escapedMime', mtime=now()" - ." WHERE gunid=x'{$this->gunid}'::bigint"; + ." WHERE gunid='{$this->gunid}'"; $res = $CC_DBC->query($sql); if (PEAR::isError($res)) { return $res; @@ -1087,7 +1121,7 @@ class StoredFile { $escapedMd5 = pg_escape_string($p_md5sum); $sql = "UPDATE ".$CC_CONFIG['filesTable'] ." SET md5='$escapedMd5', mtime=now()" - ." WHERE gunid=x'{$this->gunid}'::bigint"; + ." WHERE gunid='{$this->gunid}'"; $res = $CC_DBC->query($sql); if (PEAR::isError($res)) { return $res; @@ -1119,7 +1153,7 @@ class StoredFile { // } $sql = "SELECT to_hex(token)as token, ext " ." FROM ".$CC_CONFIG['accessTable'] - ." WHERE gunid=x'{$this->gunid}'::bigint"; + ." WHERE gunid='{$this->gunid}'"; $tokens = $CC_DBC->getAll($sql); if (is_array($tokens)) { foreach ($tokens as $i => $item) { @@ -1130,13 +1164,13 @@ class StoredFile { } } $sql = "DELETE FROM ".$CC_CONFIG['accessTable'] - ." WHERE gunid=x'{$this->gunid}'::bigint"; + ." WHERE gunid='{$this->gunid}'"; $res = $CC_DBC->query($sql); if (PEAR::isError($res)) { return $res; } $sql = "DELETE FROM ".$CC_CONFIG['filesTable'] - ." WHERE gunid=x'{$this->gunid}'::bigint"; + ." WHERE gunid='{$this->gunid}'"; $res = $CC_DBC->query($sql); if (PEAR::isError($res)) { return $res; @@ -1182,7 +1216,7 @@ class StoredFile { return ($this->currentlyaccessing > 0); } $sql = "SELECT currentlyAccessing FROM ".$CC_CONFIG['filesTable'] - ." WHERE gunid=x'$p_gunid'::bigint"; + ." WHERE gunid='$p_gunid'"; $ca = $CC_DBC->getOne($sql); if (is_null($ca)) { return PEAR::raiseError( @@ -1229,7 +1263,7 @@ class StoredFile { $p_playlistId = $this->gunid; } $sql = "SELECT editedBy FROM ".$CC_CONFIG['filesTable'] - ." WHERE gunid=x'$p_playlistId'::bigint"; + ." WHERE gunid='$p_playlistId'"; $ca = $CC_DBC->getOne($sql); if (PEAR::isError($ca)) { return $ca; @@ -1270,9 +1304,9 @@ class StoredFile { public function exists() { global $CC_CONFIG, $CC_DBC; - $sql = "SELECT to_hex(gunid) " + $sql = "SELECT gunid " ." FROM ".$CC_CONFIG['filesTable'] - ." WHERE gunid=x'{$this->gunid}'::bigint"; + ." WHERE gunid='{$this->gunid}'"; $indb = $CC_DBC->getRow($sql); if (PEAR::isError($indb)) { return $indb; @@ -1291,15 +1325,17 @@ class StoredFile { * Create new global unique id * @return string */ - public static function CreateGunid() + public static function generateGunid() { - $ip = (isset($_SERVER['SERVER_ADDR']) ? $_SERVER['SERVER_ADDR'] : ''); - $initString = microtime().$ip.rand()."org.mdlf.campcaster"; - $hash = md5($initString); - // non-negative int8 - $hsd = substr($hash, 0, 1); - $res = dechex(hexdec($hsd)>>1).substr($hash, 1, 15); - return StoredFile::NormalizeGunid($res); + return md5(uniqid("", true)); + +// $ip = (isset($_SERVER['SERVER_ADDR']) ? $_SERVER['SERVER_ADDR'] : ''); +// $initString = microtime().$ip.rand(); +// $hash = md5($initString); +// // non-negative int8 +// $hsd = substr($hash, 0, 1); +// $res = dechex(hexdec($hsd)>>1).substr($hash, 1, 15); +// return StoredFile::NormalizeGunid($res); } @@ -1308,10 +1344,10 @@ class StoredFile { * * @return string */ - public static function NormalizeGunid($p_gunid) - { - return str_pad($p_gunid, 16, "0", STR_PAD_LEFT); - } +// public static function NormalizeGunid($p_gunid) +// { +// return str_pad($p_gunid, 16, "0", STR_PAD_LEFT); +// } /** @@ -1386,7 +1422,7 @@ class StoredFile { return $this->state; } $sql = "SELECT state FROM ".$CC_CONFIG['filesTable'] - ." WHERE gunid=x'$p_gunid'::bigint"; + ." WHERE gunid='$p_gunid'"; return $CC_DBC->getOne($sql); } @@ -1405,7 +1441,7 @@ class StoredFile { return $this->name; } $sql = "SELECT name FROM ".$CC_CONFIG['filesTable'] - ." WHERE gunid=x'$p_gunid'::bigint"; + ." WHERE gunid='$p_gunid'"; return $CC_DBC->getOne($sql); } @@ -1416,18 +1452,18 @@ class StoredFile { * * @return string */ - private function _getResDir() - { - global $CC_CONFIG, $CC_DBC; - $resDir = $CC_CONFIG['storageDir']."/".substr($this->gunid, 0, 3); - //$this->gb->debugLog("$resDir"); - // see Transport::_getResDir too for resDir name create code - if (!is_dir($resDir)) { - mkdir($resDir, 02775); - chmod($resDir, 02775); - } - return $resDir; - } +// private function _getResDir() +// { +// global $CC_CONFIG, $CC_DBC; +// $resDir = $CC_CONFIG['storageDir']."/".substr($this->gunid, 0, 3); +// //$this->gb->debugLog("$resDir"); +// // see Transport::_getResDir too for resDir name create code +// if (!is_dir($resDir)) { +// mkdir($resDir, 02775); +// chmod($resDir, 02775); +// } +// return $resDir; +// } /** @@ -1472,7 +1508,6 @@ class StoredFile { private function _getAccessFileName($p_token, $p_ext='EXT') { global $CC_CONFIG; - $token = StoredFile::NormalizeGunid($p_token); return $CC_CONFIG['accessDir']."/$p_token.$p_ext"; } diff --git a/backend/propel-db/build/sql/schema.sql b/backend/propel-db/build/sql/schema.sql index a04acadbc..401ce4dd3 100644 --- a/backend/propel-db/build/sql/schema.sql +++ b/backend/propel-db/build/sql/schema.sql @@ -9,7 +9,7 @@ DROP TABLE "cc_access" CASCADE; CREATE TABLE "cc_access" ( "id" serial NOT NULL, - "gunid" INT8, + "gunid" CHAR(32), "token" INT8, "chsum" CHAR(32) default '' NOT NULL, "ext" VARCHAR(128) default '' NOT NULL, @@ -61,10 +61,11 @@ DROP TABLE "cc_files" CASCADE; CREATE TABLE "cc_files" ( "id" serial NOT NULL, - "gunid" INT8 NOT NULL, + "gunid" CHAR(32) NOT NULL, "name" VARCHAR(255) default '' NOT NULL, "mime" VARCHAR(255) default '' NOT NULL, "ftype" VARCHAR(128) default '' NOT NULL, + "filepath" TEXT default '', "state" VARCHAR(128) default 'empty' NOT NULL, "currentlyaccessing" INTEGER default 0 NOT NULL, "editedby" INTEGER, @@ -339,7 +340,7 @@ CREATE TABLE "cc_trans" "target" VARCHAR(255), "rtrtok" CHAR(16), "mdtrtok" CHAR(16), - "gunid" INT8, + "gunid" CHAR(32), "pdtoken" INT8, "url" VARCHAR(255), "localfile" VARCHAR(255), diff --git a/backend/propel-db/schema.xml b/backend/propel-db/schema.xml index 6998002ed..565137772 100644 --- a/backend/propel-db/schema.xml +++ b/backend/propel-db/schema.xml @@ -3,7 +3,7 @@ - + @@ -33,10 +33,11 @@
- + + @@ -232,7 +233,7 @@ - + diff --git a/utils/campcaster-import.php b/utils/campcaster-import.php index 34d308bb6..e4e0f5d67 100644 --- a/utils/campcaster-import.php +++ b/utils/campcaster-import.php @@ -137,7 +137,7 @@ function camp_import_audio_file($p_filepath, $p_importMode = null, $p_testOnly = // Check for non-supported file type if (!preg_match('/\.(ogg|mp3)$/i', $p_filepath, $var)) { - echo "IGNORED: $p_filepath\n"; + echo "IGNORED: [xxxxx] $p_filepath\n"; //echo " * WARNING: File extension not supported - skipping file: $p_filepath\n"; return; }