From 4b07803e0c444e5ac2dd330ed4bb5c754e8f95d6 Mon Sep 17 00:00:00 2001 From: tomas Date: Wed, 29 Sep 2004 01:51:37 +0000 Subject: [PATCH] Major changes in transport layer - generalized for metadata and search jobs. --- livesupport/modules/storageServer/Makefile | 26 +- .../modules/storageServer/var/GreenBox.php | 216 ++++++- .../modules/storageServer/var/MetaData.php | 24 +- .../storageServer/var/RawMediaData.php | 12 +- .../modules/storageServer/var/StoredFile.php | 11 +- .../modules/storageServer/var/Transport.php | 596 ++++++++++-------- .../modules/storageServer/var/conf.php | 17 +- .../storageServer/var/cron/transportCron.php | 16 + .../storageServer/var/html/gbHtmlBrowse.php | 2 +- .../modules/storageServer/var/html/gbHttp.php | 4 +- .../storageServer/var/install/getGname.php | 7 + .../storageServer/var/install/getWwwRoot.php | 6 + .../storageServer/var/install/getXrUrl.php | 6 + .../storageServer/var/install/install.php | 6 +- .../storageServer/var/tests/transTest.php | 43 +- .../storageServer/var/xmlrpc/testRunner.sh | 5 +- 16 files changed, 650 insertions(+), 347 deletions(-) create mode 100755 livesupport/modules/storageServer/var/cron/transportCron.php create mode 100644 livesupport/modules/storageServer/var/install/getGname.php create mode 100644 livesupport/modules/storageServer/var/install/getWwwRoot.php create mode 100644 livesupport/modules/storageServer/var/install/getXrUrl.php diff --git a/livesupport/modules/storageServer/Makefile b/livesupport/modules/storageServer/Makefile index 4b5817317..bbbdeb2d9 100644 --- a/livesupport/modules/storageServer/Makefile +++ b/livesupport/modules/storageServer/Makefile @@ -20,7 +20,7 @@ # # # Author : $Author: tomas $ -# Version : $Revision: 1.3 $ +# Version : $Revision: 1.4 $ # Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/Attic/Makefile,v $ # # @configure_input@ @@ -62,13 +62,15 @@ USR_LIB_DIR = ${USR_DIR}/lib DOXYGEN_CONFIG = ${ETC_DIR}/doxygen.config -HTTP_GROUP = apache -#HTTP_GROUP = nobody +GET_GNAME = install/getGname.php +WWW_ROOT := $(shell cd var/install; php -q getWwwRoot.php) +HTTP_GROUP := $(shell curl -s ${WWW_ROOT}/${GET_GNAME}) PHP_DIR = ${BASE_DIR}/var INSTALL_DIR = ${PHP_DIR}/install STOR_DIR = ${PHP_DIR}/stor ACCESS_DIR = ${PHP_DIR}/access +TRANS_DIR = ${PHP_DIR}/trans BUFF_DIR = ${STOR_DIR}/buffer TEST_RUNNER = ${PHP_DIR}/xmlrpc/testRunner.sh @@ -95,18 +97,23 @@ TEST_RUNNER = ${PHP_DIR}/xmlrpc/testRunner.sh db_init db_clean transtest all: dir_setup db_init + $(MAKE) -C ../archiveServer all -dir_setup: ${DOXYGEN_DIR} ${STOR_DIR} ${ACCESS_DIR} +dir_setup: ${DOXYGEN_DIR} ${STOR_DIR} ${ACCESS_DIR} ${TRANS_DIR} doc: ${DOXYGEN} ${DOXYGEN_CONFIG} clean: db_clean + ${RMDIR} ${STOR_DIR} + ${RMDIR} ${ACCESS_DIR} + ${RMDIR} ${TRANS_DIR} docclean: ${RMDIR} ${DOXYGEN_DIR}/html depclean: clean + $(MAKE) -C ../archiveServer clean dist: ${TAR_C} ${MODULE_NAME}${DATE}${DIST_EXT} * @@ -115,6 +122,7 @@ distclean: clean docclean check: all ${TEST_RUNNER} ${TEST_RUNNER} + $(MAKE) transtest #------------------------------------------------------------------------------- # Specific targets @@ -124,6 +132,9 @@ transtest: cd var/tests; php -q transTest.php db_init: + @echo "Settings:" + @echo "Http daemon group: ${HTTP_GROUP}" + @echo "WWW root: ${WWW_ROOT}" cd var/install; php -q install.php db_clean: @@ -136,7 +147,7 @@ ${DOXYGEN_DIR}: ${MKDIR} ${DOXYGEN_DIR} ${STOR_DIR}: - @echo "Http daemon group have set in Makefile: ${HTTP_GROUP}" + @echo "Http daemon group: ${HTTP_GROUP}" @echo "You should be in this group for easy testing" ${MKDIR} ${STOR_DIR} chown .${HTTP_GROUP} ${STOR_DIR} @@ -147,6 +158,11 @@ ${ACCESS_DIR}: chown .${HTTP_GROUP} ${ACCESS_DIR} chmod g+ws ${ACCESS_DIR} +${TRANS_DIR}: + ${MKDIR} ${TRANS_DIR} + chown .${HTTP_GROUP} ${TRANS_DIR} + chmod g+ws ${TRANS_DIR} + ${BUFF_DIR}: ${MKDIR} ${BUFF_DIR} chmod g+w ${BUFF_DIR} diff --git a/livesupport/modules/storageServer/var/GreenBox.php b/livesupport/modules/storageServer/var/GreenBox.php index 36e6ea595..362463941 100644 --- a/livesupport/modules/storageServer/var/GreenBox.php +++ b/livesupport/modules/storageServer/var/GreenBox.php @@ -23,7 +23,7 @@ Author : $Author: tomas $ - Version : $Revision: 1.3 $ + Version : $Revision: 1.4 $ Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/GreenBox.php,v $ ------------------------------------------------------------------------------*/ @@ -40,6 +40,7 @@ define('GBERR_NOTIMPL', 50); require_once "../../../alib/var/alib.php"; require_once "StoredFile.php"; +require_once "Transport.php"; /** * GreenBox class @@ -47,7 +48,7 @@ require_once "StoredFile.php"; * LiveSupport file storage module * * @author $Author: tomas $ - * @version $Revision: 1.3 $ + * @version $Revision: 1.4 $ * @see Alib */ class GreenBox extends Alib{ @@ -55,6 +56,7 @@ class GreenBox extends Alib{ var $mdataTable; var $accessTable; var $storageDir; + var $bufferDir; var $accessDir; var $rootId; var $storId; @@ -69,10 +71,13 @@ class GreenBox extends Alib{ function GreenBox(&$dbc, $config) { parent::Alib(&$dbc, $config); + $this->config = $config; $this->filesTable = $config['tblNamePrefix'].'files'; $this->mdataTable = $config['tblNamePrefix'].'mdata'; $this->accessTable= $config['tblNamePrefix'].'access'; $this->storageDir = $config['storageDir']; + $this->bufferDir = $config['bufferDir']; + $this->transDir = $config['transDir']; $this->accessDir = $config['accessDir']; $this->dbc->setErrorHandling(PEAR_ERROR_RETURN); $this->rootId = $this->getRootNode(); @@ -112,14 +117,14 @@ class GreenBox extends Alib{ * @exception PEAR::error */ function putFile($parid, $fileName, - $mediaFileLP, $mdataFileLP, $sessid='') + $mediaFileLP, $mdataFileLP, $sessid='', $gunid=NULL) { if(($res = $this->_authorize('write', $parid, $sessid)) !== TRUE) return $res; $name = "$fileName"; $id = $this->addObj($name , 'File', $parid); $ac =& StoredFile::insert( - &$this, $id, $name, $mediaFileLP, $mdataFileLP + &$this, $id, $name, $mediaFileLP, $mdataFileLP, $gunid ); if(PEAR::isError($ac)) return $ac; return $id; @@ -447,12 +452,6 @@ class GreenBox extends Alib{ GROUP BY md.gunid "); if(!is_array($res)) $res = array(); -/* - if(!(count($res)>0)) - return PEAR::raiseError( - 'GreenBox::localSearch: no items found', GBERR_NONE - ); -*/ return $res; } @@ -462,30 +461,70 @@ class GreenBox extends Alib{ * Upload file to remote repository * * @param id int, virt.file's local id + * @param gunid string, global id * @param sessid string, session id * @return string - transfer id or PEAR::error */ - function uploadFile($id, $sessid='') + function uploadFile($id, $gunid, $sessid='') { - return PEAR::raiseError( - 'GreenBox::uploadFile: not implemented', GBERR_NOTIMPL - ); + $res = $this->prepareForTransport($id, $gunid, $sessid); + if(PEAR::isError($res)) return $res; + list($mediaFile, $mdataFile, $gunid) = $res; + $tr =& new Transport(&$this->dbc, $this->config); + $res = $tr->uploadOpen($mediaFile, 'media', $sessid, $gunid); + if(PEAR::isError($res)) return $res; + $res2 = $tr->uploadOpen($mdataFile, 'metadata', $sessid, $gunid); + if(PEAR::isError($res2)) return $res2; + $res3 = $tr->getTransportStatus($res); + $res4 = $tr->getTransportStatus($res2); +# return $res; + return array($res, $res2, $res3, $res4); +# return PEAR::raiseError( +# 'GreenBox::uploadFile: not implemented', GBERR_NOTIMPL +# ); } /** * Download file from remote repository * - * @param id int, virt.file's local id - * @param parid int, destination folder local id + * @param gunid int, global unique id * @param sessid string, session id * @return string - transfer id or PEAR::error */ - function downloadFile($id, $parid, $sessid='') + function downloadFile($gunid, $sessid='') { - return PEAR::raiseError( - 'GreenBox::downloadFile: not implemented', GBERR_NOTIMPL + $tr =& new Transport(&$this->dbc, $this->config); + // get home dir if needed + $res = $tr->downloadOpen($sessid, 'media', $gunid, + $this->getSessUserId($sessid) ); + if(PEAR::isError($res)) return $res; + $res2 = $tr->downloadOpen($sessid, 'metadata', $gunid, + $this->getSessUserId($sessid) + ); + if(PEAR::isError($res)) return $res; + $res3 = $tr->getTransportStatus($res); + $res4 = $tr->getTransportStatus($res2); +# return $res; + return array($res, $res2, $res3, $res4); +# return PEAR::raiseError( +# 'GreenBox::downloadFile: not implemented', GBERR_NOTIMPL +# ); } + + + + /** + * Method for handling interupted transports via cron + * + */ + function cronJob() + { + $tr =& new Transport(&$this->dbc, $this->config); + $ru = $tr->uploadCron(); + $rd = $tr->downloadCron(&$this); + return array($ru, $rd); + } /** * Get status of asynchronous transfer @@ -502,6 +541,82 @@ class GreenBox extends Alib{ ); } + /** + * Prepare symlink to media file and create metadata file for transport + * + * @param id + * @param gunid + * @param sessid + * @return array + */ + function prepareForTransport($id, $gunid, $sessid='') + { + if(!$gunid) $gunid = $this->_gunidFromId($id); + else $id = $this->_idFromGunid($gunid); + $ac =& StoredFile::recall(&$this, '', $gunid); + if(PEAR::isError($ac)) return $ac; + $mediaTarget = $ac->_getRealRADFname(); + $mediaFile = "$gunid"; + $mdataFile = "$gunid.xml"; + @symlink($mediaTarget, $this->transDir."/$mediaFile"); + $mdata = $this->getMdata($id, $sessid); + if(PEAR::isError($mdata)) return $mdata; + if(!($fh = fopen($this->transDir."/$mdataFile", 'w'))) $res=FALSE; + else{ + $res = fwrite($fh, $mdata); + fclose($fh); + } + if($res === FALSE) return PEAR::raiseError( + "GreenBox::prepareForTransport:". + " can't write metadata tmp file ($mdataFile)" + ); + return array($mediaFile, $mdataFile, $gunid); + } + + /** + * Insert transported file and metadata into storage.
+ * TODO: cals methods from LocStor - it's not good + * + * @param sessid string - session id + * @param file string - local path to filr + * @param type string - media|metadata|search + * @param gunid string - global unique id + */ + function processTransported($sessid, $file, $type, $gunid='X') + { + switch($type){ + case 'media': + if(!file_exists($file)) break; + $res = $this->storeAudioClip($sessid, $gunid, + $file, ''); + if(PEAR::isError($res)) return $res; + @unlink($file); + break; + case 'metadata': + case 'mdata': + if(!file_exists($file)) break; + $res = $this->updateAudioClipMetadata($sessid, $gunid, + $file); + if(PEAR::isError($res)){ + // catch valid exception + if($res->getCode() == GBERR_FOBJNEX){ + $res2 = $this->storeAudioClip($sessid, $gunid, + '', $file); + if(PEAR::isError($res2)) return $res2; + }else return $res; + } + @unlink($file); + break; + case 'search': + //$this->localSearch($criteria); + return PEAR::raiseError("processTranferred: search not implemented"); + break; + default: + return PEAR::raiseError("processTranferred: unknown type ($type)"); + break; + } + } + /** * Search in central metadata database * @@ -514,6 +629,15 @@ class GreenBox extends Alib{ return PEAR::raiseError( 'GreenBox::globalSearch: not implemented', GBERR_NOTIMPL ); + /* + $srchid = md5($sessid.mtime()); + $fh = fopen($this->transDir."/$srchid", "w"); + fwrite($fh, serialize($searchData)); + fclose($fh); + $res = $tr->uploadOpen($srchid, 'search', $sessid, $gunid); + if(PEAR::isError($res)) return $res; + return $res; + */ } /** @@ -561,7 +685,6 @@ class GreenBox extends Alib{ */ function getObjIdFromRelPath($id, $relPath='.') { -# $a = $this->getDir($id, 'id, name, type, param as target', 'name'); $a = split('/', $relPath); if($this->getObjType($id)!=='Folder') $nid = $this->getparent($id); else $nid = $id; @@ -577,9 +700,7 @@ class GreenBox extends Alib{ default: $nid = $this->getObjId($item, $nid); } -# $a[$i] = array('o'=>$item, 'n'=>($nid==null ? 'NULL' : $nid)); } -# return $a; return $nid; } @@ -730,7 +851,27 @@ class GreenBox extends Alib{ $perm = $perm && $res; } if($perm) return TRUE; - return PEAR::raiseError("GreenBox::$method: access denied", GBERR_DENY); + $adesc = "[".join(',',$acts)."]"; + return PEAR::raiseError("GreenBox::$adesc: access denied", GBERR_DENY); + } + + /** + * Create fake session for downloaded files + * + * @param userid user id + * @return string sessid + */ + function _fakeSession($userid) + { + $sessid = $this->_createSessid(); + if(PEAR::isError($sessid)) return $sessid; + $login = $this->getSubjName($userid); + $r = $this->dbc->query("INSERT INTO {$this->sessTable} + (sessid, userid, login, ts) + VALUES + ('$sessid', '$userid', '$login', now())"); + if(PEAR::isError($r)) return $r; + return $sessid; } /** @@ -746,6 +887,20 @@ class GreenBox extends Alib{ ); } + /** + * Get global id from local id + * + * @param id int local id + * @return string global id + */ + function _gunidFromId($id) + { + if(!is_numeric($id)) return NULL; + return $this->dbc->getOne( + "SELECT gunid FROM {$this->filesTable} WHERE id='$id'" + ); + } + /* =============================================== test and debug methods */ /** * dump @@ -922,9 +1077,9 @@ class GreenBox extends Alib{ )"); $this->dbc->query("CREATE INDEX {$this->accessTable}_acc_idx ON {$this->accessTable} (tmpLink, sessid)"); - if(!file_exists("{$this->storageDir}/buffer")){ - mkdir("{$this->storageDir}/buffer", 02775); - chmod("{$this->storageDir}/buffer", 02775); + if(!file_exists($this->bufferDir)){ + mkdir($this->bufferDir, 02775); + chmod($this->bufferDir, 02775); // may be obsolete } $this->initData(); } @@ -960,7 +1115,6 @@ class GreenBox extends Alib{ { $dd = dir("{$this->storageDir}/$entry"); while (false !== ($ee = $dd->read())){ - //if(substr($ee, -4)=='.mp3' || substr($ee, -4)=='.xml') if(substr($ee, 0, 1)!=='.') unlink("{$this->storageDir}/$entry/$ee"); } @@ -969,12 +1123,12 @@ class GreenBox extends Alib{ } } if(is_object($d)) $d->close(); - if(file_exists("{$this->storageDir}/buffer")){ - $d = dir("{$this->storageDir}/buffer"); + if(file_exists($this->bufferDir)){ + $d = dir($this->bufferDir); while (false !== ($entry = $d->read())) if(substr($entry,0,1)!='.') - { unlink("{$this->storageDir}/buffer/$entry"); } + { unlink("{$this->bufferDir}/$entry"); } $d->close(); - rmdir("{$this->storageDir}/buffer"); + rmdir($this->bufferDir); } parent::uninstall(); } diff --git a/livesupport/modules/storageServer/var/MetaData.php b/livesupport/modules/storageServer/var/MetaData.php index eae393c3f..a2204b51f 100644 --- a/livesupport/modules/storageServer/var/MetaData.php +++ b/livesupport/modules/storageServer/var/MetaData.php @@ -23,7 +23,7 @@ Author : $Author: tomas $ - Version : $Revision: 1.2 $ + Version : $Revision: 1.3 $ Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/MetaData.php,v $ ------------------------------------------------------------------------------*/ @@ -372,11 +372,16 @@ class MetaData{ AND subjns='_G' AND subject='{$this->gunid}' "); if(PEAR::isError($row)) return $row; - if(is_null($row)) return PEAR::raiseError( - "MetaData::genXMLDoc: not exists ({$this->gunid})" - ); - $rr = $this->genXMLNode(&$domd, &$domd, $row); - if(PEAR::isError($rr)) return $rr; + if(is_null($row)){ +// return PEAR::raiseError( +// "MetaData::genXMLDoc: not exists ({$this->gunid})" +// ); + $nxn =& $domd->create_element('metadata'); + $domd->append_child(&$nxn); + }else{ + $rr = $this->genXMLNode(&$domd, &$domd, $row); + if(PEAR::isError($rr)) return $rr; + } return preg_replace("|]*)>|", "\n", $domd->dump_mem())."\n"; } @@ -403,10 +408,11 @@ class MetaData{ "); if(!is_null($uri) && $uri !== ''){ $root =& $domd->document_element(); - $root->add_namespace($uri, $row['predns']); - if($row['predns'] !== ''){ +# if($row['predns'] !== ''){ + if($row['predns'] === '') $row['predns']='d'; + $root->add_namespace($uri, $row['predns']); $nxn->set_namespace($uri, $row['predns']); - } +# } } if($row['object'] != 'NULL'){ $tn =& $domd->create_text_node($row['object']); diff --git a/livesupport/modules/storageServer/var/RawMediaData.php b/livesupport/modules/storageServer/var/RawMediaData.php index ecb6f7095..f10592816 100644 --- a/livesupport/modules/storageServer/var/RawMediaData.php +++ b/livesupport/modules/storageServer/var/RawMediaData.php @@ -23,7 +23,7 @@ Author : $Author: tomas $ - Version : $Revision: 1.2 $ + Version : $Revision: 1.3 $ Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/RawMediaData.php,v $ ------------------------------------------------------------------------------*/ @@ -62,16 +62,22 @@ class RawMediaData{ function insert($mediaFileLP) { if($this->exists) return FALSE; + // for files downloaded from archive: + if($mediaFileLP == $this->fname){ + $this->exists = TRUE; + return TRUE; + } umask(0002); if(@copy($mediaFileLP, $this->fname)){ // @chmod($this->fname, 0775); $this->exists = TRUE; return TRUE; }else{ - @unlink($this->fname); +// @unlink($this->fname); $this->exists = FALSE; return PEAR::raiseError( - "RawMediaData::insert: file save failed", GBERR_FILEIO + "RawMediaData::insert: file save failed". + " ($mediaFileLP, {$this->fname})",GBERR_FILEIO ); } } diff --git a/livesupport/modules/storageServer/var/StoredFile.php b/livesupport/modules/storageServer/var/StoredFile.php index d565ccabc..30a37b2f0 100644 --- a/livesupport/modules/storageServer/var/StoredFile.php +++ b/livesupport/modules/storageServer/var/StoredFile.php @@ -23,7 +23,7 @@ Author : $Author: tomas $ - Version : $Revision: 1.3 $ + Version : $Revision: 1.4 $ Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/StoredFile.php,v $ ------------------------------------------------------------------------------*/ @@ -230,7 +230,7 @@ class StoredFile{ { $this->dbc->query("BEGIN"); $res = $this->dbc->query("UPDATE {$this->filesTable} - SET currentlyaccessing=currentlyaccessing+1 + SET currentlyAccessing=currentlyAccessing+1 WHERE gunid='{$this->gunid}'"); if(PEAR::isError($res)){ $this->dbc->query("ROLLBACK"); return $res; } $accLinkName = $this->_getAccessFname($sessid, $this->_getExt()); @@ -261,11 +261,11 @@ class StoredFile{ { $this->dbc->query("BEGIN"); $res = $this->dbc->query("UPDATE {$this->filesTable} - SET currentlyaccessing=currentlyaccessing-1 - WHERE gunid='{$this->gunid}' AND currentlyaccessing>0" + SET currentlyAccessing=currentlyAccessing-1 + WHERE gunid='{$this->gunid}' AND currentlyAccessing>0" ); if(PEAR::isError($res)){ $this->dbc->query("ROLLBACK"); return $res; } - $ca = $this->dbc->getOne("SELECT currentlyaccessing + $ca = $this->dbc->getOne("SELECT currentlyAccessing FROM {$this->filesTable} WHERE gunid='{$this->gunid}'" ); @@ -481,6 +481,7 @@ class StoredFile{ function _getResDir() { $resDir="{$this->gb->storageDir}/".substr($this->gunid, 0, 3); + // see Transport::_getResDir too for resDir name create code if(!file_exists($resDir)){ mkdir($resDir, 02775); chmod($resDir, 02775); } return $resDir; } diff --git a/livesupport/modules/storageServer/var/Transport.php b/livesupport/modules/storageServer/var/Transport.php index d5835ebd7..9fbfefbd8 100644 --- a/livesupport/modules/storageServer/var/Transport.php +++ b/livesupport/modules/storageServer/var/Transport.php @@ -23,18 +23,11 @@ Author : $Author: tomas $ - Version : $Revision: 1.1 $ + Version : $Revision: 1.2 $ Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/Transport.php,v $ ------------------------------------------------------------------------------*/ -include("xmlrpc/xmlrpc.inc"); - -$config['archiveUrlPath'] = '/livesupport/modules/archiveServer/var'; -$config['archiveXMLRPC'] = 'xmlrpc/xrArchive.php'; -$config['archiveUrlHost'] = 'localhost'; -$config['archiveUrlPort'] = 80; -$config['archiveAccountLogin'] = 'root'; -$config['archiveAccountPass'] = 'q'; +include_once "xmlrpc/xmlrpc.inc"; /** * Class for handling file tranport between StorageServer and ArchiveServer
@@ -54,269 +47,26 @@ class Transport{ */ function Transport(&$dbc, $config) { - $this->dbc =& $dbc; - $this->config = $config; + $this->dbc =& $dbc; + $this->config = $config; $this->transTable = $config['tblNamePrefix'].'trans'; - } - - /** - * Start of download
- * - create transport record
- * - call archive.downloadOpen
- * - * @param gunid - */ - function downloadFile($gunid) - { - $res = $this->xmlrpcCall( - 'archive.login', - array( - 'login'=>$this->config['archiveAccountLogin'], - 'pass'=>$this->config['archiveAccountPass'] - ) - ); - if(PEAR::isError($res)) return $res; - $sessid = $res; - // call archive.downloadOpen - $res = $this->xmlrpcCall( - 'archive.downloadOpen', array('sessid'=>$sessid, 'gunid'=>$gunid) - ); - if(PEAR::isError($res)) return $res; - $file = $res; - // insert transport record to db - $id = $this->dbc->nextId("{$this->transTable}_id_seq"); - $res = $this->dbc->query(" - INSERT INTO {$this->transTable} - (id, direction, state, gunid, type, sessid, md5h, url, fname) - VALUES ( - $id, 'down', 'pending', '$gunid', 'file', '$sessid', - '{$file['md5h']}', '{$file['url']}', '{$file['fname']}' - ) - "); - if(PEAR::isError($res)) return $res; -#?? $this->downloadCron(); - return $id; - } - - /** - * Cron method for download.
- * Should be called periodically. - * - */ - function downloadCron() - { - // fetch all pending downloads - $rows = $this->dbc->getAll(" - SELECT id, url, md5h, fname - FROM {$this->transTable} - WHERE direction='down' AND state='pending' - "); - // for all pending downloads: - foreach($rows as $i=>$row){ - // wget the file - $res = system( - "wget -q -c --timeout={$this->timeout}". - " --waitretry={$this->waitretry}". - " -t {$this->retries} {$row['url']}", - $status - ); - // check consistency - $md5h = $this->_md5sum($row['fname']); - if($status == 0){ - if($md5h == $row['md5h']){ - // mark download as finished - $this->dbc->query(" - UPDATE {$this->transTable} - SET state='finished' - WHERE id='{$row['id']}' - "); - }else{ - @unlink($fname); - } - } - } - // fetch all finished downloads - $rows = $this->dbc->getAll(" - SELECT id, url, md5h, sessid - FROM {$this->transTable} - WHERE direction='down' AND state='finished' - "); - // for all finished downloads: - foreach($rows as $i=>$row){ - $res = $this->xmlrpcCall( - 'archive.downloadClose', - array('sessid'=>$row['sessid'], 'url'=>$row['url']) - ); - if(PEAR::isError($res)) return $res; - // close download in db TODO: or delete record? - $this->dbc->query(" - UPDATE {$this->transTable} - SET state='closed' - WHERE id='{$row['id']}' - "); - } - return TRUE; - } - - /** - * Start of upload - * - * @param fname - * @param gunid - */ - function uploadFile($fname, $gunid) - { - $res = $this->xmlrpcCall( - 'archive.login', - array( - 'login'=>$this->config['archiveAccountLogin'], - 'pass'=>$this->config['archiveAccountPass'] - ) - ); - if(PEAR::isError($res)) return $res; - $sessid = $res; - $file = $this->xmlrpcCall( - 'archive.uploadOpen', array('sessid'=>$sessid, 'gunid'=>$gunid) - ); - if(PEAR::isError($file)) return $file; - $md5h = $this->_md5sum($fname); - $id = $this->dbc->nextId("{$this->transTable}_id_seq"); - $res = $this->dbc->query(" - INSERT INTO {$this->transTable} - (id, direction, state, gunid, type, sessid, md5h, url, fname) - VALUES ( - $id, 'up', 'pending', '$gunid', 'file', '$sessid', - '$md5h', '{$file['url']}', '$fname' - ) - "); - if(PEAR::isError($res)) return $res; -#?? $this->uploadCron(); - return $id; - } - - /** - * Cron method for upload.
- * Should be called periodically. - * - */ - function uploadCron() - { - // fetch all pending uploads - $rows = $this->dbc->getAll(" - SELECT id, sessid, gunid, fname, url, md5h - FROM {$this->transTable} - WHERE direction='up' AND state='pending' - "); - // for all pending uploads: - foreach($rows as $i=>$row){ - $file = $this->uploadCheck($row['sessid'], $row['url']); - if(PEAR::isError($file)) return $file; - // test filesize - if(intval($file['size']) < filesize($row['fname'])){ - // not finished - upload next part - $res = system( - "curl -s -C {$file['size']} --max-time 600". - " --speed-time 20 --speed-limit 500". - " --connect-timeout 20". - " -T {$row['fname']} {$row['url']}", - $status - ); - }else{ - // hmmm - we are finished? strage, but we'll try to continue - $status = 0; - } - if($status == 0){ - $file = $this->uploadCheck($row['sessid'], $row['url']); - if(PEAR::isError($file)) return $file; - // test checksum - if($file['md5h'] == $row['md5h']){ - // finished - $res = $this->dbc->query(" - UPDATE {$this->transTable} SET state='finished' - WHERE id='{$row['id']}' - "); - if(PEAR::isError($res)) return $res; - }else{ - if(intval($file['size']) >= filesize($row['fname'])){ - // wrong md5 at finish - we probably have to start again - // $this->xmlrpcCall('archive.uploadReset', array()); - return PEAR::raiseError( - "Transport::uploadCron: file uploaded with bad md5" - ); - } - } - } - } - // fetch all finished uploads - $rows = $this->dbc->getAll(" - SELECT id, sessid, gunid, fname, url, md5h - FROM {$this->transTable} - WHERE direction='up' AND state='finished' - "); - // for all finished uploads: - foreach($rows as $i=>$row){ - $res = $this->xmlrpcCall( - 'archive.uploadClose', - array('sessid'=>$row['sessid'], 'url'=>$row['url']) - ); - if(PEAR::isError($res)) return $res; - // close upload in db TODO: or delete record? - $this->dbc->query(" - UPDATE {$this->transTable} SET state='closed' - WHERE id='{$row['id']}' - "); - } - return TRUE; - } - - /** - * Check state of uploaded file - * - * @param sessid - * @param url - * @return hash: md5h, size, url - */ - function uploadCheck($sessid, $url) - { - $file = $this->xmlrpcCall( - 'archive.uploadCheck', - array('sessid'=>$sessid, 'url'=>$url) - ); - return $file; - } - - /** - * Abort pending upload - * - * @param id local tranport id - */ - function uploadAbort($id) - { - $row = $this->dbc->getRow(" - SELECT id, sessid, gunid, fname, url - FROM {$this->transTable} - WHERE id='$id' - "); - if(PEAR::isError($row)) return $row; - $res = $this->xmlrpcCall('archive.uploadAbort', - array('sessid'=>$row['sessid'], 'url'=>$row['url']) - ); - if(PEAR::isError($res)) return $res; + $this->transDir = $config['transDir']; } /** * Return state of transport job * */ - function getTransportStatus($id) + function getTransportStatus($trid) { $row = $this->dbc->getRow( - "SELECT state FROM {$this->transTable} WHERE id='$id'" + "SELECT state FROM {$this->transTable} WHERE trid='$trid'" ); if(PEAR::isError($res)) return $res; return $row['state']; } + /* ======================================================= search methods */ /** * Start search in archive */ @@ -337,17 +87,327 @@ class Transport{ // not implemented yet } + /* ======================================= general file transport methods */ + + /** + * + */ + function uploadOpen($file, $type, $sessid='', $gunid='X') + { + $trid = $this->_createTrId(); + $md5h = $this->_md5sum($this->transDir."/$file"); + $id = $this->dbc->nextId("{$this->transTable}_id_seq"); + $res = $this->dbc->query(" + INSERT INTO {$this->transTable} + (id, trid, direction, state, type, + md5h, url, fname, gunid + ) + VALUES + ($id, '$trid', 'up', 'init', '$type', + '$md5h', '', '$file', '$gunid' + ) + "); + if(PEAR::isError($res)) return $res; +#?? $this->uploadCron(); + return $trid; + } + + /** + * + */ + function uploadCron() + { + // fetch all opened uploads + $rows = $this->dbc->getAll(" + SELECT * FROM {$this->transTable} + WHERE direction='up' AND state<>'closed' + "); + if(count($rows)==0) return TRUE; + $asessid = $this->loginToArchive(); + chdir($this->config['transDir']); + // for all opened uploads: + foreach($rows as $i=>$row){ + switch($row['state']){ + case"init": // ------ new uploads + $finf = $this->xmlrpcCall( 'archive.uploadOpen', + array('sessid'=>$asessid, 'trid'=>$row['trid'], + 'type'=>$row['type'] + ) + ); + if(PEAR::isError($finf)) return $finf; + $res = $this->dbc->query(" + UPDATE {$this->transTable} + SET state='pending', url='{$finf['url']}' + WHERE id='{$row['id']}' + "); + if(PEAR::isError($res)) return $res; + $row['url'] = $finf['url']; +#break; + case"pending": // ------ pending uploads + $finf = $this->uploadCheck($asessid, $row['url']); + if(PEAR::isError($finf)) return $finf; + // test filesize + if(intval($finf['size']) < filesize($row['fname'])){ + // not finished - upload next part + $res = system( + "curl -s -C {$finf['size']} --max-time 600". + " --speed-time 20 --speed-limit 500". + " --connect-timeout 20". + " -T {$row['fname']} {$row['url']}", + $status + ); + }else{ + // hmmm - we are finished? OK - continue + $status = 0; + } + if($status == 0 || $status == 18){ + $finf = $this->uploadCheck($asessid, $row['url']); + if(PEAR::isError($finf)) return $finf; + // test checksum + if($finf['md5h'] == $row['md5h']){ + // finished + $res = $this->dbc->query(" + UPDATE {$this->transTable} SET state='finished' + WHERE id='{$row['id']}' + "); + if(PEAR::isError($res)) return $res; + }else{ + if(intval($finf['size']) >= filesize($row['fname'])) + { + // wrong md5 at finish - TODO: start again + // $this->xmlrpcCall('archive.uploadReset', array()); + return PEAR::raiseError("Transport::uploadCron:". + " file uploaded with bad md5" + ); + } + } + } +#break; + case"finished": // ------ finished uploads + $res = $this->xmlrpcCall( + 'archive.uploadClose', + array('sessid'=>$asessid, + 'url'=>$row['url'], 'type'=>$row['type'], + 'gunid'=>$row['gunid'], + ) + ); + if(PEAR::isError($res)) return $res; + @unlink($this->transDir."/".$row['fname']); + // close upload in db TODO: or delete record? + $this->dbc->query(" + UPDATE {$this->transTable} SET state='closed' + WHERE id='{$row['id']}' + "); + break; + default: + echo "Transport::uploadCron: unknown state". + " '{$row['state']}' (id={$row['id']})\n"; + } // switch state + } // foreach opened + $this->logoutFromArchive($asessid); + return TRUE; + } + + /** + * Check state of uploaded file + * + * @param sessid + * @param url + * @return hash: md5h, size, url + */ + function uploadCheck($sessid, $url) + { + $finf = $this->xmlrpcCall( + 'archive.uploadCheck', + array('sessid'=>$sessid, 'url'=>$url) + ); + return $finf; + } + + /** + * + */ + function downloadOpen($sessid, $type, $gunid, $uid) + { + // insert transport record to db + $trid = $this->_createTrId(); + $id = $this->dbc->nextId("{$this->transTable}_id_seq"); + $res = $this->dbc->query(" + INSERT INTO {$this->transTable} + (id, trid, direction, state, type, + gunid, sessid, uid + ) + VALUES + ($id, '$trid', 'down', 'init', '$type', + '$gunid', '$sessid', $uid + ) + "); + if(PEAR::isError($res)) return $res; +#?? $this->downloadCron(); + return $trid; + } + + /** + * + */ + function downloadCron(&$gb) + { + // fetch all opened downloads + $rows = $this->dbc->getAll(" + SELECT * FROM {$this->transTable} + WHERE direction='down' AND state<>'closed' + "); + if(count($rows)==0) return TRUE; + $asessid = $this->loginToArchive(); + chdir($this->config['transDir']); + // for all opened downloads: + foreach($rows as $i=>$row){ + switch($row['state']){ + case"init": // ------ new downloads + // call archive.downloadOpen + $finf = $this->xmlrpcCall( + 'archive.downloadOpen', + array('sessid'=>$asessid, 'type'=>$row['type'], + 'par'=>$row['gunid'] + ) + ); + if(PEAR::isError($finf)) return $finf; + $res = $this->dbc->query(" + UPDATE {$this->transTable} + SET state='pending', url='{$finf['url']}', + md5h='{$finf['md5h']}', fname='{$finf['fname']}' + WHERE id='{$row['id']}' + "); + if(PEAR::isError($res)) return $res; + $row = array_merge($row, $finf); +#break; + case"pending": // ------ pending downloads + // wget the file + $res = system( + "wget -q -c --timeout={$this->timeout}". + " --waitretry={$this->waitretry}". + " -t {$this->retries} {$row['url']}", + $status + ); + // check consistency + $md5h = $this->_md5sum($row['fname']); + if($status == 0){ + if($md5h == $row['md5h']){ + // mark download as finished + $res = $this->dbc->query(" + UPDATE {$this->transTable} + SET state='finished' + WHERE id='{$row['id']}' + "); + if(PEAR::isError($res)) return $res; + }else{ + @unlink($row['fname']); + } + } +#break; + case"finished": // ------ finished downloads + // call archive that downloads have been finished OK + $res = $this->xmlrpcCall( + 'archive.downloadClose', + array('sessid'=>$asessid, 'url'=>$row['url']) + ); + if(PEAR::isError($res)) return $res; + // process file in fake session + $lsessid = $gb->_fakeSession($row['uid']); + if(PEAR::isError($lsessid)) return $lsessid; + $res = $gb->processTransported( + $lsessid, $row['fname'], $row['type'], $row['gunid'] + ); + if(PEAR::isError($res)) return $res; + $res = $gb->logout($lsessid); + if(PEAR::isError($res)) return $res; + + // close download in db TODO: or delete record? + $res = $this->dbc->query(" + UPDATE {$this->transTable} + SET state='closed' + WHERE id='{$row['id']}' + "); + if(PEAR::isError($res)) return $res; + break; + default: + echo "Transport::downloadCron: unknown state". + " '{$row['state']}' (id={$row['id']})\n"; + } // switch state + } // foreach opened + $this->logoutFromArchive($asessid); + return TRUE; + } + + + /* =============================================== authentication methods */ + + /** + * Login to archive server + * + * @return string sessid or error + */ + function loginToArchive() + { + $res = $this->xmlrpcCall( + 'archive.login', + array( + 'login'=>$this->config['archiveAccountLogin'], + 'pass'=>$this->config['archiveAccountPass'] + ) + ); + return $res; + } + + /** + * Logout from archive server + * + * @param sessid session id + * @return string Bye or error + */ + function logoutFromArchive($sessid) + { + $res = $this->xmlrpcCall( + 'archive.logout', + array( + 'sessid'=>$sessid, + ) + ); + return $res; + } + + /* ==================================================== auxiliary methods */ + /** + * + */ + function _createTrId() + { + return md5(microtime().$_SERVER['SERVER_ADDR'].rand(). + "org.mdlf.livesupport"); + } + + /** + * + */ + function _getResDir($gunid) + { + $resDir = $this->config['storageDir']."/".substr($gunid, 0, 3); + if(!file_exists($resDir)){ mkdir($resDir, 02775); chmod($resDir, 02775); } + return $resDir; + } + /** * XMLRPC call to archive */ function xmlrpcCall($method, $pars=array()) { + $xrp = xmlrpc_encoder($pars); $c = new xmlrpc_client( "{$this->config['archiveUrlPath']}/". "{$this->config['archiveXMLRPC']}", $this->config['archiveUrlHost'], $this->config['archiveUrlPort'] ); - $f=new xmlrpcmsg($method, array(xmlrpc_encoder($pars))); + $f=new xmlrpcmsg($method, array($xrp)); $r = $c->send($f); if ($r->faultCode()>0) { return PEAR::raiseError($r->faultString(), $r->faultCode()); @@ -367,6 +427,7 @@ class Transport{ return $arr[0]; } + /* ====================================================== install methods */ /** * Install method
* state: pending, finished, closed @@ -375,19 +436,24 @@ class Transport{ { $this->dbc->query("CREATE TABLE {$this->transTable} ( id int not null, - gunid char(32) not null, - md5h char(32) not null, - sessid char(32) not null, - url varchar(255) not null, - fname varchar(255) not null, - type varchar(128) not null, -- file | searchJob + trid char(32) not null, direction varchar(128) not null, -- down | up state varchar(128) not null, + type varchar(128) not null, -- file | searchJob + md5h char(32), + url varchar(255), + fname varchar(255), + gunid char(32), + sessid char(32), + uid int, + parid int, ts timestamp )"); $this->dbc->createSequence("{$this->transTable}_id_seq"); $this->dbc->query("CREATE UNIQUE INDEX {$this->transTable}_id_idx ON {$this->transTable} (id)"); + $this->dbc->query("CREATE INDEX {$this->transTable}_trid_idx + ON {$this->transTable} (trid)"); $this->dbc->query("CREATE INDEX {$this->transTable}_gunid_idx ON {$this->transTable} (gunid)"); } diff --git a/livesupport/modules/storageServer/var/conf.php b/livesupport/modules/storageServer/var/conf.php index 3bb89b167..d09a485d4 100644 --- a/livesupport/modules/storageServer/var/conf.php +++ b/livesupport/modules/storageServer/var/conf.php @@ -23,7 +23,7 @@ Author : $Author: tomas $ - Version : $Revision: 1.2 $ + Version : $Revision: 1.3 $ Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/conf.php,v $ ------------------------------------------------------------------------------*/ @@ -58,6 +58,21 @@ $config = array( 'editPrivs', 'write', 'read', 'classes', 'subjects' ), 'storageDir' => dirname(getcwd()).'/stor', + 'bufferDir' => dirname(getcwd()).'/stor/buffer', + 'transDir' => dirname(getcwd()).'/trans', 'accessDir' => dirname(getcwd()).'/access', + + 'storageUrlPath' => '/livesupport/modules/storageServer/var', + 'storageXMLRPC' => 'xmlrpc/xrLocStor.php', + 'storageUrlHost' => 'localhost', + 'storageUrlPort' => 80, + + 'archiveUrlPath' => '/livesupport/modules/archiveServer/var', + 'archiveXMLRPC' => 'xmlrpc/xrArchive.php', + 'archiveUrlHost' => 'localhost', + 'archiveUrlPort' => 80, + 'archiveAccountLogin' => 'root', + 'archiveAccountPass' => 'q', + ); ?> \ No newline at end of file diff --git a/livesupport/modules/storageServer/var/cron/transportCron.php b/livesupport/modules/storageServer/var/cron/transportCron.php new file mode 100755 index 000000000..2cfab1ed5 --- /dev/null +++ b/livesupport/modules/storageServer/var/cron/transportCron.php @@ -0,0 +1,16 @@ +#!/usr/bin/php -q +setFetchMode(DB_FETCHMODE_ASSOC); + +$gb = &new LocStor(&$dbc, $config); + +$res = $gb->cronJob(); + +var_dump($res); +?> \ No newline at end of file diff --git a/livesupport/modules/storageServer/var/html/gbHtmlBrowse.php b/livesupport/modules/storageServer/var/html/gbHtmlBrowse.php index d0c40e0f0..b13ee1ef5 100644 --- a/livesupport/modules/storageServer/var/html/gbHtmlBrowse.php +++ b/livesupport/modules/storageServer/var/html/gbHtmlBrowse.php @@ -77,7 +77,7 @@ $tpldata['showMenu']=true; ?> -Browser +Storage - browser