Congegni
/
sintonia
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Major changes in transport layer - generalized for metadata and search jobs.

This commit is contained in:
tomas 2004-09-29 01:51:37 +00:00
parent d086760277
commit 4b07803e0c
16 changed files with 650 additions and 347 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
livesupport/modules/storageServer/Makefile
View File

@ -20,7 +20,7 @@
#
#
# Author : $Author: tomas $
# Version : $Revision: 1.3 $
# Version : $Revision: 1.4 $
# Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/Attic/Makefile,v $
#
# @configure_input@
@ -62,13 +62,15 @@ USR_LIB_DIR = ${USR_DIR}/lib
DOXYGEN_CONFIG = ${ETC_DIR}/doxygen.config
HTTP_GROUP = apache
#HTTP_GROUP = nobody
GET_GNAME = install/getGname.php
WWW_ROOT := $(shell cd var/install; php -q getWwwRoot.php)
HTTP_GROUP := $(shell curl -s ${WWW_ROOT}/${GET_GNAME})
PHP_DIR = ${BASE_DIR}/var
INSTALL_DIR = ${PHP_DIR}/install
STOR_DIR = ${PHP_DIR}/stor
ACCESS_DIR = ${PHP_DIR}/access
TRANS_DIR = ${PHP_DIR}/trans
BUFF_DIR = ${STOR_DIR}/buffer
TEST_RUNNER = ${PHP_DIR}/xmlrpc/testRunner.sh
@ -95,18 +97,23 @@ TEST_RUNNER = ${PHP_DIR}/xmlrpc/testRunner.sh
db_init db_clean transtest
all: dir_setup db_init
$(MAKE) -C ../archiveServer all
dir_setup: ${DOXYGEN_DIR} ${STOR_DIR} ${ACCESS_DIR}
dir_setup: ${DOXYGEN_DIR} ${STOR_DIR} ${ACCESS_DIR} ${TRANS_DIR}
doc:
${DOXYGEN} ${DOXYGEN_CONFIG}
clean: db_clean
${RMDIR} ${STOR_DIR}
${RMDIR} ${ACCESS_DIR}
${RMDIR} ${TRANS_DIR}
docclean:
${RMDIR} ${DOXYGEN_DIR}/html
depclean: clean
$(MAKE) -C ../archiveServer clean
dist:
${TAR_C} ${MODULE_NAME}${DATE}${DIST_EXT} *
@ -115,6 +122,7 @@ distclean: clean docclean
check: all ${TEST_RUNNER}
${TEST_RUNNER}
$(MAKE) transtest
#-------------------------------------------------------------------------------
# Specific targets
@ -124,6 +132,9 @@ transtest:
cd var/tests; php -q transTest.php
db_init:
@echo "Settings:"
@echo "Http daemon group: ${HTTP_GROUP}"
@echo "WWW root: ${WWW_ROOT}"
cd var/install; php -q install.php
db_clean:
@ -136,7 +147,7 @@ ${DOXYGEN_DIR}:
${MKDIR} ${DOXYGEN_DIR}
${STOR_DIR}:
@echo "Http daemon group have set in Makefile: ${HTTP_GROUP}"
@echo "Http daemon group: ${HTTP_GROUP}"
@echo "You should be in this group for easy testing"
${MKDIR} ${STOR_DIR}
chown .${HTTP_GROUP} ${STOR_DIR}
@ -147,6 +158,11 @@ ${ACCESS_DIR}:
chown .${HTTP_GROUP} ${ACCESS_DIR}
chmod g+ws ${ACCESS_DIR}
${TRANS_DIR}:
${MKDIR} ${TRANS_DIR}
chown .${HTTP_GROUP} ${TRANS_DIR}
chmod g+ws ${TRANS_DIR}
${BUFF_DIR}:
${MKDIR} ${BUFF_DIR}
chmod g+w ${BUFF_DIR}

216
livesupport/modules/storageServer/var/GreenBox.php
View File

@ -23,7 +23,7 @@
Author : $Author: tomas $
Version : $Revision: 1.3 $
Version : $Revision: 1.4 $
Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/GreenBox.php,v $
------------------------------------------------------------------------------*/
@ -40,6 +40,7 @@ define('GBERR_NOTIMPL', 50);
require_once "../../../alib/var/alib.php";
require_once "StoredFile.php";
require_once "Transport.php";
/**
* GreenBox class
@ -47,7 +48,7 @@ require_once "StoredFile.php";
* LiveSupport file storage module
*
* @author $Author: tomas $
* @version $Revision: 1.3 $
* @version $Revision: 1.4 $
* @see Alib
*/
class GreenBox extends Alib{
@ -55,6 +56,7 @@ class GreenBox extends Alib{
var $mdataTable;
var $accessTable;
var $storageDir;
var $bufferDir;
var $accessDir;
var $rootId;
var $storId;
@ -69,10 +71,13 @@ class GreenBox extends Alib{
function GreenBox(&$dbc, $config)
{
parent::Alib(&$dbc, $config);
$this->config = $config;
$this->filesTable = $config['tblNamePrefix'].'files';
$this->mdataTable = $config['tblNamePrefix'].'mdata';
$this->accessTable= $config['tblNamePrefix'].'access';
$this->storageDir = $config['storageDir'];
$this->bufferDir = $config['bufferDir'];
$this->transDir = $config['transDir'];
$this->accessDir = $config['accessDir'];
$this->dbc->setErrorHandling(PEAR_ERROR_RETURN);
$this->rootId = $this->getRootNode();
@ -112,14 +117,14 @@ class GreenBox extends Alib{
* @exception PEAR::error
*/
function putFile($parid, $fileName,
$mediaFileLP, $mdataFileLP, $sessid='')
$mediaFileLP, $mdataFileLP, $sessid='', $gunid=NULL)
{
if(($res = $this->_authorize('write', $parid, $sessid)) !== TRUE)
return $res;
$name = "$fileName";
$id = $this->addObj($name , 'File', $parid);
$ac =& StoredFile::insert(
&$this, $id, $name, $mediaFileLP, $mdataFileLP
&$this, $id, $name, $mediaFileLP, $mdataFileLP, $gunid
);
if(PEAR::isError($ac)) return $ac;
return $id;
@ -447,12 +452,6 @@ class GreenBox extends Alib{
GROUP BY md.gunid
");
if(!is_array($res)) $res = array();
/*
if(!(count($res)>0))
return PEAR::raiseError(
'GreenBox::localSearch: no items found', GBERR_NONE
);
*/
return $res;
}
@ -462,30 +461,70 @@ class GreenBox extends Alib{
* Upload file to remote repository
*
* @param id int, virt.file's local id
* @param gunid string, global id
* @param sessid string, session id
* @return string - transfer id or PEAR::error
*/
function uploadFile($id, $sessid='')
function uploadFile($id, $gunid, $sessid='')
{
return PEAR::raiseError(
'GreenBox::uploadFile: not implemented', GBERR_NOTIMPL
);
$res = $this->prepareForTransport($id, $gunid, $sessid);
if(PEAR::isError($res)) return $res;
list($mediaFile, $mdataFile, $gunid) = $res;
$tr =& new Transport(&$this->dbc, $this->config);
$res = $tr->uploadOpen($mediaFile, 'media', $sessid, $gunid);
if(PEAR::isError($res)) return $res;
$res2 = $tr->uploadOpen($mdataFile, 'metadata', $sessid, $gunid);
if(PEAR::isError($res2)) return $res2;
$res3 = $tr->getTransportStatus($res);
$res4 = $tr->getTransportStatus($res2);
# return $res;
return array($res, $res2, $res3, $res4);
# return PEAR::raiseError(
# 'GreenBox::uploadFile: not implemented', GBERR_NOTIMPL
# );
}
/**
* Download file from remote repository
*
* @param id int, virt.file's local id
* @param parid int, destination folder local id
* @param gunid int, global unique id
* @param sessid string, session id
* @return string - transfer id or PEAR::error
*/
function downloadFile($id, $parid, $sessid='')
function downloadFile($gunid, $sessid='')
{
return PEAR::raiseError(
'GreenBox::downloadFile: not implemented', GBERR_NOTIMPL
$tr =& new Transport(&$this->dbc, $this->config);
// get home dir if needed
$res = $tr->downloadOpen($sessid, 'media', $gunid,
$this->getSessUserId($sessid)
);
if(PEAR::isError($res)) return $res;
$res2 = $tr->downloadOpen($sessid, 'metadata', $gunid,
$this->getSessUserId($sessid)
);
if(PEAR::isError($res)) return $res;
$res3 = $tr->getTransportStatus($res);
$res4 = $tr->getTransportStatus($res2);
# return $res;
return array($res, $res2, $res3, $res4);
# return PEAR::raiseError(
# 'GreenBox::downloadFile: not implemented', GBERR_NOTIMPL
# );
}
/**
* Method for handling interupted transports via cron
*
*/
function cronJob()
{
$tr =& new Transport(&$this->dbc, $this->config);
$ru = $tr->uploadCron();
$rd = $tr->downloadCron(&$this);
return array($ru, $rd);
}
/**
* Get status of asynchronous transfer
@ -502,6 +541,82 @@ class GreenBox extends Alib{
);
}
/**
* Prepare symlink to media file and create metadata file for transport
*
* @param id
* @param gunid
* @param sessid
* @return array
*/
function prepareForTransport($id, $gunid, $sessid='')
{
if(!$gunid) $gunid = $this->_gunidFromId($id);
else $id = $this->_idFromGunid($gunid);
$ac =& StoredFile::recall(&$this, '', $gunid);
if(PEAR::isError($ac)) return $ac;
$mediaTarget = $ac->_getRealRADFname();
$mediaFile = "$gunid";
$mdataFile = "$gunid.xml";
@symlink($mediaTarget, $this->transDir."/$mediaFile");
$mdata = $this->getMdata($id, $sessid);
if(PEAR::isError($mdata)) return $mdata;
if(!($fh = fopen($this->transDir."/$mdataFile", 'w'))) $res=FALSE;
else{
$res = fwrite($fh, $mdata);
fclose($fh);
}
if($res === FALSE) return PEAR::raiseError(
"GreenBox::prepareForTransport:".
" can't write metadata tmp file ($mdataFile)"
);
return array($mediaFile, $mdataFile, $gunid);
}
/**
* Insert transported file and metadata into storage.<br>
* TODO: cals methods from LocStor - it's not good
*
* @param sessid string - session id
* @param file string - local path to filr
* @param type string - media|metadata|search
* @param gunid string - global unique id
*/
function processTransported($sessid, $file, $type, $gunid='X')
{
switch($type){
case 'media':
if(!file_exists($file)) break;
$res = $this->storeAudioClip($sessid, $gunid,
$file, '');
if(PEAR::isError($res)) return $res;
@unlink($file);
break;
case 'metadata':
case 'mdata':
if(!file_exists($file)) break;
$res = $this->updateAudioClipMetadata($sessid, $gunid,
$file);
if(PEAR::isError($res)){
// catch valid exception
if($res->getCode() == GBERR_FOBJNEX){
$res2 = $this->storeAudioClip($sessid, $gunid,
'', $file);
if(PEAR::isError($res2)) return $res2;
}else return $res;
}
@unlink($file);
break;
case 'search':
//$this->localSearch($criteria);
return PEAR::raiseError("processTranferred: search not implemented");
break;
default:
return PEAR::raiseError("processTranferred: unknown type ($type)");
break;
}
}
/**
* Search in central metadata database
*
@ -514,6 +629,15 @@ class GreenBox extends Alib{
return PEAR::raiseError(
'GreenBox::globalSearch: not implemented', GBERR_NOTIMPL
);
/*
$srchid = md5($sessid.mtime());
$fh = fopen($this->transDir."/$srchid", "w");
fwrite($fh, serialize($searchData));
fclose($fh);
$res = $tr->uploadOpen($srchid, 'search', $sessid, $gunid);
if(PEAR::isError($res)) return $res;
return $res;
*/
}
/**
@ -561,7 +685,6 @@ class GreenBox extends Alib{
*/
function getObjIdFromRelPath($id, $relPath='.')
{
# $a = $this->getDir($id, 'id, name, type, param as target', 'name');
$a = split('/', $relPath);
if($this->getObjType($id)!=='Folder') $nid = $this->getparent($id);
else $nid = $id;
@ -577,9 +700,7 @@ class GreenBox extends Alib{
default:
$nid = $this->getObjId($item, $nid);
}
# $a[$i] = array('o'=>$item, 'n'=>($nid==null ? 'NULL' : $nid));
}
# return $a;
return $nid;
}
@ -730,7 +851,27 @@ class GreenBox extends Alib{
$perm = $perm && $res;
}
if($perm) return TRUE;
return PEAR::raiseError("GreenBox::$method: access denied", GBERR_DENY);
$adesc = "[".join(',',$acts)."]";
return PEAR::raiseError("GreenBox::$adesc: access denied", GBERR_DENY);
}
/**
* Create fake session for downloaded files
*
* @param userid user id
* @return string sessid
*/
function _fakeSession($userid)
{
$sessid = $this->_createSessid();
if(PEAR::isError($sessid)) return $sessid;
$login = $this->getSubjName($userid);
$r = $this->dbc->query("INSERT INTO {$this->sessTable}
(sessid, userid, login, ts)
VALUES
('$sessid', '$userid', '$login', now())");
if(PEAR::isError($r)) return $r;
return $sessid;
}
/**
@ -746,6 +887,20 @@ class GreenBox extends Alib{
);
}
/**
* Get global id from local id
*
* @param id int local id
* @return string global id
*/
function _gunidFromId($id)
{
if(!is_numeric($id)) return NULL;
return $this->dbc->getOne(
"SELECT gunid FROM {$this->filesTable} WHERE id='$id'"
);
}
/* =============================================== test and debug methods */
/**
* dump
@ -922,9 +1077,9 @@ class GreenBox extends Alib{
)");
$this->dbc->query("CREATE INDEX {$this->accessTable}_acc_idx
ON {$this->accessTable} (tmpLink, sessid)");
if(!file_exists("{$this->storageDir}/buffer")){
mkdir("{$this->storageDir}/buffer", 02775);
chmod("{$this->storageDir}/buffer", 02775);
if(!file_exists($this->bufferDir)){
mkdir($this->bufferDir, 02775);
chmod($this->bufferDir, 02775); // may be obsolete
}
$this->initData();
}
@ -960,7 +1115,6 @@ class GreenBox extends Alib{
{
$dd = dir("{$this->storageDir}/$entry");
while (false !== ($ee = $dd->read())){
//if(substr($ee, -4)=='.mp3' || substr($ee, -4)=='.xml')
if(substr($ee, 0, 1)!=='.')
unlink("{$this->storageDir}/$entry/$ee");
}
@ -969,12 +1123,12 @@ class GreenBox extends Alib{
}
}
if(is_object($d)) $d->close();
if(file_exists("{$this->storageDir}/buffer")){
$d = dir("{$this->storageDir}/buffer");
if(file_exists($this->bufferDir)){
$d = dir($this->bufferDir);
while (false !== ($entry = $d->read())) if(substr($entry,0,1)!='.')
{ unlink("{$this->storageDir}/buffer/$entry"); }
{ unlink("{$this->bufferDir}/$entry"); }
$d->close();
rmdir("{$this->storageDir}/buffer");
rmdir($this->bufferDir);
}
parent::uninstall();
}

24
livesupport/modules/storageServer/var/MetaData.php
View File

@ -23,7 +23,7 @@
Author : $Author: tomas $
Version : $Revision: 1.2 $
Version : $Revision: 1.3 $
Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/MetaData.php,v $
------------------------------------------------------------------------------*/
@ -372,11 +372,16 @@ class MetaData{
AND subjns='_G' AND subject='{$this->gunid}'
");
if(PEAR::isError($row)) return $row;
if(is_null($row)) return PEAR::raiseError(
"MetaData::genXMLDoc: not exists ({$this->gunid})"
);
$rr = $this->genXMLNode(&$domd, &$domd, $row);
if(PEAR::isError($rr)) return $rr;
if(is_null($row)){
// return PEAR::raiseError(
// "MetaData::genXMLDoc: not exists ({$this->gunid})"
// );
$nxn =& $domd->create_element('metadata');
$domd->append_child(&$nxn);
}else{
$rr = $this->genXMLNode(&$domd, &$domd, $row);
if(PEAR::isError($rr)) return $rr;
}
return preg_replace("|</([^>]*)>|", "</\\1>\n", $domd->dump_mem())."\n";
}
@ -403,10 +408,11 @@ class MetaData{
");
if(!is_null($uri) && $uri !== ''){
$root =& $domd->document_element();
$root->add_namespace($uri, $row['predns']);
if($row['predns'] !== ''){
# if($row['predns'] !== ''){
if($row['predns'] === '') $row['predns']='d';
$root->add_namespace($uri, $row['predns']);
$nxn->set_namespace($uri, $row['predns']);
}
# }
}
if($row['object'] != 'NULL'){
$tn =& $domd->create_text_node($row['object']);

12
livesupport/modules/storageServer/var/RawMediaData.php
View File

@ -23,7 +23,7 @@
Author : $Author: tomas $
Version : $Revision: 1.2 $
Version : $Revision: 1.3 $
Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/RawMediaData.php,v $
------------------------------------------------------------------------------*/
@ -62,16 +62,22 @@ class RawMediaData{
function insert($mediaFileLP)
{
if($this->exists) return FALSE;
// for files downloaded from archive:
if($mediaFileLP == $this->fname){
$this->exists = TRUE;
return TRUE;
}
umask(0002);
if(@copy($mediaFileLP, $this->fname)){
// @chmod($this->fname, 0775);
$this->exists = TRUE;
return TRUE;
}else{
@unlink($this->fname);
// @unlink($this->fname);
$this->exists = FALSE;
return PEAR::raiseError(
"RawMediaData::insert: file save failed", GBERR_FILEIO
"RawMediaData::insert: file save failed".
" ($mediaFileLP, {$this->fname})",GBERR_FILEIO
);
}
}

11
livesupport/modules/storageServer/var/StoredFile.php
View File

@ -23,7 +23,7 @@
Author : $Author: tomas $
Version : $Revision: 1.3 $
Version : $Revision: 1.4 $
Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/StoredFile.php,v $
------------------------------------------------------------------------------*/
@ -230,7 +230,7 @@ class StoredFile{
{
$this->dbc->query("BEGIN");
$res = $this->dbc->query("UPDATE {$this->filesTable}
SET currentlyaccessing=currentlyaccessing+1
SET currentlyAccessing=currentlyAccessing+1
WHERE gunid='{$this->gunid}'");
if(PEAR::isError($res)){ $this->dbc->query("ROLLBACK"); return $res; }
$accLinkName = $this->_getAccessFname($sessid, $this->_getExt());
@ -261,11 +261,11 @@ class StoredFile{
{
$this->dbc->query("BEGIN");
$res = $this->dbc->query("UPDATE {$this->filesTable}
SET currentlyaccessing=currentlyaccessing-1
WHERE gunid='{$this->gunid}' AND currentlyaccessing>0"
SET currentlyAccessing=currentlyAccessing-1
WHERE gunid='{$this->gunid}' AND currentlyAccessing>0"
);
if(PEAR::isError($res)){ $this->dbc->query("ROLLBACK"); return $res; }
$ca = $this->dbc->getOne("SELECT currentlyaccessing
$ca = $this->dbc->getOne("SELECT currentlyAccessing
FROM {$this->filesTable}
WHERE gunid='{$this->gunid}'"
);
@ -481,6 +481,7 @@ class StoredFile{
function _getResDir()
{
$resDir="{$this->gb->storageDir}/".substr($this->gunid, 0, 3);
// see Transport::_getResDir too for resDir name create code
if(!file_exists($resDir)){ mkdir($resDir, 02775); chmod($resDir, 02775); }
return $resDir;
}

596
livesupport/modules/storageServer/var/Transport.php
View File

@ -23,18 +23,11 @@
Author : $Author: tomas $
Version : $Revision: 1.1 $
Version : $Revision: 1.2 $
Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/Transport.php,v $
------------------------------------------------------------------------------*/
include("xmlrpc/xmlrpc.inc");
$config['archiveUrlPath'] = '/livesupport/modules/archiveServer/var';
$config['archiveXMLRPC'] = 'xmlrpc/xrArchive.php';
$config['archiveUrlHost'] = 'localhost';
$config['archiveUrlPort'] = 80;
$config['archiveAccountLogin'] = 'root';
$config['archiveAccountPass'] = 'q';
include_once "xmlrpc/xmlrpc.inc";
/**
* Class for handling file tranport between StorageServer and ArchiveServer<br>
@ -54,269 +47,26 @@ class Transport{
*/
function Transport(&$dbc, $config)
{
$this->dbc =& $dbc;
$this->config = $config;
$this->dbc =& $dbc;
$this->config = $config;
$this->transTable = $config['tblNamePrefix'].'trans';
}
/**
* Start of download<br>
* - create transport record<br>
* - call archive.downloadOpen<br>
*
* @param gunid
*/
function downloadFile($gunid)
{
$res = $this->xmlrpcCall(
'archive.login',
array(
'login'=>$this->config['archiveAccountLogin'],
'pass'=>$this->config['archiveAccountPass']
)
);
if(PEAR::isError($res)) return $res;
$sessid = $res;
// call archive.downloadOpen
$res = $this->xmlrpcCall(
'archive.downloadOpen', array('sessid'=>$sessid, 'gunid'=>$gunid)
);
if(PEAR::isError($res)) return $res;
$file = $res;
// insert transport record to db
$id = $this->dbc->nextId("{$this->transTable}_id_seq");
$res = $this->dbc->query("
INSERT INTO {$this->transTable}
(id, direction, state, gunid, type, sessid, md5h, url, fname)
VALUES (
$id, 'down', 'pending', '$gunid', 'file', '$sessid',
'{$file['md5h']}', '{$file['url']}', '{$file['fname']}'
)
");
if(PEAR::isError($res)) return $res;
#?? $this->downloadCron();
return $id;
}
/**
* Cron method for download.<br>
* Should be called periodically.
*
*/
function downloadCron()
{
// fetch all pending downloads
$rows = $this->dbc->getAll("
SELECT id, url, md5h, fname
FROM {$this->transTable}
WHERE direction='down' AND state='pending'
");
// for all pending downloads:
foreach($rows as $i=>$row){
// wget the file
$res = system(
"wget -q -c --timeout={$this->timeout}".
" --waitretry={$this->waitretry}".
" -t {$this->retries} {$row['url']}",
$status
);
// check consistency
$md5h = $this->_md5sum($row['fname']);
if($status == 0){
if($md5h == $row['md5h']){
// mark download as finished
$this->dbc->query("
UPDATE {$this->transTable}
SET state='finished'
WHERE id='{$row['id']}'
");
}else{
@unlink($fname);
}
}
}
// fetch all finished downloads
$rows = $this->dbc->getAll("
SELECT id, url, md5h, sessid
FROM {$this->transTable}
WHERE direction='down' AND state='finished'
");
// for all finished downloads:
foreach($rows as $i=>$row){
$res = $this->xmlrpcCall(
'archive.downloadClose',
array('sessid'=>$row['sessid'], 'url'=>$row['url'])
);
if(PEAR::isError($res)) return $res;
// close download in db TODO: or delete record?
$this->dbc->query("
UPDATE {$this->transTable}
SET state='closed'
WHERE id='{$row['id']}'
");
}
return TRUE;
}
/**
* Start of upload
*
* @param fname
* @param gunid
*/
function uploadFile($fname, $gunid)
{
$res = $this->xmlrpcCall(
'archive.login',
array(
'login'=>$this->config['archiveAccountLogin'],
'pass'=>$this->config['archiveAccountPass']
)
);
if(PEAR::isError($res)) return $res;
$sessid = $res;
$file = $this->xmlrpcCall(
'archive.uploadOpen', array('sessid'=>$sessid, 'gunid'=>$gunid)
);
if(PEAR::isError($file)) return $file;
$md5h = $this->_md5sum($fname);
$id = $this->dbc->nextId("{$this->transTable}_id_seq");
$res = $this->dbc->query("
INSERT INTO {$this->transTable}
(id, direction, state, gunid, type, sessid, md5h, url, fname)
VALUES (
$id, 'up', 'pending', '$gunid', 'file', '$sessid',
'$md5h', '{$file['url']}', '$fname'
)
");
if(PEAR::isError($res)) return $res;
#?? $this->uploadCron();
return $id;
}
/**
* Cron method for upload.<br>
* Should be called periodically.
*
*/
function uploadCron()
{
// fetch all pending uploads
$rows = $this->dbc->getAll("
SELECT id, sessid, gunid, fname, url, md5h
FROM {$this->transTable}
WHERE direction='up' AND state='pending'
");
// for all pending uploads:
foreach($rows as $i=>$row){
$file = $this->uploadCheck($row['sessid'], $row['url']);
if(PEAR::isError($file)) return $file;
// test filesize
if(intval($file['size']) < filesize($row['fname'])){
// not finished - upload next part
$res = system(
"curl -s -C {$file['size']} --max-time 600".
" --speed-time 20 --speed-limit 500".
" --connect-timeout 20".
" -T {$row['fname']} {$row['url']}",
$status
);
}else{
// hmmm - we are finished? strage, but we'll try to continue
$status = 0;
}
if($status == 0){
$file = $this->uploadCheck($row['sessid'], $row['url']);
if(PEAR::isError($file)) return $file;
// test checksum
if($file['md5h'] == $row['md5h']){
// finished
$res = $this->dbc->query("
UPDATE {$this->transTable} SET state='finished'
WHERE id='{$row['id']}'
");
if(PEAR::isError($res)) return $res;
}else{
if(intval($file['size']) >= filesize($row['fname'])){
// wrong md5 at finish - we probably have to start again
// $this->xmlrpcCall('archive.uploadReset', array());
return PEAR::raiseError(
"Transport::uploadCron: file uploaded with bad md5"
);
}
}
}
}
// fetch all finished uploads
$rows = $this->dbc->getAll("
SELECT id, sessid, gunid, fname, url, md5h
FROM {$this->transTable}
WHERE direction='up' AND state='finished'
");
// for all finished uploads:
foreach($rows as $i=>$row){
$res = $this->xmlrpcCall(
'archive.uploadClose',
array('sessid'=>$row['sessid'], 'url'=>$row['url'])
);
if(PEAR::isError($res)) return $res;
// close upload in db TODO: or delete record?
$this->dbc->query("
UPDATE {$this->transTable} SET state='closed'
WHERE id='{$row['id']}'
");
}
return TRUE;
}
/**
* Check state of uploaded file
*
* @param sessid
* @param url
* @return hash: md5h, size, url
*/
function uploadCheck($sessid, $url)
{
$file = $this->xmlrpcCall(
'archive.uploadCheck',
array('sessid'=>$sessid, 'url'=>$url)
);
return $file;
}
/**
* Abort pending upload
*
* @param id local tranport id
*/
function uploadAbort($id)
{
$row = $this->dbc->getRow("
SELECT id, sessid, gunid, fname, url
FROM {$this->transTable}
WHERE id='$id'
");
if(PEAR::isError($row)) return $row;
$res = $this->xmlrpcCall('archive.uploadAbort',
array('sessid'=>$row['sessid'], 'url'=>$row['url'])
);
if(PEAR::isError($res)) return $res;
$this->transDir = $config['transDir'];
}
/**
* Return state of transport job
*
*/
function getTransportStatus($id)
function getTransportStatus($trid)
{
$row = $this->dbc->getRow(
"SELECT state FROM {$this->transTable} WHERE id='$id'"
"SELECT state FROM {$this->transTable} WHERE trid='$trid'"
);
if(PEAR::isError($res)) return $res;
return $row['state'];
}
/* ======================================================= search methods */
/**
* Start search in archive
*/
@ -337,17 +87,327 @@ class Transport{
// not implemented yet
}
/* ======================================= general file transport methods */
/**
*
*/
function uploadOpen($file, $type, $sessid='', $gunid='X')
{
$trid = $this->_createTrId();
$md5h = $this->_md5sum($this->transDir."/$file");
$id = $this->dbc->nextId("{$this->transTable}_id_seq");
$res = $this->dbc->query("
INSERT INTO {$this->transTable}
(id, trid, direction, state, type,
md5h, url, fname, gunid
)
VALUES
($id, '$trid', 'up', 'init', '$type',
'$md5h', '', '$file', '$gunid'
)
");
if(PEAR::isError($res)) return $res;
#?? $this->uploadCron();
return $trid;
}
/**
*
*/
function uploadCron()
{
// fetch all opened uploads
$rows = $this->dbc->getAll("
SELECT * FROM {$this->transTable}
WHERE direction='up' AND state<>'closed'
");
if(count($rows)==0) return TRUE;
$asessid = $this->loginToArchive();
chdir($this->config['transDir']);
// for all opened uploads:
foreach($rows as $i=>$row){
switch($row['state']){
case"init": // ------ new uploads
$finf = $this->xmlrpcCall( 'archive.uploadOpen',
array('sessid'=>$asessid, 'trid'=>$row['trid'],
'type'=>$row['type']
)
);
if(PEAR::isError($finf)) return $finf;
$res = $this->dbc->query("
UPDATE {$this->transTable}
SET state='pending', url='{$finf['url']}'
WHERE id='{$row['id']}'
");
if(PEAR::isError($res)) return $res;
$row['url'] = $finf['url'];
#break;
case"pending": // ------ pending uploads
$finf = $this->uploadCheck($asessid, $row['url']);
if(PEAR::isError($finf)) return $finf;
// test filesize
if(intval($finf['size']) < filesize($row['fname'])){
// not finished - upload next part
$res = system(
"curl -s -C {$finf['size']} --max-time 600".
" --speed-time 20 --speed-limit 500".
" --connect-timeout 20".
" -T {$row['fname']} {$row['url']}",
$status
);
}else{
// hmmm - we are finished? OK - continue
$status = 0;
}
if($status == 0 || $status == 18){
$finf = $this->uploadCheck($asessid, $row['url']);
if(PEAR::isError($finf)) return $finf;
// test checksum
if($finf['md5h'] == $row['md5h']){
// finished
$res = $this->dbc->query("
UPDATE {$this->transTable} SET state='finished'
WHERE id='{$row['id']}'
");
if(PEAR::isError($res)) return $res;
}else{
if(intval($finf['size']) >= filesize($row['fname']))
{
// wrong md5 at finish - TODO: start again
// $this->xmlrpcCall('archive.uploadReset', array());
return PEAR::raiseError("Transport::uploadCron:".
" file uploaded with bad md5"
);
}
}
}
#break;
case"finished": // ------ finished uploads
$res = $this->xmlrpcCall(
'archive.uploadClose',
array('sessid'=>$asessid,
'url'=>$row['url'], 'type'=>$row['type'],
'gunid'=>$row['gunid'],
)
);
if(PEAR::isError($res)) return $res;
@unlink($this->transDir."/".$row['fname']);
// close upload in db TODO: or delete record?
$this->dbc->query("
UPDATE {$this->transTable} SET state='closed'
WHERE id='{$row['id']}'
");
break;
default:
echo "Transport::uploadCron: unknown state".
" '{$row['state']}' (id={$row['id']})\n";
} // switch state
} // foreach opened
$this->logoutFromArchive($asessid);
return TRUE;
}
/**
* Check state of uploaded file
*
* @param sessid
* @param url
* @return hash: md5h, size, url
*/
function uploadCheck($sessid, $url)
{
$finf = $this->xmlrpcCall(
'archive.uploadCheck',
array('sessid'=>$sessid, 'url'=>$url)
);
return $finf;
}
/**
*
*/
function downloadOpen($sessid, $type, $gunid, $uid)
{
// insert transport record to db
$trid = $this->_createTrId();
$id = $this->dbc->nextId("{$this->transTable}_id_seq");
$res = $this->dbc->query("
INSERT INTO {$this->transTable}
(id, trid, direction, state, type,
gunid, sessid, uid
)
VALUES
($id, '$trid', 'down', 'init', '$type',
'$gunid', '$sessid', $uid
)
");
if(PEAR::isError($res)) return $res;
#?? $this->downloadCron();
return $trid;
}
/**
*
*/
function downloadCron(&$gb)
{
// fetch all opened downloads
$rows = $this->dbc->getAll("
SELECT * FROM {$this->transTable}
WHERE direction='down' AND state<>'closed'
");
if(count($rows)==0) return TRUE;
$asessid = $this->loginToArchive();
chdir($this->config['transDir']);
// for all opened downloads:
foreach($rows as $i=>$row){
switch($row['state']){
case"init": // ------ new downloads
// call archive.downloadOpen
$finf = $this->xmlrpcCall(
'archive.downloadOpen',
array('sessid'=>$asessid, 'type'=>$row['type'],
'par'=>$row['gunid']
)
);
if(PEAR::isError($finf)) return $finf;
$res = $this->dbc->query("
UPDATE {$this->transTable}
SET state='pending', url='{$finf['url']}',
md5h='{$finf['md5h']}', fname='{$finf['fname']}'
WHERE id='{$row['id']}'
");
if(PEAR::isError($res)) return $res;
$row = array_merge($row, $finf);
#break;
case"pending": // ------ pending downloads
// wget the file
$res = system(
"wget -q -c --timeout={$this->timeout}".
" --waitretry={$this->waitretry}".
" -t {$this->retries} {$row['url']}",
$status
);
// check consistency
$md5h = $this->_md5sum($row['fname']);
if($status == 0){
if($md5h == $row['md5h']){
// mark download as finished
$res = $this->dbc->query("
UPDATE {$this->transTable}
SET state='finished'
WHERE id='{$row['id']}'
");
if(PEAR::isError($res)) return $res;
}else{
@unlink($row['fname']);
}
}
#break;
case"finished": // ------ finished downloads
// call archive that downloads have been finished OK
$res = $this->xmlrpcCall(
'archive.downloadClose',
array('sessid'=>$asessid, 'url'=>$row['url'])
);
if(PEAR::isError($res)) return $res;
// process file in fake session
$lsessid = $gb->_fakeSession($row['uid']);
if(PEAR::isError($lsessid)) return $lsessid;
$res = $gb->processTransported(
$lsessid, $row['fname'], $row['type'], $row['gunid']
);
if(PEAR::isError($res)) return $res;
$res = $gb->logout($lsessid);
if(PEAR::isError($res)) return $res;
// close download in db TODO: or delete record?
$res = $this->dbc->query("
UPDATE {$this->transTable}
SET state='closed'
WHERE id='{$row['id']}'
");
if(PEAR::isError($res)) return $res;
break;
default:
echo "Transport::downloadCron: unknown state".
" '{$row['state']}' (id={$row['id']})\n";
} // switch state
} // foreach opened
$this->logoutFromArchive($asessid);
return TRUE;
}
/* =============================================== authentication methods */
/**
* Login to archive server
*
* @return string sessid or error
*/
function loginToArchive()
{
$res = $this->xmlrpcCall(
'archive.login',
array(
'login'=>$this->config['archiveAccountLogin'],
'pass'=>$this->config['archiveAccountPass']
)
);
return $res;
}
/**
* Logout from archive server
*
* @param sessid session id
* @return string Bye or error
*/
function logoutFromArchive($sessid)
{
$res = $this->xmlrpcCall(
'archive.logout',
array(
'sessid'=>$sessid,
)
);
return $res;
}
/* ==================================================== auxiliary methods */
/**
*
*/
function _createTrId()
{
return md5(microtime().$_SERVER['SERVER_ADDR'].rand().
"org.mdlf.livesupport");
}
/**
*
*/
function _getResDir($gunid)
{
$resDir = $this->config['storageDir']."/".substr($gunid, 0, 3);
if(!file_exists($resDir)){ mkdir($resDir, 02775); chmod($resDir, 02775); }
return $resDir;
}
/**
* XMLRPC call to archive
*/
function xmlrpcCall($method, $pars=array())
{
$xrp = xmlrpc_encoder($pars);
$c = new xmlrpc_client(
"{$this->config['archiveUrlPath']}/".
"{$this->config['archiveXMLRPC']}",
$this->config['archiveUrlHost'], $this->config['archiveUrlPort']
);
$f=new xmlrpcmsg($method, array(xmlrpc_encoder($pars)));
$f=new xmlrpcmsg($method, array($xrp));
$r = $c->send($f);
if ($r->faultCode()>0) {
return PEAR::raiseError($r->faultString(), $r->faultCode());
@ -367,6 +427,7 @@ class Transport{
return $arr[0];
}
/* ====================================================== install methods */
/**
* Install method<br>
* state: pending, finished, closed
@ -375,19 +436,24 @@ class Transport{
{
$this->dbc->query("CREATE TABLE {$this->transTable} (
id int not null,
gunid char(32) not null,
md5h char(32) not null,
sessid char(32) not null,
url varchar(255) not null,
fname varchar(255) not null,
type varchar(128) not null, -- file | searchJob
trid char(32) not null,
direction varchar(128) not null, -- down | up
state varchar(128) not null,
type varchar(128) not null, -- file | searchJob
md5h char(32),
url varchar(255),
fname varchar(255),
gunid char(32),
sessid char(32),
uid int,
parid int,
ts timestamp
)");
$this->dbc->createSequence("{$this->transTable}_id_seq");
$this->dbc->query("CREATE UNIQUE INDEX {$this->transTable}_id_idx
ON {$this->transTable} (id)");
$this->dbc->query("CREATE INDEX {$this->transTable}_trid_idx
ON {$this->transTable} (trid)");
$this->dbc->query("CREATE INDEX {$this->transTable}_gunid_idx
ON {$this->transTable} (gunid)");
}

17
livesupport/modules/storageServer/var/conf.php
View File

@ -23,7 +23,7 @@
Author : $Author: tomas $
Version : $Revision: 1.2 $
Version : $Revision: 1.3 $
Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/conf.php,v $
------------------------------------------------------------------------------*/
@ -58,6 +58,21 @@ $config = array(
'editPrivs', 'write', 'read', 'classes', 'subjects'
),
'storageDir' => dirname(getcwd()).'/stor',
'bufferDir' => dirname(getcwd()).'/stor/buffer',
'transDir' => dirname(getcwd()).'/trans',
'accessDir' => dirname(getcwd()).'/access',
'storageUrlPath' => '/livesupport/modules/storageServer/var',
'storageXMLRPC' => 'xmlrpc/xrLocStor.php',
'storageUrlHost' => 'localhost',
'storageUrlPort' => 80,
'archiveUrlPath' => '/livesupport/modules/archiveServer/var',
'archiveXMLRPC' => 'xmlrpc/xrArchive.php',
'archiveUrlHost' => 'localhost',
'archiveUrlPort' => 80,
'archiveAccountLogin' => 'root',
'archiveAccountPass' => 'q',
);
?>

16
livesupport/modules/storageServer/var/cron/transportCron.php Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/php -q
<?php
require_once "../conf.php";
require_once "DB.php";
require_once '../LocStor.php';
PEAR::setErrorHandling(PEAR_ERROR_RETURN);
$dbc = DB::connect($config['dsn'], TRUE);
$dbc->setFetchMode(DB_FETCHMODE_ASSOC);
$gb = &new LocStor(&$dbc, $config);
$res = $gb->cronJob();
var_dump($res);
?>

2
livesupport/modules/storageServer/var/html/gbHtmlBrowse.php
View File

@ -77,7 +77,7 @@ $tpldata['showMenu']=true;
?>
<html><head>
<title>Browser</title>
<title>Storage - browser</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<link rel="stylesheet" type="text/css" href="default.css">
<style type="text/css">

4
livesupport/modules/storageServer/var/html/gbHttp.php
View File

@ -42,8 +42,8 @@ switch($_REQUEST['act']){
// --- files ---
case"upload":
$tmpgunid = md5(microtime().$_SERVER['SERVER_ADDR'].rand()."org.mdlf.livesupport");
$ntmp = "{$gb->storageDir}/buffer/$tmpgunid";
# $ntmp = tempnam(""{$gb->storageDir}/buffer", 'gbTmp_');
$ntmp = "{$gb->bufferDir}/$tmpgunid";
# $ntmp = tempnam(""{$gb->bufferDir}", 'gbTmp_');
$mdtmp = "";
move_uploaded_file($_FILES['mediafile']['tmp_name'], $ntmp); chmod($ntmp, 0664);
if($_FILES['mdatafile']['tmp_name']){

7
livesupport/modules/storageServer/var/install/getGname.php Normal file
View File

@ -0,0 +1,7 @@
<?
header("Content-type: text/plain");
$gid = posix_getgid();
$egid = posix_getegid();
$info = posix_getgrgid($egid);
echo $info['name'];
?>

6
livesupport/modules/storageServer/var/install/getWwwRoot.php Normal file
View File

@ -0,0 +1,6 @@
<?
header("Content-type: text/plain");
require "../conf.php";
echo "http://{$config['storageUrlHost']}:{$config['storageUrlPort']}".
"{$config['storageUrlPath']}";
?>

6
livesupport/modules/storageServer/var/install/getXrUrl.php Normal file
View File

@ -0,0 +1,6 @@
<?
header("Content-type: text/plain");
require "../conf.php";
echo "http://{$config['storageUrlHost']}:{$config['storageUrlPort']}".
"{$config['storageUrlPath']}/{$config['storageXMLRPC']}";
?>

6
livesupport/modules/storageServer/var/install/install.php
View File

@ -23,7 +23,7 @@
Author : $Author: tomas $
Version : $Revision: 1.3 $
Version : $Revision: 1.4 $
Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/install/install.php,v $
------------------------------------------------------------------------------*/
@ -53,10 +53,6 @@ if(PEAR::isError($dbc)){
$dbc->setFetchMode(DB_FETCHMODE_ASSOC);
$gb = &new GreenBox(&$dbc, $config);
echo "\nIMPORTANT settings:\n";
echo " Makefile: HTTP_GROUP - group of running http daemon\n";
echo " var/xmlrpc/testRunner.sh: XMLRPC - URL of XMLRPC server\n\n";
echo "Storage: Install ...\n";
$dbc->setErrorHandling(PEAR_ERROR_RETURN);
$gb->uninstall();

43
livesupport/modules/storageServer/var/tests/transTest.php
View File

@ -1,30 +1,37 @@
<?php
require_once "../conf.php";
require_once "DB.php";
require_once "../Transport.php";
<?
header("Content-type: text/plain");
echo "\nTransport test\n\n";
require_once '../conf.php';
require_once 'DB.php';
require_once '../LocStor.php';
#PEAR::setErrorHandling(PEAR_ERROR_PRINT, "%s<hr>\n");
PEAR::setErrorHandling(PEAR_ERROR_RETURN);
$dbc = DB::connect($config['dsn'], TRUE);
$dbc->setFetchMode(DB_FETCHMODE_ASSOC);
$gb = &new LocStor(&$dbc, $config);
$tr =& new Transport(&$dbc, $config);
$gunid = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
$mediaFile = '../tests/ex1.mp3';
$mdataFile = '../tests/mdata1.xml';
$gunid = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
echo"login: ".($sessid = $gb->login('root', 'q'))."\n";
#$r = $tr->install(); var_dump($r);
#$r = $tr->uninstall(); var_dump($r);
#$r = $tr->getTransportStatus(); var_dump($r);
echo"Store:\n";
$r = $gb->storeAudioClip($sessid, $gunid, $mediaFile, $mdataFile); var_dump($r);
echo"Upload:\n"; $r = $gb->uploadFile('', $gunid, $sessid); var_dump($r);
echo "uploadFile:\n";
$r = $tr->uploadFile('ex2.wav', $gunid); var_dump($r);
echo "uploadCron:\n";
$r = $tr->uploadCron(); var_dump($r);
echo"Cron:\n"; $r = $gb->cronJob(); var_dump($r);
echo "downloadFile:\n";
$r = $tr->downloadFile($gunid); var_dump($r);
echo "downloadCron:\n";
$r = $tr->downloadCron(); var_dump($r);
echo"Delete:\n"; $r = $gb->deleteAudioClip($sessid, $gunid); var_dump($r);
#$r = $tr->uploadAbort(1); var_dump($r);
echo"Download:\n"; $r = $gb->downloadFile($gunid, $sessid); var_dump($r);
echo"Cron:\n"; $r = $gb->cronJob(); var_dump($r);
echo"logout: ".($r = $gb->logout($sessid))."\n";
echo"\n";
?>

5
livesupport/modules/storageServer/var/xmlrpc/testRunner.sh
View File

@ -23,7 +23,7 @@
#
#
# Author : $Author: tomas $
# Version : $Revision: 1.2 $
# Version : $Revision: 1.3 $
# Location : $Source: /home/paul/cvs2svn-livesupport/newcvsrepo/livesupport/modules/storageServer/var/xmlrpc/testRunner.sh,v $
#-------------------------------------------------------------------------------
@ -31,7 +31,8 @@ COMM=$1
shift
GUNID=$1
XMLRPC=http://localhost:80/livesupport/modules/storageServer/var/xmlrpc/xrLocStor.php
#XMLRPC=http://localhost:80/livesupport/modules/storageServer/var/xmlrpc/xrLocStor.php
XMLRPC=`cd var/install; php -q getXrUrl.php`
echo "XMLRPC server URL (check it in troubles):"
echo $XMLRPC