Congegni
/
sintonia
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactored the import script. Now you dont need to specify whether something is a directory or a file, and you can list multiple files/directories on the command line. It also works straight from PHP so you can do: php import.php ...if you want.

This commit is contained in:
paul 2007-01-11 23:28:05 +00:00
parent 623be37d8a
commit 36b6fa83e0
8 changed files with 138 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
campcaster/src/modules/storageAdmin/bin/import.sh
View File

@ -24,9 +24,9 @@
# Author : $Author$ # Author : $Author$
# Version : $Revision$ # Version : $Revision$
# Location : $URL$ # Location : $URL$
#-------------------------------------------------------------------------------
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
# This script imports audio files to Campcaster storageServer. #-------------------------------------------------------------------------------
# This script imports audio files to the Campcaster storageServer.
# #
# To get usage help, try the -h option # To get usage help, try the -h option
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
@ -34,7 +34,6 @@
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
# Determine directories, files # Determine directories, files
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
reldir=`dirname $0`/.. reldir=`dirname $0`/..
phpdir=ls_storageAdmin_phppart_dir phpdir=ls_storageAdmin_phppart_dir
if [ "$phpdir" == "ls_storageAdmin_phppart_dir" ] if [ "$phpdir" == "ls_storageAdmin_phppart_dir" ]
@ -43,102 +42,9 @@ then
fi fi
filelistpathname=. filelistpathname=.
#-------------------------------------------------------------------------------
# Print the usage information for this script.
#-------------------------------------------------------------------------------
printUsage()
{
echo "Campcaster import script.";
echo "parameters:";
echo "";
echo " -d, --directory The source directory, required;";
echo " will be readed recursively.";
echo " -l, --list The filename with list of absolute filepaths";
echo " (newline-separated).";
echo " -f, --file The filename - import of one file";
echo " -t, --test Test only - co not import, show analyze";
echo " -h, --help Print this message and exit.";
echo "";
echo "Usage:";
echo " $0 -d <directory>";
echo " $0 -l <listfile>";
echo " $0 -f <filename>";
echo " $0 -h";
}
#-------------------------------------------------------------------------------
# Process command line parameters
#-------------------------------------------------------------------------------
unset srcabsdir
unset filelistpathname
unset filepathname
unset phpSw
CMD=${0##*/}
opts=$(getopt -o d:l:f:th -l directory:,list:,file:,test,help -n $CMD -- "$@") || exit 1
eval set -- "$opts"
while true; do
case "$1" in
-d|--directory)
srcdir=$2;
test -d "$srcdir" || { echo "Directory not found ($srcdir)."; exit 1; }
srcabsdir=`cd "$srcdir"; pwd`
shift; shift;;
-l|--list)
filelist=$2;
test -f "$filelist" || { echo "File not found ($filelist)."; exit 1; }
filelistbasename=`basename "$filelist"`
filelistdir=`dirname "$filelist"`
filelistabsdir=`cd "$filelistdir"; pwd`
filelistpathname="$filelistabsdir/$filelistbasename"
shift; shift;;
-f|--file)
file=$2;
test -f "$file" || { echo "File not found ($file)."; exit 1; }
filebasename=`basename "$file"`
filedir=`dirname "$file"`
fileabsdir=`cd "$filedir"; pwd`
filepathname="$fileabsdir/$filebasename"
shift; shift;;
-t|--test)
phpSw="-t"
shift;;
-h|--help)
printUsage;
exit 0;;
--)
shift;
break;;
*)
echo "Unrecognized option $1.";
printUsage;
exit 1;
esac
done
if [ "x$srcabsdir" == "x" -a "x$filelist" == "x" -a "x$file" == "x" ]; then
echo "Directory, filelist or file required.";
printUsage;
exit 1;
fi
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
# Do recursive import # Do recursive import
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
cd $phpdir cd $phpdir
if [ -f "$filelistpathname" ]; then php -q import.php $@ || exit 1
cat "$filelistpathname" | php -q import.php $phpSw || exit 1
elif [ -d "$srcabsdir" ]; then
find "$srcabsdir" -type f | php -q import.php $phpSw || exit 1
elif [ -f "$filepathname" ]; then
echo "$filepathname" | php -q import.php $phpSw || exit 1
else
echo "Warning: not a directory: $srcabsdir"
fi
#-------------------------------------------------------------------------------
# Say goodbye
#-------------------------------------------------------------------------------
echo "Import completed."

6
campcaster/src/modules/storageAdmin/var/backup.php
View File

@ -7,12 +7,12 @@ define('NSPACE', 'lse');
header("Content-type: text/plain"); header("Content-type: text/plain");
require_once('conf.php'); require_once('conf.php');
require_once("$storageServerPath/var/conf.php"); require_once("$STORAGE_SERVER_PATH/var/conf.php");
require_once('DB.php'); require_once('DB.php');
require_once("XML/Util.php"); require_once("XML/Util.php");
require_once("XML/Beautifier.php"); require_once("XML/Beautifier.php");
require_once("$storageServerPath/var/BasicStor.php"); require_once("$STORAGE_SERVER_PATH/var/BasicStor.php");
require_once("$storageServerPath/var/Prefs.php"); require_once("$STORAGE_SERVER_PATH/var/Prefs.php");
PEAR::setErrorHandling(PEAR_ERROR_RETURN); PEAR::setErrorHandling(PEAR_ERROR_RETURN);
$CC_DBC = DB::connect($CC_CONFIG['dsn'], TRUE); $CC_DBC = DB::connect($CC_CONFIG['dsn'], TRUE);

2
campcaster/src/modules/storageAdmin/var/conf.php
View File

@ -4,5 +4,5 @@
* @version $Revision$ * @version $Revision$
*/ */
$storageServerPath = dirname(__FILE__)."/../../storageServer"; $STORAGE_SERVER_PATH = dirname(__FILE__)."/../../storageServer";
?> ?>

2
campcaster/src/modules/storageAdmin/var/conf.php.template
View File

@ -3,5 +3,5 @@
* @author $Author$ * @author $Author$
* @version $Revision$ * @version $Revision$
*/ */
$storageServerPath = 'ls_storageServer'; $STORAGE_SERVER_PATH = 'ls_storageServer';
?> ?>

2
campcaster/src/modules/storageAdmin/var/dumpDbSchema.php
View File

@ -1,6 +1,6 @@
<? <?
require_once('conf.php'); require_once('conf.php');
require_once("$storageServerPath/var/conf.php"); require_once("$STORAGE_SERVER_PATH/var/conf.php");
header("Conten-type: text/plain"); header("Conten-type: text/plain");
$dbname = $CC_CONFIG['dsn']['database']; $dbname = $CC_CONFIG['dsn']['database'];
$dbuser = $CC_CONFIG['dsn']['username']; $dbuser = $CC_CONFIG['dsn']['username'];

2
campcaster/src/modules/storageAdmin/var/getStorPath.php
View File

@ -5,7 +5,7 @@
*/ */
header("Content-type: text/plain"); header("Content-type: text/plain");
require_once('conf.php'); require_once('conf.php');
require_once("$storageServerPath/var/conf.php"); require_once("$STORAGE_SERVER_PATH/var/conf.php");
echo $CC_CONFIG['storageDir']."\n"; echo $CC_CONFIG['storageDir']."\n";
?> ?>

184
campcaster/src/modules/storageAdmin/var/import.php
View File

@ -13,30 +13,24 @@
*/ */
ini_set('memory_limit', '64M'); ini_set('memory_limit', '64M');
set_time_limit(0); set_time_limit(0);
//header("Content-type: text/plain"); error_reporting(E_ALL);
echo "===========================\n";
echo "StorageServer Import Script\n";
echo "===========================\n";
$start = intval(date('U'));
require_once('conf.php'); require_once('conf.php');
require_once("$storageServerPath/var/conf.php"); require_once("$STORAGE_SERVER_PATH/var/conf.php");
require_once('DB.php'); require_once('DB.php');
require_once("$storageServerPath/var/GreenBox.php"); require_once("$STORAGE_SERVER_PATH/var/GreenBox.php");
require_once('Console/Getopt.php');
require_once('File/Find.php');
PEAR::setErrorHandling(PEAR_ERROR_RETURN); function printUsage()
$CC_DBC = DB::connect($CC_CONFIG['dsn'], TRUE); {
if (PEAR::isError($CC_DBC)) { echo "Parameters:\n";
echo "ERROR: ".$CC_DBC->getMessage()." ".$CC_DBC->getUserInfo()."\n"; echo "\n";
exit(1); echo " -t, --test Test only - do not import, show analyze\n";
echo " -h, --help Print this message and exit.\n";
echo "\n";
} }
$CC_DBC->setFetchMode(DB_FETCHMODE_ASSOC);
$gb = new GreenBox();
$testonly = (isset($argv[1]) && $argv[1] == '-t');
$g_errors = 0;
$filecount = 0;
/** /**
* Print error to the screen and keep a count of number of errors. * Print error to the screen and keep a count of number of errors.
@ -57,75 +51,140 @@ function import_err($p_pearErrorObj, $txt='')
$g_errors++; $g_errors++;
} }
/**
* Import a file or directory into the storage database.
* If it is a directory, files will be imported recursively.
*
* @param string $p_filepath
* You can pass in a directory or file name.
* @param GreenBox $p_greenbox
* @param int $p_parentId
* @param boolean $p_testOnly
*
* @return int
*/
function camp_import_audio_file($p_filepath, $p_greenbox, $p_parentId,
$p_testOnly = false)
{
global $STORAGE_SERVER_PATH;
$fileCount = 0;
$p_filepath = realpath(rtrim($p_filepath));
$r = M2tree::GetObjId('import', $gb->storId); if (!file_exists($p_filepath)) {
if (PEAR::isError($r)) { echo " * WARNING: File does not exist: $p_filepath\n";
echo "ERROR: ".$r->getMessage()." ".$r->getUserInfo()."\n"; return 0;
exit(1);
}
if (is_null($r)) {
$r = BasicStor::bsCreateFolder($gb->storId, 'import');
if (PEAR::isError($r)) {
echo "ERROR: ".$r->getMessage()." ".$r->getUserInfo()."\n";
exit(1);
} }
}
$parid = $r;
if (is_dir($p_filepath)) {
$stdin = fopen('php://stdin', 'r'); list(,$fileList) = File_Find::maptree($p_filepath);
while ($filename = fgets($stdin, 2048)) { foreach ($fileList as $tmpFile) {
$filename = rtrim($filename); $fileCount += camp_import_audio_file($tmpFile, $p_greenbox, $p_parentId, $p_testOnly);
if (!preg_match('/\.(ogg|mp3)$/i', $filename, $var)) { }
// echo "File extension not supported - skipping file\n"; return $fileCount;
continue;
} }
echo "Importing: $filename\n";
$md5sum = md5_file($filename); // Check for non-supported file type
//echo " * MD5: $md5sum\n"; if (!preg_match('/\.(ogg|mp3)$/i', $p_filepath, $var)) {
//echo " * WARNING: File extension not supported - skipping file: $p_filepath\n";
return 0;
}
echo "Importing: $p_filepath\n";
$md5sum = md5_file($p_filepath);
// Look up md5sum in database // Look up md5sum in database
$file = StoredFile::RecallByMd5($md5sum); $duplicate = StoredFile::RecallByMd5($md5sum);
if ($file) { if ($duplicate) {
echo " * File already exists in the database.\n"; echo " * File already exists in the database.\n";
continue; return 0;
} }
$mdata = camp_get_audio_metadata($filename, $testonly); $metadata = camp_get_audio_metadata($p_filepath, $p_testOnly);
if (PEAR::isError($mdata)) { if (PEAR::isError($metadata)) {
import_err($mdata); import_err($metadata);
continue; return 0;
} }
unset($mdata['audio']); unset($metadata['audio']);
unset($mdata['playtime_seconds']); unset($metadata['playtime_seconds']);
if (!$testonly) { if (!$p_testOnly) {
$r = $gb->bsPutFile($parid, $mdata['ls:filename'], "$filename", "$storageServerPath/var/emptyMdata.xml", NULL, 'audioclip', 'file', FALSE); $r = $p_greenbox->bsPutFile($p_parentId, $metadata['ls:filename'], "$p_filepath", "$STORAGE_SERVER_PATH/var/emptyMdata.xml", NULL, 'audioclip', 'file', FALSE);
if (PEAR::isError($r)) { if (PEAR::isError($r)) {
import_err($r, "Error in bsPutFile()"); import_err($r, "Error in bsPutFile()");
echo var_export($mdata)."\n"; echo var_export($metadata)."\n";
continue; return 0;
} }
$id = $r; $id = $r;
$r = $gb->bsSetMetadataBatch($id, $mdata); $r = $p_greenbox->bsSetMetadataBatch($id, $metadata);
if (PEAR::isError($r)) { if (PEAR::isError($r)) {
import_err($r, "Error in bsSetMetadataBatch()"); import_err($r, "Error in bsSetMetadataBatch()");
echo var_export($mdata)."\n"; echo var_export($metadata)."\n";
continue; return 0;
} }
} else { } else {
var_dump($infoFromFile); var_dump($infoFromFile);
echo "======================= "; echo "======================= ";
var_dump($mdata); var_dump($metadata);
echo "======================= "; echo "======================= ";
} }
echo " * OK\n"; echo " * OK\n";
$filecount++; $fileCount++;
return $fileCount;
} }
fclose($stdin); echo "========================\n";
echo "Campcaster Import Script\n";
echo "========================\n";
$g_errors = 0;
$start = intval(date('U'));
PEAR::setErrorHandling(PEAR_ERROR_RETURN);
$CC_DBC = DB::connect($CC_CONFIG['dsn'], TRUE);
if (PEAR::isError($CC_DBC)) {
echo "ERROR: ".$CC_DBC->getMessage()." ".$CC_DBC->getUserInfo()."\n";
exit(1);
}
$CC_DBC->setFetchMode(DB_FETCHMODE_ASSOC);
$parsedCommandLine = Console_Getopt::getopt($argv, "th", array("test", "help"));
$cmdLineOptions = $parsedCommandLine[0];
if (count($parsedCommandLine[1]) == 0) {
printUsage();
exit;
}
//print_r($parsedCommandLine);
$files = $parsedCommandLine[1];
$testonly = FALSE;
foreach ($cmdLineOptions as $tmpValue) {
$optionName = $tmpValue[0];
$optionValue = $tmpValue[1];
switch ($optionName) {
case "h":
case '--help':
printUsage();
exit;
case "t":
case "--test":
$testonly = TRUE;
break;
}
}
$greenbox = new GreenBox();
$parentId = M2tree::GetObjId(M2tree::GetRootNode());
$filecount = 0;
//print_r($files);
//echo "\n";
if (is_array($files)) {
foreach ($files as $filepath) {
$filecount += camp_import_audio_file($filepath, $greenbox, $parentId, $testonly);
}
}
$end = intval(date('U')); $end = intval(date('U'));
$time = $end - $start; $time = $end - $start;
if ($time > 0) { if ($time > 0) {
@ -133,5 +192,10 @@ if ($time > 0) {
} else { } else {
$speed = "N/A"; $speed = "N/A";
} }
echo " Files ".($testonly ? "analyzed" : "imported").": $filecount, in $time s, $speed files/s, errors: $g_errors\n";
echo "==========================================================================\n";
echo " * Files ".($testonly ? "analyzed" : "imported").": $filecount in $time s = $speed files/second, errors: $g_errors\n";
echo " * Import completed.\n";
?> ?>

8
campcaster/src/modules/storageAdmin/var/restore.php
View File

@ -9,12 +9,12 @@ define('VERBOSE', FALSE);
header("Content-type: text/plain"); header("Content-type: text/plain");
require_once 'conf.php'; require_once 'conf.php';
require_once "$storageServerPath/var/conf.php"; require_once "$STORAGE_SERVER_PATH/var/conf.php";
require_once 'DB.php'; require_once 'DB.php';
require_once "XML/Util.php"; require_once "XML/Util.php";
require_once "XML/Beautifier.php"; require_once "XML/Beautifier.php";
require_once "$storageServerPath/var/BasicStor.php"; require_once "$STORAGE_SERVER_PATH/var/BasicStor.php";
require_once "$storageServerPath/var/Prefs.php"; require_once "$STORAGE_SERVER_PATH/var/Prefs.php";
/* =========================================================== misc functions */ /* =========================================================== misc functions */
function ls_restore_processObject($el) function ls_restore_processObject($el)
@ -104,7 +104,7 @@ $pr = new Prefs($bs);
$dbxml = file_get_contents($argv[1]); $dbxml = file_get_contents($argv[1]);
$tmpdir = $argv[2]; $tmpdir = $argv[2];
require_once("$storageServerPath/var/XmlParser.php"); require_once("$STORAGE_SERVER_PATH/var/XmlParser.php");
$parser = new XmlParser($dbxml); $parser = new XmlParser($dbxml);
if ($parser->isError()) { if ($parser->isError()) {
return PEAR::raiseError( return PEAR::raiseError(