adding zend project folders into old campcaster.

library/Zend/Search/Lucene/Analysis/Analyzer.php

@@ -0,0 +1,175 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: Analyzer.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** User land classes and interfaces turned on by Zend/Search/Analyzer.php file inclusion. */
+/** @todo Section should be removed with ZF 2.0 release as obsolete                      */
+if (!defined('ZEND_SEARCH_LUCENE_COMMON_ANALYZER_PROCESSED')) {
+    /** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8 */
+    require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php';
+
+    /** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8_CaseInsensitive */
+    require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8/CaseInsensitive.php';
+
+    /** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num */
+    require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php';
+
+    /** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive */
+    require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num/CaseInsensitive.php';
+
+    /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text */
+    require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php';
+
+    /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive */
+    require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php';
+
+    /** Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum */
+    require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php';
+
+    /** Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive */
+    require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php';
+}
+
+
+/**
+ * An Analyzer is used to analyze text.
+ * It thus represents a policy for extracting index terms from text.
+ *
+ * Note:
+ * Lucene Java implementation is oriented to streams. It provides effective work
+ * with a huge documents (more then 20Mb).
+ * But engine itself is not oriented such documents.
+ * Thus Zend_Search_Lucene analysis API works with data strings and sets (arrays).
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+abstract class Zend_Search_Lucene_Analysis_Analyzer
+{
+    /**
+     * The Analyzer implementation used by default.
+     *
+     * @var Zend_Search_Lucene_Analysis_Analyzer
+     */
+    private static $_defaultImpl;
+
+    /**
+     * Input string
+     *
+     * @var string
+     */
+    protected $_input = null;
+
+    /**
+     * Input string encoding
+     *
+     * @var string
+     */
+    protected $_encoding = '';
+
+    /**
+     * Tokenize text to a terms
+     * Returns array of Zend_Search_Lucene_Analysis_Token objects
+     *
+     * Tokens are returned in UTF-8 (internal Zend_Search_Lucene encoding)
+     *
+     * @param string $data
+     * @return array
+     */
+    public function tokenize($data, $encoding = '')
+    {
+        $this->setInput($data, $encoding);
+
+        $tokenList = array();
+        while (($nextToken = $this->nextToken()) !== null) {
+            $tokenList[] = $nextToken;
+        }
+
+        return $tokenList;
+    }
+
+
+    /**
+     * Tokenization stream API
+     * Set input
+     *
+     * @param string $data
+     */
+    public function setInput($data, $encoding = '')
+    {
+        $this->_input    = $data;
+        $this->_encoding = $encoding;
+        $this->reset();
+    }
+
+    /**
+     * Reset token stream
+     */
+    abstract public function reset();
+
+    /**
+     * Tokenization stream API
+     * Get next token
+     * Returns null at the end of stream
+     *
+     * Tokens are returned in UTF-8 (internal Zend_Search_Lucene encoding)
+     *
+     * @return Zend_Search_Lucene_Analysis_Token|null
+     */
+    abstract public function nextToken();
+
+
+
+
+    /**
+     * Set the default Analyzer implementation used by indexing code.
+     *
+     * @param Zend_Search_Lucene_Analysis_Analyzer $similarity
+     */
+    public static function setDefault(Zend_Search_Lucene_Analysis_Analyzer $analyzer)
+    {
+        self::$_defaultImpl = $analyzer;
+    }
+
+
+    /**
+     * Return the default Analyzer implementation used by indexing code.
+     *
+     * @return Zend_Search_Lucene_Analysis_Analyzer
+     */
+    public static function getDefault()
+    {
+        /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive */
+        require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php';
+
+        if (!self::$_defaultImpl instanceof Zend_Search_Lucene_Analysis_Analyzer) {
+            self::$_defaultImpl = new Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive();
+        }
+
+        return self::$_defaultImpl;
+    }
+}
+

library/Zend/Search/Lucene/Analysis/Analyzer/Common.php

@@ -0,0 +1,92 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: Common.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** Define constant used to provide correct file processing order    */
+/** @todo Section should be removed with ZF 2.0 release as obsolete  */
+define('ZEND_SEARCH_LUCENE_COMMON_ANALYZER_PROCESSED', true);
+
+
+/** Zend_Search_Lucene_Analysis_Analyzer */
+require_once 'Zend/Search/Lucene/Analysis/Analyzer.php';
+
+/** Zend_Search_Lucene_Analysis_Token */
+require_once 'Zend/Search/Lucene/Analysis/Token.php';
+
+/** Zend_Search_Lucene_Analysis_TokenFilter */
+require_once 'Zend/Search/Lucene/Analysis/TokenFilter.php';
+
+
+/**
+ * Common implementation of the Zend_Search_Lucene_Analysis_Analyzer interface.
+ * There are several standard standard subclasses provided by Zend_Search_Lucene/Analysis
+ * subpackage: Zend_Search_Lucene_Analysis_Analyzer_Common_Text, ZSearchHTMLAnalyzer, ZSearchXMLAnalyzer.
+ *
+ * @todo ZSearchHTMLAnalyzer and ZSearchXMLAnalyzer implementation
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+abstract class Zend_Search_Lucene_Analysis_Analyzer_Common extends Zend_Search_Lucene_Analysis_Analyzer
+{
+    /**
+     * The set of Token filters applied to the Token stream.
+     * Array of Zend_Search_Lucene_Analysis_TokenFilter objects.
+     *
+     * @var array
+     */
+    private $_filters = array();
+
+    /**
+     * Add Token filter to the Analyzer
+     *
+     * @param Zend_Search_Lucene_Analysis_TokenFilter $filter
+     */
+    public function addFilter(Zend_Search_Lucene_Analysis_TokenFilter $filter)
+    {
+        $this->_filters[] = $filter;
+    }
+
+    /**
+     * Apply filters to the token. Can return null when the token was removed.
+     *
+     * @param Zend_Search_Lucene_Analysis_Token $token
+     * @return Zend_Search_Lucene_Analysis_Token
+     */
+    public function normalize(Zend_Search_Lucene_Analysis_Token $token)
+    {
+        foreach ($this->_filters as $filter) {
+            $token = $filter->normalize($token);
+
+            // resulting token can be null if the filter removes it
+            if ($token === null) {
+                return null;
+            }
+        }
+
+        return $token;
+    }
+}
+

library/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php

@@ -0,0 +1,96 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: Text.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common */
+require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+class Zend_Search_Lucene_Analysis_Analyzer_Common_Text extends Zend_Search_Lucene_Analysis_Analyzer_Common
+{
+    /**
+     * Current position in a stream
+     *
+     * @var integer
+     */
+    private $_position;
+
+    /**
+     * Reset token stream
+     */
+    public function reset()
+    {
+        $this->_position = 0;
+
+        if ($this->_input === null) {
+            return;
+        }
+
+        // convert input into ascii
+        if (PHP_OS != 'AIX') {
+            $this->_input = iconv($this->_encoding, 'ASCII//TRANSLIT', $this->_input);
+        }
+        $this->_encoding = 'ASCII';
+    }
+
+    /**
+     * Tokenization stream API
+     * Get next token
+     * Returns null at the end of stream
+     *
+     * @return Zend_Search_Lucene_Analysis_Token|null
+     */
+    public function nextToken()
+    {
+        if ($this->_input === null) {
+            return null;
+        }
+
+
+        do {
+            if (! preg_match('/[a-zA-Z]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) {
+                // It covers both cases a) there are no matches (preg_match(...) === 0)
+                // b) error occured (preg_match(...) === FALSE)
+                return null;
+            }
+
+            $str = $match[0][0];
+            $pos = $match[0][1];
+            $endpos = $pos + strlen($str);
+
+            $this->_position = $endpos;
+
+            $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($str, $pos, $endpos));
+        } while ($token === null); // try again if token is skipped
+
+        return $token;
+    }
+}
+

library/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php

@@ -0,0 +1,47 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: CaseInsensitive.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common_Text */
+require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php';
+
+/** Zend_Search_Lucene_Analysis_TokenFilter_LowerCase */
+require_once 'Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+class Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive extends Zend_Search_Lucene_Analysis_Analyzer_Common_Text
+{
+    public function __construct()
+    {
+        $this->addFilter(new Zend_Search_Lucene_Analysis_TokenFilter_LowerCase());
+    }
+}
+

library/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php

@@ -0,0 +1,95 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: TextNum.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common */
+require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+class Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum extends Zend_Search_Lucene_Analysis_Analyzer_Common
+{
+    /**
+     * Current position in a stream
+     *
+     * @var integer
+     */
+    private $_position;
+
+    /**
+     * Reset token stream
+     */
+    public function reset()
+    {
+        $this->_position = 0;
+
+        if ($this->_input === null) {
+            return;
+        }
+
+        // convert input into ascii
+        if (PHP_OS != 'AIX') {
+            $this->_input = iconv($this->_encoding, 'ASCII//TRANSLIT', $this->_input);
+        }
+        $this->_encoding = 'ASCII';
+    }
+
+    /**
+     * Tokenization stream API
+     * Get next token
+     * Returns null at the end of stream
+     *
+     * @return Zend_Search_Lucene_Analysis_Token|null
+     */
+    public function nextToken()
+    {
+        if ($this->_input === null) {
+            return null;
+        }
+
+        do {
+            if (! preg_match('/[a-zA-Z0-9]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) {
+                // It covers both cases a) there are no matches (preg_match(...) === 0)
+                // b) error occured (preg_match(...) === FALSE)
+                return null;
+            }
+
+            $str = $match[0][0];
+            $pos = $match[0][1];
+            $endpos = $pos + strlen($str);
+
+            $this->_position = $endpos;
+
+            $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($str, $pos, $endpos));
+        } while ($token === null); // try again if token is skipped
+
+        return $token;
+    }
+}
+

library/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php

@@ -0,0 +1,47 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: CaseInsensitive.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum */
+require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php';
+
+/** Zend_Search_Lucene_Analysis_TokenFilter_LowerCase */
+require_once 'Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+class Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive extends Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum
+{
+    public function __construct()
+    {
+        $this->addFilter(new Zend_Search_Lucene_Analysis_TokenFilter_LowerCase());
+    }
+}
+

library/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php

@@ -0,0 +1,126 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: Utf8.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common */
+require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8 extends Zend_Search_Lucene_Analysis_Analyzer_Common
+{
+    /**
+     * Current char position in an UTF-8 stream
+     *
+     * @var integer
+     */
+    private $_position;
+
+    /**
+     * Current binary position in an UTF-8 stream
+     *
+     * @var integer
+     */
+    private $_bytePosition;
+
+    /**
+     * Object constructor
+     *
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function __construct()
+    {
+        if (@preg_match('/\pL/u', 'a') != 1) {
+            // PCRE unicode support is turned off
+            require_once 'Zend/Search/Lucene/Exception.php';
+            throw new Zend_Search_Lucene_Exception('Utf8 analyzer needs PCRE unicode support to be enabled.');
+        }
+    }
+
+    /**
+     * Reset token stream
+     */
+    public function reset()
+    {
+        $this->_position     = 0;
+        $this->_bytePosition = 0;
+
+        // convert input into UTF-8
+        if (strcasecmp($this->_encoding, 'utf8' ) != 0  &&
+            strcasecmp($this->_encoding, 'utf-8') != 0 ) {
+                $this->_input = iconv($this->_encoding, 'UTF-8', $this->_input);
+                $this->_encoding = 'UTF-8';
+        }
+    }
+
+    /**
+     * Tokenization stream API
+     * Get next token
+     * Returns null at the end of stream
+     *
+     * @return Zend_Search_Lucene_Analysis_Token|null
+     */
+    public function nextToken()
+    {
+        if ($this->_input === null) {
+            return null;
+        }
+
+        do {
+            if (! preg_match('/[\p{L}]+/u', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_bytePosition)) {
+                // It covers both cases a) there are no matches (preg_match(...) === 0)
+                // b) error occured (preg_match(...) === FALSE)
+                return null;
+            }
+
+            // matched string
+            $matchedWord = $match[0][0];
+
+            // binary position of the matched word in the input stream
+            $binStartPos = $match[0][1];
+
+            // character position of the matched word in the input stream
+            $startPos = $this->_position +
+                        iconv_strlen(substr($this->_input,
+                                            $this->_bytePosition,
+                                            $binStartPos - $this->_bytePosition),
+                                     'UTF-8');
+            // character postion of the end of matched word in the input stream
+            $endPos = $startPos + iconv_strlen($matchedWord, 'UTF-8');
+
+            $this->_bytePosition = $binStartPos + strlen($matchedWord);
+            $this->_position     = $endPos;
+
+            $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($matchedWord, $startPos, $endPos));
+        } while ($token === null); // try again if token is skipped
+
+        return $token;
+    }
+}
+

library/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8/CaseInsensitive.php

@@ -0,0 +1,49 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: CaseInsensitive.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8 */
+require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php';
+
+/** Zend_Search_Lucene_Analysis_TokenFilter_LowerCaseUtf8 */
+require_once 'Zend/Search/Lucene/Analysis/TokenFilter/LowerCaseUtf8.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8_CaseInsensitive extends Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8
+{
+    public function __construct()
+    {
+        parent::__construct();
+
+        $this->addFilter(new Zend_Search_Lucene_Analysis_TokenFilter_LowerCaseUtf8());
+    }
+}
+

library/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php

@@ -0,0 +1,126 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: Utf8Num.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common */
+require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num extends Zend_Search_Lucene_Analysis_Analyzer_Common
+{
+    /**
+     * Current char position in an UTF-8 stream
+     *
+     * @var integer
+     */
+    private $_position;
+
+    /**
+     * Current binary position in an UTF-8 stream
+     *
+     * @var integer
+     */
+    private $_bytePosition;
+
+    /**
+     * Object constructor
+     *
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function __construct()
+    {
+        if (@preg_match('/\pL/u', 'a') != 1) {
+            // PCRE unicode support is turned off
+            require_once 'Zend/Search/Lucene/Exception.php';
+            throw new Zend_Search_Lucene_Exception('Utf8Num analyzer needs PCRE unicode support to be enabled.');
+        }
+    }
+
+    /**
+     * Reset token stream
+     */
+    public function reset()
+    {
+        $this->_position     = 0;
+        $this->_bytePosition = 0;
+
+        // convert input into UTF-8
+        if (strcasecmp($this->_encoding, 'utf8' ) != 0  &&
+            strcasecmp($this->_encoding, 'utf-8') != 0 ) {
+                $this->_input = iconv($this->_encoding, 'UTF-8', $this->_input);
+                $this->_encoding = 'UTF-8';
+        }
+    }
+
+    /**
+     * Tokenization stream API
+     * Get next token
+     * Returns null at the end of stream
+     *
+     * @return Zend_Search_Lucene_Analysis_Token|null
+     */
+    public function nextToken()
+    {
+        if ($this->_input === null) {
+            return null;
+        }
+
+        do {
+            if (! preg_match('/[\p{L}\p{N}]+/u', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_bytePosition)) {
+                // It covers both cases a) there are no matches (preg_match(...) === 0)
+                // b) error occured (preg_match(...) === FALSE)
+                return null;
+            }
+
+            // matched string
+            $matchedWord = $match[0][0];
+
+            // binary position of the matched word in the input stream
+            $binStartPos = $match[0][1];
+
+            // character position of the matched word in the input stream
+            $startPos = $this->_position +
+                        iconv_strlen(substr($this->_input,
+                                            $this->_bytePosition,
+                                            $binStartPos - $this->_bytePosition),
+                                     'UTF-8');
+            // character postion of the end of matched word in the input stream
+            $endPos = $startPos + iconv_strlen($matchedWord, 'UTF-8');
+
+            $this->_bytePosition = $binStartPos + strlen($matchedWord);
+            $this->_position     = $endPos;
+
+            $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($matchedWord, $startPos, $endPos));
+        } while ($token === null); // try again if token is skipped
+
+        return $token;
+    }
+}
+

library/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num/CaseInsensitive.php

@@ -0,0 +1,49 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: CaseInsensitive.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num */
+require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php';
+
+/** Zend_Search_Lucene_Analysis_TokenFilter_LowerCaseUtf8 */
+require_once 'Zend/Search/Lucene/Analysis/TokenFilter/LowerCaseUtf8.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive extends Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num
+{
+    public function __construct()
+    {
+        parent::__construct();
+
+        $this->addFilter(new Zend_Search_Lucene_Analysis_TokenFilter_LowerCaseUtf8());
+    }
+}
+

library/Zend/Search/Lucene/Analysis/Token.php

@@ -0,0 +1,154 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: Token.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Analysis_Token
+{
+    /**
+     * The text of the term.
+     *
+     * @var string
+     */
+    private $_termText;
+
+    /**
+     * Start in source text.
+     *
+     * @var integer
+     */
+    private $_startOffset;
+
+    /**
+     * End in source text
+     *
+     * @var integer
+     */
+    private $_endOffset;
+
+    /**
+     * The position of this token relative to the previous Token.
+     *
+     * The default value is one.
+     *
+     * Some common uses for this are:
+     * Set it to zero to put multiple terms in the same position.  This is
+     * useful if, e.g., a word has multiple stems.  Searches for phrases
+     * including either stem will match.  In this case, all but the first stem's
+     * increment should be set to zero: the increment of the first instance
+     * should be one.  Repeating a token with an increment of zero can also be
+     * used to boost the scores of matches on that token.
+     *
+     * Set it to values greater than one to inhibit exact phrase matches.
+     * If, for example, one does not want phrases to match across removed stop
+     * words, then one could build a stop word filter that removes stop words and
+     * also sets the increment to the number of stop words removed before each
+     * non-stop word.  Then exact phrase queries will only match when the terms
+     * occur with no intervening stop words.
+     *
+     * @var integer
+     */
+    private $_positionIncrement;
+
+
+    /**
+     * Object constructor
+     *
+     * @param string  $text
+     * @param integer $start
+     * @param integer $end
+     * @param string  $type
+     */
+    public function __construct($text, $start, $end)
+    {
+        $this->_termText    = $text;
+        $this->_startOffset = $start;
+        $this->_endOffset   = $end;
+
+        $this->_positionIncrement = 1;
+    }
+
+
+    /**
+     * positionIncrement setter
+     *
+     * @param integer $positionIncrement
+     */
+    public function setPositionIncrement($positionIncrement)
+    {
+        $this->_positionIncrement = $positionIncrement;
+    }
+
+    /**
+     * Returns the position increment of this Token.
+     *
+     * @return integer
+     */
+    public function getPositionIncrement()
+    {
+        return $this->_positionIncrement;
+    }
+
+    /**
+     * Returns the Token's term text.
+     *
+     * @return string
+     */
+    public function getTermText()
+    {
+        return $this->_termText;
+    }
+
+    /**
+     * Returns this Token's starting offset, the position of the first character
+     * corresponding to this token in the source text.
+     *
+     * Note:
+     * The difference between getEndOffset() and getStartOffset() may not be equal
+     * to strlen(Zend_Search_Lucene_Analysis_Token::getTermText()), as the term text may have been altered
+     * by a stemmer or some other filter.
+     *
+     * @return integer
+     */
+    public function getStartOffset()
+    {
+        return $this->_startOffset;
+    }
+
+    /**
+     * Returns this Token's ending offset, one greater than the position of the
+     * last character corresponding to this token in the source text.
+     *
+     * @return integer
+     */
+    public function getEndOffset()
+    {
+        return $this->_endOffset;
+    }
+}
+

library/Zend/Search/Lucene/Analysis/TokenFilter.php

@@ -0,0 +1,47 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: TokenFilter.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** Zend_Search_Lucene_Analysis_Token */
+require_once 'Zend/Search/Lucene/Analysis/Token.php';
+
+
+/**
+ * Token filter converts (normalizes) Token ore removes it from a token stream.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+abstract class Zend_Search_Lucene_Analysis_TokenFilter
+{
+    /**
+     * Normalize Token or remove it (if null is returned)
+     *
+     * @param Zend_Search_Lucene_Analysis_Token $srcToken
+     * @return Zend_Search_Lucene_Analysis_Token
+     */
+    abstract public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken);
+}
+

library/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php

@@ -0,0 +1,58 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: LowerCase.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** Zend_Search_Lucene_Analysis_TokenFilter */
+require_once 'Zend/Search/Lucene/Analysis/TokenFilter.php';
+
+
+/**
+ * Lower case Token filter.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+class Zend_Search_Lucene_Analysis_TokenFilter_LowerCase extends Zend_Search_Lucene_Analysis_TokenFilter
+{
+    /**
+     * Normalize Token or remove it (if null is returned)
+     *
+     * @param Zend_Search_Lucene_Analysis_Token $srcToken
+     * @return Zend_Search_Lucene_Analysis_Token
+     */
+    public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken)
+    {
+        $newToken = new Zend_Search_Lucene_Analysis_Token(
+                                     strtolower( $srcToken->getTermText() ),
+                                     $srcToken->getStartOffset(),
+                                     $srcToken->getEndOffset());
+
+        $newToken->setPositionIncrement($srcToken->getPositionIncrement());
+
+        return $newToken;
+    }
+}
+

library/Zend/Search/Lucene/Analysis/TokenFilter/LowerCaseUtf8.php

@@ -0,0 +1,70 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: LowerCaseUtf8.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** Zend_Search_Lucene_Analysis_TokenFilter */
+require_once 'Zend/Search/Lucene/Analysis/TokenFilter.php';
+
+
+/**
+ * Lower case Token filter.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+class Zend_Search_Lucene_Analysis_TokenFilter_LowerCaseUtf8 extends Zend_Search_Lucene_Analysis_TokenFilter
+{
+    /**
+     * Object constructor
+     */
+    public function __construct()
+    {
+        if (!function_exists('mb_strtolower')) {
+            // mbstring extension is disabled
+            require_once 'Zend/Search/Lucene/Exception.php';
+            throw new Zend_Search_Lucene_Exception('Utf8 compatible lower case filter needs mbstring extension to be enabled.');
+        }
+    }
+
+    /**
+     * Normalize Token or remove it (if null is returned)
+     *
+     * @param Zend_Search_Lucene_Analysis_Token $srcToken
+     * @return Zend_Search_Lucene_Analysis_Token
+     */
+    public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken)
+    {
+        $newToken = new Zend_Search_Lucene_Analysis_Token(
+                                     mb_strtolower($srcToken->getTermText(), 'UTF-8'),
+                                     $srcToken->getStartOffset(),
+                                     $srcToken->getEndOffset());
+
+        $newToken->setPositionIncrement($srcToken->getPositionIncrement());
+
+        return $newToken;
+    }
+}
+

library/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php

@@ -0,0 +1,69 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: ShortWords.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+
+/** Zend_Search_Lucene_Analysis_TokenFilter */
+require_once 'Zend/Search/Lucene/Analysis/TokenFilter.php';
+
+
+/**
+ * Token filter that removes short words. What is short word can be configured with constructor.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+class Zend_Search_Lucene_Analysis_TokenFilter_ShortWords extends Zend_Search_Lucene_Analysis_TokenFilter
+{
+    /**
+     * Minimum allowed term length
+     * @var integer
+     */
+    private $length;
+
+    /**
+     * Constructs new instance of this filter.
+     *
+     * @param integer $short  minimum allowed length of term which passes this filter (default 2)
+     */
+    public function __construct($length = 2) {
+        $this->length = $length;
+    }
+
+    /**
+     * Normalize Token or remove it (if null is returned)
+     *
+     * @param Zend_Search_Lucene_Analysis_Token $srcToken
+     * @return Zend_Search_Lucene_Analysis_Token
+     */
+    public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken) {
+        if (strlen($srcToken->getTermText()) < $this->length) {
+            return null;
+        } else {
+            return $srcToken;
+        }
+    }
+}
+

library/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php

@@ -0,0 +1,101 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to license@zend.com so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ * @version    $Id: StopWords.php 20096 2010-01-06 02:05:09Z bkarwin $
+ */
+
+/** Zend_Search_Lucene_Analysis_TokenFilter */
+require_once 'Zend/Search/Lucene/Analysis/TokenFilter.php';
+
+/**
+ * Token filter that removes stop words. These words must be provided as array (set), example:
+ * $stopwords = array('the' => 1, 'an' => '1');
+ *
+ * We do recommend to provide all words in lowercase and concatenate this class after the lowercase filter.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+class Zend_Search_Lucene_Analysis_TokenFilter_StopWords extends Zend_Search_Lucene_Analysis_TokenFilter
+{
+    /**
+     * Stop Words
+     * @var array
+     */
+    private $_stopSet;
+
+    /**
+     * Constructs new instance of this filter.
+     *
+     * @param array $stopwords array (set) of words that will be filtered out
+     */
+    public function __construct($stopwords = array()) {
+        $this->_stopSet = array_flip($stopwords);
+    }
+
+    /**
+     * Normalize Token or remove it (if null is returned)
+     *
+     * @param Zend_Search_Lucene_Analysis_Token $srcToken
+     * @return Zend_Search_Lucene_Analysis_Token
+     */
+    public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken) {
+        if (array_key_exists($srcToken->getTermText(), $this->_stopSet)) {
+            return null;
+        } else {
+            return $srcToken;
+        }
+    }
+
+    /**
+     * Fills stopwords set from a text file. Each line contains one stopword, lines with '#' in the first
+     * column are ignored (as comments).
+     *
+     * You can call this method one or more times. New stopwords are always added to current set.
+     *
+     * @param string $filepath full path for text file with stopwords
+     * @throws Zend_Search_Exception When the file doesn`t exists or is not readable.
+     */
+    public function loadFromFile($filepath = null) {
+        if (! $filepath || ! file_exists($filepath)) {
+            require_once 'Zend/Search/Lucene/Exception.php';
+            throw new Zend_Search_Lucene_Exception('You have to provide valid file path');
+        }
+        $fd = fopen($filepath, "r");
+        if (! $fd) {
+            require_once 'Zend/Search/Lucene/Exception.php';
+            throw new Zend_Search_Lucene_Exception('Cannot open file ' . $filepath);
+        }
+        while (!feof ($fd)) {
+            $buffer = trim(fgets($fd));
+            if (strlen($buffer) > 0 && $buffer[0] != '#') {
+                $this->_stopSet[$buffer] = 1;
+            }
+        }
+        if (!fclose($fd)) {
+            require_once 'Zend/Search/Lucene/Exception.php';
+            throw new Zend_Search_Lucene_Exception('Cannot close file ' . $filepath);
+        }
+    }
+}
+