diff --git a/Makefile b/Makefile index 4b1e7ec55..c1d281546 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ pear: (cd SeedDMS_Core/; pear package) (cd SeedDMS_Lucene/; pear package) (cd SeedDMS_Preview/; pear package) + (cd SeedDMS_SQLiteFTS/; pear package) webdav: mkdir -p tmp/seeddms-webdav-$(VERSION) diff --git a/SeedDMS_Lucene/Lucene/Search.php b/SeedDMS_Lucene/Lucene/Search.php index d51c800ca..857d23ceb 100644 --- a/SeedDMS_Lucene/Lucene/Search.php +++ b/SeedDMS_Lucene/Lucene/Search.php @@ -42,6 +42,17 @@ class SeedDMS_Lucene_Search { $this->version = '3.0.0'; } /* }}} */ + /** + * Get document from index + * + * @param object $index lucene index + * @return object instance of SeedDMS_Lucene_Document of false + */ + function getDocument($id) { /* {{{ */ + $hits = $this->index->find('document_id:'.$id); + return $hits ? $hits[0] : false; + } /* }}} */ + /** * Search in index * diff --git a/SeedDMS_SQLiteFTS/SQLiteFTS.php b/SeedDMS_SQLiteFTS/SQLiteFTS.php new file mode 100644 index 000000000..0a3b710e8 --- /dev/null +++ b/SeedDMS_SQLiteFTS/SQLiteFTS.php @@ -0,0 +1,44 @@ + diff --git a/SeedDMS_SQLiteFTS/SQLiteFTS/Document.php b/SeedDMS_SQLiteFTS/SQLiteFTS/Document.php new file mode 100644 index 000000000..48c4e789c --- /dev/null +++ b/SeedDMS_SQLiteFTS/SQLiteFTS/Document.php @@ -0,0 +1,58 @@ + + * @copyright Copyright (C) 2010, Uwe Steinmann + * @version Release: @package_version@ + */ + + +/** + * Class for managing a document. + * + * @category DMS + * @package SeedDMS_SQLiteFTS + * @version @version@ + * @author Uwe Steinmann + * @copyright Copyright (C) 2011, Uwe Steinmann + * @version Release: @package_version@ + */ +class SeedDMS_SQLiteFTS_Document { + + /** + * @var integer $id id of document + * @access protected + */ + public $id; + + /** + * @var array $fields fields + * @access protected + */ + protected $fields; + + public function addField($key, $value) { /* {{{ */ + if($key == 'document_id') { + $this->id = $this->fields[$key] = (int) $value; + } else { + if(isset($this->fields[$key])) + $this->fields[$key] .= ' '.$value; + else + $this->fields[$key] = $value; + } + } /* }}} */ + + public function getFieldValue($key) { /* {{{ */ + if(isset($this->fields[$key])) + return $this->fields[$key]; + else + return false; + } /* }}} */ + +} +?> diff --git a/SeedDMS_SQLiteFTS/SQLiteFTS/IndexedDocument.php b/SeedDMS_SQLiteFTS/SQLiteFTS/IndexedDocument.php new file mode 100644 index 000000000..455b3dd77 --- /dev/null +++ b/SeedDMS_SQLiteFTS/SQLiteFTS/IndexedDocument.php @@ -0,0 +1,140 @@ + + * @copyright Copyright (C) 2010, Uwe Steinmann + * @version Release: @package_version@ + */ + +/** + * @uses SeedDMS_SQLiteFTS_Document + */ +require_once('Document.php'); + + +/** + * Class for managing an indexed document. + * + * @category DMS + * @package SeedDMS_SQLiteFTS + * @version @version@ + * @author Uwe Steinmann + * @copyright Copyright (C) 2011, Uwe Steinmann + * @version Release: @package_version@ + */ +class SeedDMS_SQLiteFTS_IndexedDocument extends SeedDMS_SQLiteFTS_Document { + + static function execWithTimeout($cmd, $timeout=2) { /* {{{ */ + $descriptorspec = array( + 0 => array("pipe", "r"), + 1 => array("pipe", "w"), + 2 => array("pipe", "w") + ); + $pipes = array(); + + $timeout += time(); + $process = proc_open($cmd, $descriptorspec, $pipes); + if (!is_resource($process)) { + throw new Exception("proc_open failed on: " . $cmd); + } + + $output = ''; + do { + $timeleft = $timeout - time(); + $read = array($pipes[1]); + stream_select($read, $write = NULL, $exeptions = NULL, $timeleft, NULL); + + if (!empty($read)) { + $output .= fread($pipes[1], 8192); + } + } while (!feof($pipes[1]) && $timeleft > 0); + + if ($timeleft <= 0) { + proc_terminate($process); + throw new Exception("command timeout on: " . $cmd); + } else { + return $output; + } + } /* }}} */ + + /** + * Constructor. Creates our indexable document and adds all + * necessary fields to it using the passed in document + */ + public function __construct($dms, $document, $convcmd=null, $nocontent=false, $timeout=5) { + $_convcmd = array( + 'application/pdf' => 'pdftotext -enc UTF-8 -nopgbrk %s - |sed -e \'s/ [a-zA-Z0-9.]\{1\} / /g\' -e \'s/[0-9.]//g\'', + 'application/msword' => 'catdoc %s', + 'application/vnd.ms-excel' => 'ssconvert -T Gnumeric_stf:stf_csv -S %s fd://1', + 'audio/mp3' => "id3 -l -R %s | egrep '(Title|Artist|Album)' | sed 's/^[^:]*: //g'", + 'audio/mpeg' => "id3 -l -R %s | egrep '(Title|Artist|Album)' | sed 's/^[^:]*: //g'", + 'text/plain' => 'cat %s', + ); + if($convcmd) { + $_convcmd = $convcmd; + } + + $version = $document->getLatestContent(); + $this->addField('document_id', $document->getID()); + if($version) { + $this->addField('mimetype', $version->getMimeType()); + $this->addField('origfilename', $version->getOriginalFileName()); + if(!$nocontent) + $this->addField('created', $version->getDate(), 'unindexed'); + if($attributes = $version->getAttributes()) { + foreach($attributes as $attribute) { + $attrdef = $attribute->getAttributeDefinition(); + if($attrdef->getValueSet() != '') + $this->addField('attr_'.str_replace(' ', '_', $attrdef->getName()), $attribute->getValue()); + else + $this->addField('attr_'.str_replace(' ', '_', $attrdef->getName()), $attribute->getValue()); + } + } + } + $this->addField('title', $document->getName()); + if($categories = $document->getCategories()) { + $names = array(); + foreach($categories as $cat) { + $names[] = $cat->getName(); + } + $this->addField('category', implode(' ', $names)); + } + if($attributes = $document->getAttributes()) { + foreach($attributes as $attribute) { + $attrdef = $attribute->getAttributeDefinition(); + if($attrdef->getValueSet() != '') + $this->addField('attr_'.str_replace(' ', '_', $attrdef->getName()), $attribute->getValue()); + else + $this->addField('attr_'.str_replace(' ', '_', $attrdef->getName()), $attribute->getValue()); + } + } + + $owner = $document->getOwner(); + $this->addField('owner', $owner->getLogin()); + if($keywords = $document->getKeywords()) { + $this->addField('keywords', $keywords); + } + if($comment = $document->getComment()) { + $this->addField('comment', $comment); + } + if($version && !$nocontent) { + $path = $dms->contentDir . $version->getPath(); + $content = ''; + $fp = null; + $mimetype = $version->getMimeType(); + if(isset($_convcmd[$mimetype])) { + $cmd = sprintf($_convcmd[$mimetype], $path); + $content = self::execWithTimeout($cmd); + if($content) { + $this->addField('content', $content, 'unstored'); + } + } + } + } +} +?> diff --git a/SeedDMS_SQLiteFTS/SQLiteFTS/Indexer.php b/SeedDMS_SQLiteFTS/SQLiteFTS/Indexer.php new file mode 100644 index 000000000..337298035 --- /dev/null +++ b/SeedDMS_SQLiteFTS/SQLiteFTS/Indexer.php @@ -0,0 +1,251 @@ + + * @copyright Copyright (C) 2010, Uwe Steinmann + * @version Release: @package_version@ + */ + + +/** + * Class for managing a SQLiteFTS index. + * + * @category DMS + * @package SeedDMS_Lucene + * @version @version@ + * @author Uwe Steinmann + * @copyright Copyright (C) 2011, Uwe Steinmann + * @version Release: @package_version@ + */ +class SeedDMS_SQLiteFTS_Indexer { + /** + * @var object $index sqlite index + * @access protected + */ + protected $_conn; + + /** + * Constructor + * + */ + function __construct($indexerDir) { /* {{{ */ + $this->_conn = new PDO('sqlite:'.$indexerDir.'/index.db'); + } /* }}} */ + + /** + * Open an existing index + * + * @param string $indexerDir directory on disk containing the index + */ + static function open($indexerDir) { /* {{{ */ + if(file_exists($indexerDir.'/index.db')) { + return new SeedDMS_SQLiteFTS_Indexer($indexerDir); + } else + return self::create($indexerDir); + } /* }}} */ + + /** + * Create a new index + * + * @param string $indexerDir directory on disk containing the index + */ + static function create($indexerDir) { /* {{{ */ + if(!@unlink($indexerDir.'/index.db')) + return null; + $index = new SeedDMS_SQLiteFTS_Indexer($indexerDir); + $sql = 'CREATE VIRTUAL TABLE docs USING fts4(title, comment, keywords, category, owner, content, created, notindexed=created, matchinfo=fts3)'; + $res = $index->_conn->exec($sql); + if($res === false) { + return null; + } + $sql = 'CREATE VIRTUAL TABLE docs_terms USING fts4aux(docs);'; + $res = $index->_conn->exec($sql); + if($res === false) { + return null; + } + return($index); + } /* }}} */ + + /** + * Do some initialization + * + */ + static function init($stopWordsFile='') { /* {{{ */ + } /* }}} */ + + /** + * Add document to index + * + * @param object $doc indexed document of class + * SeedDMS_SQLiteFTS_IndexedDocument + * @return boolean false in case of an error, otherwise true + */ + function addDocument($doc) { /* {{{ */ + if(!$this->_conn) + return false; + + $sql = "INSERT INTO docs (docid, title, comment, keywords, category, owner, content, created) VALUES(".$doc->getFieldValue('document_id').", ".$this->_conn->quote($doc->getFieldValue('title')).", ".$this->_conn->quote($doc->getFieldValue('comment')).", ".$this->_conn->quote($doc->getFieldValue('keywords')).", ".$this->_conn->quote($doc->getFieldValue('category')).", ".$this->_conn->quote($doc->getFieldValue('owner')).", ".$this->_conn->quote($doc->getFieldValue('content')).", ".time().")"; + $res = $this->_conn->exec($sql); + if($res === false) { + var_dump($this->_conn->errorInfo()); + } + return $res; + } /* }}} */ + + /** + * Remove document from index + * + * @param object $doc indexed document of class + * SeedDMS_SQLiteFTS_IndexedDocument + * @return boolean false in case of an error, otherwise true + */ + public function delete($id) { /* {{{ */ + if(!$this->_conn) + return false; + + $sql = "DELETE FROM docs WHERE docid=".(int) $id; + $res = $this->_conn->exec($sql); + return $res; + } /* }}} */ + + /** + * Check if document was deleted + * + * Just for compatibility with lucene. + * + * @return boolean always false + */ + public function isDeleted($id) { /* {{{ */ + return false; + } /* }}} */ + + /** + * Find documents in index + * + * @param object $doc indexed document of class + * SeedDMS_SQLiteFTS_IndexedDocument + * @return boolean false in case of an error, otherwise true + */ + public function find($query) { /* {{{ */ + if(!$this->_conn) + return false; + + $sql = "SELECT docid FROM docs WHERE docs MATCH ".$this->_conn->quote($query); + $res = $this->_conn->query($sql); + $hits = array(); + if($res) { + foreach($res as $rec) { + $hit = new SeedDMS_SQLiteFTS_QueryHit($this); + $hit->id = $rec['docid']; + $hits[] = $hit; + } + } + return $hits; + } /* }}} */ + + /** + * Get a single document from index + * + * @param integer $id id of document + * @return boolean false in case of an error, otherwise true + */ + public function findById($id) { /* {{{ */ + if(!$this->_conn) + return false; + + $sql = "SELECT docid FROM docs WHERE docid=".(int) $id; + $res = $this->_conn->query($sql); + $hits = array(); + if($res) { + while($rec = $res->fetch(PDO::FETCH_ASSOC)) { + $hit = new SeedDMS_SQLiteFTS_QueryHit($this); + $hit->id = $rec['docid']; + $hits[] = $hit; + } + } + return $hits; + } /* }}} */ + + /** + * Get a single document from index + * + * @param integer $id id of document + * @return boolean false in case of an error, otherwise true + */ + public function getDocument($id) { /* {{{ */ + if(!$this->_conn) + return false; + + $sql = "SELECT title, comment, owner, keywords, category, created FROM docs WHERE docid=".(int) $id; + $res = $this->_conn->query($sql); + $doc = false; + if($res) { + $rec = $res->fetch(PDO::FETCH_ASSOC); + $doc = new SeedDMS_SQLiteFTS_Document(); + $doc->addField('title', $rec['title']); + $doc->addField('comment', $rec['comment']); + $doc->addField('keywords', $rec['keywords']); + $doc->addField('category', $rec['category']); + $doc->addField('owner', $rec['owner']); + $doc->addField('created', $rec['created']); + } + return $doc; + } /* }}} */ + + /** + * Return list of terms in index + * + * This function does nothing! + */ + public function terms() { /* {{{ */ + if(!$this->_conn) + return false; + + $sql = "SELECT term, col, occurrences FROM docs_terms WHERE col!='*' ORDER BY col"; + $res = $this->_conn->query($sql); + $terms = array(); + if($res) { + while($rec = $res->fetch(PDO::FETCH_ASSOC)) { + $term = new SeedDMS_SQLiteFTS_Term($rec['term'], $rec['col'], $rec['occurrences']); + $terms[] = $term; + } + } + return $terms; + } /* }}} */ + + /** + * Return list of documents in index + * + */ + public function count() { /* {{{ */ + $sql = "SELECT count(*) c FROM docs"; + $res = $this->_conn->query($sql); + if($res) { + $rec = $res->fetch(PDO::FETCH_ASSOC); + return $rec['c']; + } + return 0; + } /* }}} */ + + /** + * Commit changes + * + * This function does nothing! + */ + function commit() { /* {{{ */ + } /* }}} */ + + /** + * Optimize index + * + * This function does nothing! + */ + function optimize() { /* {{{ */ + } /* }}} */ +} +?> diff --git a/SeedDMS_SQLiteFTS/SQLiteFTS/QueryHit.php b/SeedDMS_SQLiteFTS/SQLiteFTS/QueryHit.php new file mode 100644 index 000000000..d87486118 --- /dev/null +++ b/SeedDMS_SQLiteFTS/SQLiteFTS/QueryHit.php @@ -0,0 +1,65 @@ + + * @copyright Copyright (C) 2010, Uwe Steinmann + * @version Release: @package_version@ + */ + + +/** + * Class for managing a query hit. + * + * @category DMS + * @package SeedDMS_SQLiteFTS + * @version @version@ + * @author Uwe Steinmann + * @copyright Copyright (C) 2011, Uwe Steinmann + * @version Release: @package_version@ + */ +class SeedDMS_SQLiteFTS_QueryHit { + + /** + * @var SeedDMS_SQliteFTS_Indexer $index + * @access protected + */ + protected $_index; + + /** + * @var SeedDMS_SQliteFTS_Document $document + * @access protected + */ + protected $_document; + + /** + * @var integer $id id of document + * @access public + */ + public $id; + + /** + * + */ + public function __construct(SeedDMS_SQLiteFTS_Indexer $index) { /* {{{ */ + $this->_index = $index; + } /* }}} */ + + /** + * Return the document associated with this hit + * + * @return SeedDMS_SQLiteFTS_Document + */ + public function getDocument() { /* {{{ */ + if (!$this->_document instanceof SeedDMS_SQLiteFTS_Document) { + $this->_document = $this->_index->getDocument($this->id); + } + + return $this->_document; + } /* }}} */ +} +?> diff --git a/SeedDMS_SQLiteFTS/SQLiteFTS/Search.php b/SeedDMS_SQLiteFTS/SQLiteFTS/Search.php new file mode 100644 index 000000000..1226eaded --- /dev/null +++ b/SeedDMS_SQLiteFTS/SQLiteFTS/Search.php @@ -0,0 +1,94 @@ + + * @copyright Copyright (C) 2010, Uwe Steinmann + * @version Release: @package_version@ + */ + + +/** + * Class for searching in a SQlite FTS index. + * + * @category DMS + * @package SeedDMS_Lucene + * @version @version@ + * @author Uwe Steinmann + * @copyright Copyright (C) 2011, Uwe Steinmann + * @version Release: @package_version@ + */ +class SeedDMS_SQliteFTS_Search { + /** + * @var object $index SQlite FTS index + * @access protected + */ + protected $index; + + /** + * Create a new instance of the search + * + * @param object $index SQlite FTS index + * @return object instance of SeedDMS_SQliteFTS_Search + */ + function __construct($index) { /* {{{ */ + $this->index = $index; + $this->version = '@package_version@'; + if($this->version[0] == '@') + $this->version = '3.0.0'; + } /* }}} */ + + /** + * Get hit from index + * + * @param object $index lucene index + * @return object instance of SeedDMS_Lucene_Document of false + */ + function getDocument($id) { /* {{{ */ + $hits = $this->index->findById((int) $id); + return $hits ? $hits[0] : false; + } /* }}} */ + + /** + * Search in index + * + * @param object $index SQlite FTS index + * @return object instance of SeedDMS_Lucene_Search + */ + function search($term, $owner, $status='', $categories=array(), $fields=array()) { /* {{{ */ + $querystr = ''; + if($fields) { + } else { + if($term) + $querystr .= trim($term); + } + if($owner) { + if($querystr) + $querystr .= ' AND '; + $querystr .= 'owner:'.$owner; + } + if($categories) { + if($querystr) + $querystr .= ' AND '; + $querystr .= 'category:'; + $querystr .= implode(' OR category:', $categories); + $querystr .= ''; + } +// echo $querystr; + try { + $hits = $this->index->find($querystr); + $recs = array(); + foreach($hits as $hit) { + $recs[] = array('id'=>$hit->id, 'document_id'=>$hit->id); + } + return $recs; + } catch (Exception $e) { + return false; + } + } /* }}} */ +} +?> diff --git a/SeedDMS_SQLiteFTS/SQLiteFTS/Term.php b/SeedDMS_SQLiteFTS/SQLiteFTS/Term.php new file mode 100644 index 000000000..eb87f1a6b --- /dev/null +++ b/SeedDMS_SQLiteFTS/SQLiteFTS/Term.php @@ -0,0 +1,64 @@ + + * @copyright Copyright (C) 2010, Uwe Steinmann + * @version Release: @package_version@ + */ + + +/** + * Class for managing a term. + * + * @category DMS + * @package SeedDMS_SQLiteFTS + * @version @version@ + * @author Uwe Steinmann + * @copyright Copyright (C) 2011, Uwe Steinmann + * @version Release: @package_version@ + */ +class SeedDMS_SQLiteFTS_Term { + + /** + * @var string $text + * @access public + */ + public $text; + + /** + * @var string $field + * @access public + */ + public $field; + + /** + * @var integer $occurrence + * @access public + */ + public $_occurrence; + + /** + * + */ + public function __construct($term, $col, $occurrence) { /* {{{ */ + $this->text = $term; + $fields = array( + 0 => 'title', + 1 => 'comment', + 2 => 'keywords', + 3 => 'category', + 4 => 'owner', + 5 => 'content', + 6 => 'created' + ); + $this->field = $fields[$col]; + $this->_occurrence = $occurrence; + } /* }}} */ + +} +?> diff --git a/SeedDMS_SQLiteFTS/package.xml b/SeedDMS_SQLiteFTS/package.xml new file mode 100644 index 000000000..614df9222 --- /dev/null +++ b/SeedDMS_SQLiteFTS/package.xml @@ -0,0 +1,67 @@ + + + SeedDMS_SQLiteFTS + pear.php.net + Fulltext search based on sqlite for SeedDMS + SeedDMS is a web based document management system (DMS). This is + the fulltext search engine for it, based on SQLite FTS. + + Uwe Steinmann + steinm + uwe@steinmann.cx + yes + + 2015-08-10 + + + 1.0.0 + 1.0.0 + + + stable + stable + + GPL License + +initial release + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 4.3.0 + + + 1.5.4 + + + + + + + diff --git a/SeedDMS_SQLiteFTS/tests/Index.php b/SeedDMS_SQLiteFTS/tests/Index.php new file mode 100644 index 000000000..e69de29bb diff --git a/controllers/class.RemoveDocument.php b/controllers/class.RemoveDocument.php index c50fc5eac..282300e05 100644 --- a/controllers/class.RemoveDocument.php +++ b/controllers/class.RemoveDocument.php @@ -28,6 +28,7 @@ class SeedDMS_Controller_RemoveDocument extends SeedDMS_Controller_Common { $settings = $this->params['settings']; $document = $this->params['document']; $index = $this->params['index']; + $indexconf = $this->params['indexconf']; $folder = $document->getFolder(); @@ -48,12 +49,13 @@ class SeedDMS_Controller_RemoveDocument extends SeedDMS_Controller_Common { } /* Remove the document from the fulltext index */ - if($index && $hits = $index->find('document_id:'.$documentid)) { - $hit = $hits[0]; - $index->delete($hit->id); - $index->commit(); + if($index) { + $lucenesearch = new $indexconf['Search']($index); + if($hit = $lucenesearch->getDocument($documentid)) { + $index->delete($hit->id); + $index->commit(); + } } - } } diff --git a/controllers/class.RemoveFolder.php b/controllers/class.RemoveFolder.php index 5b38a5adc..d8ae53128 100644 --- a/controllers/class.RemoveFolder.php +++ b/controllers/class.RemoveFolder.php @@ -28,6 +28,7 @@ class SeedDMS_Controller_RemoveFolder extends SeedDMS_Controller_Common { $settings = $this->params['settings']; $folder = $this->params['folder']; $index = $this->params['index']; + $indexconf = $this->params['indexconf']; /* Get the document id and name before removing the document */ $foldername = $folder->getName(); @@ -41,23 +42,19 @@ class SeedDMS_Controller_RemoveFolder extends SeedDMS_Controller_Common { /* Register a callback which removes each document from the fulltext index * The callback must return true other the removal will be canceled. */ - if($settings->_enableFullSearch) { - if(!empty($settings->_luceneClassDir)) - require_once($settings->_luceneClassDir.'/Lucene.php'); - else - require_once('SeedDMS/Lucene.php'); - - $index = SeedDMS_Lucene_Indexer::open($settings->_luceneDir); - function removeFromIndex($index, $document) { - if($hits = $index->find('document_id:'.$document->getId())) { - $hit = $hits[0]; - $index->delete($hit->id); - $index->commit(); - } - return true; + function removeFromIndex($arr, $document) { + $index = $arr[0]; + $indexconf = $arr[1]; + $lucenesearch = new $indexconf['Search']($index); + if($hit = $lucenesearch->getDocument($document->getID())) { + $index->delete($hit->id); + $index->commit(); } - $dms->setCallback('onPreRemoveDocument', 'removeFromIndex', $index); + return true; } + if($index) + $dms->setCallback('onPreRemoveDocument', 'removeFromIndex', array($index, $indexconf)); + if (!$folder->remove()) { return false; } else { diff --git a/inc/inc.ClassSettings.php b/inc/inc.ClassSettings.php index 2972f2fd2..9a791ffea 100644 --- a/inc/inc.ClassSettings.php +++ b/inc/inc.ClassSettings.php @@ -89,6 +89,8 @@ class Settings { /* {{{ */ var $_stopWordsFile = null; // enable/disable lucene fulltext search var $_enableFullSearch = true; + // fulltext search engine + var $_fullSearchEngine = 'lucene'; // contentOffsetDirTo var $_contentOffsetDir = "1048576"; // Maximum number of sub-directories per parent directory @@ -354,6 +356,7 @@ class Settings { /* {{{ */ $this->_enableLanguageSelector = Settings::boolVal($tab["enableLanguageSelector"]); $this->_enableThemeSelector = Settings::boolVal($tab["enableThemeSelector"]); $this->_enableFullSearch = Settings::boolVal($tab["enableFullSearch"]); + $this->_fullSearchEngine = strval($tab["fullSearchEngine"]); $this->_stopWordsFile = strval($tab["stopWordsFile"]); $this->_sortUsersInList = strval($tab["sortUsersInList"]); $this->_sortFoldersDefault = strval($tab["sortFoldersDefault"]); @@ -643,6 +646,7 @@ class Settings { /* {{{ */ $this->setXMLAttributValue($node, "enableLanguageSelector", $this->_enableLanguageSelector); $this->setXMLAttributValue($node, "enableThemeSelector", $this->_enableThemeSelector); $this->setXMLAttributValue($node, "enableFullSearch", $this->_enableFullSearch); + $this->setXMLAttributValue($node, "fullSearchEngine", $this->_fullSearchEngine); $this->setXMLAttributValue($node, "expandFolderTree", $this->_expandFolderTree); $this->setXMLAttributValue($node, "stopWordsFile", $this->_stopWordsFile); $this->setXMLAttributValue($node, "sortUsersInList", $this->_sortUsersInList); diff --git a/inc/inc.Settings.php b/inc/inc.Settings.php index 46e7a7738..b855a068d 100644 --- a/inc/inc.Settings.php +++ b/inc/inc.Settings.php @@ -92,8 +92,32 @@ if (get_magic_quotes_gpc()) { unset($process); } +if($settings->_enableFullSearch) { + if($settings->_fullSearchEngine == 'sqlitefts') { + $indexconf = array( + 'Indexer' => 'SeedDMS_SQLiteFTS_Indexer', + 'Search' => 'SeedDMS_SQLiteFTS_Search', + 'IndexedDocument' => 'SeedDMS_SQLiteFTS_IndexedDocument' + ); + + require_once('SeedDMS/SQLiteFTS.php'); + } else { + $indexconf = array( + 'Indexer' => 'SeedDMS_Lucene_Indexer', + 'Search' => 'SeedDMS_Lucene_Search', + 'IndexedDocument' => 'SeedDMS_Lucene_IndexedDocument' + ); + + if(!empty($settings->_luceneClassDir)) + require_once($settings->_luceneClassDir.'/Lucene.php'); + else + require_once('SeedDMS/Lucene.php'); + } +} + /* Add root Dir. Needed because the view classes are included * relative to it. */ ini_set('include_path', $settings->_rootDir. PATH_SEPARATOR .ini_get('include_path')); + ?> diff --git a/op/op.AddDocument.php b/op/op.AddDocument.php index 53271a599..c836f80f7 100644 --- a/op/op.AddDocument.php +++ b/op/op.AddDocument.php @@ -303,15 +303,10 @@ for ($file_num=0;$file_num_enableFullSearch) { - if(!empty($settings->_luceneClassDir)) - require_once($settings->_luceneClassDir.'/Lucene.php'); - else - require_once('SeedDMS/Lucene.php'); - - $index = SeedDMS_Lucene_Indexer::open($settings->_luceneDir); + $index = $indexconf['Indexer']::open($settings->_luceneDir); if($index) { - SeedDMS_Lucene_Indexer::init($settings->_stopWordsFile); - $index->addDocument(new SeedDMS_Lucene_IndexedDocument($dms, $document, isset($settings->_converters['fulltext']) ? $settings->_converters['fulltext'] : null, true)); + $indexconf['Indexer']::init($settings->_stopWordsFile); + $index->addDocument(new $indexconf['IndexedDocument']($dms, $document, isset($settings->_converters['fulltext']) ? $settings->_converters['fulltext'] : null, true)); } } diff --git a/op/op.Ajax.php b/op/op.Ajax.php index bb2d84937..109300e7f 100644 --- a/op/op.Ajax.php +++ b/op/op.Ajax.php @@ -338,6 +338,17 @@ switch($command) { if($document) { if ($document->getAccessMode($user) >= M_READWRITE) { if($document->remove()) { + /* Remove the document from the fulltext index */ + if($settings->_enableFullSearch) { + $index = $indexconf['Indexer']::open($settings->_luceneDir); + if($index) { + $lucenesearch = new $indexconf['Search']($index); + if($hit = $lucenesearch->getDocument($_REQUEST['id'])) { + $index->delete($hit->id); + $index->commit(); + } + } + } header('Content-Type', 'application/json'); echo json_encode(array('success'=>true, 'message'=>'', 'data'=>'')); } else { diff --git a/op/op.RemoveDocument.php b/op/op.RemoveDocument.php index 369c9df66..62a3bf65c 100644 --- a/op/op.RemoveDocument.php +++ b/op/op.RemoveDocument.php @@ -58,11 +58,7 @@ if($document->isLocked()) { } if($settings->_enableFullSearch) { - if(!empty($settings->_luceneClassDir)) - require_once($settings->_luceneClassDir.'/Lucene.php'); - else - require_once('SeedDMS/Lucene.php'); - $index = SeedDMS_Lucene_Indexer::open($settings->_luceneDir); + $index = $indexconf['Indexer']::open($settings->_luceneDir); } else { $index = null; } @@ -74,6 +70,7 @@ $docname = $document->getName(); $controller->setParam('document', $document); $controller->setParam('index', $index); +$controller->setParam('indexconf', $indexconf); if(!$controller->run()) { UI::exitError(getMLText("document_title", array("documentname" => getMLText("invalid_doc_id"))),getMLText("error_occured")); } diff --git a/op/op.RemoveFolder.php b/op/op.RemoveFolder.php index 3810e15b3..d579f15ee 100644 --- a/op/op.RemoveFolder.php +++ b/op/op.RemoveFolder.php @@ -54,6 +54,12 @@ if ($folder->getAccessMode($user) < M_ALL) { UI::exitError(getMLText("folder_title", array("foldername" => $folder->getName())),getMLText("access_denied")); } +if($settings->_enableFullSearch) { + $index = $indexconf['Indexer']::open($settings->_luceneDir); +} else { + $index = null; +} + /* save this for notification later on */ $nl = $folder->getNotifyList(); $parent=$folder->getParent(); @@ -61,6 +67,7 @@ $foldername = $folder->getName(); $controller->setParam('folder', $folder); $controller->setParam('index', $index); +$controller->setParam('indexconf', $indexconf); if(!$controller->run()) { UI::exitError(getMLText("folder_title", array("foldername" => getMLText("invalid_folder_id"))),getMLText("invalid_folder_id")); } diff --git a/op/op.Search.php b/op/op.Search.php index 569f98350..9db68a7fe 100644 --- a/op/op.Search.php +++ b/op/op.Search.php @@ -114,25 +114,13 @@ if(isset($_GET["fullsearch"]) && $_GET["fullsearch"]) { } } - $pageNumber=1; - if (isset($_GET["pg"])) { - if (is_numeric($_GET["pg"]) && $_GET["pg"]>0) { - $pageNumber = (integer)$_GET["pg"]; - } - else if (!strcasecmp($_GET["pg"], "all")) { - $pageNumber = "all"; - } - } - $startTime = getTime(); if($settings->_enableFullSearch) { - if(!empty($settings->_luceneClassDir)) - require_once($settings->_luceneClassDir.'/Lucene.php'); - else - require_once('SeedDMS/Lucene.php'); + if($settings->_fullSearchEngine == 'lucene') { + Zend_Search_Lucene_Search_QueryParser::setDefaultEncoding('utf-8'); + } } - Zend_Search_Lucene_Search_QueryParser::setDefaultEncoding('utf-8'); if(strlen($query) < 4 && strpos($query, '*')) { $session->setSplashMsg(array('type'=>'error', 'msg'=>getMLText('splash_invalid_searchterm'))); $resArr = array(); @@ -142,8 +130,8 @@ if(isset($_GET["fullsearch"]) && $_GET["fullsearch"]) { $entries = array(); $searchTime = 0; } else { - $index = Zend_Search_Lucene::open($settings->_luceneDir); - $lucenesearch = new SeedDMS_Lucene_Search($index); + $index = $indexconf['Indexer']::open($settings->_luceneDir); + $lucenesearch = new $indexconf['Search']($index); $hits = $lucenesearch->search($query, $owner ? $owner->getLogin() : '', '', $categorynames); if($hits === false) { $session->setSplashMsg(array('type'=>'error', 'msg'=>getMLText('splash_invalid_searchterm'))); diff --git a/op/op.Settings.php b/op/op.Settings.php index e2bd3ea91..c4098987c 100644 --- a/op/op.Settings.php +++ b/op/op.Settings.php @@ -69,6 +69,7 @@ if ($action == "saveSettings") $settings->_enableEmail =getBoolValue("enableEmail"); $settings->_enableUsersView = getBoolValue("enableUsersView"); $settings->_enableFullSearch = getBoolValue("enableFullSearch"); + $settings->_fullSearchEngine = $_POST["fullSearchEngine"]; $settings->_enableClipboard = getBoolValue("enableClipboard"); $settings->_enableDropUpload = getBoolValue("enableDropUpload"); $settings->_enableFolderTree = getBoolValue("enableFolderTree"); diff --git a/out/out.IndexInfo.php b/out/out.IndexInfo.php index fc97d1f96..2bb6a4342 100644 --- a/out/out.IndexInfo.php +++ b/out/out.IndexInfo.php @@ -36,12 +36,7 @@ if(!$settings->_enableFullSearch) { UI::exitError(getMLText("admin_tools"),getMLText("fulltextsearch_disabled")); } -if(!empty($settings->_luceneClassDir)) - require_once($settings->_luceneClassDir.'/Lucene.php'); -else - require_once('SeedDMS/Lucene.php'); - -$index = SeedDMS_Lucene_Indexer::open($settings->_luceneDir); +$index = $indexconf['Indexer']::open($settings->_luceneDir); if(!$index) { UI::exitError(getMLText("admin_tools"),getMLText("no_fulltextindex")); } diff --git a/out/out.Indexer.php b/out/out.Indexer.php index ce4ab48f7..15fff7d26 100644 --- a/out/out.Indexer.php +++ b/out/out.Indexer.php @@ -37,25 +37,20 @@ if(!$settings->_enableFullSearch) { UI::exitError(getMLText("admin_tools"),getMLText("fulltextsearch_disabled")); } -if(!empty($settings->_luceneClassDir)) - require_once($settings->_luceneClassDir.'/Lucene.php'); -else - require_once('SeedDMS/Lucene.php'); - if(isset($_GET['create']) && $_GET['create'] == 1) { if(isset($_GET['confirm']) && $_GET['confirm'] == 1) { - $index = SeedDMS_Lucene_Indexer::create($settings->_luceneDir); - SeedDMS_Lucene_Indexer::init($settings->_stopWordsFile); + $index = $indexconf['Indexer']::create($settings->_luceneDir); + $indexconf['Indexer']::init($settings->_stopWordsFile); } else { header('Location: out.CreateIndex.php'); exit; } } else { - $index = SeedDMS_Lucene_Indexer::open($settings->_luceneDir); + $index = $indexconf['Indexer']::open($settings->_luceneDir); if(!$index) { UI::exitError(getMLText("admin_tools"),getMLText("no_fulltextindex")); } - SeedDMS_Lucene_Indexer::init($settings->_stopWordsFile); + $indexconf['Indexer']::init($settings->_stopWordsFile); } if (!isset($_GET["folderid"]) || !is_numeric($_GET["folderid"]) || intval($_GET["folderid"])<1) { @@ -67,7 +62,7 @@ else { $folder = $dms->getFolder($folderid); $tmp = explode('.', basename($_SERVER['SCRIPT_FILENAME'])); -$view = UI::factory($theme, $tmp[1], array('dms'=>$dms, 'user'=>$user, 'index'=>$index, 'recreate'=>(isset($_GET['create']) && $_GET['create']==1), 'folder'=>$folder, 'converters'=>$settings->_converters['fulltext'], 'timeout'=>$settings->_cmdTimeout)); +$view = UI::factory($theme, $tmp[1], array('dms'=>$dms, 'user'=>$user, 'index'=>$index, 'indexconf'=>$indexconf, 'recreate'=>(isset($_GET['create']) && $_GET['create']==1), 'folder'=>$folder, 'converters'=>$settings->_converters['fulltext'], 'timeout'=>$settings->_cmdTimeout)); if($view) { $view->show(); exit; diff --git a/utils/indexer.php b/utils/indexer.php index e7b6de1f4..d8170469a 100644 --- a/utils/indexer.php +++ b/utils/indexer.php @@ -52,26 +52,46 @@ if(isset($settings->_extraPath)) ini_set('include_path', $settings->_extraPath. PATH_SEPARATOR .ini_get('include_path')); require_once("SeedDMS/Core.php"); -require_once("SeedDMS/Lucene.php"); +if($settings->_fullSearchEngine == 'sqlitefts') { + $indexconf = array( + 'Indexer' => 'SeedDMS_SQLiteFTS_Indexer', + 'Search' => 'SeedDMS_SQLiteFTS_Search', + 'IndexedDocument' => 'SeedDMS_SQLiteFTS_IndexedDocument' + ); -function tree($dms, $index, $folder, $indent='') { + require_once('SeedDMS/SQLiteFTS.php'); +} else { + $indexconf = array( + 'Indexer' => 'SeedDMS_Lucene_Indexer', + 'Search' => 'SeedDMS_Lucene_Search', + 'IndexedDocument' => 'SeedDMS_Lucene_IndexedDocument' + ); + + require_once('SeedDMS/Lucene.php'); +} + +function tree($dms, $index, $indexconf, $folder, $indent='') { /* {{{ */ global $settings; echo $indent."D ".$folder->getName()."\n"; $subfolders = $folder->getSubFolders(); foreach($subfolders as $subfolder) { - tree($dms, $index, $subfolder, $indent.' '); + tree($dms, $index, $indexconf, $subfolder, $indent.' '); } $documents = $folder->getDocuments(); foreach($documents as $document) { echo $indent." ".$document->getId().":".$document->getName()." "; - if(!($hits = $index->find('document_id:'.$document->getId()))) { - $index->addDocument(new SeedDMS_Lucene_IndexedDocument($dms, $document, isset($settings->_converters['fulltext']) ? $settings->_converters['fulltext'] : null)); - echo " (Document added)\n"; + $lucenesearch = new $indexconf['Search']($index); + if(!($hit = $lucenesearch->getDocument($document->getId()))) { + try { + $index->addDocument(new $indexconf['IndexedDocument']($dms, $document, isset($settings->_converters['fulltext']) ? $settings->_converters['fulltext'] : null, false)); + echo " (Document added)\n"; + } catch(Exception $e) { + echo " (Timeout)\n"; + } } else { - $hit = $hits[0]; try { $created = (int) $hit->getDocument()->getFieldValue('created'); - } catch (Zend_Search_Lucene_Exception $e) { + } catch (Exception $e) { $created = 0; } $content = $document->getLatestContent(); @@ -79,33 +99,42 @@ function tree($dms, $index, $folder, $indent='') { echo " (Document unchanged)\n"; } else { if($index->delete($hit->id)) { - $index->addDocument(new SeedDMS_Lucene_IndexedDocument($dms, $document, $settings->_converters['fulltext'] ? $settings->_converters['fulltext'] : null)); - echo " (Document updated)\n"; + try { + $index->addDocument(new $indexconf['IndexedDocument']($dms, $document, isset($settings->_converters['fulltext']) ? $settings->_converters['fulltext'] : null, false)); + echo " (Document updated)\n"; + } catch(Exception $e) { + echo " (Timeout)\n"; + } } } } } -} +} /* }}} */ $db = new SeedDMS_Core_DatabaseAccess($settings->_dbDriver, $settings->_dbHostname, $settings->_dbUser, $settings->_dbPass, $settings->_dbDatabase); $db->connect() or die ("Could not connect to db-server \"" . $settings->_dbHostname . "\""); $dms = new SeedDMS_Core_DMS($db, $settings->_contentDir.$settings->_contentOffsetDir); if(!$dms->checkVersion()) { - echo "Database update needed."; - exit; + echo "Database update needed.\n"; + exit(1); } $dms->setRootFolderID($settings->_rootFolderID); if($recreate) - $index = Zend_Search_Lucene::create($settings->_luceneDir); + $index = $indexconf['Indexer']::create($settings->_luceneDir); else - $index = Zend_Search_Lucene::open($settings->_luceneDir); -SeedDMS_Lucene_Indexer::init($settings->_stopWordsFile); + $index = $indexconf['Indexer']::open($settings->_luceneDir); +if(!$index) { + echo "Could not create index.\n"; + exit(1); +} + +$indexconf['Indexer']::init($settings->_stopWordsFile); $folder = $dms->getFolder($settings->_rootFolderID); -tree($dms, $index, $folder); +tree($dms, $index, $indexconf, $folder); $index->commit(); $index->optimize(); diff --git a/utils/xmldump.php b/utils/xmldump.php index 4ad58bf7b..f872a5e19 100644 --- a/utils/xmldump.php +++ b/utils/xmldump.php @@ -134,7 +134,7 @@ function dumplog($version, $type, $logs, $indent) { /* {{{ */ echo $indent." ".$a['status']."\n"; echo $indent." ".wrapWithCData($a['comment'])."\n"; echo $indent." ".$a['date']."\n"; - if($a['file']) { + if(!empty($a['file'])) { $filename = $dms->contentDir . $document->getDir().'r'.(int) $a[$type2.'LogID']; if(file_exists($filename)) { echo $indent." getName())."\n"; $subfolders = $folder->getSubFolders(); foreach($subfolders as $subfolder) { - $this->tree($dms, $index, $subfolder, $indent.' '); + $this->tree($dms, $index, $indexconf, $subfolder, $indent.' '); } $documents = $folder->getDocuments(); foreach($documents as $document) { echo $indent." ".$document->getId().":".htmlspecialchars($document->getName())." "; /* If the document wasn't indexed before then just add it */ - if(!($hits = $index->find('document_id:'.$document->getId()))) { + $lucenesearch = new $indexconf['Search']($index); + if(!($hit = $lucenesearch->getDocument($document->getId()))) { try { - $index->addDocument(new SeedDMS_Lucene_IndexedDocument($dms, $document, $this->converters ? $this->converters : null, false, $this->timeout)); + $index->addDocument(new $indexconf['IndexedDocument']($dms, $document, $this->converters ? $this->converters : null, false, $this->timeout)); echo "(document added)"; } catch(Exception $e) { echo $indent."(adding document failed '".$e->getMessage()."')"; } } else { - $hit = $hits[0]; /* Check if the attribute created is set or has a value older * than the lasted content. Documents without such an attribute * where added when a new document was added to the dms. In such @@ -58,7 +58,7 @@ class SeedDMS_View_Indexer extends SeedDMS_Bootstrap_Style { */ try { $created = (int) $hit->getDocument()->getFieldValue('created'); - } catch (Zend_Search_Lucene_Exception $e) { + } catch (/* Zend_Search_Lucene_ */Exception $e) { $created = 0; } $content = $document->getLatestContent(); @@ -67,7 +67,7 @@ class SeedDMS_View_Indexer extends SeedDMS_Bootstrap_Style { } else { $index->delete($hit->id); try { - $index->addDocument(new SeedDMS_Lucene_IndexedDocument($dms, $document, $this->converters ? $this->converters : null, false, $this->timeout)); + $index->addDocument(new $indexconf['IndexedDocument']($dms, $document, $this->converters ? $this->converters : null, false, $this->timeout)); echo $indent."(document updated)"; } catch(Exception $e) { print_r($e); @@ -83,6 +83,7 @@ class SeedDMS_View_Indexer extends SeedDMS_Bootstrap_Style { $dms = $this->params['dms']; $user = $this->params['user']; $index = $this->params['index']; + $indexconf = $this->params['indexconf']; $recreate = $this->params['recreate']; $folder = $this->params['folder']; $this->converters = $this->params['converters']; @@ -95,7 +96,7 @@ class SeedDMS_View_Indexer extends SeedDMS_Bootstrap_Style { $this->contentHeading(getMLText("update_fulltext_index")); echo "
";
-		$this->tree($dms, $index, $folder);
+		$this->tree($dms, $index, $indexconf, $folder);
 		echo "
"; $index->commit(); diff --git a/views/bootstrap/class.Settings.php b/views/bootstrap/class.Settings.php index 454fac8b3..c0a3ea709 100644 --- a/views/bootstrap/class.Settings.php +++ b/views/bootstrap/class.Settings.php @@ -162,6 +162,15 @@ if(!is_writeable($settings->_configFilePath)) { : _enableFullSearch) echo "checked" ?> /> + "> + : + + + + "> :