gnh1201/seeddms-code
1
0
Fork 0
mirror of https://git.code.sf.net/p/seeddms/code synced 2025-02-11 17:44:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

use new fulltext index interface with support for sqlitefts

Browse Source
This commit is contained in:
Uwe Steinmann 2015-08-10 21:43:15 +02:00
parent b0413a0d46
commit 8ffcfab6e4
9 changed files with 95 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
op/op.AddDocument.php
View File

@ -293,15 +293,10 @@ for ($file_num=0;$file_num<count($_FILES["userfile"]["tmp_name"]);$file_num++){
} }
} }
if($settings->_enableFullSearch) { if($settings->_enableFullSearch) {
if(!empty($settings->_luceneClassDir)) $index = $indexconf['Indexer']::open($settings->_luceneDir);
require_once($settings->_luceneClassDir.'/Lucene.php');
else
require_once('SeedDMS/Lucene.php');
$index = SeedDMS_Lucene_Indexer::open($settings->_luceneDir);
if($index) { if($index) {
SeedDMS_Lucene_Indexer::init($settings->_stopWordsFile); $indexconf['Indexer']::init($settings->_stopWordsFile);
$index->addDocument(new SeedDMS_Lucene_IndexedDocument($dms, $document, isset($settings->_converters['fulltext']) ? $settings->_converters['fulltext'] : null, true)); $index->addDocument(new $indexconf['IndexedDocument']($dms, $document, isset($settings->_converters['fulltext']) ? $settings->_converters['fulltext'] : null, true));
} }
} }

11
op/op.Ajax.php
View File

@ -319,6 +319,17 @@ switch($command) {
if($document) { if($document) {
if ($document->getAccessMode($user) >= M_READWRITE) { if ($document->getAccessMode($user) >= M_READWRITE) {
if($document->remove()) { if($document->remove()) {
/* Remove the document from the fulltext index */
if($settings->_enableFullSearch) {
$index = $indexconf['Indexer']::open($settings->_luceneDir);
if($index) {
$lucenesearch = new $indexconf['Search']($index);
if($hit = $lucenesearch->getDocument($_REQUEST['id'])) {
$index->delete($hit->id);
$index->commit();
}
}
}
header('Content-Type', 'application/json'); header('Content-Type', 'application/json');
echo json_encode(array('success'=>true, 'message'=>'', 'data'=>'')); echo json_encode(array('success'=>true, 'message'=>'', 'data'=>''));
} else { } else {

17
op/op.RemoveDocument.php
View File

@ -62,16 +62,13 @@ if (!$document->remove()) {
/* Remove the document from the fulltext index */ /* Remove the document from the fulltext index */
if($settings->_enableFullSearch) { if($settings->_enableFullSearch) {
if(!empty($settings->_luceneClassDir)) $index = $indexconf['Indexer']::open($settings->_luceneDir);
require_once($settings->_luceneClassDir.'/Lucene.php'); if($index) {
else $lucenesearch = new $indexconf['Search']($index);
require_once('SeedDMS/Lucene.php'); if($hit = $lucenesearch->getDocument($documentid)) {
$index->delete($hit->id);
$index = SeedDMS_Lucene_Indexer::open($settings->_luceneDir); $index->commit();
if($index && $hits = $index->find('document_id:'.$documentid)) { }
$hit = $hits[0];
$index->delete($hit->id);
$index->commit();
} }
} }

18
op/op.RemoveFolder.php
View File

@ -54,21 +54,19 @@ $parent=$folder->getParent();
* The callback must return true other the removal will be canceled. * The callback must return true other the removal will be canceled.
*/ */
if($settings->_enableFullSearch) { if($settings->_enableFullSearch) {
if(!empty($settings->_luceneClassDir)) function removeFromIndex($arr, $document) {
require_once($settings->_luceneClassDir.'/Lucene.php'); $index = $arr[0];
else $indexconf = $arr[1];
require_once('SeedDMS/Lucene.php'); $lucenesearch = new $indexconf['Search']($index);
if($hit = $lucenesearch->getDocument($document->getID())) {
$index = SeedDMS_Lucene_Indexer::open($settings->_luceneDir);
function removeFromIndex($index, $document) {
if($hits = $index->find('document_id:'.$document->getId())) {
$hit = $hits[0];
$index->delete($hit->id); $index->delete($hit->id);
$index->commit(); $index->commit();
} }
return true; return true;
} }
$dms->setCallback('onPreRemoveDocument', 'removeFromIndex', $index); $index = $indexconf['Indexer']::open($settings->_luceneDir);
if($index)
$dms->setCallback('onPreRemoveDocument', 'removeFromIndex', array($index, $indexconf));
} }
$nl = $folder->getNotifyList(); $nl = $folder->getNotifyList();

22
op/op.Search.php
View File

@ -112,25 +112,13 @@ if(isset($_GET["fullsearch"]) && $_GET["fullsearch"]) {
} }
} }
$pageNumber=1;
if (isset($_GET["pg"])) {
if (is_numeric($_GET["pg"]) && $_GET["pg"]>0) {
$pageNumber = (integer)$_GET["pg"];
}
else if (!strcasecmp($_GET["pg"], "all")) {
$pageNumber = "all";
}
}
$startTime = getTime(); $startTime = getTime();
if($settings->_enableFullSearch) { if($settings->_enableFullSearch) {
if(!empty($settings->_luceneClassDir)) if($settings->_fullSearchEngine == 'lucene') {
require_once($settings->_luceneClassDir.'/Lucene.php'); Zend_Search_Lucene_Search_QueryParser::setDefaultEncoding('utf-8');
else }
require_once('SeedDMS/Lucene.php');
} }
Zend_Search_Lucene_Search_QueryParser::setDefaultEncoding('utf-8');
if(strlen($query) < 4 && strpos($query, '*')) { if(strlen($query) < 4 && strpos($query, '*')) {
$session->setSplashMsg(array('type'=>'error', 'msg'=>getMLText('splash_invalid_searchterm'))); $session->setSplashMsg(array('type'=>'error', 'msg'=>getMLText('splash_invalid_searchterm')));
$resArr = array(); $resArr = array();
@ -140,8 +128,8 @@ if(isset($_GET["fullsearch"]) && $_GET["fullsearch"]) {
$entries = array(); $entries = array();
$searchTime = 0; $searchTime = 0;
} else { } else {
$index = Zend_Search_Lucene::open($settings->_luceneDir); $index = $indexconf['Indexer']::open($settings->_luceneDir);
$lucenesearch = new SeedDMS_Lucene_Search($index); $lucenesearch = new $indexconf['Search']($index);
$hits = $lucenesearch->search($query, $owner ? $owner->getLogin() : '', '', $categorynames); $hits = $lucenesearch->search($query, $owner ? $owner->getLogin() : '', '', $categorynames);
if($hits === false) { if($hits === false) {
$session->setSplashMsg(array('type'=>'error', 'msg'=>getMLText('splash_invalid_searchterm'))); $session->setSplashMsg(array('type'=>'error', 'msg'=>getMLText('splash_invalid_searchterm')));

7
out/out.IndexInfo.php
View File

@ -34,12 +34,7 @@ if(!$settings->_enableFullSearch) {
UI::exitError(getMLText("admin_tools"),getMLText("fulltextsearch_disabled")); UI::exitError(getMLText("admin_tools"),getMLText("fulltextsearch_disabled"));
} }
if(!empty($settings->_luceneClassDir)) $index = $indexconf['Indexer']::open($settings->_luceneDir);
require_once($settings->_luceneClassDir.'/Lucene.php');
else
require_once('SeedDMS/Lucene.php');
$index = SeedDMS_Lucene_Indexer::open($settings->_luceneDir);
if(!$index) { if(!$index) {
UI::exitError(getMLText("admin_tools"),getMLText("no_fulltextindex")); UI::exitError(getMLText("admin_tools"),getMLText("no_fulltextindex"));
} }

15
out/out.Indexer.php
View File

@ -35,25 +35,20 @@ if(!$settings->_enableFullSearch) {
UI::exitError(getMLText("admin_tools"),getMLText("fulltextsearch_disabled")); UI::exitError(getMLText("admin_tools"),getMLText("fulltextsearch_disabled"));
} }
if(!empty($settings->_luceneClassDir))
require_once($settings->_luceneClassDir.'/Lucene.php');
else
require_once('SeedDMS/Lucene.php');
if(isset($_GET['create']) && $_GET['create'] == 1) { if(isset($_GET['create']) && $_GET['create'] == 1) {
if(isset($_GET['confirm']) && $_GET['confirm'] == 1) { if(isset($_GET['confirm']) && $_GET['confirm'] == 1) {
$index = SeedDMS_Lucene_Indexer::create($settings->_luceneDir); $index = $indexconf['Indexer']::create($settings->_luceneDir);
SeedDMS_Lucene_Indexer::init($settings->_stopWordsFile); $indexconf['Indexer']::init($settings->_stopWordsFile);
} else { } else {
header('Location: out.CreateIndex.php'); header('Location: out.CreateIndex.php');
exit; exit;
} }
} else { } else {
$index = SeedDMS_Lucene_Indexer::open($settings->_luceneDir); $index = $indexconf['Indexer']::open($settings->_luceneDir);
if(!$index) { if(!$index) {
UI::exitError(getMLText("admin_tools"),getMLText("no_fulltextindex")); UI::exitError(getMLText("admin_tools"),getMLText("no_fulltextindex"));
} }
SeedDMS_Lucene_Indexer::init($settings->_stopWordsFile); $indexconf['Indexer']::init($settings->_stopWordsFile);
} }
if (!isset($_GET["folderid"]) || !is_numeric($_GET["folderid"]) || intval($_GET["folderid"])<1) { if (!isset($_GET["folderid"]) || !is_numeric($_GET["folderid"]) || intval($_GET["folderid"])<1) {
@ -65,7 +60,7 @@ else {
$folder = $dms->getFolder($folderid); $folder = $dms->getFolder($folderid);
$tmp = explode('.', basename($_SERVER['SCRIPT_FILENAME'])); $tmp = explode('.', basename($_SERVER['SCRIPT_FILENAME']));
$view = UI::factory($theme, $tmp[1], array('dms'=>$dms, 'user'=>$user, 'index'=>$index, 'recreate'=>(isset($_GET['create']) && $_GET['create']==1), 'folder'=>$folder, 'converters'=>$settings->_converters['fulltext'], 'timeout'=>$settings->_cmdTimeout)); $view = UI::factory($theme, $tmp[1], array('dms'=>$dms, 'user'=>$user, 'index'=>$index, 'indexconf'=>$indexconf, 'recreate'=>(isset($_GET['create']) && $_GET['create']==1), 'folder'=>$folder, 'converters'=>$settings->_converters['fulltext'], 'timeout'=>$settings->_cmdTimeout));
if($view) { if($view) {
$view->show(); $view->show();
exit; exit;

63
utils/indexer.php
View File

@ -52,26 +52,46 @@ if(isset($settings->_extraPath))
ini_set('include_path', $settings->_extraPath. PATH_SEPARATOR .ini_get('include_path')); ini_set('include_path', $settings->_extraPath. PATH_SEPARATOR .ini_get('include_path'));
require_once("SeedDMS/Core.php"); require_once("SeedDMS/Core.php");
require_once("SeedDMS/Lucene.php"); if($settings->_fullSearchEngine == 'sqlitefts') {
$indexconf = array(
'Indexer' => 'SeedDMS_SQLiteFTS_Indexer',
'Search' => 'SeedDMS_SQLiteFTS_Search',
'IndexedDocument' => 'SeedDMS_SQLiteFTS_IndexedDocument'
);
function tree($dms, $index, $folder, $indent='') { require_once('SeedDMS/SQLiteFTS.php');
} else {
$indexconf = array(
'Indexer' => 'SeedDMS_Lucene_Indexer',
'Search' => 'SeedDMS_Lucene_Search',
'IndexedDocument' => 'SeedDMS_Lucene_IndexedDocument'
);
require_once('SeedDMS/Lucene.php');
}
function tree($dms, $index, $indexconf, $folder, $indent='') { /* {{{ */
global $settings; global $settings;
echo $indent."D ".$folder->getName()."\n"; echo $indent."D ".$folder->getName()."\n";
$subfolders = $folder->getSubFolders(); $subfolders = $folder->getSubFolders();
foreach($subfolders as $subfolder) { foreach($subfolders as $subfolder) {
tree($dms, $index, $subfolder, $indent.' '); tree($dms, $index, $indexconf, $subfolder, $indent.' ');
} }
$documents = $folder->getDocuments(); $documents = $folder->getDocuments();
foreach($documents as $document) { foreach($documents as $document) {
echo $indent." ".$document->getId().":".$document->getName()." "; echo $indent." ".$document->getId().":".$document->getName()." ";
if(!($hits = $index->find('document_id:'.$document->getId()))) { $lucenesearch = new $indexconf['Search']($index);
$index->addDocument(new SeedDMS_Lucene_IndexedDocument($dms, $document, isset($settings->_converters['fulltext']) ? $settings->_converters['fulltext'] : null)); if(!($hit = $lucenesearch->getDocument($document->getId()))) {
echo " (Document added)\n"; try {
$index->addDocument(new $indexconf['IndexedDocument']($dms, $document, isset($settings->_converters['fulltext']) ? $settings->_converters['fulltext'] : null, false));
echo " (Document added)\n";
} catch(Exception $e) {
echo " (Timeout)\n";
}
} else { } else {
$hit = $hits[0];
try { try {
$created = (int) $hit->getDocument()->getFieldValue('created'); $created = (int) $hit->getDocument()->getFieldValue('created');
} catch (Zend_Search_Lucene_Exception $e) { } catch (Exception $e) {
$created = 0; $created = 0;
} }
$content = $document->getLatestContent(); $content = $document->getLatestContent();
@ -79,33 +99,42 @@ function tree($dms, $index, $folder, $indent='') {
echo " (Document unchanged)\n"; echo " (Document unchanged)\n";
} else { } else {
if($index->delete($hit->id)) { if($index->delete($hit->id)) {
$index->addDocument(new SeedDMS_Lucene_IndexedDocument($dms, $document, $settings->_converters['fulltext'] ? $settings->_converters['fulltext'] : null)); try {
echo " (Document updated)\n"; $index->addDocument(new $indexconf['IndexedDocument']($dms, $document, isset($settings->_converters['fulltext']) ? $settings->_converters['fulltext'] : null, false));
echo " (Document updated)\n";
} catch(Exception $e) {
echo " (Timeout)\n";
}
} }
} }
} }
} }
} } /* }}} */
$db = new SeedDMS_Core_DatabaseAccess($settings->_dbDriver, $settings->_dbHostname, $settings->_dbUser, $settings->_dbPass, $settings->_dbDatabase); $db = new SeedDMS_Core_DatabaseAccess($settings->_dbDriver, $settings->_dbHostname, $settings->_dbUser, $settings->_dbPass, $settings->_dbDatabase);
$db->connect() or die ("Could not connect to db-server \"" . $settings->_dbHostname . "\""); $db->connect() or die ("Could not connect to db-server \"" . $settings->_dbHostname . "\"");
$dms = new SeedDMS_Core_DMS($db, $settings->_contentDir.$settings->_contentOffsetDir); $dms = new SeedDMS_Core_DMS($db, $settings->_contentDir.$settings->_contentOffsetDir);
if(!$dms->checkVersion()) { if(!$dms->checkVersion()) {
echo "Database update needed."; echo "Database update needed.\n";
exit; exit(1);
} }
$dms->setRootFolderID($settings->_rootFolderID); $dms->setRootFolderID($settings->_rootFolderID);
if($recreate) if($recreate)
$index = Zend_Search_Lucene::create($settings->_luceneDir); $index = $indexconf['Indexer']::create($settings->_luceneDir);
else else
$index = Zend_Search_Lucene::open($settings->_luceneDir); $index = $indexconf['Indexer']::open($settings->_luceneDir);
SeedDMS_Lucene_Indexer::init($settings->_stopWordsFile); if(!$index) {
echo "Could not create index.\n";
exit(1);
}
$indexconf['Indexer']::init($settings->_stopWordsFile);
$folder = $dms->getFolder($settings->_rootFolderID); $folder = $dms->getFolder($settings->_rootFolderID);
tree($dms, $index, $folder); tree($dms, $index, $indexconf, $folder);
$index->commit(); $index->commit();
$index->optimize(); $index->optimize();

17
views/bootstrap/class.Indexer.php
View File

@ -31,26 +31,26 @@ require_once("class.Bootstrap.php");
*/ */
class SeedDMS_View_Indexer extends SeedDMS_Bootstrap_Style { class SeedDMS_View_Indexer extends SeedDMS_Bootstrap_Style {
function tree($dms, $index, $folder, $indent='') { /* {{{ */ function tree($dms, $index, $indexconf, $folder, $indent='') { /* {{{ */
set_time_limit(30); set_time_limit(30);
echo $indent."D ".htmlspecialchars($folder->getName())."\n"; echo $indent."D ".htmlspecialchars($folder->getName())."\n";
$subfolders = $folder->getSubFolders(); $subfolders = $folder->getSubFolders();
foreach($subfolders as $subfolder) { foreach($subfolders as $subfolder) {
$this->tree($dms, $index, $subfolder, $indent.' '); $this->tree($dms, $index, $indexconf, $subfolder, $indent.' ');
} }
$documents = $folder->getDocuments(); $documents = $folder->getDocuments();
foreach($documents as $document) { foreach($documents as $document) {
echo $indent." ".$document->getId().":".htmlspecialchars($document->getName())." "; echo $indent." ".$document->getId().":".htmlspecialchars($document->getName())." ";
/* If the document wasn't indexed before then just add it */ /* If the document wasn't indexed before then just add it */
if(!($hits = $index->find('document_id:'.$document->getId()))) { $lucenesearch = new $indexconf['Search']($index);
if(!($hit = $lucenesearch->getDocument($document->getId()))) {
try { try {
$index->addDocument(new SeedDMS_Lucene_IndexedDocument($dms, $document, $this->converters ? $this->converters : null, false, $this->timeout)); $index->addDocument(new $indexconf['IndexedDocument']($dms, $document, $this->converters ? $this->converters : null, false, $this->timeout));
echo "(document added)"; echo "(document added)";
} catch(Exception $e) { } catch(Exception $e) {
echo $indent."(adding document failed '".$e->getMessage()."')"; echo $indent."(adding document failed '".$e->getMessage()."')";
} }
} else { } else {
$hit = $hits[0];
/* Check if the attribute created is set or has a value older /* Check if the attribute created is set or has a value older
* than the lasted content. Documents without such an attribute * than the lasted content. Documents without such an attribute
* where added when a new document was added to the dms. In such * where added when a new document was added to the dms. In such
@ -58,7 +58,7 @@ class SeedDMS_View_Indexer extends SeedDMS_Bootstrap_Style {
*/ */
try { try {
$created = (int) $hit->getDocument()->getFieldValue('created'); $created = (int) $hit->getDocument()->getFieldValue('created');
} catch (Zend_Search_Lucene_Exception $e) { } catch (/* Zend_Search_Lucene_ */Exception $e) {
$created = 0; $created = 0;
} }
$content = $document->getLatestContent(); $content = $document->getLatestContent();
@ -67,7 +67,7 @@ class SeedDMS_View_Indexer extends SeedDMS_Bootstrap_Style {
} else { } else {
$index->delete($hit->id); $index->delete($hit->id);
try { try {
$index->addDocument(new SeedDMS_Lucene_IndexedDocument($dms, $document, $this->converters ? $this->converters : null, false, $this->timeout)); $index->addDocument(new $indexconf['IndexedDocument']($dms, $document, $this->converters ? $this->converters : null, false, $this->timeout));
echo $indent."(document updated)"; echo $indent."(document updated)";
} catch(Exception $e) { } catch(Exception $e) {
print_r($e); print_r($e);
@ -83,6 +83,7 @@ class SeedDMS_View_Indexer extends SeedDMS_Bootstrap_Style {
$dms = $this->params['dms']; $dms = $this->params['dms'];
$user = $this->params['user']; $user = $this->params['user'];
$index = $this->params['index']; $index = $this->params['index'];
$indexconf = $this->params['indexconf'];
$recreate = $this->params['recreate']; $recreate = $this->params['recreate'];
$folder = $this->params['folder']; $folder = $this->params['folder'];
$this->converters = $this->params['converters']; $this->converters = $this->params['converters'];
@ -95,7 +96,7 @@ class SeedDMS_View_Indexer extends SeedDMS_Bootstrap_Style {
$this->contentHeading(getMLText("update_fulltext_index")); $this->contentHeading(getMLText("update_fulltext_index"));
echo "<pre>"; echo "<pre>";
$this->tree($dms, $index, $folder); $this->tree($dms, $index, $indexconf, $folder);
echo "</pre>"; echo "</pre>";
$index->commit(); $index->commit();