From 602a02722b431c5afbb489b6320cb01e578c9c84 Mon Sep 17 00:00:00 2001 From: Uwe Steinmann Date: Wed, 1 Mar 2017 15:49:38 +0100 Subject: [PATCH 01/12] decrement items in queue after displaying bars --- views/bootstrap/class.Indexer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/views/bootstrap/class.Indexer.php b/views/bootstrap/class.Indexer.php index aa3a459ce..91e9e2784 100644 --- a/views/bootstrap/class.Indexer.php +++ b/views/bootstrap/class.Indexer.php @@ -90,10 +90,10 @@ function check_queue() { } }, complete: function(xhr, textstatus) { - queue_count--; // Substract request to the counter $('.queue-bar').css('width', (queue_count*100/MAX_REQUESTS)+'%'); $('.total-bar').css('width', (100 - (funcArray.length+queue_count)*100/docstoindex)+'%'); $('.total-bar').text(Math.round(100 - (funcArray.length+queue_count)*100/docstoindex)+' %'); + queue_count--; // Substract request to the counter if(funcArray.length+queue_count == 0) $('.total-bar').addClass('bar-success'); } From 77eab44dc10b3ad8d8b3d1d5ba647ba96c558b63 Mon Sep 17 00:00:00 2001 From: Uwe Steinmann Date: Wed, 1 Mar 2017 15:56:00 +0100 Subject: [PATCH 02/12] new version, catch exception in execWithTimeout() --- SeedDMS_Lucene/Lucene/IndexedDocument.php | 17 ++++---------- SeedDMS_Lucene/package.xml | 28 ++++++++++++++++++----- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/SeedDMS_Lucene/Lucene/IndexedDocument.php b/SeedDMS_Lucene/Lucene/IndexedDocument.php index 874bfd149..027536dae 100644 --- a/SeedDMS_Lucene/Lucene/IndexedDocument.php +++ b/SeedDMS_Lucene/Lucene/IndexedDocument.php @@ -127,19 +127,12 @@ class SeedDMS_Lucene_IndexedDocument extends Zend_Search_Lucene_Document { $mimetype = $version->getMimeType(); if(isset($_convcmd[$mimetype])) { $cmd = sprintf($_convcmd[$mimetype], $path); - $content = self::execWithTimeout($cmd, $timeout); - /* - $fp = popen($cmd, 'r'); - if($fp) { - $content = ''; - while(!feof($fp)) { - $content .= fread($fp, 2048); + try { + $content = self::execWithTimeout($cmd, $timeout); + if($content) { + $this->addField(Zend_Search_Lucene_Field::UnStored('content', $content, 'utf-8')); } - pclose($fp); - } - */ - if($content) { - $this->addField(Zend_Search_Lucene_Field::UnStored('content', $content, 'utf-8')); + } catch (Exception $e) { } } } diff --git a/SeedDMS_Lucene/package.xml b/SeedDMS_Lucene/package.xml index 3f5e3d45c..774715d7b 100644 --- a/SeedDMS_Lucene/package.xml +++ b/SeedDMS_Lucene/package.xml @@ -11,11 +11,11 @@ uwe@steinmann.cx yes - 2016-04-28 - + 2017-03-01 + - 1.1.9 - 1.1.7 + 1.1.10 + 1.1.10 stable @@ -23,8 +23,7 @@ GPL License -pass variables to stream_select() to fullfill strict standards. -make all functions in Indexer.php static +catch exception in execWithTimeout() @@ -235,5 +234,22 @@ add command for indexing postѕcript files set last parameter of stream_select() to 200000 micro sec. in case the timeout in sec. is set to 0 + + 2016-04-28 + + + 1.1.9 + 1.1.7 + + + stable + stable + + GPL License + +pass variables to stream_select() to fullfill strict standards. +make all functions in Indexer.php static + + From fab21da3d8ef8b050789642dcd70ce40edcd5dcf Mon Sep 17 00:00:00 2001 From: Uwe Steinmann Date: Wed, 1 Mar 2017 15:56:12 +0100 Subject: [PATCH 03/12] new version, catch exception in execWithTimeout() --- .../SQLiteFTS/IndexedDocument.php | 9 ++++--- SeedDMS_SQLiteFTS/package.xml | 26 +++++++++++++++---- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/SeedDMS_SQLiteFTS/SQLiteFTS/IndexedDocument.php b/SeedDMS_SQLiteFTS/SQLiteFTS/IndexedDocument.php index ed664da77..d7c178b0f 100644 --- a/SeedDMS_SQLiteFTS/SQLiteFTS/IndexedDocument.php +++ b/SeedDMS_SQLiteFTS/SQLiteFTS/IndexedDocument.php @@ -133,9 +133,12 @@ class SeedDMS_SQLiteFTS_IndexedDocument extends SeedDMS_SQLiteFTS_Document { $mimetype = $version->getMimeType(); if(isset($_convcmd[$mimetype])) { $cmd = sprintf($_convcmd[$mimetype], $path); - $content = self::execWithTimeout($cmd, $timeout); - if($content) { - $this->addField('content', $content, 'unstored'); + try { + $content = self::execWithTimeout($cmd, $timeout); + if($content) { + $this->addField('content', $content, 'unstored'); + } + } catch (Exception $e) { } } } diff --git a/SeedDMS_SQLiteFTS/package.xml b/SeedDMS_SQLiteFTS/package.xml index 9cb06a0d0..62167bb8e 100644 --- a/SeedDMS_SQLiteFTS/package.xml +++ b/SeedDMS_SQLiteFTS/package.xml @@ -11,11 +11,11 @@ uwe@steinmann.cx yes - 2016-03-29 - + 2017-03-01 + - 1.0.6 - 1.0.1 + 1.0.7 + 1.0.7 stable @@ -23,7 +23,7 @@ GPL License -fix calculation of timeout (see bug #269) +catch exception in execWithTimeout() @@ -162,5 +162,21 @@ make it work with sqlite3 < 3.8.0 set last parameter of stream_select() to 200000 micro sec. in case the timeout in sec. is set to 0 + + 2016-03-29 + + + 1.0.6 + 1.0.1 + + + stable + stable + + GPL License + +fix calculation of timeout (see bug #269) + + From 2cfc080382efbc1b529142c87c77c25dd857b0a5 Mon Sep 17 00:00:00 2001 From: Uwe Steinmann Date: Wed, 1 Mar 2017 16:18:27 +0100 Subject: [PATCH 04/12] better handling of timeout --- SeedDMS_Lucene/Lucene/IndexedDocument.php | 15 ++++++++------- SeedDMS_SQLiteFTS/SQLiteFTS/IndexedDocument.php | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/SeedDMS_Lucene/Lucene/IndexedDocument.php b/SeedDMS_Lucene/Lucene/IndexedDocument.php index 027536dae..54dbb2676 100644 --- a/SeedDMS_Lucene/Lucene/IndexedDocument.php +++ b/SeedDMS_Lucene/Lucene/IndexedDocument.php @@ -31,24 +31,25 @@ class SeedDMS_Lucene_IndexedDocument extends Zend_Search_Lucene_Document { 2 => array("pipe", "w") ); $pipes = array(); - - $timeout += time(); + + $timeout += time(); $process = proc_open($cmd, $descriptorspec, $pipes); if (!is_resource($process)) { throw new Exception("proc_open failed on: " . $cmd); } $output = ''; + $timeleft = $timeout - time(); + $read = array($pipes[1]); + $write = NULL; + $exeptions = NULL; do { - $timeleft = $timeout - time(); - $read = array($pipes[1]); - $write = NULL; - $exeptions = NULL; stream_select($read, $write, $exeptions, $timeleft, 200000); if (!empty($read)) { $output .= fread($pipes[1], 8192); - } + } + $timeleft = $timeout - time(); } while (!feof($pipes[1]) && $timeleft > 0); if ($timeleft <= 0) { diff --git a/SeedDMS_SQLiteFTS/SQLiteFTS/IndexedDocument.php b/SeedDMS_SQLiteFTS/SQLiteFTS/IndexedDocument.php index d7c178b0f..5d2914074 100644 --- a/SeedDMS_SQLiteFTS/SQLiteFTS/IndexedDocument.php +++ b/SeedDMS_SQLiteFTS/SQLiteFTS/IndexedDocument.php @@ -37,7 +37,7 @@ class SeedDMS_SQLiteFTS_IndexedDocument extends SeedDMS_SQLiteFTS_Document { ); $pipes = array(); - $timeout += time(); + $timeout += time(); $process = proc_open($cmd, $descriptorspec, $pipes); if (!is_resource($process)) { throw new Exception("proc_open failed on: " . $cmd); @@ -53,7 +53,7 @@ class SeedDMS_SQLiteFTS_IndexedDocument extends SeedDMS_SQLiteFTS_Document { if (!empty($read)) { $output .= fread($pipes[1], 8192); - } + } $timeleft = $timeout - time(); } while (!feof($pipes[1]) && $timeleft > 0); From b1df3b460b0bd08272f32bbf3ff77c21e83f21e8 Mon Sep 17 00:00:00 2001 From: Uwe Steinmann Date: Wed, 1 Mar 2017 16:59:58 +0100 Subject: [PATCH 05/12] update queue-bar after adding new tasks --- views/bootstrap/class.Indexer.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/views/bootstrap/class.Indexer.php b/views/bootstrap/class.Indexer.php index 91e9e2784..78e6bf93a 100644 --- a/views/bootstrap/class.Indexer.php +++ b/views/bootstrap/class.Indexer.php @@ -62,6 +62,7 @@ function check_queue() { data: {command: 'indexdocument', id: docid}, beforeSend: function() { queue_count++; // Add request to the counter + $('.queue-bar').css('width', (queue_count*100/MAX_REQUESTS)+'%'); }, error: function(xhr, textstatus) { noty({ @@ -90,10 +91,10 @@ function check_queue() { } }, complete: function(xhr, textstatus) { + queue_count--; // Substract request to the counter $('.queue-bar').css('width', (queue_count*100/MAX_REQUESTS)+'%'); $('.total-bar').css('width', (100 - (funcArray.length+queue_count)*100/docstoindex)+'%'); $('.total-bar').text(Math.round(100 - (funcArray.length+queue_count)*100/docstoindex)+' %'); - queue_count--; // Substract request to the counter if(funcArray.length+queue_count == 0) $('.total-bar').addClass('bar-success'); } From ebb0c71a3fa6d06dc5e38cf2e57d00ca104f4fd5 Mon Sep 17 00:00:00 2001 From: Uwe Steinmann Date: Thu, 2 Mar 2017 06:58:02 +0100 Subject: [PATCH 06/12] allow '*' in mimetype --- op/op.Settings.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/op/op.Settings.php b/op/op.Settings.php index 0ff28d117..a82d7b0c7 100644 --- a/op/op.Settings.php +++ b/op/op.Settings.php @@ -188,7 +188,7 @@ if ($action == "saveSettings") // SETTINGS - ADVANCED - INDEX CMD $settings->_converters['fulltext'] = $_POST["converters"]; - $newmimetype = preg_replace('#[^A-Za-z0-9_/+.-]+#', '', $_POST["converters_newmimetype"]); + $newmimetype = preg_replace('#[^A-Za-z0-9_/+.-*]+#', '', $_POST["converters_newmimetype"]); if($newmimetype && trim($_POST["converters_newcmd"])) { $settings->_converters['fulltext'][$newmimetype] = trim($_POST["converters_newcmd"]); } From 787fb4777b27be55b71430496d0eb334642a6eb7 Mon Sep 17 00:00:00 2001 From: Uwe Steinmann Date: Thu, 2 Mar 2017 07:16:26 +0100 Subject: [PATCH 07/12] new version 1.2.2 --- SeedDMS_Preview/package.xml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/SeedDMS_Preview/package.xml b/SeedDMS_Preview/package.xml index 6422fe77c..df8b0a5d7 100644 --- a/SeedDMS_Preview/package.xml +++ b/SeedDMS_Preview/package.xml @@ -11,10 +11,10 @@ uwe@steinmann.cx yes - 2016-11-15 - + 2017-03-02 + - 1.2.1 + 1.2.2 1.2.0 @@ -23,7 +23,7 @@ GPL License -setConverters() overrides exiting converters +commands can be set for mimetypes 'xxxx/*' and '*' @@ -254,5 +254,21 @@ check if cache dir exists before deleting it in deleteDocumentPreviews() add new previewer which converts document to pdf instead of png + + 2016-11-15 + + + 1.2.1 + 1.2.0 + + + stable + stable + + GPL License + +setConverters() overrides exiting converters + + From d563333f05dd5710a07be56d1e51fd5078108996 Mon Sep 17 00:00:00 2001 From: Uwe Steinmann Date: Thu, 2 Mar 2017 07:16:42 +0100 Subject: [PATCH 08/12] commands can be assigned to mimetype 'xxx/*' and '*' --- SeedDMS_Preview/Preview/Previewer.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/SeedDMS_Preview/Preview/Previewer.php b/SeedDMS_Preview/Preview/Previewer.php index 8c9b67713..aebfe7bbb 100644 --- a/SeedDMS_Preview/Preview/Previewer.php +++ b/SeedDMS_Preview/Preview/Previewer.php @@ -106,9 +106,15 @@ class SeedDMS_Preview_Previewer extends SeedDMS_Preview_Base { $target = $this->previewDir.$dir.md5($infile).'-'.$width; if($target != '' && (!file_exists($target.'.png') || filectime($target.'.png') < filectime($infile))) { $cmd = ''; + $mimeparts = explode('/', $mimetype, 2); if(isset($this->converters[$mimetype])) { $cmd = str_replace(array('%w', '%f', '%o'), array($width, $infile, $target.'.png'), $this->converters[$mimetype]); + } elseif(isset($this->converters[$mimeparts[0].'/*'])) { + $cmd = str_replace(array('%w', '%f', '%o'), array($width, $infile, $target.'.png'), $this->converters[$mimeparts[0].'/*']); + } elseif(isset($this->converters['*'])) { + $cmd = str_replace(array('%w', '%f', '%o'), array($width, $infile, $target.'.png'), $this->converters['*']); } + /* switch($mimetype) { case "image/png": @@ -131,7 +137,6 @@ class SeedDMS_Preview_Previewer extends SeedDMS_Preview_Base { } */ if($cmd) { - //exec($cmd); try { self::execWithTimeout($cmd, $this->timeout); } catch(Exception $e) { From fbfa2a21323c7ad33389acf7310f4e2e11f5b8ff Mon Sep 17 00:00:00 2001 From: Uwe Steinmann Date: Thu, 2 Mar 2017 07:17:42 +0100 Subject: [PATCH 09/12] allow '*' in mimetype for preview command --- op/op.Settings.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/op/op.Settings.php b/op/op.Settings.php index e549fd26a..238a8a807 100644 --- a/op/op.Settings.php +++ b/op/op.Settings.php @@ -183,7 +183,7 @@ if ($action == "saveSettings") // SETTINGS - ADVANCED - INDEX CMD $settings->_converters['fulltext'] = $_POST["converters"]; - $newmimetype = preg_replace('#[^A-Za-z0-9_/+.-]+#', '', $_POST["converters_newmimetype"]); + $newmimetype = preg_replace('#[^A-Za-z0-9_/+.-*]+#', '', $_POST["converters_newmimetype"]); if($newmimetype && trim($_POST["converters_newcmd"])) { $settings->_converters['fulltext'][$newmimetype] = trim($_POST["converters_newcmd"]); } From 5cbaf0e7a83f5529a14e3b2befc428fc5dc4e9f9 Mon Sep 17 00:00:00 2001 From: Uwe Steinmann Date: Thu, 2 Mar 2017 07:59:49 +0100 Subject: [PATCH 10/12] replace %m in converter cmd with mimetype --- SeedDMS_Preview/Preview/Previewer.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SeedDMS_Preview/Preview/Previewer.php b/SeedDMS_Preview/Preview/Previewer.php index aebfe7bbb..970fcdb95 100644 --- a/SeedDMS_Preview/Preview/Previewer.php +++ b/SeedDMS_Preview/Preview/Previewer.php @@ -108,11 +108,11 @@ class SeedDMS_Preview_Previewer extends SeedDMS_Preview_Base { $cmd = ''; $mimeparts = explode('/', $mimetype, 2); if(isset($this->converters[$mimetype])) { - $cmd = str_replace(array('%w', '%f', '%o'), array($width, $infile, $target.'.png'), $this->converters[$mimetype]); + $cmd = str_replace(array('%w', '%f', '%o', '%m'), array($width, $infile, $target.'.png', $mimetype), $this->converters[$mimetype]); } elseif(isset($this->converters[$mimeparts[0].'/*'])) { - $cmd = str_replace(array('%w', '%f', '%o'), array($width, $infile, $target.'.png'), $this->converters[$mimeparts[0].'/*']); + $cmd = str_replace(array('%w', '%f', '%o', '%m'), array($width, $infile, $target.'.png', $mimetype), $this->converters[$mimeparts[0].'/*']); } elseif(isset($this->converters['*'])) { - $cmd = str_replace(array('%w', '%f', '%o'), array($width, $infile, $target.'.png'), $this->converters['*']); + $cmd = str_replace(array('%w', '%f', '%o', '%m'), array($width, $infile, $target.'.png', $mimetype), $this->converters['*']); } /* From e5ee075a2dc0f53d8acae042500826c7b93b2cc5 Mon Sep 17 00:00:00 2001 From: Uwe Steinmann Date: Thu, 2 Mar 2017 08:00:28 +0100 Subject: [PATCH 11/12] allow 'xxx/*' and '*' as wild card mimetypes --- SeedDMS_Preview/Preview/PdfPreviewer.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/SeedDMS_Preview/Preview/PdfPreviewer.php b/SeedDMS_Preview/Preview/PdfPreviewer.php index 10005afc7..f3c618b9f 100644 --- a/SeedDMS_Preview/Preview/PdfPreviewer.php +++ b/SeedDMS_Preview/Preview/PdfPreviewer.php @@ -89,8 +89,13 @@ class SeedDMS_Preview_PdfPreviewer extends SeedDMS_Preview_Base { $target = $this->previewDir.$dir.md5($infile); if($target != '' && (!file_exists($target.'.pdf') || filectime($target.'.pdf') < filectime($infile))) { $cmd = ''; + $mimeparts = explode('/', $mimetype, 2); if(isset($this->converters[$mimetype])) { - $cmd = str_replace(array('%f', '%o'), array($infile, $target.'.pdf'), $this->converters[$mimetype]); + $cmd = str_replace(array('%f', '%o', '%m'), array($infile, $target.'.pdf', $mimetype), $this->converters[$mimetype]); + } elseif(isset($this->converters[$mimeparts[0].'/*'])) { + $cmd = str_replace(array('%f', '%o', '%m'), array($infile, $target.'.pdf', $mimetype), $this->converters[$mimeparts[0].'/*']); + } elseif(isset($this->converters['*'])) { + $cmd = str_replace(array('%f', '%o', '%m'), array($infile, $target.'.pdf', $mimetype), $this->converters['*']); } if($cmd) { try { From 9e1000506d523a5fc297716a3dea9debfd523068 Mon Sep 17 00:00:00 2001 From: Uwe Steinmann Date: Thu, 2 Mar 2017 08:21:19 +0100 Subject: [PATCH 12/12] add note for 1.2.2 --- SeedDMS_Preview/package.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/SeedDMS_Preview/package.xml b/SeedDMS_Preview/package.xml index df8b0a5d7..71eb627ba 100644 --- a/SeedDMS_Preview/package.xml +++ b/SeedDMS_Preview/package.xml @@ -24,6 +24,7 @@ GPL License commands can be set for mimetypes 'xxxx/*' and '*' +pass mimetype as parameter '%m' to converter