You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2020/04/14 15:23:14 UTC
[tika] branch branch_1x updated (ca6bf65 -> f51ae0a)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git.
from ca6bf65 TIKA-3090 -- extract doc security from ooxml
new 86cdbb1 add optional postgres dialect for comparison reports; improve initialization of ref tables in tika-eval
new f51ae0a TIKA-3091 prevent npe in PDFParserConfig by initializing three parameters with default values.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../tika/eval/batch/EvalConsumerBuilder.java | 37 +++++++++-------
...rison-reports.xml => comparison-reports-pg.xml} | 49 +++++++++++-----------
.../src/main/resources/comparison-reports.xml | 3 +-
.../java/org/apache/tika/parser/pdf/PDFParser.java | 11 +++++
.../apache/tika/parser/pdf/PDFParserConfig.java | 28 ++++++++++++-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 10 +++++
6 files changed, 97 insertions(+), 41 deletions(-)
copy tika-eval/src/main/resources/{comparison-reports.xml => comparison-reports-pg.xml} (97%)
[tika] 02/02: TIKA-3091 prevent npe in PDFParserConfig by
initializing three parameters with default values.
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit f51ae0aef10a052308d6830fcefad77347bc5ebd
Author: tallison <ta...@apache.org>
AuthorDate: Tue Apr 14 11:09:52 2020 -0400
TIKA-3091 prevent npe in PDFParserConfig by initializing
three parameters with default values.
# Conflicts:
# tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
---
.../java/org/apache/tika/parser/pdf/PDFParser.java | 11 +++++++++
.../apache/tika/parser/pdf/PDFParserConfig.java | 28 ++++++++++++++++++++--
.../org/apache/tika/parser/pdf/PDFParserTest.java | 10 ++++++++
3 files changed, 47 insertions(+), 2 deletions(-)
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
index 2e637e0..6d8b5b1 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
@@ -524,6 +524,17 @@ public class PDFParser extends AbstractParser implements Initializable {
}
@Field
+ void setAverageCharTolerance(float averageCharTolerance) {
+ defaultConfig.setAverageCharTolerance(averageCharTolerance);
+ }
+
+ @Field
+ void setSpacingTolerance(float spacingTolerance) {
+ defaultConfig.setSpacingTolerance(spacingTolerance);
+ }
+
+
+ @Field
void setCatchIntermediateExceptions(boolean catchIntermediateExceptions) {
defaultConfig.setCatchIntermediateIOExceptions(catchIntermediateExceptions);
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
index b5d6824..da8b309 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
@@ -114,10 +114,16 @@ public class PDFParserConfig implements Serializable {
private boolean extractMarkedContent = false;
//The character width-based tolerance value used to estimate where spaces in text should be added
- private Float averageCharTolerance;
+ //Default taken from PDFBox.
+ private Float averageCharTolerance = 0.5f;
//The space width-based tolerance value used to estimate where spaces in text should be added
- private Float spacingTolerance;
+ //Default taken from PDFBox.
+ private Float spacingTolerance = 0.3f;
+
+ // The multiplication factor for line height to decide when a new paragraph starts.
+ //Default taken from PDFBox.
+ private Float dropThreshold = 2.5f;
//If the PDF has an XFA element, process only that and skip extracting
//content from elsewhere in the document.
@@ -238,6 +244,10 @@ public class PDFParserConfig implements Serializable {
setSetKCMS(getBooleanProp(props.getProperty("setKCMS"), false));
+ setAverageCharTolerance(getFloatProp(props.getProperty("averageCharTolerance"), averageCharTolerance));
+ setSpacingTolerance(getFloatProp(props.getProperty("spacingTolerance"), spacingTolerance));
+ setDropThreshold(getFloatProp(props.getProperty("dropThreshold"), dropThreshold));
+
boolean checkExtractAccessPermission = getBooleanProp(props.getProperty("checkExtractAccessPermission"), false);
boolean allowExtractionForAccessibility = getBooleanProp(props.getProperty("allowExtractionForAccessibility"), true);
@@ -287,6 +297,9 @@ public class PDFParserConfig implements Serializable {
if (getSpacingTolerance() != null) {
pdf2XHTML.setSpacingTolerance(getSpacingTolerance());
}
+ if (getDropThreshold() != null) {
+ pdf2XHTML.setDropThreshold(dropThreshold);
+ }
pdf2XHTML.setSuppressDuplicateOverlappingText(getSuppressDuplicateOverlappingText());
}
@@ -513,6 +526,14 @@ public class PDFParserConfig implements Serializable {
this.spacingTolerance = spacingTolerance;
}
+ public Float getDropThreshold() {
+ return dropThreshold;
+ }
+
+ public void setDropThreshold(float dropThreshold) {
+ this.dropThreshold = dropThreshold;
+ }
+
public AccessChecker getAccessChecker() {
return accessChecker;
}
@@ -824,6 +845,7 @@ public class PDFParserConfig implements Serializable {
if (getCatchIntermediateIOExceptions() != config.getCatchIntermediateIOExceptions()) return false;
if (!getAverageCharTolerance().equals(config.getAverageCharTolerance())) return false;
if (!getSpacingTolerance().equals(config.getSpacingTolerance())) return false;
+ if (!getDropThreshold().equals(config.getDropThreshold())) return false;
if (!getOcrStrategy().equals(config.getOcrStrategy())) return false;
if (getOcrImageType() != config.getOcrImageType()) return false;
if (!getOcrImageFormatName().equals(config.getOcrImageFormatName())) return false;
@@ -844,6 +866,7 @@ public class PDFParserConfig implements Serializable {
result = 31 * result + (getExtractUniqueInlineImagesOnly() ? 1 : 0);
result = 31 * result + getAverageCharTolerance().hashCode();
result = 31 * result + getSpacingTolerance().hashCode();
+ result = 31 * result + getDropThreshold().hashCode();
result = 31 * result + (getIfXFAExtractOnlyXFA() ? 1 : 0);
result = 31 * result + ocrStrategy.hashCode();
result = 31 * result + getOcrDPI();
@@ -869,6 +892,7 @@ public class PDFParserConfig implements Serializable {
", extractUniqueInlineImagesOnly=" + extractUniqueInlineImagesOnly +
", averageCharTolerance=" + averageCharTolerance +
", spacingTolerance=" + spacingTolerance +
+ ", dropThreshold=" + dropThreshold +
", ifXFAExtractOnlyXFA=" + ifXFAExtractOnlyXFA +
", ocrStrategy=" + ocrStrategy +
", ocrDPI=" + ocrDPI +
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index edcd513..4e2e3c5 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -1528,6 +1528,16 @@ public class PDFParserTest extends TikaTest {
}
+ @Test
+ public void testNPEInPDFParserConfig() {
+ //TIKA-3091
+ PDFParserConfig config = new PDFParserConfig();
+ //don't care about values; want to make sure no NPE is thrown
+ String txt = config.toString();
+ config.hashCode();
+ config.equals(new PDFParserConfig());
+ }
+
@Test //TIKA-3041
@Ignore("turn back on if we add file from PDFBOX-52")
public void testPDFBox52() throws Exception {
[tika] 01/02: add optional postgres dialect for comparison reports;
improve initialization of ref tables in tika-eval
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 86cdbb11a53e040afbce4bbc91d3d1b95efa8f4c
Author: tallison <ta...@apache.org>
AuthorDate: Tue Apr 14 10:57:25 2020 -0400
add optional postgres dialect for comparison reports;
improve initialization of ref tables in tika-eval
---
.../tika/eval/batch/EvalConsumerBuilder.java | 37 +++++++++-------
...rison-reports.xml => comparison-reports-pg.xml} | 49 +++++++++++-----------
.../src/main/resources/comparison-reports.xml | 3 +-
3 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
index 694b05e..b50d4a1 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
@@ -19,12 +19,11 @@ package org.apache.tika.eval.batch;
import java.io.IOException;
import java.sql.Connection;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
@@ -41,9 +40,13 @@ import org.apache.tika.eval.io.ExtractReader;
import org.apache.tika.eval.io.ExtractReaderException;
import org.apache.tika.eval.io.IDBWriter;
import org.apache.tika.util.PropsUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public abstract class EvalConsumerBuilder {
- private AtomicInteger count = new AtomicInteger(0);
+
+ private static final Logger LOG = LoggerFactory.getLogger(EvalConsumerBuilder.class);
+
protected ArrayBlockingQueue<FileResource> queue;
Map<String, String> localAttrs;
JDBCUtil dbUtil;
@@ -75,7 +78,7 @@ public abstract class EvalConsumerBuilder {
//step 3. create mime buffer
this.mimeBuffer = new MimeBuffer(dbUtil.getConnection(), TikaConfig.getDefaultConfig());
- //step 4. populate the reference tabless
+ //step 4. populate the reference tables
populateRefTables();
return mimeBuffer;
@@ -100,21 +103,27 @@ public abstract class EvalConsumerBuilder {
protected abstract void addErrorLogTablePairs(DBConsumersManager manager);
public void populateRefTables() throws IOException, SQLException {
- //test for one ref table. If it exists, don't populate ref tables
- //TODO: test one at a time
- boolean tableExists = false;
+ boolean refTablesPopulated = true;
try (Connection connection = dbUtil.getConnection()) {
- Set<String> tables = dbUtil.getTables(connection);
- if (tables.contains(
- AbstractProfiler.REF_PARSE_ERROR_TYPES.getName().toLowerCase(Locale.US)
- )) {
- tableExists = true;
+ for (TableInfo tableInfo : getRefTableInfos()) {
+ int rows = 0;
+ try (ResultSet rs = connection.createStatement().executeQuery("select * from "+
+ tableInfo.getName())) {
+ while (rs.next()) {
+ rows++;
+ }
+ }
+ if (rows == 0) {
+ refTablesPopulated = false;
+ break;
+ }
+
}
} catch (SQLException e) {
//swallow
}
-
- if (tableExists) {
+ if (refTablesPopulated) {
+ LOG.info("ref tables are already populated");
return;
}
diff --git a/tika-eval/src/main/resources/comparison-reports.xml b/tika-eval/src/main/resources/comparison-reports-pg.xml
similarity index 97%
copy from tika-eval/src/main/resources/comparison-reports.xml
copy to tika-eval/src/main/resources/comparison-reports-pg.xml
index e84454a..5bcf88e 100644
--- a/tika-eval/src/main/resources/comparison-reports.xml
+++ b/tika-eval/src/main/resources/comparison-reports-pg.xml
@@ -25,24 +25,24 @@
<before>
<sql>drop table if exists md5_multiples_tmp_a</sql>
- <sql>create table md5_multiples_tmp_a (MD5 char(32), cnt int)
+ <sql>create table md5_multiples_tmp_a (MD5, cnt)
as
- select md5, count(1) cnt
+ select md5, count(1) as cnt
from profiles_a
where md5 is not null
group by md5
- having cnt > 1
+ having count(1) > 1
order by cnt desc
</sql>
<sql>drop table if exists md5_multiples_tmp_b</sql>
- <sql>create table md5_multiples_tmp_b (MD5 char(32), cnt int)
+ <sql>create table md5_multiples_tmp_b (MD5, cnt)
as
select md5, count(1) cnt
from profiles_b
where md5 is not null
group by md5
- having cnt > 1
+ having count(1) > 1
order by cnt desc
</sql>
<!-- build mime indexes -->
@@ -133,12 +133,12 @@
create table token_counts_compared
(mime_id_a integer,
mime_id_b integer,
- num_tokens_a long default 0,
- num_tokens_b long default 0,
- num_alphabetic_tokens_a long default 0,
- num_alphabetic_tokens_b long default 0,
- num_common_tokens_a long default 0,
- num_common_tokens_b long default 0
+ num_tokens_a bigint default 0,
+ num_tokens_b bigint default 0,
+ num_alphabetic_tokens_a bigint default 0,
+ num_alphabetic_tokens_b bigint default 0,
+ num_common_tokens_a bigint default 0,
+ num_common_tokens_b bigint default 0
);
</sql>
<sql>
@@ -674,7 +674,7 @@
mime_id_b integer,
total_a bigint,
total_b bigint,
- prcnt_increase double
+ prcnt_increase double precision
);
</sql>
<sql>
@@ -704,7 +704,7 @@
group by mime_id_a, mime_id_b)
</sql>
<sql>
- update parse_time_compared ptc set prcnt_increase=(
+ update parse_time_compared ptc set prcnt_increase=(100.0 *
cast(total_b as decimal)/cast(total_a as decimal))
where total_a > 0;
</sql>
@@ -913,7 +913,7 @@
join mimes m on m.mime_id=p.mime_id
join ref_parse_exception_types r on
r.parse_exception_id=e.parse_exception_id
- group by p.mime_id, parse_exception_description
+ group by m.mime_string, parse_exception_description
order by MIME_TYPE, EXCEPTION_TYPE
</sql>
</report>
@@ -932,7 +932,7 @@
join mimes m on m.mime_id=p.mime_id
join ref_parse_exception_types r on
r.parse_exception_id=e.parse_exception_id
- group by p.mime_id, parse_exception_description
+ group by m.mime_string, parse_exception_description
order by MIME_TYPE, EXCEPTION_TYPE
</sql>
</report>
@@ -963,7 +963,7 @@
left join exceptions_a ea on ca.id = ea.id
where eb.orig_stack_trace is not null
and ea.orig_stack_trace is null
- order by ca.num_common_tokens - ifnull(cb.num_common_tokens,0) desc
+ order by ca.num_common_tokens - coalesce(cb.num_common_tokens,0) desc
</sql>
</report>
@@ -1004,7 +1004,7 @@
from exceptions_a ea
left join exceptions_b eb on ea.id = eb.id
join profiles_a pa on pa.id=ea.id
- join profiles_b pb on pb.id=pa.id //this ensures that files were actually processed in both runs
+ join profiles_b pb on pb.id=pa.id --this ensures that files were actually processed in both runs
join containers c on pa.container_id=c.container_id
join mimes ma on ma.mime_id=pa.mime_id
join mimes mb on mb.mime_id=pb.mime_id
@@ -1211,7 +1211,7 @@
from exceptions_a e
join ref_parse_exception_types t on
t.parse_exception_id=e.parse_exception_id
- group by e.parse_exception_id
+ group by t.parse_exception_description
</sql>
</report>
<report reportName="parseExceptionTypesB"
@@ -1223,7 +1223,7 @@
from exceptions_b e
join ref_parse_exception_types t on
t.parse_exception_id=e.parse_exception_id
- group by e.parse_exception_id
+ group by t.parse_exception_description
</sql>
</report>
@@ -1245,8 +1245,8 @@
ca.num_common_tokens as NUM_COMMON_TOKENS_A,
cb.common_tokens_lang as COMMON_TOKENS_LANG_B,
cb.num_common_tokens as NUM_COMMON_TOKENS_B,
- ifnull(cb.num_common_tokens,0)-
- ifnull(ca.num_common_tokens, 0) as NUM_COMMON_TOKENS_DIFF_IN_B,
+ coalesce(cb.num_common_tokens,0)-
+ coalesce(ca.num_common_tokens, 0) as NUM_COMMON_TOKENS_DIFF_IN_B,
ca.top_n_tokens as TOP_N_TOKENS_A,
cb.top_n_tokens as TOP_N_TOKENS_B,
ca.unicode_char_blocks as UNICODE_CHAR_BLOCKS_A,
@@ -1295,8 +1295,8 @@
ca.num_common_tokens as NUM_COMMON_TOKENS_A,
cb.common_tokens_lang as COMMON_TOKENS_LANG_B,
cb.num_common_tokens as NUM_COMMON_TOKENS_B,
- ifnull(cb.num_common_tokens,0)-
- ifnull(ca.num_common_tokens, 0) as NUM_COMMON_TOKENS_DIFF_IN_B,
+ coalesce(cb.num_common_tokens,0)-
+ coalesce(ca.num_common_tokens, 0) as NUM_COMMON_TOKENS_DIFF_IN_B,
ca.top_n_tokens as TOP_N_TOKENS_A,
cb.top_n_tokens as TOP_N_TOKENS_B,
ca.unicode_char_blocks as UNICODE_CHAR_BLOCKS_A,
@@ -1332,7 +1332,7 @@
num_tokens_a, num_tokens_b,
num_alphabetic_tokens_a, num_alphabetic_tokens_b,
num_common_tokens_a, num_common_tokens_b,
- ifnull(num_common_tokens_b, 0)-ifnull(num_common_tokens_a, 0) as change_in_common_tokens_b
+ coalesce(num_common_tokens_b, 0)-coalesce(num_common_tokens_a, 0) as change_in_common_tokens_b
from token_counts_compared tcc
join mimes ma on tcc.mime_id_a = ma.mime_id
join mimes mb on tcc.mime_id_b = mb.mime_id
@@ -1722,6 +1722,7 @@
from parse_time_compared ptc
join mimes ma on ptc.mime_id_a=ma.mime_id
join mimes mb on ptc.mime_id_b=mb.mime_id
+ where TOTAL_A > 1000 AND TOTAL_B > 1000 -- only show comparisons if > a second
order by prcnt_increase desc
</sql>
</report>
diff --git a/tika-eval/src/main/resources/comparison-reports.xml b/tika-eval/src/main/resources/comparison-reports.xml
index e84454a..e23ec5e 100644
--- a/tika-eval/src/main/resources/comparison-reports.xml
+++ b/tika-eval/src/main/resources/comparison-reports.xml
@@ -704,7 +704,7 @@
group by mime_id_a, mime_id_b)
</sql>
<sql>
- update parse_time_compared ptc set prcnt_increase=(
+ update parse_time_compared ptc set prcnt_increase=(100.0 *
cast(total_b as decimal)/cast(total_a as decimal))
where total_a > 0;
</sql>
@@ -1722,6 +1722,7 @@
from parse_time_compared ptc
join mimes ma on ptc.mime_id_a=ma.mime_id
join mimes mb on ptc.mime_id_b=mb.mime_id
+ where TOTAL_A > 1000 AND TOTAL_B > 1000 -- only show comparisons if > a second
order by prcnt_increase desc
</sql>
</report>