You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2016/11/30 21:28:55 UTC
tika git commit: TIKA-2187 -- change default behavior in experimental
.docx parser to ignore deleted text. Allow configuration of including or not
including text from .doc files.
Repository: tika
Updated Branches:
refs/heads/sax_docx d19e4725f -> fe20ecd83
TIKA-2187 -- change default behavior in experimental .docx parser to ignore deleted text. Allow configuration of including or not including text from .doc files.
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/fe20ecd8
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/fe20ecd8
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/fe20ecd8
Branch: refs/heads/sax_docx
Commit: fe20ecd83ea43e5ec6ad0e9fded9d803cb011251
Parents: d19e472
Author: tballison <ta...@mitre.org>
Authored: Wed Nov 30 16:28:47 2016 -0500
Committer: tballison <ta...@mitre.org>
Committed: Wed Nov 30 16:28:47 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 5 ++++-
.../tika/parser/microsoft/OfficeParserConfig.java | 2 +-
.../tika/parser/microsoft/WordExtractor.java | 8 +++++++-
.../tika/parser/microsoft/WordParserTest.java | 14 ++++++++++++++
.../microsoft/ooxml/xwpf/SXWPFExtractorTest.java | 8 ++++----
.../ooxml/xwpf/ml2006/Word2006MLParserTest.java | 8 ++++----
.../resources/test-documents/testWORD_2006ml.doc | Bin 0 -> 265728 bytes
7 files changed, 34 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/fe20ecd8/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2462e1a..4d2c573 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,7 +1,10 @@
Release 1.15 - ??
+ * Change default behavior in experimental .docx parser to ignore
+ deleted text to align with .doc (TIKA-2187).
+
* Added experimental SAX parser for .docx files. To select this parser,
- set useSAXDocxExtractor(true) on OfficeConfig (TIKA-1321).
+ set useSAXDocxExtractor(true) on OfficeParserConfig (TIKA-1321).
* Change default behavior to parse embedded documents even if the user
forgets to specify a Parser.class in the ParseContext (TIKA-2096).
http://git-wip-us.apache.org/repos/asf/tika/blob/fe20ecd8/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java
index 55f4673..f3cdbfe 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java
@@ -21,7 +21,7 @@ import java.io.Serializable;
public class OfficeParserConfig implements Serializable {
- private boolean includeDeletedContent = true;
+ private boolean includeDeletedContent = false;
private boolean includeMoveFromContent = false;
private boolean useSAXDocxExtractor = false;
http://git-wip-us.apache.org/repos/asf/tika/blob/fe20ecd8/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java
index 54ba55b..6fd8f8e 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java
@@ -78,6 +78,7 @@ public class WordExtractor extends AbstractPOIFSExtractor {
fixedParagraphStyles.put("HTML Preformatted", new TagAndStyle("pre", null));
}
+ private final boolean extractDeletedContent;
// True if we are currently in the named style tag:
private boolean curStrikeThrough;
private boolean curBold;
@@ -88,6 +89,7 @@ public class WordExtractor extends AbstractPOIFSExtractor {
public WordExtractor(ParseContext context, Metadata metadata) {
super(context);
this.metadata = metadata;
+ extractDeletedContent = context.get(OfficeParserConfig.class).getIncludeDeletedContent();
}
private static int countParagraphs(Range... ranges) {
@@ -654,7 +656,11 @@ public class WordExtractor extends AbstractPOIFSExtractor {
* @return true if character run should be included in extraction.
*/
private boolean isRendered(final CharacterRun cr) {
- return cr == null || !cr.isMarkedDeleted();
+ if (cr == null) {
+ return false;
+ }
+ return !cr.isMarkedDeleted() ||
+ (cr.isMarkedDeleted() && extractDeletedContent);
}
public static class TagAndStyle {
http://git-wip-us.apache.org/repos/asf/tika/blob/fe20ecd8/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java
index bfb7ca1..c7808d0 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java
@@ -534,5 +534,19 @@ public class WordParserTest extends TikaTest {
List<Metadata> metadataList = getRecursiveMetadata("testWORD_macros.doc");
assertContainsAtLeast(minExpected, metadataList);
}
+
+ @Test
+ public void testDeleted() throws Exception {
+ //test classic behavior
+ assertNotContained("frog", getXML("testWORD_2006ml.doc").xml);
+
+ //now test inclusion of deleted text
+ ParseContext context = new ParseContext();
+ OfficeParserConfig officeParserConfig = new OfficeParserConfig();
+ officeParserConfig.setIncludeDeletedContent(true);
+ context.set(OfficeParserConfig.class, officeParserConfig);
+ XMLResult r = getXML("testWORD_2006ml.doc", context);
+ assertContains("frog", r.xml);
+ }
}
http://git-wip-us.apache.org/repos/asf/tika/blob/fe20ecd8/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/SXWPFExtractorTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/SXWPFExtractorTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/SXWPFExtractorTest.java
index f5512cb..06f0eed 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/SXWPFExtractorTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/SXWPFExtractorTest.java
@@ -145,8 +145,8 @@ public class SXWPFExtractorTest extends TikaTest {
assertContains("Odd page footer", content);
- //test default includes deleted
- assertContains("frog", content);
+ //test default does not include deleted
+ assertNotContained("frog", content);
assertContains("Mattmann", content);
@@ -159,13 +159,13 @@ public class SXWPFExtractorTest extends TikaTest {
public void testSkipDeleted() throws Exception {
ParseContext pc = new ParseContext();
OfficeParserConfig officeParserConfig = new OfficeParserConfig();
- officeParserConfig.setIncludeDeletedContent(false);
+ officeParserConfig.setIncludeDeletedContent(true);
officeParserConfig.setUseSAXDocxExtractor(true);
officeParserConfig.setIncludeMoveFromContent(true);
pc.set(OfficeParserConfig.class, officeParserConfig);
XMLResult r = getXML("testWORD_2006ml.docx", pc);
- assertNotContained("frog", r.xml);
+ assertContains("frog", r.xml);
assertContainsCount("Second paragraph", r.xml, 2);
}
http://git-wip-us.apache.org/repos/asf/tika/blob/fe20ecd8/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParserTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParserTest.java
index 79f1890..c77b0fa 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParserTest.java
@@ -140,8 +140,8 @@ public class Word2006MLParserTest extends TikaTest {
assertContains("Odd page footer", content);
- //test default includes deleted
- assertContains("frog", content);
+ //test default ignores deleted
+ assertNotContained("frog", content);
assertContains("Mattmann", content);
@@ -157,12 +157,12 @@ public class Word2006MLParserTest extends TikaTest {
public void testSkipDeletedAndMoveFrom() throws Exception {
ParseContext pc = new ParseContext();
OfficeParserConfig officeParserConfig = new OfficeParserConfig();
- officeParserConfig.setIncludeDeletedContent(false);
+ officeParserConfig.setIncludeDeletedContent(true);
officeParserConfig.setIncludeMoveFromContent(true);
pc.set(OfficeParserConfig.class, officeParserConfig);
XMLResult r = getXML("testWORD_2006ml.xml", pc);
- assertNotContained("frog", r.xml);
+ assertContains("frog", r.xml);
assertContainsCount("Second paragraph", r.xml, 2);
}
http://git-wip-us.apache.org/repos/asf/tika/blob/fe20ecd8/tika-parsers/src/test/resources/test-documents/testWORD_2006ml.doc
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/resources/test-documents/testWORD_2006ml.doc b/tika-parsers/src/test/resources/test-documents/testWORD_2006ml.doc
new file mode 100644
index 0000000..c8f509a
Binary files /dev/null and b/tika-parsers/src/test/resources/test-documents/testWORD_2006ml.doc differ