You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2016/12/07 20:32:49 UTC
tika git commit: TIKA-2191 - step 6(?) add list numbering,
bookmarks and styles
Repository: tika
Updated Branches:
refs/heads/master 5425d02a1 -> 3ee9fd5bf
TIKA-2191 - step 6(?) add list numbering, bookmarks and styles
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/3ee9fd5b
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/3ee9fd5b
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/3ee9fd5b
Branch: refs/heads/master
Commit: 3ee9fd5bf3df913dc8d3cf8cf76da433bb7f9e17
Parents: 5425d02
Author: tballison <ta...@mitre.org>
Authored: Wed Dec 7 15:32:37 2016 -0500
Committer: tballison <ta...@mitre.org>
Committed: Wed Dec 7 15:32:37 2016 -0500
----------------------------------------------------------------------
.../ooxml/SXWPFWordExtractorDecorator.java | 32 +++---
.../ooxml/xwpf/XWPFDocumentXMLBodyHandler.java | 49 ++++++++-
.../ooxml/xwpf/XWPFEventBasedWordExtractor.java | 12 ++-
.../microsoft/ooxml/xwpf/XWPFNumberingShim.java | 37 +++++++
.../ooxml/xwpf/XWPFParagraphProperties.java | 56 ++++++++++
.../microsoft/ooxml/xwpf/XWPFStylesShim.java | 37 +++++++
.../ooxml/xwpf/XWPFTikaBodyPartHandler.java | 89 +++++++++++++--
.../ooxml/xwpf/ml2006/BodyPartHandler.java | 2 +-
.../microsoft/ooxml/SXWPFExtractorTest.java | 107 +++++++------------
.../ooxml/xwpf/ml2006/Word2006MLParserTest.java | 6 +-
10 files changed, 330 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/3ee9fd5b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
index 43fca3b..7c20a07 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
@@ -34,11 +34,14 @@ import org.apache.poi.openxml4j.opc.PackageRelationshipCollection;
import org.apache.poi.openxml4j.opc.internal.FileHelper;
import org.apache.poi.xwpf.usermodel.XWPFNumbering;
import org.apache.poi.xwpf.usermodel.XWPFRelation;
+import org.apache.poi.xwpf.usermodel.XWPFStyles;
import org.apache.tika.exception.TikaException;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.microsoft.OfficeParserConfig;
import org.apache.tika.parser.microsoft.ooxml.xwpf.XWPFDocumentXMLBodyHandler;
import org.apache.tika.parser.microsoft.ooxml.xwpf.XWPFEventBasedWordExtractor;
+import org.apache.tika.parser.microsoft.ooxml.xwpf.XWPFNumberingShim;
+import org.apache.tika.parser.microsoft.ooxml.xwpf.XWPFStylesShim;
import org.apache.tika.parser.microsoft.ooxml.xwpf.XWPFTikaBodyPartHandler;
import org.apache.tika.sax.EmbeddedContentHandler;
import org.apache.tika.sax.OfflineContentHandler;
@@ -111,8 +114,8 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
private void handleDocumentPart(PackagePart documentPart, XHTMLContentHandler xhtml) throws IOException, SAXException {
//load the numbering/list manager and styles from the main document part
XWPFNumbering numbering = loadNumbering(documentPart);
- XWPFListManager xwpfListManager = new XWPFListManager(numbering);
- //TODO: XWPFStyles styles = loadStyles(documentPart);
+ XWPFListManager listManager = new XWPFListManager(numbering);
+ XWPFStyles styles = loadStyles(documentPart);
//headers
try {
@@ -120,7 +123,7 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
if (headersPRC != null) {
for (int i = 0; i < headersPRC.size(); i++) {
PackagePart header = documentPart.getRelatedPart(headersPRC.getRelationship(i));
- handlePart(header, xwpfListManager, xhtml);
+ handlePart(header, styles, listManager, xhtml);
}
}
} catch (InvalidFormatException e) {
@@ -128,7 +131,7 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
}
//main document
- handlePart(documentPart, xwpfListManager, xhtml);
+ handlePart(documentPart, styles, listManager, xhtml);
//for now, just dump other components at end
for (XWPFRelation rel : new XWPFRelation[]{
@@ -142,7 +145,7 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
if (prc != null) {
for (int i = 0; i < prc.size(); i++) {
PackagePart packagePart = documentPart.getRelatedPart(prc.getRelationship(i));
- handlePart(packagePart, xwpfListManager, xhtml);
+ handlePart(packagePart, styles, listManager, xhtml);
}
}
} catch (InvalidFormatException e) {
@@ -151,8 +154,8 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
}
}
- private void handlePart(PackagePart packagePart,
- XWPFListManager xwpfListManager, XHTMLContentHandler xhtml) throws IOException, SAXException {
+ private void handlePart(PackagePart packagePart, XWPFStyles styles,
+ XWPFListManager listManager, XHTMLContentHandler xhtml) throws IOException, SAXException {
Map<String, String> linkedRelationships = loadLinkedRelationships(packagePart);
try (InputStream stream = packagePart.getInputStream()) {
@@ -160,7 +163,7 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
new CloseShieldInputStream(stream),
new OfflineContentHandler(new EmbeddedContentHandler(
new XWPFDocumentXMLBodyHandler(
- new XWPFTikaBodyPartHandler(xhtml, xwpfListManager,
+ new XWPFTikaBodyPartHandler(xhtml, styles, listManager,
context.get(OfficeParserConfig.class)), linkedRelationships))));
} catch (TikaException e) {
//swallow
@@ -213,7 +216,7 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
}
return linkedRelationships;
}
-/*
+
private XWPFStyles loadStyles(PackagePart packagePart) {
try {
PackageRelationshipCollection stylesParts =
@@ -223,11 +226,12 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
if (stylesRelationShip == null) {
return null;
}
- PackagePart stylesPart = opcPackage.getPart(stylesRelationShip);
+ PackagePart stylesPart = packagePart.getRelatedPart(stylesRelationShip);
if (stylesPart == null) {
return null;
}
- return new XWPFStyles(stylesPart);
+
+ return new XWPFStylesShim(stylesPart);
}
} catch (IOException|OpenXML4JException e) {
//swallow
@@ -235,7 +239,7 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
return null;
}
-*/
+
private XWPFNumbering loadNumbering(PackagePart packagePart) {
try {
PackageRelationshipCollection numberingParts = packagePart.getRelationshipsByType(XWPFRelation.NUMBERING.getRelation());
@@ -244,11 +248,11 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
if (numberingRelationShip == null) {
return null;
}
- PackagePart numberingPart = opcPackage.getPart(numberingRelationShip);
+ PackagePart numberingPart = packagePart.getRelatedPart(numberingRelationShip);
if (numberingPart == null) {
return null;
}
- return new XWPFNumbering(numberingPart);
+ return new XWPFNumberingShim(numberingPart);
}
} catch (IOException | OpenXML4JException e) {
//swallow
http://git-wip-us.apache.org/repos/asf/tika/blob/3ee9fd5b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFDocumentXMLBodyHandler.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFDocumentXMLBodyHandler.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFDocumentXMLBodyHandler.java
index b2e74d1..92f4bfe 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFDocumentXMLBodyHandler.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFDocumentXMLBodyHandler.java
@@ -77,6 +77,7 @@ public class XWPFDocumentXMLBodyHandler extends DefaultHandler {
private String picDescription = null;
private String picRId = null;
private String picFilename = null;
+ private boolean lastStartElementWasP;
//alternate content can be embedded in itself.
//need to track depth.
@@ -86,6 +87,7 @@ public class XWPFDocumentXMLBodyHandler extends DefaultHandler {
private EditType editType = EditType.NONE;
private XWPFRunProperties currRunProperties = new XWPFRunProperties();
+ private XWPFParagraphProperties currPProperties = new XWPFParagraphProperties();
public XWPFDocumentXMLBodyHandler(XWPFBodyContentsHandler bodyContentsHandler,
Map<String, String> hyperlinks) {
@@ -112,6 +114,12 @@ public class XWPFDocumentXMLBodyHandler extends DefaultHandler {
@Override
public void startElement(String uri, String localName, String qName, Attributes atts) throws SAXException {
+
+ if (lastStartElementWasP && ! "pPr".equals(localName)) {
+ bodyContentsHandler.startParagraph(currPProperties);
+ }
+ lastStartElementWasP = false;
+
if (uri != null && uri.equals(MC_NS)) {
if (localName.equals("Choice")) {
inACChoiceDepth++;
@@ -166,7 +174,7 @@ public class XWPFDocumentXMLBodyHandler extends DefaultHandler {
if (uri == null || uri.equals(W_NS)) {
if (localName.equals("p")) {
- bodyContentsHandler.startParagraph();
+ lastStartElementWasP = true;
pDepth++;
} else if (localName.equals("r")) {
inR = true;
@@ -204,21 +212,43 @@ public class XWPFDocumentXMLBodyHandler extends DefaultHandler {
String hyperlink = null;
if (hyperlinkId != null) {
hyperlink = linkedRelationships.get(hyperlinkId);
+ bodyContentsHandler.hyperlinkStart(hyperlink);
+ } else {
+ String anchor = atts.getValue(W_NS, "anchor");
+ if (anchor != null) {
+ anchor = "#"+anchor;
+ }
+ bodyContentsHandler.hyperlinkStart(anchor);
}
- bodyContentsHandler.hyperlinkStart(hyperlink);
} else if (localName.equals("footnoteReference")) {
String id = atts.getValue(W_NS, "id");
bodyContentsHandler.footnoteReference(id);
} else if (localName.equals("endnoteReference")) {
String id = atts.getValue(W_NS, "id");
bodyContentsHandler.endnoteReference(id);
- } /*else if (localName.equals("headerReference")) {
+ } else if ("pStyle".equals(localName)) {
+ String styleId = atts.getValue(W_NS, "val");
+ currPProperties.setStyleID(styleId);
+ } else if (inNumPr && "ilvl".equals(localName)) {
+ currPProperties.setIlvl(getIntVal(atts));
+ } else if (inNumPr && "numId".equals(localName)) {
+ currPProperties.setNumId(getIntVal(atts));
+ } else if ("bookmarkStart".equals(localName)) {
+ String name = atts.getValue(W_NS, "name");
+ String id = atts.getValue(W_NS, "id");
+ bodyContentsHandler.startBookmark(id, name);
+ } else if ("bookmarkEnd".equals(localName)) {
+ String id = atts.getValue(W_NS, "id");
+ bodyContentsHandler.endBookmark(id);
+ }
+ /*else if (localName.equals("headerReference")) {
//TODO
} else if (localName.equals("footerReference")) {
//TODO
} else if (localName.equals("commentRangeEnd")) {
//TODO
}*/
+
}
}
@@ -298,6 +328,9 @@ public class XWPFDocumentXMLBodyHandler extends DefaultHandler {
bodyContentsHandler.hyperlinkEnd();
} else if ("pict".equals(localName)) {
handlePict();
+ } else if ("pPr".equals(localName)) {
+ bodyContentsHandler.startParagraph(currPProperties);
+ currPProperties.reset();
}
}
}
@@ -348,11 +381,15 @@ public class XWPFDocumentXMLBodyHandler extends DefaultHandler {
void run(XWPFRunProperties runProperties, String contents);
+ /**
+ *
+ * @param link the link; can be null
+ */
void hyperlinkStart(String link);
void hyperlinkEnd();
- void startParagraph();
+ void startParagraph(XWPFParagraphProperties paragraphProperties);
void endParagraph();
@@ -387,5 +424,9 @@ public class XWPFDocumentXMLBodyHandler extends DefaultHandler {
void embeddedOLERef(String refId);
void embeddedPicRef(String picFileName, String picDescription);
+
+ void startBookmark(String id, String name);
+
+ void endBookmark(String id);
}
}
http://git-wip-us.apache.org/repos/asf/tika/blob/3ee9fd5b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFEventBasedWordExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFEventBasedWordExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFEventBasedWordExtractor.java
index ee6bb85..3cae6d9 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFEventBasedWordExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFEventBasedWordExtractor.java
@@ -275,7 +275,7 @@ public class XWPFEventBasedWordExtractor extends POIXMLTextExtractor {
}
@Override
- public void startParagraph() {
+ public void startParagraph(XWPFParagraphProperties paragraphProperties) {
//no-op
}
@@ -363,6 +363,16 @@ public class XWPFEventBasedWordExtractor extends POIXMLTextExtractor {
public void embeddedPicRef(String picFileName, String picDescription) {
//no-op
}
+
+ @Override
+ public void startBookmark(String id, String name) {
+ //no-op
+ }
+
+ @Override
+ public void endBookmark(String id) {
+ //no-op
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tika/blob/3ee9fd5b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFNumberingShim.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFNumberingShim.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFNumberingShim.java
new file mode 100644
index 0000000..2a07e3e
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFNumberingShim.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.parser.microsoft.ooxml.xwpf;
+
+import java.io.IOException;
+
+import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
+import org.apache.poi.openxml4j.opc.PackagePart;
+import org.apache.poi.xwpf.usermodel.XWPFNumbering;
+
+/**
+ * Stub class of POI's XWPFNumbering because onDocumentRead() is protected
+ */
+public class XWPFNumberingShim extends XWPFNumbering {
+
+ public XWPFNumberingShim(PackagePart part) throws IOException, OpenXML4JException {
+ super(part);
+ onDocumentRead();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/3ee9fd5b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFParagraphProperties.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFParagraphProperties.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFParagraphProperties.java
new file mode 100644
index 0000000..fd2b022
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFParagraphProperties.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.parser.microsoft.ooxml.xwpf;
+
+
+public class XWPFParagraphProperties {
+
+ private String styleId;
+ private int ilvl = -1;
+ private int numId = -1;
+
+ String getStyleID() {
+ return styleId;
+ }
+
+ void setStyleID(String styleId) {
+ this.styleId = styleId;
+ }
+
+ void reset() {
+ styleId = null;
+ ilvl = -1;
+ numId = -1;
+ }
+
+ public void setIlvl(int ilvl) {
+ this.ilvl = ilvl;
+ }
+
+ public void setNumId(int numId) {
+ this.numId = numId;
+ }
+
+ public int getIlvl() {
+ return ilvl;
+ }
+
+ public int getNumId() {
+ return numId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/3ee9fd5b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFStylesShim.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFStylesShim.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFStylesShim.java
new file mode 100644
index 0000000..579c29a
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFStylesShim.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.parser.microsoft.ooxml.xwpf;
+
+import java.io.IOException;
+
+import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
+import org.apache.poi.openxml4j.opc.PackagePart;
+import org.apache.poi.xwpf.usermodel.XWPFStyles;
+
+/**
+ * Stub class of POI's XWPFStyles because onDocumentRead() is protected
+ */
+public class XWPFStylesShim extends XWPFStyles {
+
+ public XWPFStylesShim(PackagePart part) throws IOException, OpenXML4JException {
+ super(part);
+ onDocumentRead();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/3ee9fd5b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFTikaBodyPartHandler.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFTikaBodyPartHandler.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFTikaBodyPartHandler.java
index cd28583..759457a 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFTikaBodyPartHandler.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFTikaBodyPartHandler.java
@@ -18,9 +18,13 @@
package org.apache.tika.parser.microsoft.ooxml.xwpf;
+import java.math.BigInteger;
import java.util.Date;
+import org.apache.poi.xwpf.usermodel.XWPFStyle;
+import org.apache.poi.xwpf.usermodel.XWPFStyles;
import org.apache.tika.parser.microsoft.OfficeParserConfig;
+import org.apache.tika.parser.microsoft.WordExtractor;
import org.apache.tika.parser.microsoft.ooxml.XWPFListManager;
import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.SAXException;
@@ -35,14 +39,22 @@ public class XWPFTikaBodyPartHandler implements XWPFDocumentXMLBodyHandler.XWPFB
private final XWPFListManager listManager;
private final boolean includeDeletedText;
private final boolean includeMoveFromText;
+ private final XWPFStyles styles;
private int pDepth = 0; //paragraph depth
+ private int tableDepth = 0;//table depth
+ private int sdtDepth = 0;//
private boolean isItalics = false;
private boolean isBold = false;
private boolean wroteHyperlinkStart = false;
- public XWPFTikaBodyPartHandler(XHTMLContentHandler xhtml, XWPFListManager listManager, OfficeParserConfig parserConfig) {
+ //will need to replace this with a stack
+ //if we're marking more that the first level <p/> element
+ private String paragraphTag = null;
+
+ public XWPFTikaBodyPartHandler(XHTMLContentHandler xhtml, XWPFStyles styles, XWPFListManager listManager, OfficeParserConfig parserConfig) {
this.xhtml = xhtml;
+ this.styles = styles;
this.listManager = listManager;
this.includeDeletedText = parserConfig.getIncludeDeletedContent();
this.includeMoveFromText = parserConfig.getIncludeMoveFromContent();
@@ -109,14 +121,41 @@ public class XWPFTikaBodyPartHandler implements XWPFDocumentXMLBodyHandler.XWPFB
}
@Override
- public void startParagraph() {
- if (pDepth == 0) {
+ public void startParagraph(XWPFParagraphProperties paragraphProperties) {
+ if (pDepth == 0 && tableDepth == 0 && sdtDepth == 0) {
+ paragraphTag = "p";
+ String styleClass = null;
+ //TIKA-2144 check that styles is not null
+ if (paragraphProperties.getStyleID() != null && styles != null) {
+ XWPFStyle style = styles.getStyle(
+ paragraphProperties.getStyleID()
+ );
+ if (style != null && style.getName() != null) {
+ WordExtractor.TagAndStyle tas = WordExtractor.buildParagraphTagAndStyle(
+ style.getName(), false);
+ paragraphTag = tas.getTag();
+ styleClass = tas.getStyleClass();
+ }
+ }
+
+
try {
- xhtml.startElement("p");
+ if (styleClass == null) {
+ xhtml.startElement(paragraphTag);
+ } else {
+ xhtml.startElement(paragraphTag, "class", styleClass);
+ }
} catch (SAXException e) {
}
}
+
+ try {
+ writeParagraphNumber(paragraphProperties.getNumId(),
+ paragraphProperties.getIlvl(), listManager, xhtml);
+ } catch (SAXException e) {
+
+ }
pDepth++;
}
@@ -124,8 +163,9 @@ public class XWPFTikaBodyPartHandler implements XWPFDocumentXMLBodyHandler.XWPFB
public void endParagraph() {
try {
closeStyleTags();
- if (pDepth == 1) {
- xhtml.endElement("p");
+ if (pDepth == 1 && tableDepth == 0 && sdtDepth == 0) {
+ xhtml.endElement(paragraphTag);
+ paragraphTag = null;
} else {
xhtml.characters(NEWLINE, 0, 1);
}
@@ -139,6 +179,7 @@ public class XWPFTikaBodyPartHandler implements XWPFDocumentXMLBodyHandler.XWPFB
public void startTable() {
try {
xhtml.startElement("table");
+ tableDepth++;
} catch (SAXException e) {
}
@@ -148,6 +189,7 @@ public class XWPFTikaBodyPartHandler implements XWPFDocumentXMLBodyHandler.XWPFB
public void endTable() {
try {
xhtml.endElement("table");
+ tableDepth--;
} catch (SAXException e) {
}
@@ -193,6 +235,7 @@ public class XWPFTikaBodyPartHandler implements XWPFDocumentXMLBodyHandler.XWPFB
public void startSDT() {
try {
closeStyleTags();
+ sdtDepth++;
} catch (SAXException e) {
}
@@ -200,7 +243,7 @@ public class XWPFTikaBodyPartHandler implements XWPFDocumentXMLBodyHandler.XWPFB
@Override
public void endSDT() {
- //no-op
+ sdtDepth--;
}
@Override
@@ -286,6 +329,24 @@ public class XWPFTikaBodyPartHandler implements XWPFDocumentXMLBodyHandler.XWPFB
}
}
+ @Override
+ public void startBookmark(String id, String name) {
+ //skip bookmarks within hyperlinks
+ if (name != null && ! wroteHyperlinkStart) {
+ try {
+ xhtml.startElement("a", "name", name);
+ xhtml.endElement("a");
+ } catch (SAXException e) {
+
+ }
+ }
+ }
+
+ @Override
+ public void endBookmark(String id) {
+ //no-op
+ }
+
private void closeStyleTags() throws SAXException {
if (isItalics) {
xhtml.endElement("i");
@@ -296,4 +357,18 @@ public class XWPFTikaBodyPartHandler implements XWPFDocumentXMLBodyHandler.XWPFB
isBold = false;
}
}
+
+ private void writeParagraphNumber(int numId, int ilvl,
+ XWPFListManager listManager,
+ XHTMLContentHandler xhtml) throws SAXException {
+
+ if (ilvl < 0 || numId < 0 || listManager == null) {
+ return;
+ }
+ String number = listManager.getFormattedNumber(BigInteger.valueOf(numId), ilvl);
+ if (number != null) {
+ xhtml.characters(number);
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/tika/blob/3ee9fd5b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/BodyPartHandler.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/BodyPartHandler.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/BodyPartHandler.java
index 4d04b2b..1b25683 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/BodyPartHandler.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/BodyPartHandler.java
@@ -37,7 +37,7 @@ class BodyPartHandler extends XWPFDocumentXMLBodyHandler implements PartHandler
public BodyPartHandler(String contentType, XHTMLContentHandler xhtml,
RelationshipsManager relationshipsManager,
OfficeParserConfig officeParserConfig) {
- super(new XWPFTikaBodyPartHandler(xhtml, null, officeParserConfig),
+ super(new XWPFTikaBodyPartHandler(xhtml, null, null, officeParserConfig),
new HashMap<String, String>());
this.contentType = contentType;
}
http://git-wip-us.apache.org/repos/asf/tika/blob/3ee9fd5b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXWPFExtractorTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXWPFExtractorTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXWPFExtractorTest.java
index 6064be2..70f977b 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXWPFExtractorTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXWPFExtractorTest.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.ByteArrayOutputStream;
-import java.io.File;
import java.io.InputStream;
import java.io.PrintStream;
import java.util.Arrays;
@@ -39,7 +38,6 @@ import org.apache.tika.metadata.OfficeOpenXMLCore;
import org.apache.tika.metadata.OfficeOpenXMLExtended;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.AutoDetectParser;
-import org.apache.tika.parser.EmptyParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.PasswordProvider;
@@ -47,7 +45,6 @@ import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.parser.microsoft.OfficeParserConfig;
import org.apache.tika.sax.BodyContentHandler;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import org.xml.sax.ContentHandler;
@@ -97,11 +94,10 @@ public class SXWPFExtractorTest extends TikaTest {
assertContains("My Document Title", content);
assertContains("My Document Subtitle", content);
- assertContains("<p>\tHeading1\t3</p>", content);
+ assertContains("<p class=\"toc_1\">\t<a href=\"#_Toc467647605\">Heading1\t3</a></p>", content);
- //TODO: integrate numbering
- assertContains("Really basic 2.", content);
+ assertContains("2. Really basic 2.", content);
assertContainsCount("This is a text box", content, 1);
@@ -112,11 +108,11 @@ public class SXWPFExtractorTest extends TikaTest {
assertContains("<p>This is 10 spaces</p>", content);
//caption
- assertContains("<p>Table 1: Table1 Caption</p>", content);
+ assertContains("<p class=\"table_of_figures\">\t<a href=\"#_Toc467647797\">Table 1: Table1 Caption\t2</a></p>", content);
//embedded table
//TODO: figure out how to handle embedded tables in html
- assertContains("<p>Embedded table r1c1</p>", content);
+ assertContains("<td>Embedded table r1c1", content);
//shape
assertContainsCount("<p>This is text within a shape", content, 1);
@@ -190,13 +186,13 @@ public class SXWPFExtractorTest extends TikaTest {
public void testWord() throws Exception {
XMLResult xmlResult = getXML("testWORD.docx", parseContext);
- assertEquals(
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- xmlResult.metadata.get(Metadata.CONTENT_TYPE));
- assertEquals("Sample Word Document", xmlResult.metadata.get(TikaCoreProperties.TITLE));
- assertEquals("Keith Bennett", xmlResult.metadata.get(TikaCoreProperties.CREATOR));
- assertEquals("Keith Bennett", xmlResult.metadata.get(Metadata.AUTHOR));
- assertTrue(xmlResult.xml.contains("Sample Word Document"));
+ assertEquals(
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ xmlResult.metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("Sample Word Document", xmlResult.metadata.get(TikaCoreProperties.TITLE));
+ assertEquals("Keith Bennett", xmlResult.metadata.get(TikaCoreProperties.CREATOR));
+ assertEquals("Keith Bennett", xmlResult.metadata.get(Metadata.AUTHOR));
+ assertTrue(xmlResult.xml.contains("Sample Word Document"));
}
@@ -208,10 +204,10 @@ public class SXWPFExtractorTest extends TikaTest {
@Test
public void testWordFootnote() throws Exception {
XMLResult xmlResult = getXML("footnotes.docx", parseContext);
- assertEquals(
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- xmlResult.metadata.get(Metadata.CONTENT_TYPE));
- assertTrue(xmlResult.xml.contains("snoska"));
+ assertEquals(
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ xmlResult.metadata.get(Metadata.CONTENT_TYPE));
+ assertTrue(xmlResult.xml.contains("snoska"));
}
@@ -231,15 +227,17 @@ public class SXWPFExtractorTest extends TikaTest {
assertEquals("Keith Bennett", metadata.get(TikaCoreProperties.CREATOR));
assertEquals("Keith Bennett", metadata.get(Metadata.AUTHOR));
assertTrue(xml.contains("Sample Word Document"));
-
+ System.out.println(xml);
// Check that custom headings came through
-//TODO: assertTrue(xml.contains("<h1 class=\"title\">"));
+ assertTrue(xml.contains("<h1 class=\"title\">"));
// Regular headings
-//TODO: assertTrue(xml.contains("<h1>Heading Level 1</h1>"));
-//TODO: assertTrue(xml.contains("<h2>Heading Level 2</h2>"));
+ assertTrue(xml.contains("<h1>Heading Level 1</h1>"));
+ assertTrue(xml.contains("<h2>Heading Level 2</h2>"));
// Headings with anchor tags in them
-//TODO: assertTrue(xml.contains("<h3><a name=\"OnLevel3\" />Heading Level 3</h3>"));
+ //TODO: still not getting bookmarks
+ assertTrue(xml.contains("<h3>Heading Level 3<a name=\"OnLevel3\" /></h3>"));
+// assertTrue(xml.contains("<h3>Heading Level 3</h3>"));
// Bold and italic
assertTrue(xml.contains("<b>BOLD</b>"));
assertTrue(xml.contains("<i>ITALIC</i>"));
@@ -249,17 +247,17 @@ public class SXWPFExtractorTest extends TikaTest {
// Links
assertTrue(xml.contains("<a href=\"http://tika.apache.org/\">Tika</a>"));
// Anchor links
-//TODO: assertContains("<a href=\"#OnMainHeading\">The Main Heading Bookmark</a>", xml);
+ assertContains("<a href=\"#OnMainHeading\">The Main Heading Bookmark</a>", xml);
// Paragraphs with other styles
-//TODO: assertTrue(xml.contains("<p class=\"signature\">This one"));
+ assertTrue(xml.contains("<p class=\"signature\">This one"));
result = getXML("testWORD_3imgs.docx", parseContext);
xml = result.xml;
// Images 2-4 (there is no 1!)
-//TODO: assertTrue("Image not found in:\n" + xml, xml.contains("<img src=\"embedded:image2.png\" alt=\"A description...\" />"));
-//TODO: assertTrue("Image not found in:\n" + xml, xml.contains("<img src=\"embedded:image3.jpeg\" alt=\"A description...\" />"));
-//TODO: assertTrue("Image not found in:\n" + xml, xml.contains("<img src=\"embedded:image4.png\" alt=\"A description...\" />"));
+ assertTrue("Image not found in:\n" + xml, xml.contains("<img src=\"embedded:image2.png\" alt=\"A description...\" />"));
+ assertTrue("Image not found in:\n" + xml, xml.contains("<img src=\"embedded:image3.jpeg\" alt=\"A description...\" />"));
+ assertTrue("Image not found in:\n" + xml, xml.contains("<img src=\"embedded:image4.png\" alt=\"A description...\" />"));
// Text too
assertTrue(xml.contains("<p>The end!</p>"));
@@ -295,11 +293,11 @@ public class SXWPFExtractorTest extends TikaTest {
assertEquals(2, metadataList.size());
Metadata m = metadataList.get(0);
String mainContent = m.get(RecursiveParserWrapper.TIKA_CONTENT);
- assertEquals(
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- m.get(Metadata.CONTENT_TYPE));
- // Check that custom headings came through
- assertTrue(mainContent.contains("<img"));
+ assertEquals(
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ m.get(Metadata.CONTENT_TYPE));
+ // Check that custom headings came through
+ assertTrue(mainContent.contains("<img"));
}
@Test
@@ -312,8 +310,8 @@ public class SXWPFExtractorTest extends TikaTest {
assertEquals(3, metadataList.size());
String content = metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT);
for (int i = 1; i < 4; i++) {
- assertContains("header"+i+"_pic", content);
- assertContains("footer"+i+"_pic", content);
+ assertContains("header" + i + "_pic", content);
+ assertContains("footer" + i + "_pic", content);
}
assertContains("body_pic.jpg", content);
assertContains("sdt_pic.jpg", content);
@@ -515,12 +513,12 @@ public class SXWPFExtractorTest extends TikaTest {
public void testMissingText() throws Exception {
XMLResult xmlResult = getXML("testWORD_missing_text.docx", parseContext);
- assertEquals(
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- xmlResult.metadata.get(Metadata.CONTENT_TYPE));
- assertContains("BigCompany", xmlResult.xml);
- assertContains("Seasoned", xmlResult.xml);
- assertContains("Rich_text_in_cell", xmlResult.xml);
+ assertEquals(
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ xmlResult.metadata.get(Metadata.CONTENT_TYPE));
+ assertContains("BigCompany", xmlResult.xml);
+ assertContains("Seasoned", xmlResult.xml);
+ assertContains("Rich_text_in_cell", xmlResult.xml);
}
@@ -596,7 +594,6 @@ public class SXWPFExtractorTest extends TikaTest {
}
@Test
- @Ignore("TODO -- paragraph list numbers")
public void testDOCXParagraphNumbering() throws Exception {
String xml = getXML("testWORD_numbered_list.docx", parseContext).xml;
//SAX parser is getting this. DOM parser is not!
@@ -625,7 +622,6 @@ public class SXWPFExtractorTest extends TikaTest {
}
@Test
- @Ignore("TODO -- paragraph list numbers")
public void testDOCXOverrideParagraphNumbering() throws Exception {
String xml = getXML("testWORD_override_list_numbering.docx").xml;
@@ -695,7 +691,6 @@ public class SXWPFExtractorTest extends TikaTest {
public void testBoldHyperlink() throws Exception {
//TIKA-1255
String xml = getXML("testWORD_boldHyperlink.docx", parseContext).xml;
- System.out.println(xml);
xml = xml.replaceAll("\\s+", " ");
assertContains("<a href=\"http://tika.apache.org/\">hyper <b>link</b></a>", xml);
assertContains("<a href=\"http://tika.apache.org/\"><b>hyper</b> link</a>; bold", xml);
@@ -738,26 +733,4 @@ public class SXWPFExtractorTest extends TikaTest {
assertEquals(16, metadataList.size());
}
- @Test
- public void iterate() throws Exception {
- ParseContext context = new ParseContext();
- context.set(Parser.class, EmptyParser.INSTANCE);
- for (File f : getResourceAsFile("/test-documents").listFiles()) {
- if (! f.getName().equals("testWORD_embeded.docx")) {
- continue;
- }
- if (f.getName().endsWith("docx") || f.getName().endsWith(".docm")) {
- try {
- XMLResult r = getXML(f.getName(), context);
- if (r.xml.contains("<img")) {
- System.out.println(f.getName());
- }
- System.out.println(r.xml);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/tika/blob/3ee9fd5b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParserTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParserTest.java
index c77b0fa..4e4bf7b 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParserTest.java
@@ -66,7 +66,7 @@ public class Word2006MLParserTest extends TikaTest {
assertContains("My Document Title", content);
assertContains("My Document Subtitle", content);
- assertContains("<p>\tHeading1\t3</p>", content);
+ assertContains("<p>\t<a href=\"#_Toc467647605\">Heading1 3</a></p>", content);
//TODO: integrate numbering
@@ -81,11 +81,11 @@ public class Word2006MLParserTest extends TikaTest {
assertContains("<p>This is 10 spaces</p>", content);
//caption
- assertContains("<p>Table 1: Table1 Caption</p>", content);
+ assertContains("<p>\t<a href=\"#_Toc467647797\">Table 1: Table1 Caption\t2</a>", content);
//embedded table
//TODO: figure out how to handle embedded tables in html
- assertContains("<p>Embedded table r1c1</p>", content);
+ assertContains("<td>Embedded table r1c1", content);
//shape
assertContainsCount("<p>This is text within a shape", content, 1);