You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/01/18 02:12:53 UTC
tika git commit: TIKA-2242 fix style markup in ODT
Repository: tika
Updated Branches:
refs/heads/2.x 45a9b77d6 -> 4374bcecf
TIKA-2242 fix style markup in ODT
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/4374bcec
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/4374bcec
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/4374bcec
Branch: refs/heads/2.x
Commit: 4374bcecf4600664998cd08bd1c6b044b9acded7
Parents: 45a9b77
Author: tballison <ta...@mitre.org>
Authored: Tue Jan 17 21:12:41 2017 -0500
Committer: tballison <ta...@mitre.org>
Committed: Tue Jan 17 21:12:41 2017 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../parser/odf/OpenDocumentContentParser.java | 178 +++++++++++++------
.../apache/tika/parser/odf/ODFParserTest.java | 9 +
.../resources/test-documents/testODTStyles2.odt | Bin 0 -> 17383 bytes
4 files changed, 133 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/4374bcec/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index fc4ef6d..1e25d96 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -17,6 +17,8 @@ Release 2.0 - ???
Release 1.15 -???
+ * Improve style tags in ODT (TIKA-2242).
+
* Add container detection for embedded MSEquation files (TIKA-2238).
* Add parsing of JBIG2 and extraction of JBIG2 from PDFs when
http://git-wip-us.apache.org/repos/asf/tika/blob/4374bcec/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
index a149dd8..021bb62 100644
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
+++ b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
@@ -56,6 +56,15 @@ public class OpenDocumentContentParser extends AbstractParser {
public boolean italic;
public boolean bold;
public boolean underlined;
+
+ @Override
+ public String toString() {
+ return "TextStyle{" +
+ "italic=" + italic +
+ ", bold=" + bold +
+ ", underlined=" + underlined +
+ '}';
+ }
}
private static class ListStyle implements Style {
@@ -76,13 +85,26 @@ public class OpenDocumentContentParser extends AbstractParser {
private int nodeDepth = 0;
private int completelyFiltered = 0;
private Stack<String> headingStack = new Stack<String>();
+ private Map<String, TextStyle> paragraphTextStyleMap = new HashMap<String, TextStyle>();
private Map<String, TextStyle> textStyleMap = new HashMap<String, TextStyle>();
private Map<String, ListStyle> listStyleMap = new HashMap<String, ListStyle>();
- private TextStyle textStyle;
- private TextStyle lastTextStyle;
+ private String currParagraphStyleName; //paragraph style name
+ private TextStyle currTextStyle; //this is the text style for particular spans/paragraphs
+ private String currTextStyleName;
+
private Stack<ListStyle> listStyleStack = new Stack<ListStyle>();
private ListStyle listStyle;
+ // True if we are currently in the named style:
+ private boolean curUnderlined;
+ private boolean curBold;
+ private boolean curItalic;
+
+ //have we written the start style tags
+ //yet for the current text style
+ boolean hasWrittenStartStyleTags = false;
+
+
private OpenDocumentElementMappingContentHandler(ContentHandler handler,
Map<QName, TargetElement> mappings) {
super(handler, mappings);
@@ -95,7 +117,10 @@ public class OpenDocumentContentParser extends AbstractParser {
// only forward content of tags from text:-namespace
if (completelyFiltered == 0 && nodeDepth > 0
&& textNodeStack.get(nodeDepth - 1)) {
- lazyEndSpan();
+ if (!hasWrittenStartStyleTags) {
+ updateStyleTags();
+ hasWrittenStartStyleTags = true;
+ }
super.characters(ch, start, length);
}
}
@@ -165,59 +190,87 @@ public class OpenDocumentContentParser extends AbstractParser {
if (name == null) {
return;
}
+ currTextStyle = textStyleMap.get(name);
+ hasWrittenStartStyleTags = false;
+ }
- TextStyle style = textStyleMap.get(name);
- if (style == null) {
- return;
- }
+ private void startParagraph(String styleName) throws SAXException {
- // End tags that refer to no longer valid styles
- if (!style.underlined && lastTextStyle != null && lastTextStyle.underlined) {
- handler.endElement(XHTML, "u", "u");
- }
- if (!style.italic && lastTextStyle != null && lastTextStyle.italic) {
- handler.endElement(XHTML, "i", "i");
- }
- if (!style.bold && lastTextStyle != null && lastTextStyle.bold) {
- handler.endElement(XHTML, "b", "b");
+ handler.startElement(XHTML, "p", "p", EMPTY_ATTRIBUTES);
+ if (styleName != null) {
+ currTextStyle = paragraphTextStyleMap.get(styleName);
}
+ hasWrittenStartStyleTags = false;
+ }
+
+ private void updateStyleTags() throws SAXException {
- // Start tags for new styles
- if (style.bold && (lastTextStyle == null || !lastTextStyle.bold)) {
- handler.startElement(XHTML, "b", "b", EMPTY_ATTRIBUTES);
+ if (currTextStyle == null) {
+ closeStyleTags();
+ return;
}
- if (style.italic && (lastTextStyle == null || !lastTextStyle.italic)) {
- handler.startElement(XHTML, "i", "i", EMPTY_ATTRIBUTES);
+ if (currTextStyle.bold != curBold) {
+ // Enforce nesting -- must close s and i tags
+ if (curUnderlined) {
+ handler.endElement(XHTML, "u", "u");
+ curUnderlined = false;
+ }
+ if (curItalic) {
+ handler.endElement(XHTML, "i", "i");
+ curItalic = false;
+ }
+ if (currTextStyle.bold) {
+ handler.startElement(XHTML, "b", "b", EMPTY_ATTRIBUTES);
+ } else {
+ handler.endElement(XHTML, "b", "b");
+ }
+ curBold = currTextStyle.bold;
}
- if (style.underlined && (lastTextStyle == null || !lastTextStyle.underlined)) {
- handler.startElement(XHTML, "u", "u", EMPTY_ATTRIBUTES);
+
+ if (currTextStyle.italic != curItalic) {
+ // Enforce nesting -- must close s tag
+ if (curUnderlined) {
+ handler.endElement(XHTML, "u", "u");
+ curUnderlined = false;
+ }
+ if (currTextStyle.italic) {
+ handler.startElement(XHTML, "i", "i", EMPTY_ATTRIBUTES);
+ } else {
+ handler.endElement(XHTML, "i", "i");
+ }
+ curItalic = currTextStyle.italic;
}
- textStyle = style;
- lastTextStyle = null;
+ if (currTextStyle.underlined != curUnderlined) {
+ if (currTextStyle.underlined) {
+ handler.startElement(XHTML, "u", "u", EMPTY_ATTRIBUTES);
+ } else {
+ handler.endElement(XHTML, "u", "u");
+ }
+ curUnderlined = currTextStyle.underlined;
+ }
}
private void endSpan() throws SAXException {
- lastTextStyle = textStyle;
- textStyle = null;
- }
- private void lazyEndSpan() throws SAXException {
- if (lastTextStyle == null) {
- return;
- }
+ }
- if (lastTextStyle.underlined) {
- handler.endElement(XHTML, "u", "u");
+ private void closeStyleTags() throws SAXException {
+ // Close any still open style tags
+ if (curUnderlined) {
+ handler.endElement(XHTML,"u", "u");
+ curUnderlined = false;
}
- if (lastTextStyle.italic) {
- handler.endElement(XHTML, "i", "i");
+ if (curItalic) {
+ handler.endElement(XHTML,"i", "i");
+ curItalic = false;
}
- if (lastTextStyle.bold) {
- handler.endElement(XHTML, "b", "b");
+ if (curBold) {
+ handler.endElement(XHTML,"b", "b");
+ curBold = false;
}
-
- lastTextStyle = null;
+ currTextStyle = null;
+ hasWrittenStartStyleTags = false;
}
@Override
@@ -235,29 +288,31 @@ public class OpenDocumentContentParser extends AbstractParser {
if (STYLE_NS.equals(namespaceURI) && "style".equals(localName)) {
String family = attrs.getValue(STYLE_NS, "family");
if ("text".equals(family)) {
- textStyle = new TextStyle();
- String name = attrs.getValue(STYLE_NS, "name");
- textStyleMap.put(name, textStyle);
+ currTextStyle = new TextStyle();
+ currTextStyleName = attrs.getValue(STYLE_NS, "name");
+ } else if ("paragraph".equals(family)) {
+ currTextStyle = new TextStyle();
+ currParagraphStyleName = attrs.getValue(STYLE_NS, "name");
}
} else if (TEXT_NS.equals(namespaceURI) && "list-style".equals(localName)) {
listStyle = new ListStyle();
String name = attrs.getValue(STYLE_NS, "name");
listStyleMap.put(name, listStyle);
- } else if (textStyle != null && STYLE_NS.equals(namespaceURI)
+ } else if (currTextStyle != null && STYLE_NS.equals(namespaceURI)
&& "text-properties".equals(localName)) {
String fontStyle = attrs.getValue(FORMATTING_OBJECTS_NS, "font-style");
if ("italic".equals(fontStyle) || "oblique".equals(fontStyle)) {
- textStyle.italic = true;
+ currTextStyle.italic = true;
}
String fontWeight = attrs.getValue(FORMATTING_OBJECTS_NS, "font-weight");
if ("bold".equals(fontWeight) || "bolder".equals(fontWeight)
|| (fontWeight != null && Character.isDigit(fontWeight.charAt(0))
&& Integer.valueOf(fontWeight) > 500)) {
- textStyle.bold = true;
+ currTextStyle.bold = true;
}
String underlineStyle = attrs.getValue(STYLE_NS, "text-underline-style");
- if (underlineStyle != null) {
- textStyle.underlined = true;
+ if (underlineStyle != null && !underlineStyle.equals("none")) {
+ currTextStyle.underlined = true;
}
} else if (listStyle != null && TEXT_NS.equals(namespaceURI)) {
if ("list-level-style-bullet".equals(localName)) {
@@ -286,6 +341,8 @@ public class OpenDocumentContentParser extends AbstractParser {
startList(attrs.getValue(TEXT_NS, "style-name"));
} else if (TEXT_NS.equals(namespaceURI) && "span".equals(localName)) {
startSpan(attrs.getValue(TEXT_NS, "style-name"));
+ } else if (TEXT_NS.equals(namespaceURI) && "p".equals(localName)) {
+ startParagraph(attrs.getValue(TEXT_NS, "style-name"));
} else if (TEXT_NS.equals(namespaceURI) && "s".equals(localName)) {
handler.characters(SPACE, 0, 1);
} else {
@@ -299,7 +356,15 @@ public class OpenDocumentContentParser extends AbstractParser {
String namespaceURI, String localName, String qName)
throws SAXException {
if (STYLE_NS.equals(namespaceURI) && "style".equals(localName)) {
- textStyle = null;
+ if (currTextStyle != null && currTextStyleName != null) {
+ textStyleMap.put(currTextStyleName, currTextStyle);
+ currTextStyleName = null;
+ currTextStyle = null;
+ } else if (currTextStyle != null && currParagraphStyleName != null) {
+ paragraphTextStyleMap.put(currParagraphStyleName, currTextStyle);
+ currParagraphStyleName = null;
+ currTextStyle = null;
+ }
} else if (TEXT_NS.equals(namespaceURI) && "list-style".equals(localName)) {
listStyle = null;
}
@@ -314,11 +379,12 @@ public class OpenDocumentContentParser extends AbstractParser {
} else if (TEXT_NS.equals(namespaceURI) && "list".equals(localName)) {
endList();
} else if (TEXT_NS.equals(namespaceURI) && "span".equals(localName)) {
- endSpan();
+ currTextStyle = null;
+ hasWrittenStartStyleTags = false;
+ } else if (TEXT_NS.equals(namespaceURI) && "p".equals(localName)) {
+ closeStyleTags();
+ handler.endElement(XHTML, "p", "p");
} else {
- if (TEXT_NS.equals(namespaceURI) && "p".equals(localName)) {
- lazyEndSpan();
- }
super.endElement(namespaceURI, localName, qName);
}
@@ -493,9 +559,9 @@ public class OpenDocumentContentParser extends AbstractParser {
SAXParser parser = context.getSAXParser();
parser.parse(
- new CloseShieldInputStream(stream),
- new OfflineContentHandler(
- new NSNormalizerContentHandler(dh)));
+ new CloseShieldInputStream(stream),
+ new OfflineContentHandler(
+ new NSNormalizerContentHandler(dh)));
}
}
http://git-wip-us.apache.org/repos/asf/tika/blob/4374bcec/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
index 15f0c74..3c6c966 100644
--- a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
+++ b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
@@ -337,4 +337,13 @@ public class ODFParserTest extends TikaTest {
String xml = getXML("testODTNoMeta.odt").xml;
assertContains("Test text", xml);
}
+
+ @Test //TIKA-2242
+ public void testParagraphLevelFontStyles() throws Exception {
+ String xml = getXML("testODTStyles2.odt").xml;
+ //test text span font-style properties
+ assertContains("<p><b>name</b>, advocaat", xml);
+ //test paragraph's font-style properties
+ assertContains("<p><b>Publicatie Onbekwaamverklaring", xml);
+ }
}
http://git-wip-us.apache.org/repos/asf/tika/blob/4374bcec/tika-test-resources/src/test/resources/test-documents/testODTStyles2.odt
----------------------------------------------------------------------
diff --git a/tika-test-resources/src/test/resources/test-documents/testODTStyles2.odt b/tika-test-resources/src/test/resources/test-documents/testODTStyles2.odt
new file mode 100644
index 0000000..f55d4de
Binary files /dev/null and b/tika-test-resources/src/test/resources/test-documents/testODTStyles2.odt differ