You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/01/18 02:12:53 UTC

tika git commit: TIKA-2242 fix style markup in ODT

Repository: tika
Updated Branches:
  refs/heads/2.x 45a9b77d6 -> 4374bcecf


TIKA-2242  fix style markup in ODT


Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/4374bcec
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/4374bcec
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/4374bcec

Branch: refs/heads/2.x
Commit: 4374bcecf4600664998cd08bd1c6b044b9acded7
Parents: 45a9b77
Author: tballison <ta...@mitre.org>
Authored: Tue Jan 17 21:12:41 2017 -0500
Committer: tballison <ta...@mitre.org>
Committed: Tue Jan 17 21:12:41 2017 -0500

----------------------------------------------------------------------
 CHANGES.txt                                     |   2 +
 .../parser/odf/OpenDocumentContentParser.java   | 178 +++++++++++++------
 .../apache/tika/parser/odf/ODFParserTest.java   |   9 +
 .../resources/test-documents/testODTStyles2.odt | Bin 0 -> 17383 bytes
 4 files changed, 133 insertions(+), 56 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tika/blob/4374bcec/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index fc4ef6d..1e25d96 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -17,6 +17,8 @@ Release 2.0 - ???
 
 Release 1.15 -???
 
+  * Improve style tags in ODT (TIKA-2242).
+
   * Add container detection for embedded MSEquation files (TIKA-2238).
 
   * Add parsing of JBIG2 and extraction of JBIG2 from PDFs when

http://git-wip-us.apache.org/repos/asf/tika/blob/4374bcec/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
index a149dd8..021bb62 100644
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
+++ b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
@@ -56,6 +56,15 @@ public class OpenDocumentContentParser extends AbstractParser {
         public boolean italic;
         public boolean bold;
         public boolean underlined;
+
+        @Override
+        public String toString() {
+            return "TextStyle{" +
+                    "italic=" + italic +
+                    ", bold=" + bold +
+                    ", underlined=" + underlined +
+                    '}';
+        }
     }
 
     private static class ListStyle implements Style {
@@ -76,13 +85,26 @@ public class OpenDocumentContentParser extends AbstractParser {
         private int nodeDepth = 0;
         private int completelyFiltered = 0;
         private Stack<String> headingStack = new Stack<String>();
+        private Map<String, TextStyle> paragraphTextStyleMap = new HashMap<String, TextStyle>();
         private Map<String, TextStyle> textStyleMap = new HashMap<String, TextStyle>();
         private Map<String, ListStyle> listStyleMap = new HashMap<String, ListStyle>();
-        private TextStyle textStyle;
-        private TextStyle lastTextStyle;
+        private String currParagraphStyleName; //paragraph style name
+        private TextStyle currTextStyle; //this is the text style for particular spans/paragraphs
+        private String currTextStyleName;
+
         private Stack<ListStyle> listStyleStack = new Stack<ListStyle>();
         private ListStyle listStyle;
 
+        // True if we are currently in the named style:
+        private boolean curUnderlined;
+        private boolean curBold;
+        private boolean curItalic;
+
+        //have we written the start style tags
+        //yet for the current text style
+        boolean hasWrittenStartStyleTags = false;
+
+
         private OpenDocumentElementMappingContentHandler(ContentHandler handler,
                                                          Map<QName, TargetElement> mappings) {
             super(handler, mappings);
@@ -95,7 +117,10 @@ public class OpenDocumentContentParser extends AbstractParser {
             // only forward content of tags from text:-namespace
             if (completelyFiltered == 0 && nodeDepth > 0
                     && textNodeStack.get(nodeDepth - 1)) {
-                lazyEndSpan();
+                if (!hasWrittenStartStyleTags) {
+                    updateStyleTags();
+                    hasWrittenStartStyleTags = true;
+                }
                 super.characters(ch, start, length);
             }
         }
@@ -165,59 +190,87 @@ public class OpenDocumentContentParser extends AbstractParser {
             if (name == null) {
                 return;
             }
+            currTextStyle = textStyleMap.get(name);
+            hasWrittenStartStyleTags = false;
+        }
 
-            TextStyle style = textStyleMap.get(name);
-            if (style == null) {
-                return;
-            }
+        private void startParagraph(String styleName) throws SAXException {
 
-            // End tags that refer to no longer valid styles
-            if (!style.underlined && lastTextStyle != null && lastTextStyle.underlined) {
-                handler.endElement(XHTML, "u", "u");
-            }
-            if (!style.italic && lastTextStyle != null && lastTextStyle.italic) {
-                handler.endElement(XHTML, "i", "i");
-            }
-            if (!style.bold && lastTextStyle != null && lastTextStyle.bold) {
-                handler.endElement(XHTML, "b", "b");
+            handler.startElement(XHTML, "p", "p", EMPTY_ATTRIBUTES);
+            if (styleName != null) {
+                currTextStyle = paragraphTextStyleMap.get(styleName);
             }
+            hasWrittenStartStyleTags = false;
+        }
+
+        private void updateStyleTags() throws SAXException {
 
-            // Start tags for new styles
-            if (style.bold && (lastTextStyle == null || !lastTextStyle.bold)) {
-                handler.startElement(XHTML, "b", "b", EMPTY_ATTRIBUTES);
+            if (currTextStyle == null) {
+                closeStyleTags();
+                return;
             }
-            if (style.italic && (lastTextStyle == null || !lastTextStyle.italic)) {
-                handler.startElement(XHTML, "i", "i", EMPTY_ATTRIBUTES);
+            if (currTextStyle.bold != curBold) {
+                // Enforce nesting -- must close s and i tags
+                if (curUnderlined) {
+                    handler.endElement(XHTML, "u", "u");
+                    curUnderlined = false;
+                }
+                if (curItalic) {
+                    handler.endElement(XHTML, "i", "i");
+                    curItalic = false;
+                }
+                if (currTextStyle.bold) {
+                    handler.startElement(XHTML, "b", "b", EMPTY_ATTRIBUTES);
+                } else {
+                    handler.endElement(XHTML, "b", "b");
+                }
+                curBold = currTextStyle.bold;
             }
-            if (style.underlined && (lastTextStyle == null || !lastTextStyle.underlined)) {
-                handler.startElement(XHTML, "u", "u", EMPTY_ATTRIBUTES);
+
+            if (currTextStyle.italic != curItalic) {
+                // Enforce nesting -- must close s tag
+                if (curUnderlined) {
+                    handler.endElement(XHTML, "u", "u");
+                    curUnderlined = false;
+                }
+                if (currTextStyle.italic) {
+                    handler.startElement(XHTML, "i", "i", EMPTY_ATTRIBUTES);
+                } else {
+                    handler.endElement(XHTML, "i", "i");
+                }
+                curItalic = currTextStyle.italic;
             }
 
-            textStyle = style;
-            lastTextStyle = null;
+            if (currTextStyle.underlined != curUnderlined) {
+                if (currTextStyle.underlined) {
+                    handler.startElement(XHTML, "u", "u", EMPTY_ATTRIBUTES);
+                } else {
+                    handler.endElement(XHTML, "u", "u");
+                }
+                curUnderlined = currTextStyle.underlined;
+            }
         }
 
         private void endSpan() throws SAXException {
-            lastTextStyle = textStyle;
-            textStyle = null;
-        }
 
-        private void lazyEndSpan() throws SAXException {
-            if (lastTextStyle == null) {
-                return;
-            }
+        }
 
-            if (lastTextStyle.underlined) {
-                handler.endElement(XHTML, "u", "u");
+        private void closeStyleTags() throws SAXException {
+            // Close any still open style tags
+            if (curUnderlined) {
+                handler.endElement(XHTML,"u", "u");
+                curUnderlined = false;
             }
-            if (lastTextStyle.italic) {
-                handler.endElement(XHTML, "i", "i");
+            if (curItalic) {
+                handler.endElement(XHTML,"i", "i");
+                curItalic = false;
             }
-            if (lastTextStyle.bold) {
-                handler.endElement(XHTML, "b", "b");
+            if (curBold) {
+                handler.endElement(XHTML,"b", "b");
+                curBold = false;
             }
-
-            lastTextStyle = null;
+            currTextStyle = null;
+            hasWrittenStartStyleTags = false;
         }
 
         @Override
@@ -235,29 +288,31 @@ public class OpenDocumentContentParser extends AbstractParser {
             if (STYLE_NS.equals(namespaceURI) && "style".equals(localName)) {
                 String family = attrs.getValue(STYLE_NS, "family");
                 if ("text".equals(family)) {
-                    textStyle = new TextStyle();
-                    String name = attrs.getValue(STYLE_NS, "name");
-                    textStyleMap.put(name, textStyle);
+                    currTextStyle = new TextStyle();
+                    currTextStyleName = attrs.getValue(STYLE_NS, "name");
+                } else if ("paragraph".equals(family)) {
+                    currTextStyle = new TextStyle();
+                    currParagraphStyleName = attrs.getValue(STYLE_NS, "name");
                 }
             } else if (TEXT_NS.equals(namespaceURI) && "list-style".equals(localName)) {
                 listStyle = new ListStyle();
                 String name = attrs.getValue(STYLE_NS, "name");
                 listStyleMap.put(name, listStyle);
-            } else if (textStyle != null && STYLE_NS.equals(namespaceURI)
+            } else if (currTextStyle != null && STYLE_NS.equals(namespaceURI)
                     && "text-properties".equals(localName)) {
                 String fontStyle = attrs.getValue(FORMATTING_OBJECTS_NS, "font-style");
                 if ("italic".equals(fontStyle) || "oblique".equals(fontStyle)) {
-                    textStyle.italic = true;
+                    currTextStyle.italic = true;
                 }
                 String fontWeight = attrs.getValue(FORMATTING_OBJECTS_NS, "font-weight");
                 if ("bold".equals(fontWeight) || "bolder".equals(fontWeight)
                         || (fontWeight != null && Character.isDigit(fontWeight.charAt(0))
                         && Integer.valueOf(fontWeight) > 500)) {
-                    textStyle.bold = true;
+                    currTextStyle.bold = true;
                 }
                 String underlineStyle = attrs.getValue(STYLE_NS, "text-underline-style");
-                if (underlineStyle != null) {
-                    textStyle.underlined = true;
+                if (underlineStyle != null && !underlineStyle.equals("none")) {
+                    currTextStyle.underlined = true;
                 }
             } else if (listStyle != null && TEXT_NS.equals(namespaceURI)) {
                 if ("list-level-style-bullet".equals(localName)) {
@@ -286,6 +341,8 @@ public class OpenDocumentContentParser extends AbstractParser {
                     startList(attrs.getValue(TEXT_NS, "style-name"));
                 } else if (TEXT_NS.equals(namespaceURI) && "span".equals(localName)) {
                     startSpan(attrs.getValue(TEXT_NS, "style-name"));
+                } else if (TEXT_NS.equals(namespaceURI) && "p".equals(localName)) {
+                    startParagraph(attrs.getValue(TEXT_NS, "style-name"));
                 } else if (TEXT_NS.equals(namespaceURI) && "s".equals(localName)) {
                     handler.characters(SPACE, 0, 1);
                 } else {
@@ -299,7 +356,15 @@ public class OpenDocumentContentParser extends AbstractParser {
                 String namespaceURI, String localName, String qName)
                 throws SAXException {
             if (STYLE_NS.equals(namespaceURI) && "style".equals(localName)) {
-                textStyle = null;
+                if (currTextStyle != null && currTextStyleName != null) {
+                    textStyleMap.put(currTextStyleName, currTextStyle);
+                    currTextStyleName = null;
+                    currTextStyle = null;
+                } else if (currTextStyle != null && currParagraphStyleName != null) {
+                    paragraphTextStyleMap.put(currParagraphStyleName, currTextStyle);
+                    currParagraphStyleName = null;
+                    currTextStyle = null;
+                }
             } else if (TEXT_NS.equals(namespaceURI) && "list-style".equals(localName)) {
                 listStyle = null;
             }
@@ -314,11 +379,12 @@ public class OpenDocumentContentParser extends AbstractParser {
                 } else if (TEXT_NS.equals(namespaceURI) && "list".equals(localName)) {
                     endList();
                 } else if (TEXT_NS.equals(namespaceURI) && "span".equals(localName)) {
-                    endSpan();
+                    currTextStyle = null;
+                    hasWrittenStartStyleTags = false;
+                } else if (TEXT_NS.equals(namespaceURI) && "p".equals(localName)) {
+                    closeStyleTags();
+                    handler.endElement(XHTML, "p", "p");
                 } else {
-                    if (TEXT_NS.equals(namespaceURI) && "p".equals(localName)) {
-                        lazyEndSpan();
-                    }
                     super.endElement(namespaceURI, localName, qName);
                 }
 
@@ -493,9 +559,9 @@ public class OpenDocumentContentParser extends AbstractParser {
 
         SAXParser parser = context.getSAXParser();
         parser.parse(
-            new CloseShieldInputStream(stream),
-            new OfflineContentHandler(
-                    new NSNormalizerContentHandler(dh)));
+                new CloseShieldInputStream(stream),
+                new OfflineContentHandler(
+                        new NSNormalizerContentHandler(dh)));
     }
 
 }

http://git-wip-us.apache.org/repos/asf/tika/blob/4374bcec/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
index 15f0c74..3c6c966 100644
--- a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
+++ b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
@@ -337,4 +337,13 @@ public class ODFParserTest extends TikaTest {
         String xml = getXML("testODTNoMeta.odt").xml;
         assertContains("Test text", xml);
     }
+
+    @Test //TIKA-2242
+    public void testParagraphLevelFontStyles() throws Exception {
+        String xml = getXML("testODTStyles2.odt").xml;
+        //test text span font-style properties
+        assertContains("<p><b>name</b>, advocaat", xml);
+        //test paragraph's font-style properties
+        assertContains("<p><b>Publicatie Onbekwaamverklaring", xml);
+    }
 }

http://git-wip-us.apache.org/repos/asf/tika/blob/4374bcec/tika-test-resources/src/test/resources/test-documents/testODTStyles2.odt
----------------------------------------------------------------------
diff --git a/tika-test-resources/src/test/resources/test-documents/testODTStyles2.odt b/tika-test-resources/src/test/resources/test-documents/testODTStyles2.odt
new file mode 100644
index 0000000..f55d4de
Binary files /dev/null and b/tika-test-resources/src/test/resources/test-documents/testODTStyles2.odt differ