You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/01/04 17:32:08 UTC
[3/3] tika git commit: TIKA-2230 -- add paragraph markup to
WordPerfect parsers
TIKA-2230 -- add paragraph markup to WordPerfect parsers
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/940e6f48
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/940e6f48
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/940e6f48
Branch: refs/heads/master
Commit: 940e6f483ec87725870b3c9d62ed0b2ab9a5811e
Parents: 6dc442d
Author: tballison <ta...@mitre.org>
Authored: Wed Jan 4 12:31:47 2017 -0500
Committer: tballison <ta...@mitre.org>
Committed: Wed Jan 4 12:31:47 2017 -0500
----------------------------------------------------------------------
.../wordperfect/WP5DocumentAreaExtractor.java | 15 ++++----
.../wordperfect/WP6DocumentAreaExtractor.java | 15 ++++----
.../wordperfect/WPDocumentAreaExtractor.java | 37 ++++++++++++++++----
.../parser/wordperfect/WordPerfectParser.java | 2 --
.../parser/wordperfect/WordPerfectTest.java | 7 ++--
5 files changed, 53 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
index b95490e..bd59725 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
@@ -18,6 +18,9 @@ package org.apache.tika.parser.wordperfect;
import java.io.IOException;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.SAXException;
+
/**
* Extracts WordPerfect Document Area text from a WordPerfect document
* version 5.x.
@@ -25,16 +28,16 @@ import java.io.IOException;
*/
class WP5DocumentAreaExtractor extends WPDocumentAreaExtractor {
- protected void extract(int c, WPInputStream in, StringBuilder out)
- throws IOException {
+ protected void extract(int c, WPInputStream in, StringBuilder out, XHTMLContentHandler xhtml)
+ throws IOException, SAXException {
// 0-31: control characters
if (c == 10) {
- out.append('\n'); // hard return ("Enter")
+ endParagraph(out, xhtml);// hard return ("Enter")
} else if (c == 11) {
out.append(' '); // soft page break
} else if (c == 12) {
- out.append('\n'); // hard page break
+ endParagraph(out, xhtml);// hard page break
} else if (c == 13) {
out.append(' '); // soft return (line wrap)
@@ -44,11 +47,11 @@ class WP5DocumentAreaExtractor extends WPDocumentAreaExtractor {
// 128-191: single-byte functions
} else if (c == 140) {
- out.append('\n'); // combination hard return/soft page (WP5.1)
+ endParagraph(out, xhtml);// combination hard return/soft page (WP5.1)
} else if (c >= 144 && c <= 149) {
out.append(' '); // deletable/invisible soft return/page
} else if (c == 153) {
- out.append('\4'); // Dormant Hard return (WP5.1)
+ endParagraph(out, xhtml);// Dormant Hard return (WP5.1)
} else if (c == 160) {
out.append('\u00A0'); // Hard space
} else if (c >= 169 && c <= 171) {
http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
index 659ad17..862c858 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
@@ -18,6 +18,9 @@ package org.apache.tika.parser.wordperfect;
import java.io.IOException;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.SAXException;
+
/**
* Extracts WordPerfect Document Area text from a WordPerfect document
* version 6+.
@@ -25,8 +28,8 @@ import java.io.IOException;
*/
class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
- protected void extract(int c, WPInputStream in, StringBuilder out)
- throws IOException {
+ protected void extract(int c, WPInputStream in, StringBuilder out, XHTMLContentHandler xhtml)
+ throws IOException, SAXException {
if (c > 0 && c <= 32) {
out.append(WP6Charsets.DEFAULT_EXTENDED_INTL_CHARS[c]);
} else if (c >= 33 && c <= 126) {
@@ -38,7 +41,7 @@ class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
} else if (c == 129) {
out.append('-'); // Hard hyphen
} else if (c == 135 || c == 137) {
- out.append('\n'); // Dormant Hard return
+ endParagraph(out, xhtml); // Dormant Hard return
} else if (c == 138) {
// skip to closing pair surrounding page number
skipUntilChar(in, 139);
@@ -46,7 +49,7 @@ class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
// end of cell
out.append('\t');
} else if (c >= 180 && c <= 207) {
- out.append('\n');
+ endParagraph(out, xhtml);
} else if (c >= 208 && c <= 239) {
// Variable-Length Multi-Byte Functions
int subgroup = in.readWP();
@@ -63,11 +66,11 @@ class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
// end of cell
out.append('\t');
} else if (subgroup >= 4 && subgroup <= 19) {
- out.append('\n');
+ endParagraph(out, xhtml);
} else if (subgroup >= 20 && subgroup <= 22) {
out.append(' ');
} else if (subgroup >= 23 && subgroup <= 28) {
- out.append('\n');
+ endParagraph(out, xhtml);
}
} else if (c == 213) {
out.append(' ');
http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
index 002ecee..ee3980e 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
@@ -27,26 +27,51 @@ import org.xml.sax.SAXException;
*/
abstract class WPDocumentAreaExtractor {
+ boolean startedP = false;
public void extract(WPInputStream in, XHTMLContentHandler xhtml)
throws IOException, SAXException {
int chunk = 4096;
StringBuilder out = new StringBuilder(chunk);
int c;
while ((c = in.read()) != -1) {
- extract(c, in, out);
+ extract(c, in, out, xhtml);
if (out.length() >= chunk) {
xhtml.characters(out.toString());
out.setLength(0);
}
}
- xhtml.characters(out.toString());
- out.setLength(0);
+ endParagraph(out, xhtml);
}
protected abstract void extract(
- int c, WPInputStream in, StringBuilder out) throws IOException;
-
-
+ int c, WPInputStream in, StringBuilder out, XHTMLContentHandler xhtml) throws IOException, SAXException;
+
+
+ protected void lazilyStartParagraph(XHTMLContentHandler xhtml) throws SAXException {
+ if (! startedP) {
+ xhtml.startElement("p");
+ }
+ startedP = true;
+ }
+
+ /**
+ * This assumes that the <p> was started before you get here.
+ * And the user is required to close the <p> at the end of the document.
+ *
+ * These are currently handled by {@link #extract(WPInputStream, XHTMLContentHandler)}.
+ *
+ * @param xhtml
+ * @throws SAXException
+ */
+ protected void endParagraph(StringBuilder buffer, XHTMLContentHandler xhtml) throws SAXException {
+ lazilyStartParagraph(xhtml);
+
+ xhtml.characters(buffer.toString());
+ buffer.setLength(0);
+ xhtml.endElement("p");
+ startedP = false;
+ }
+
// Skips until the given character is encountered.
protected int skipUntilChar(WPInputStream in, int targetChar)
throws IOException {
http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
index 4c6d445..a18fee1 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
@@ -90,11 +90,9 @@ public class WordPerfectParser extends AbstractParser {
}
xhtml.startDocument();
- xhtml.startElement("p");
getDocumentAreaExtractor(prefixArea).extract(in, xhtml);
- xhtml.endElement("p");
xhtml.endDocument();
}
http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
index 89abca0..0da8959 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
@@ -34,7 +34,7 @@ public class WordPerfectTest extends TikaTest {
assertEquals(WordPerfectParser.WP_6_x.toString(),
r.metadata.get(Metadata.CONTENT_TYPE));
assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
- assertContains("test test", r.xml);
+ assertContains("<p>AND FURTHER</p>", r.xml);
}
@Test
@@ -44,7 +44,7 @@ public class WordPerfectTest extends TikaTest {
assertEquals(WordPerfectParser.WP_5_0.toString(),
r.metadata.get(Metadata.CONTENT_TYPE));
assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
- assertContains("Surrounded by her family", r.xml);
+ assertContains("<p>Surrounded by her family", r.xml);
}
@Test
@@ -54,6 +54,7 @@ public class WordPerfectTest extends TikaTest {
assertEquals(WordPerfectParser.WP_5_1.toString(),
r.metadata.get(Metadata.CONTENT_TYPE));
assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
- assertContains("Summer is the peak travel time", r.xml);
+ assertContains("<p>STUDY RESULTS: Existing condition", r.xml);
+ assertContains("Seattle nonstop flights.</p>", r.xml);
}
}