You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/01/04 17:32:06 UTC
[1/3] tika git commit: TIKA-2228 - WordPerfect parser update to
handle 5.x from Pascal Essiembre. This closes #142.
Repository: tika
Updated Branches:
refs/heads/master ef1d9077b -> 940e6f483
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6FileHeader.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6FileHeader.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6FileHeader.java
deleted file mode 100644
index 6a95335..0000000
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6FileHeader.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/* Copyright 2015-2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.wordperfect;
-
-
-/**
- * WordPerfect file header.
- * @author Pascal Essiembre
- */
-class WP6FileHeader {
-
- // Normal header
- private String fileId;
- private long docAreaPointer;
- private int productType;
- private int fileType;
- private int majorVersion;
- private int minorVersion;
- private boolean encrypted;
- private int indexAreaPointer;
-
- // Extended header
- private long fileSize;
-
- public WP6FileHeader() {
- super();
- }
-
- public String getFileId() {
- return fileId;
- }
-
- public void setFileId(String fileId) {
- this.fileId = fileId;
- }
-
- public long getDocAreaPointer() {
- return docAreaPointer;
- }
-
- public void setDocAreaPointer(long docAreaPointer) {
- this.docAreaPointer = docAreaPointer;
- }
-
- public int getProductType() {
- return productType;
- }
-
- public void setProductType(int productType) {
- this.productType = productType;
- }
-
- public int getFileType() {
- return fileType;
- }
-
- public void setFileType(int fileType) {
- this.fileType = fileType;
- }
-
- public int getMajorVersion() {
- return majorVersion;
- }
-
- public void setMajorVersion(int majorVersion) {
- this.majorVersion = majorVersion;
- }
-
- public int getMinorVersion() {
- return minorVersion;
- }
-
- public void setMinorVersion(int minorVersion) {
- this.minorVersion = minorVersion;
- }
-
- public boolean isEncrypted() {
- return encrypted;
- }
-
- public void setEncrypted(boolean encrypted) {
- this.encrypted = encrypted;
- }
-
- public int getIndexAreaPointer() {
- return indexAreaPointer;
- }
-
- public void setIndexAreaPointer(int indexAreaPointer) {
- this.indexAreaPointer = indexAreaPointer;
- }
-
- public long getFileSize() {
- return fileSize;
- }
- public void setFileSize(long fileSize) {
- this.fileSize = fileSize;
- }
-
- @Override
- public String toString() {
- return "WP6FileHeader{" +
- "fileId='" + fileId + '\'' +
- ", docAreaPointer=" + docAreaPointer +
- ", productType=" + productType +
- ", fileType=" + fileType +
- ", majorVersion=" + majorVersion +
- ", minorVersion=" + minorVersion +
- ", encrypted=" + encrypted +
- ", indexAreaPointer=" + indexAreaPointer +
- ", fileSize=" + fileSize +
- '}';
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result
- + (int) (docAreaPointer ^ (docAreaPointer >>> 32));
- result = prime * result + (encrypted ? 1231 : 1237);
- result = prime * result + ((fileId == null) ? 0 : fileId.hashCode());
- result = prime * result + (int) (fileSize ^ (fileSize >>> 32));
- result = prime * result + fileType;
- result = prime * result + indexAreaPointer;
- result = prime * result + majorVersion;
- result = prime * result + minorVersion;
- result = prime * result + productType;
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (!(obj instanceof WP6FileHeader)) {
- return false;
- }
- WP6FileHeader other = (WP6FileHeader) obj;
- if (docAreaPointer != other.docAreaPointer) {
- return false;
- }
- if (encrypted != other.encrypted) {
- return false;
- }
- if (fileId == null) {
- if (other.fileId != null) {
- return false;
- }
- } else if (!fileId.equals(other.fileId)) {
- return false;
- }
- if (fileSize != other.fileSize) {
- return false;
- }
- if (fileType != other.fileType) {
- return false;
- }
- if (indexAreaPointer != other.indexAreaPointer) {
- return false;
- }
- if (majorVersion != other.majorVersion) {
- return false;
- }
- if (minorVersion != other.minorVersion) {
- return false;
- }
- if (productType != other.productType) {
- return false;
- }
- return true;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6TextExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6TextExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6TextExtractor.java
deleted file mode 100644
index 8c03d81..0000000
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6TextExtractor.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/* Copyright 2015-2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.wordperfect;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.tika.exception.EncryptedDocumentException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.exception.UnsupportedFormatException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.WordPerfect;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.sax.XHTMLContentHandler;
-import org.xml.sax.SAXException;
-
-/**
- * Extracts text from a WordPerfect document according to WP6 File Format.
- * This format appears to be compatible with more recent versions too.
- * @author Pascal Essiembre
- */
-class WP6TextExtractor {
-
- public void extract(
- InputStream input, XHTMLContentHandler xhtml, Metadata metadata)
- throws IOException, SAXException, TikaException {
- WPInputStream in = new WPInputStream(input);
-
- WP6FileHeader header = parseFileHeader(in);
-
- applyMetadata(header, metadata);
-
- if (header.getMajorVersion() == 0) {
- MediaType version = WordPerfectParser.WP_UNK;
- if (header.getMinorVersion() == 0) {
- version = WordPerfectParser.WP_5_0;
- } else if (header.getMinorVersion() == 1) {
- version = WordPerfectParser.WP_5_1;
- }
- metadata.set(Metadata.CONTENT_TYPE, version.toString());
- throw new UnsupportedFormatException("Parser doesn't support this version:"+version.toString());
- }
-
- if (header.getMajorVersion() != 2) {
- metadata.set(Metadata.CONTENT_TYPE, WordPerfectParser.WP_UNK.toString());
- throw new UnsupportedFormatException("Parser doesn't recognize this version");
- }
-
- if (header.isEncrypted()) {
- throw new EncryptedDocumentException();
- }
-
- // For text extraction we can safely ignore WP Index Area and
- // Packet Data Area and jump right away to Document Area.
- extractDocumentText(in, header.getDocAreaPointer(), xhtml);
-
- }
-
- private void applyMetadata(WP6FileHeader header, Metadata metadata) {
- metadata.set(WordPerfect.FILE_SIZE,
- Long.toString(header.getFileSize()));
- metadata.set(WordPerfect.FILE_ID, header.getFileId());
- metadata.set(WordPerfect.PRODUCT_TYPE, header.getProductType());
- metadata.set(WordPerfect.FILE_TYPE, header.getFileType());
- metadata.set(WordPerfect.MAJOR_VERSION, header.getMajorVersion());
- metadata.set(WordPerfect.MINOR_VERSION, header.getMinorVersion());
- metadata.set(WordPerfect.ENCRYPTED,
- Boolean.toString(header.isEncrypted()));
- }
-
- private void extractDocumentText(
- WPInputStream in, long offset, XHTMLContentHandler xhtml)
- throws IOException, SAXException {
- xhtml.startElement("p");
-
- // Move to offset (for some reason skip() did not work).
- for (int i = 0; i < offset; i++) {
- in.readWPByte();
- }
-
- int chunk = 4096;
- StringBuilder out = new StringBuilder(chunk);
-
- int c;
- while ((c = in.read()) != -1) {
- if (c > 0 && c <= 32) {
- out.append(WP6Constants.DEFAULT_EXTENDED_INTL_CHARS[c]);
- } else if (c >= 33 && c <= 126) {
- out.append((char) c);
- } else if (c == 128) {
- out.append(' '); // Soft space
- } else if (c == 129) {
- out.append('\u00A0'); // Hard space
- } else if (c == 129) {
- out.append('-'); // Hard hyphen
- } else if (c == 135 || c == 137) {
- out.append('\n'); // Dormant Hard return
- } else if (c == 138) {
- // skip to closing pair surrounding page number
- skipUntilChar(in, 139);
- } else if (c == 198) {
- // end of cell
- out.append('\t');
- } else if (c >= 180 && c <= 207) {
- out.append('\n');
- } else if (c >= 208 && c <= 239) {
- // Variable-Length Multi-Byte Functions
- int subgroup = in.readWP();
- int functionSize = in.readWPShort();
- for (int i = 0; i < functionSize - 4; i++) {
- in.readWP();
- }
-
- // End-of-Line group
- if (c == 208) {
- if (subgroup >= 1 && subgroup <= 3) {
- out.append(' ');
- } else if (subgroup == 10) {
- // end of cell
- out.append('\t');
- } else if (subgroup >= 4 && subgroup <= 19) {
- out.append('\n');
- } else if (subgroup >= 20 && subgroup <= 22) {
- out.append(' ');
- } else if (subgroup >= 23 && subgroup <= 28) {
- out.append('\n');
- }
- } else if (c == 213) {
- out.append(' ');
- } else if (c == 224) {
- out.append('\t');
- }
- //TODO Are there functions containing data? Like footnotes?
-
- } else if (c == 240) {
- // extended char
- int charval = in.readWP();
- int charset = in.readWP();
- in.readWP(); // closing character
-
- //TODO implement all charsets
- if (charset == 4 || charset == 5) {
- out.append(
- WP6Constants.EXTENDED_CHARSETS[charset][charval]);
- } else {
- out.append("[TODO:charset" + charset + "]");
- }
- } else if (c >= 241 && c <= 254) {
- skipUntilChar(in, c);
- } else if (c == 255) {
- skipUntilChar(in, c);
- }
-
- if (out.length() >= chunk) {
- xhtml.characters(out.toString());
- out.setLength(0);
- }
- }
-
- // Ignored codes above 127:
-
- // 130,131,133: soft hyphens
- // 134: invisible return in line
- // 136: soft end of center/align
- // 140: style separator mark
- // 141,142: start/end of text to skip
- // 143: exited hyphenation
- // 144: cancel hyphenation
- // 145-151: match functions
- // 152-179: unknown/ignored
- // 255: reserved, cannot be used
-
- xhtml.characters(out.toString());
- out.setLength(0);
- xhtml.endElement("p");
- }
-
- // Skips until the given character is encountered.
- private int skipUntilChar(WPInputStream in, int targetChar)
- throws IOException {
- int count = 0;
- int c;
- while ((c = in.read()) != -1) {
- count++;
- if (c == targetChar) {
- return count;
- }
- }
- return count;
- }
-
- private WP6FileHeader parseFileHeader(WPInputStream in)
- throws IOException {
- WP6FileHeader header = new WP6FileHeader();
-
- // File header
- in.mark(30);
- header.setFileId(in.readWPString(4)); // 1-4
- header.setDocAreaPointer(in.readWPLong()); // 5-8
- header.setProductType(in.readWP()); // 9
- header.setFileType(in.readWPChar()); // 10
- header.setMajorVersion(in.readWP()); // 11
- header.setMinorVersion(in.readWP()); // 12
- header.setEncrypted(in.readWPShort() != 0); // 13-14
- header.setIndexAreaPointer(in.readWPShort()); // 15-16
- try {
- in.skip(4); // 4 reserved bytes: skip // 17-20
- header.setFileSize(in.readWPLong()); // 21-24
- } catch (IOException e) {
- // May fail if not extended error, which is fine.
- }
- in.reset();
-
- //TODO header may be shared between corel products, so move validation
- //specific to each product elsewhere?
- //TODO convert to logs only, and let it fail elsewhere?
-// if (!WP6Constants.WP6_FILE_ID.equals(header.getFileId())) {
-// throw new IOException("Not a WordPerfect file. File must start "
-// + "with " + WP6Constants.WP6_FILE_ID + " but was "
-// + header.getFileId());
-// }
-// if (WP6Constants.WP6_PRODUCT_TYPE != header.getProductType()) {
-// throw new IOException("Not a WordPerfect file. Product type "
-// + "must be " + WP6Constants.WP6_PRODUCT_TYPE + " but was "
-// + header.getProductType());
-// }
- //TODO perform file type validation?
- return header;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
new file mode 100644
index 0000000..002ecee
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+import java.io.IOException;
+
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.SAXException;
+
+/**
+ * Extracts WordPerfect Document Area text from a WordPerfect document.
+ * @author Pascal Essiembre
+ */
+abstract class WPDocumentAreaExtractor {
+
+ public void extract(WPInputStream in, XHTMLContentHandler xhtml)
+ throws IOException, SAXException {
+ int chunk = 4096;
+ StringBuilder out = new StringBuilder(chunk);
+ int c;
+ while ((c = in.read()) != -1) {
+ extract(c, in, out);
+ if (out.length() >= chunk) {
+ xhtml.characters(out.toString());
+ out.setLength(0);
+ }
+ }
+ xhtml.characters(out.toString());
+ out.setLength(0);
+ }
+
+ protected abstract void extract(
+ int c, WPInputStream in, StringBuilder out) throws IOException;
+
+
+ // Skips until the given character is encountered.
+ protected int skipUntilChar(WPInputStream in, int targetChar)
+ throws IOException {
+ int count = 0;
+ int c;
+ while ((c = in.read()) != -1) {
+ count++;
+ if (c == targetChar) {
+ return count;
+ }
+ }
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPInputStream.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPInputStream.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPInputStream.java
index 67c3200..9b01aa5 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPInputStream.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPInputStream.java
@@ -1,8 +1,10 @@
-/* Copyright 2015-2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@@ -12,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.tika.parser.wordperfect;
import java.io.BufferedInputStream;
@@ -24,6 +27,7 @@ import org.apache.commons.lang.StringUtils;
/**
* {@link InputStream} wrapper adding WordPerfect-specific byte-reading methods.
+ * Applies to both 5.x and 6+ documents.
* @author Pascal Essiembre
*/
class WPInputStream extends InputStream {
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixArea.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixArea.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixArea.java
new file mode 100644
index 0000000..5c08528
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixArea.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+
+/**
+ * Section of a WordPerfect file called "Prefix Area" containing
+ * information about the specific file format/version.
+ * @author Pascal Essiembre
+ */
+class WPPrefixArea {
+
+ public static final String WP_FILE_ID = "�WPC";
+ public static final int WP_PRODUCT_TYPE = 1;
+
+ // WordPerfect 5.x header values
+ public static final int WP5_FILE_TYPE = 10;
+ public static final int WP5_MAJOR_VERSION = 0;
+ public static final int WP5_0_MINOR_VERSION = 0;
+ public static final int WP5_1_MINOR_VERSION = 1; // also for WPWin 5.2
+
+ // WordPerfect 6+ header values
+ public static final int WP6_FILE_TYPE = 36;
+ public static final int WP6_MAJOR_VERSION = 2;
+ public static final int WP6_MINOR_VERSION = 0;
+
+
+ // Normal header
+ private String fileId;
+ private long docAreaPointer;
+ private int productType;
+ private int fileType;
+ private int majorVersion;
+ private int minorVersion;
+ private boolean encrypted;
+ private int indexAreaPointer;
+
+ // Extended header
+ private long fileSize;
+
+ public WPPrefixArea() {
+ super();
+ }
+
+ public String getFileId() {
+ return fileId;
+ }
+
+ public void setFileId(String fileId) {
+ this.fileId = fileId;
+ }
+
+ public long getDocAreaPointer() {
+ return docAreaPointer;
+ }
+
+ public void setDocAreaPointer(long docAreaPointer) {
+ this.docAreaPointer = docAreaPointer;
+ }
+
+ public int getProductType() {
+ return productType;
+ }
+
+ public void setProductType(int productType) {
+ this.productType = productType;
+ }
+
+ public int getFileType() {
+ return fileType;
+ }
+
+ public void setFileType(int fileType) {
+ this.fileType = fileType;
+ }
+
+ public int getMajorVersion() {
+ return majorVersion;
+ }
+
+ public void setMajorVersion(int majorVersion) {
+ this.majorVersion = majorVersion;
+ }
+
+ public int getMinorVersion() {
+ return minorVersion;
+ }
+
+ public void setMinorVersion(int minorVersion) {
+ this.minorVersion = minorVersion;
+ }
+
+ public boolean isEncrypted() {
+ return encrypted;
+ }
+
+ public void setEncrypted(boolean encrypted) {
+ this.encrypted = encrypted;
+ }
+
+ public int getIndexAreaPointer() {
+ return indexAreaPointer;
+ }
+
+ public void setIndexAreaPointer(int indexAreaPointer) {
+ this.indexAreaPointer = indexAreaPointer;
+ }
+
+ public long getFileSize() {
+ return fileSize;
+ }
+ public void setFileSize(long fileSize) {
+ this.fileSize = fileSize;
+ }
+
+ @Override
+ public String toString() {
+ return "WP6FileHeader{" +
+ "fileId='" + fileId + '\'' +
+ ", docAreaPointer=" + docAreaPointer +
+ ", productType=" + productType +
+ ", fileType=" + fileType +
+ ", majorVersion=" + majorVersion +
+ ", minorVersion=" + minorVersion +
+ ", encrypted=" + encrypted +
+ ", indexAreaPointer=" + indexAreaPointer +
+ ", fileSize=" + fileSize +
+ '}';
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result
+ + (int) (docAreaPointer ^ (docAreaPointer >>> 32));
+ result = prime * result + (encrypted ? 1231 : 1237);
+ result = prime * result + ((fileId == null) ? 0 : fileId.hashCode());
+ result = prime * result + (int) (fileSize ^ (fileSize >>> 32));
+ result = prime * result + fileType;
+ result = prime * result + indexAreaPointer;
+ result = prime * result + majorVersion;
+ result = prime * result + minorVersion;
+ result = prime * result + productType;
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof WPPrefixArea)) {
+ return false;
+ }
+ WPPrefixArea other = (WPPrefixArea) obj;
+ if (docAreaPointer != other.docAreaPointer) {
+ return false;
+ }
+ if (encrypted != other.encrypted) {
+ return false;
+ }
+ if (fileId == null) {
+ if (other.fileId != null) {
+ return false;
+ }
+ } else if (!fileId.equals(other.fileId)) {
+ return false;
+ }
+ if (fileSize != other.fileSize) {
+ return false;
+ }
+ if (fileType != other.fileType) {
+ return false;
+ }
+ if (indexAreaPointer != other.indexAreaPointer) {
+ return false;
+ }
+ if (majorVersion != other.majorVersion) {
+ return false;
+ }
+ if (minorVersion != other.minorVersion) {
+ return false;
+ }
+ if (productType != other.productType) {
+ return false;
+ }
+ return true;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixAreaExtractor.java
new file mode 100644
index 0000000..1f587f2
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixAreaExtractor.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+import java.io.IOException;
+
+/**
+ * Extracts WordPerfect Prefix Area data from a WordPerfect document.
+ * Applies to both 5.x and 6+ documents.
+ * @author Pascal Essiembre
+ */
+final class WPPrefixAreaExtractor {
+
+ private WPPrefixAreaExtractor() {
+ super();
+ }
+
+ //WP5.x:
+ // Prefix Area: 16 bytes (standard header)
+
+ //WP6.x:
+ // Prefix Area: 30 bytes (16 standard header + 14 index header)
+
+ public static WPPrefixArea extract(WPInputStream in)
+ throws IOException {
+ WPPrefixArea prefixArea = new WPPrefixArea();
+
+ in.mark(30);
+ prefixArea.setFileId(in.readWPString(4)); // 1-4
+ prefixArea.setDocAreaPointer(in.readWPLong()); // 5-8
+ prefixArea.setProductType(in.readWP()); // 9
+ prefixArea.setFileType(in.readWPChar()); // 10
+ prefixArea.setMajorVersion(in.readWP()); // 11
+ prefixArea.setMinorVersion(in.readWP()); // 12
+ prefixArea.setEncrypted(in.readWPShort() != 0); // 13-14
+ prefixArea.setIndexAreaPointer(in.readWPShort()); // 15-16
+
+ // only applies to 6.x:
+ prefixArea.setFileSize(-1);
+ if (prefixArea.getMajorVersion() == WPPrefixArea.WP6_MAJOR_VERSION) {
+ try {
+ in.skip(4); // 4 reserved bytes: skip // 17-20
+ prefixArea.setFileSize(in.readWPLong()); // 21-24
+ } catch (IOException e) {
+ // May fail if no index header, which is fine.
+ }
+ }
+
+ in.reset();
+
+ return prefixArea;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
index 5cd81e2..4c6d445 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
@@ -1,8 +1,10 @@
-/* Copyright 2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@@ -16,11 +18,16 @@ package org.apache.tika.parser.wordperfect;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.HashSet;
import java.util.Set;
+import org.apache.tika.exception.EncryptedDocumentException;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.exception.UnsupportedFormatException;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.WordPerfect;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AbstractParser;
import org.apache.tika.parser.ParseContext;
@@ -49,7 +56,8 @@ public class WordPerfectParser extends AbstractParser {
final static MediaType WP_6_x = new MediaType(WP_BASE, "version", "6.x");
private static final Set<MediaType> SUPPORTED_TYPES =
- Collections.unmodifiableSet(Collections.singleton(WP_6_x));
+ Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
+ WP_5_0, WP_5_1, WP_6_x)));
@Override
public Set<MediaType> getSupportedTypes(ParseContext context) {
@@ -60,17 +68,87 @@ public class WordPerfectParser extends AbstractParser {
public void parse(InputStream stream, ContentHandler handler,
Metadata metadata, ParseContext context)
throws IOException, SAXException, TikaException {
- if (metadata.get(Metadata.CONTENT_TYPE) == null) {
- metadata.set(Metadata.CONTENT_TYPE, "application/wordperfect");
+
+ WPInputStream wpStream = new WPInputStream(stream);
+ WPPrefixArea prefixArea = WPPrefixAreaExtractor.extract(wpStream);
+
+ ensureFileSupport(prefixArea, metadata);
+
+ applyMetadata(prefixArea, metadata);
+
+ extractDocumentArea(prefixArea, wpStream,
+ new XHTMLContentHandler(handler, metadata));
+ }
+
+ private void extractDocumentArea(WPPrefixArea prefixArea,
+ WPInputStream in, XHTMLContentHandler xhtml)
+ throws SAXException, IOException {
+
+ // Move to offset (for some reason skip() did not work).
+ for (int i = 0; i < prefixArea.getDocAreaPointer(); i++) {
+ in.readWPByte();
}
- XHTMLContentHandler xhtml =
- new XHTMLContentHandler(handler, metadata);
xhtml.startDocument();
+ xhtml.startElement("p");
- WP6TextExtractor extractor = new WP6TextExtractor();
- extractor.extract(stream, xhtml, metadata);
-
+ getDocumentAreaExtractor(prefixArea).extract(in, xhtml);
+
+ xhtml.endElement("p");
xhtml.endDocument();
}
+
+ private void ensureFileSupport(WPPrefixArea pa, Metadata metadata)
+ throws UnsupportedFormatException, EncryptedDocumentException {
+ if (pa.getMajorVersion() != WPPrefixArea.WP5_MAJOR_VERSION
+ && pa.getMajorVersion() != WPPrefixArea.WP6_MAJOR_VERSION) {
+ metadata.set(Metadata.CONTENT_TYPE, WP_UNK.toString());
+ throw new UnsupportedFormatException(
+ "Parser doesn't recognize this major version: "
+ + pa.getMajorVersion());
+ }
+ if (pa.isEncrypted()) {
+ throw new EncryptedDocumentException();
+ }
+ }
+
+ private void applyMetadata(WPPrefixArea pa, Metadata metadata) {
+ // Should we force the more precise media type if only the base
+ // form is found? Or shall we store a friendly representation
+ // of the version in a new field?
+ if (metadata.get(Metadata.CONTENT_TYPE) == null) {
+ if (pa.getMajorVersion() == WPPrefixArea.WP6_MAJOR_VERSION) {
+ metadata.set(Metadata.CONTENT_TYPE, WP_6_x.toString());
+ } else if (pa.getMinorVersion() ==
+ WPPrefixArea.WP5_0_MINOR_VERSION) {
+ metadata.set(Metadata.CONTENT_TYPE, WP_5_0.toString());
+ } else if (pa.getMinorVersion() ==
+ WPPrefixArea.WP5_1_MINOR_VERSION) {
+ metadata.set(Metadata.CONTENT_TYPE, WP_5_1.toString());
+ } else {
+ metadata.set(Metadata.CONTENT_TYPE, WP_BASE.toString());
+ }
+ }
+
+ metadata.set(WordPerfect.FILE_ID, pa.getFileId());
+ metadata.set(WordPerfect.PRODUCT_TYPE, pa.getProductType());
+ metadata.set(WordPerfect.FILE_TYPE, pa.getFileType());
+ metadata.set(WordPerfect.MAJOR_VERSION, pa.getMajorVersion());
+ metadata.set(WordPerfect.MINOR_VERSION, pa.getMinorVersion());
+ metadata.set(WordPerfect.ENCRYPTED, Boolean.toString(pa.isEncrypted()));
+
+ if (pa.getFileSize() > -1) {
+ metadata.set(
+ WordPerfect.FILE_SIZE, Long.toString(pa.getFileSize()));
+ }
+ }
+
+ private WPDocumentAreaExtractor getDocumentAreaExtractor(
+ WPPrefixArea prefixArea) {
+ if (prefixArea.getMajorVersion() == WPPrefixArea.WP6_MAJOR_VERSION) {
+ return new WP6DocumentAreaExtractor();
+ }
+ // we can safely assume v5 as exception would have been thrown
+ return new WP5DocumentAreaExtractor();
+ }
}
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/QuattroProTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/QuattroProTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/QuattroProTest.java
index cd43a84..687caf1 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/QuattroProTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/QuattroProTest.java
@@ -1,8 +1,10 @@
-/* Copyright 2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WPInputStreamTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WPInputStreamTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WPInputStreamTest.java
index d204e0c..878388e 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WPInputStreamTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WPInputStreamTest.java
@@ -30,8 +30,7 @@ public class WPInputStreamTest {
@Test
public void testReadByte() throws Exception {
- WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
- try {
+ try (WPInputStream wpInputStream = emptyWPStream()) {
wpInputStream.readWPByte();
fail("should have thrown EOF");
} catch (EOFException e) {
@@ -42,8 +41,7 @@ public class WPInputStreamTest {
@Test
public void testReadShort() throws Exception {
- WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
- try {
+ try (WPInputStream wpInputStream = emptyWPStream()) {
wpInputStream.readWPShort();
fail("should have thrown EOF");
} catch (EOFException e) {
@@ -54,8 +52,7 @@ public class WPInputStreamTest {
@Test
public void testReadChar() throws Exception {
- WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
- try {
+ try (WPInputStream wpInputStream = emptyWPStream()) {
wpInputStream.readWPChar();
fail("should have thrown EOF");
} catch (EOFException e) {
@@ -65,8 +62,7 @@ public class WPInputStreamTest {
@Test
public void testReadHex() throws Exception {
- WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
- try {
+ try (WPInputStream wpInputStream = emptyWPStream()) {
wpInputStream.readWPHex();
fail("should have thrown EOF");
} catch (EOFException e) {
@@ -76,8 +72,7 @@ public class WPInputStreamTest {
@Test
public void testReadHexString() throws Exception {
- WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
- try {
+ try (WPInputStream wpInputStream = emptyWPStream()) {
wpInputStream.readWPHexString(10);
fail("should have thrown EOF");
} catch (EOFException e) {
@@ -87,8 +82,7 @@ public class WPInputStreamTest {
@Test
public void testReadLong() throws Exception {
- WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
- try {
+ try (WPInputStream wpInputStream = emptyWPStream()) {
wpInputStream.readWPLong();
fail("should have thrown EOF");
} catch (EOFException e) {
@@ -99,8 +93,7 @@ public class WPInputStreamTest {
@Test
public void testReadString() throws Exception {
- WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
- try {
+ try (WPInputStream wpInputStream = emptyWPStream()) {
wpInputStream.readWPString(10);
fail("should have thrown EOF");
} catch (EOFException e) {
@@ -110,8 +103,7 @@ public class WPInputStreamTest {
@Test
public void testReadArr() throws Exception {
- WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
- try {
+ try (WPInputStream wpInputStream = emptyWPStream()) {
byte[] buffer = new byte[10];
wpInputStream.read(buffer);
} catch (EOFException e) {
@@ -121,8 +113,7 @@ public class WPInputStreamTest {
@Test
public void testReadArrOffset() throws Exception {
- WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
- try {
+ try (WPInputStream wpInputStream = emptyWPStream()) {
byte[] buffer = new byte[10];
wpInputStream.read(buffer, 0, 2);
} catch (EOFException e) {
@@ -130,5 +121,7 @@ public class WPInputStreamTest {
}
}
-
+ private WPInputStream emptyWPStream() {
+ return new WPInputStream(new ByteArrayInputStream(new byte[0]));
+ }
}
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
index 8eafc18..89abca0 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
@@ -1,8 +1,10 @@
-/* Copyright 2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@@ -26,31 +28,32 @@ import org.junit.Test;
*/
public class WordPerfectTest extends TikaTest {
-
@Test
public void testWordPerfectParser() throws Exception {
-
XMLResult r = getXML("testWordPerfect.wpd");
- assertEquals("application/vnd.wordperfect; version=6.x",
+ assertEquals(WordPerfectParser.WP_6_x.toString(),
r.metadata.get(Metadata.CONTENT_TYPE));
assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
assertContains("test test", r.xml);
}
-
@Test
public void testVersion50() throws Exception {
//test file "testWordPerfect_5_0.wp" is from govdocs1: 126546.wp
XMLResult r = getXML("testWordPerfect_5_0.wp");
- assertEquals(WordPerfectParser.WP_5_0.toString(), r.metadata.get(Metadata.CONTENT_TYPE));
- assertEquals("org.apache.tika.parser.EmptyParser", r.metadata.get("X-Parsed-By"));
+ assertEquals(WordPerfectParser.WP_5_0.toString(),
+ r.metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
+ assertContains("Surrounded by her family", r.xml);
}
@Test
public void testVersion51() throws Exception {
//testfile "testWordperfect_5_1.wp is from govdocs1: 758750.wp
XMLResult r = getXML("testWordPerfect_5_1.wp");
- assertEquals(WordPerfectParser.WP_5_1.toString(), r.metadata.get(Metadata.CONTENT_TYPE));
- assertEquals("org.apache.tika.parser.EmptyParser", r.metadata.get("X-Parsed-By"));
+ assertEquals(WordPerfectParser.WP_5_1.toString(),
+ r.metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
+ assertContains("Summer is the peak travel time", r.xml);
}
}
[3/3] tika git commit: TIKA-2230 -- add paragraph markup to
WordPerfect parsers
Posted by ta...@apache.org.
TIKA-2230 -- add paragraph markup to WordPerfect parsers
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/940e6f48
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/940e6f48
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/940e6f48
Branch: refs/heads/master
Commit: 940e6f483ec87725870b3c9d62ed0b2ab9a5811e
Parents: 6dc442d
Author: tballison <ta...@mitre.org>
Authored: Wed Jan 4 12:31:47 2017 -0500
Committer: tballison <ta...@mitre.org>
Committed: Wed Jan 4 12:31:47 2017 -0500
----------------------------------------------------------------------
.../wordperfect/WP5DocumentAreaExtractor.java | 15 ++++----
.../wordperfect/WP6DocumentAreaExtractor.java | 15 ++++----
.../wordperfect/WPDocumentAreaExtractor.java | 37 ++++++++++++++++----
.../parser/wordperfect/WordPerfectParser.java | 2 --
.../parser/wordperfect/WordPerfectTest.java | 7 ++--
5 files changed, 53 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
index b95490e..bd59725 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
@@ -18,6 +18,9 @@ package org.apache.tika.parser.wordperfect;
import java.io.IOException;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.SAXException;
+
/**
* Extracts WordPerfect Document Area text from a WordPerfect document
* version 5.x.
@@ -25,16 +28,16 @@ import java.io.IOException;
*/
class WP5DocumentAreaExtractor extends WPDocumentAreaExtractor {
- protected void extract(int c, WPInputStream in, StringBuilder out)
- throws IOException {
+ protected void extract(int c, WPInputStream in, StringBuilder out, XHTMLContentHandler xhtml)
+ throws IOException, SAXException {
// 0-31: control characters
if (c == 10) {
- out.append('\n'); // hard return ("Enter")
+ endParagraph(out, xhtml);// hard return ("Enter")
} else if (c == 11) {
out.append(' '); // soft page break
} else if (c == 12) {
- out.append('\n'); // hard page break
+ endParagraph(out, xhtml);// hard page break
} else if (c == 13) {
out.append(' '); // soft return (line wrap)
@@ -44,11 +47,11 @@ class WP5DocumentAreaExtractor extends WPDocumentAreaExtractor {
// 128-191: single-byte functions
} else if (c == 140) {
- out.append('\n'); // combination hard return/soft page (WP5.1)
+ endParagraph(out, xhtml);// combination hard return/soft page (WP5.1)
} else if (c >= 144 && c <= 149) {
out.append(' '); // deletable/invisible soft return/page
} else if (c == 153) {
- out.append('\4'); // Dormant Hard return (WP5.1)
+ endParagraph(out, xhtml);// Dormant Hard return (WP5.1)
} else if (c == 160) {
out.append('\u00A0'); // Hard space
} else if (c >= 169 && c <= 171) {
http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
index 659ad17..862c858 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
@@ -18,6 +18,9 @@ package org.apache.tika.parser.wordperfect;
import java.io.IOException;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.SAXException;
+
/**
* Extracts WordPerfect Document Area text from a WordPerfect document
* version 6+.
@@ -25,8 +28,8 @@ import java.io.IOException;
*/
class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
- protected void extract(int c, WPInputStream in, StringBuilder out)
- throws IOException {
+ protected void extract(int c, WPInputStream in, StringBuilder out, XHTMLContentHandler xhtml)
+ throws IOException, SAXException {
if (c > 0 && c <= 32) {
out.append(WP6Charsets.DEFAULT_EXTENDED_INTL_CHARS[c]);
} else if (c >= 33 && c <= 126) {
@@ -38,7 +41,7 @@ class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
} else if (c == 129) {
out.append('-'); // Hard hyphen
} else if (c == 135 || c == 137) {
- out.append('\n'); // Dormant Hard return
+ endParagraph(out, xhtml); // Dormant Hard return
} else if (c == 138) {
// skip to closing pair surrounding page number
skipUntilChar(in, 139);
@@ -46,7 +49,7 @@ class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
// end of cell
out.append('\t');
} else if (c >= 180 && c <= 207) {
- out.append('\n');
+ endParagraph(out, xhtml);
} else if (c >= 208 && c <= 239) {
// Variable-Length Multi-Byte Functions
int subgroup = in.readWP();
@@ -63,11 +66,11 @@ class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
// end of cell
out.append('\t');
} else if (subgroup >= 4 && subgroup <= 19) {
- out.append('\n');
+ endParagraph(out, xhtml);
} else if (subgroup >= 20 && subgroup <= 22) {
out.append(' ');
} else if (subgroup >= 23 && subgroup <= 28) {
- out.append('\n');
+ endParagraph(out, xhtml);
}
} else if (c == 213) {
out.append(' ');
http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
index 002ecee..ee3980e 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
@@ -27,26 +27,51 @@ import org.xml.sax.SAXException;
*/
abstract class WPDocumentAreaExtractor {
+ boolean startedP = false;
public void extract(WPInputStream in, XHTMLContentHandler xhtml)
throws IOException, SAXException {
int chunk = 4096;
StringBuilder out = new StringBuilder(chunk);
int c;
while ((c = in.read()) != -1) {
- extract(c, in, out);
+ extract(c, in, out, xhtml);
if (out.length() >= chunk) {
xhtml.characters(out.toString());
out.setLength(0);
}
}
- xhtml.characters(out.toString());
- out.setLength(0);
+ endParagraph(out, xhtml);
}
protected abstract void extract(
- int c, WPInputStream in, StringBuilder out) throws IOException;
-
-
+ int c, WPInputStream in, StringBuilder out, XHTMLContentHandler xhtml) throws IOException, SAXException;
+
+
+ protected void lazilyStartParagraph(XHTMLContentHandler xhtml) throws SAXException {
+ if (! startedP) {
+ xhtml.startElement("p");
+ }
+ startedP = true;
+ }
+
+ /**
+ * This assumes that the <p> was started before you get here.
+ * And the user is required to close the <p> at the end of the document.
+ *
+ * These are currently handled by {@link #extract(WPInputStream, XHTMLContentHandler)}.
+ *
+ * @param xhtml
+ * @throws SAXException
+ */
+ protected void endParagraph(StringBuilder buffer, XHTMLContentHandler xhtml) throws SAXException {
+ lazilyStartParagraph(xhtml);
+
+ xhtml.characters(buffer.toString());
+ buffer.setLength(0);
+ xhtml.endElement("p");
+ startedP = false;
+ }
+
// Skips until the given character is encountered.
protected int skipUntilChar(WPInputStream in, int targetChar)
throws IOException {
http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
index 4c6d445..a18fee1 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
@@ -90,11 +90,9 @@ public class WordPerfectParser extends AbstractParser {
}
xhtml.startDocument();
- xhtml.startElement("p");
getDocumentAreaExtractor(prefixArea).extract(in, xhtml);
- xhtml.endElement("p");
xhtml.endDocument();
}
http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
index 89abca0..0da8959 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
@@ -34,7 +34,7 @@ public class WordPerfectTest extends TikaTest {
assertEquals(WordPerfectParser.WP_6_x.toString(),
r.metadata.get(Metadata.CONTENT_TYPE));
assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
- assertContains("test test", r.xml);
+ assertContains("<p>AND FURTHER</p>", r.xml);
}
@Test
@@ -44,7 +44,7 @@ public class WordPerfectTest extends TikaTest {
assertEquals(WordPerfectParser.WP_5_0.toString(),
r.metadata.get(Metadata.CONTENT_TYPE));
assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
- assertContains("Surrounded by her family", r.xml);
+ assertContains("<p>Surrounded by her family", r.xml);
}
@Test
@@ -54,6 +54,7 @@ public class WordPerfectTest extends TikaTest {
assertEquals(WordPerfectParser.WP_5_1.toString(),
r.metadata.get(Metadata.CONTENT_TYPE));
assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
- assertContains("Summer is the peak travel time", r.xml);
+ assertContains("<p>STUDY RESULTS: Existing condition", r.xml);
+ assertContains("Seattle nonstop flights.</p>", r.xml);
}
}
[2/3] tika git commit: TIKA-2228 - WordPerfect parser update to
handle 5.x from Pascal Essiembre. This closes #142.
Posted by ta...@apache.org.
TIKA-2228 - WordPerfect parser update to handle 5.x from Pascal Essiembre. This closes #142.
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/6dc442da
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/6dc442da
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/6dc442da
Branch: refs/heads/master
Commit: 6dc442da50dbb22bbf2b73076d5080f32af03067
Parents: ef1d907
Author: tballison <ta...@mitre.org>
Authored: Wed Jan 4 11:54:26 2017 -0500
Committer: tballison <ta...@mitre.org>
Committed: Wed Jan 4 11:54:26 2017 -0500
----------------------------------------------------------------------
.../parser/wordperfect/QPWTextExtractor.java | 12 +-
.../parser/wordperfect/QuattroProParser.java | 3 +-
.../tika/parser/wordperfect/WP5Charsets.java | 205 +++++++++
.../wordperfect/WP5DocumentAreaExtractor.java | 78 ++++
.../tika/parser/wordperfect/WP6Charsets.java | 447 +++++++++++++++++++
.../tika/parser/wordperfect/WP6Constants.java | 432 ------------------
.../wordperfect/WP6DocumentAreaExtractor.java | 104 +++++
.../tika/parser/wordperfect/WP6FileHeader.java | 191 --------
.../parser/wordperfect/WP6TextExtractor.java | 243 ----------
.../wordperfect/WPDocumentAreaExtractor.java | 63 +++
.../tika/parser/wordperfect/WPInputStream.java | 14 +-
.../tika/parser/wordperfect/WPPrefixArea.java | 209 +++++++++
.../wordperfect/WPPrefixAreaExtractor.java | 67 +++
.../parser/wordperfect/WordPerfectParser.java | 104 ++++-
.../tika/parser/wordperfect/QuattroProTest.java | 12 +-
.../parser/wordperfect/WPInputStreamTest.java | 31 +-
.../parser/wordperfect/WordPerfectTest.java | 29 +-
17 files changed, 1316 insertions(+), 928 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QPWTextExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QPWTextExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QPWTextExtractor.java
index 1f5f68f..0bf9731 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QPWTextExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QPWTextExtractor.java
@@ -1,8 +1,10 @@
-/* Copyright 2015-2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QuattroProParser.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QuattroProParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QuattroProParser.java
index 8b29b4d..64895a4 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QuattroProParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QuattroProParser.java
@@ -1,5 +1,4 @@
-/* Copyright 2016 Norconex Inc.
- *
+/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5Charsets.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5Charsets.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5Charsets.java
new file mode 100644
index 0000000..6db8afa
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5Charsets.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+/**
+ * WordPerfect 5.x constant values used for mapping WordPerfect charsets to
+ * unicode equivalents when possible.
+ * @author Pascal Essiembre
+ */
+final class WP5Charsets {
+
+ private static final Logger LOG = LogManager.getLogger(WP5Charsets.class);
+
+ /**
+ * Extended character sets used when fixed-length multi-byte functions
+ * with a byte value of 192 (0xC0) are found in a WordPerfect document.
+ * Those character set codes may be specific to WordPerfect
+ * file specifications and may or may not be considered standard
+ * outside WordPerfect. Applies to version 5.x.
+ */
+ public static final char[][] EXTENDED_CHARSETS = new char[][] {
+ // WP Charset 0: ASCII (same as WP6)
+ WP6Charsets.EXTENDED_CHARSETS[0],
+ // WP Charset 1: Multinational 1 (same as WP6)
+ WP6Charsets.EXTENDED_CHARSETS[1],
+ // WP Charset 2: Multinational 2 (28 chars)
+ {
+ '\u0323','\u0324','\u02da','\u0325','\u02bc','\u032d','\u2017','\u005f',
+ '\u0138','\u032e','\u033e','\u2018','\u0020','\u02bd','\u02db','\u0327',
+ '\u0321','\u0322','\u030d','\u2019','\u0329','\u0020','\u0621','\u02be',
+ '\u0306','\u0310','\u2032','\u2034'
+ },
+ // WP Charset 3: Box Drawing (same as WP6)
+ WP6Charsets.EXTENDED_CHARSETS[3],
+ // WP Charset 4: Typographic Symbols (same as WP6)
+ WP6Charsets.EXTENDED_CHARSETS[4],
+ // WP Charset 5: Iconic Symbol (35 chars)
+ {
+ '\u2665','\u2666','\u2663','\u2660','\u2642','\u2640','\u263c','\u263a',
+ '\u263b','\u266a','\u266c','\u25ac','\u2302','\u203c','\u221a','\u21a8',
+ '\u2310','\u2319','\u25d8','\u25d9','\u21b5','\u261e','\u261c','\u2713',
+ '\u2610','\u2612','\u2639','\u266f','\u266d','\u266e','\u260e','\u231a',
+ '\u231b','\u2104','\u23b5'
+ },
+ // WP Charset 6: Math/Scientific (same as WP6)
+ WP6Charsets.EXTENDED_CHARSETS[6],
+ // WP Charset 7 Math/Scientific Extended (same as WP6)
+ WP6Charsets.EXTENDED_CHARSETS[7],
+ // WP Charset 8: Greek (210 chars)
+ {
+ '\u0391','\u03b1','\u0392','\u03b2','\u0392','\u03d0','\u0393','\u03b3',
+ '\u0394','\u03b4','\u0395','\u03b5','\u0396','\u03b6','\u0397','\u03b7',
+ '\u0398','\u03b8','\u0399','\u03b9','\u039a','\u03ba','\u039b','\u03bb',
+ '\u039c','\u03bc','\u039d','\u03bd','\u039e','\u03be','\u039f','\u03bf',
+ '\u03a0','\u03c0','\u03a1','\u03c1','\u03a3','\u03c3','\u03f9','\u03db',
+ '\u03a4','\u03c4','\u03a5','\u03c5','\u03a6','\u03d5','\u03a7','\u03c7',
+ '\u03a8','\u03c8','\u03a9','\u03c9','\u03ac','\u03ad','\u03ae','\u03af',
+ '\u03ca','\u03cc','\u03cd','\u03cb','\u03ce','\u03b5','\u03d1','\u03f0',
+ '\u03d6','\u1fe5','\u03d2','\u03c6','\u03c9','\u037e','\u0387','\u0384',
+ '\u00a8','\u0385','\u1fed','\u1fef','\u1fc0','\u1fbd','\u1fbf','\u1fbe',
+ '\u1fce','\u1fde','\u1fcd','\u1fdd','\u1fcf','\u1fdf','\u0384','\u1fef',
+ '\u1fc0','\u1fbd','\u1fbf','\u1fce','\u1fde','\u1fcd','\u1fdd','\u1fcf',
+ '\u1fdf','\u1f70','\u1fb6','\u1fb3','\u1fb4','\u1fb7','\u1f00','\u1f04',
+ '\u1f02','\u1f06','\u1f80','\u1f84','\u1f86','\u1f01','\u1f05','\u1f03',
+ '\u1f07','\u1f81','\u1f85','\u1f87','\u1f72','\u1f10','\u1f14','\u1f13',
+ '\u1f11','\u1f15','\u1f13','\u1f74','\u1fc6','\u1fc3','\u1fc4','\u1fc2',
+ '\u1fc7','\u1f20','\u1f24','\u1f22','\u1f26','\u1f90','\u1f94','\u1f96',
+ '\u1f21','\u1f25','\u1f23','\u1f27','\u1f91','\u1f95','\u1f97','\u1f76',
+ '\u1fd6','\u0390','\u1fd2','\u1f30','\u1f34','\u1f32','\u1f36','\u1f31',
+ '\u1f35','\u1f33','\u1f37','\u1f78','\u1f40','\u1f44','\u1f42','\u1f41',
+ '\u1f45','\u1f43','\u1f7a','\u1fe6','\u03b0','\u1fe3','\u1f50','\u1f54',
+ '\u1f52','\u1f56','\u1f51','\u1f55','\u1f53','\u1f57','\u1f7c','\u1ff6',
+ '\u1ff3','\u1ff4','\u1ff2','\u1ff7','\u1f60','\u1f64','\u1f62','\u1f66',
+ '\u1fa0','\u1fa4','\u1fa6','\u1f61','\u1f65','\u1f63','\u1f67','\u1fa1',
+ '\u1fa5','\u1fa7','\u0374','\u0375','\u03db','\u03dd','\u03d9','\u03e1',
+ '\u0386','\u0388','\u0389','\u038a','\u038c','\u038e','\u038f','\u03aa',
+ '\u03ab','\u1fe5'
+ },
+ // WP Charset 9: Hebrew (119 chars)
+ {
+ '\u05d0','\u05d1','\u05d2','\u05d3','\u05d4','\u05d5','\u05d6','\u05d7',
+ '\u05d8','\u05d9','\u05da','\u05db','\u05dc','\u05dd','\u05de','\u05df',
+ '\u05e0','\u05e1','\u05e2','\u05e3','\u05e4','\u05e5','\u05e6','\u05e7',
+ '\u05e8','\u05e9','\u05ea','\u05be','\u05c0','\u05c3','\u05f3','\u05f4',
+ '\u05b0','\u05b1','\u05b2','\u05b3','\u05b4','\u05b5','\u05b6','\u05b7',
+ '\u05b8','\u05b9','\u05ba','\u05bb','\u05bc','\u05bd','\u05bf','\u05b7',
+ '\ufbe1','\u05f0','\u05f1','\u05f2','\u0591','\u0596','\u05ad','\u05a4',
+ '\u059a','\u059b','\u05a3','\u05a5','\u05a6','\u05a7','\u09aa','\u0592',
+ '\u0593','\u0594','\u0595','\u0597','\u0598','\u0599','\u05a8','\u059c',
+ '\u059d','\u059e','\u05a1','\u05a9','\u05a0','\u059f','\u05ab','\u05ac',
+ '\u05af','\u05c4','\u0544','\u05d0','\ufb31','\ufb32','\ufb33','\ufb34',
+ '\ufb35','\ufb4b','\ufb36','\u05d7','\ufb38','\ufb39','\ufb3b','\ufb3a',
+ '\u05da','\u05da','\u05da','\u05da','\u05da','\u05da','\ufb3c','\ufb3e',
+ '\ufb40','\u05df','\ufb41','\ufb44','\ufb46','\ufb47','\ufb2b','\ufb2d',
+ '\ufb2a','\ufb2c','\ufb4a','\ufb4c','\ufb4e','\ufb1f','\ufb1d'
+ },
+ // WP Charset 10: Cyrillic (150 chars)
+ {
+ '\u0410','\u0430','\u0411','\u0431','\u0412','\u0432','\u0413','\u0433',
+ '\u0414','\u0434','\u0415','\u0435','\u0401','\u0451','\u0416','\u0436',
+ '\u0417','\u0437','\u0418','\u0438','\u0419','\u0439','\u041a','\u043a',
+ '\u041b','\u043b','\u041c','\u043c','\u041d','\u043d','\u041e','\u043e',
+ '\u041f','\u043f','\u0420','\u0440','\u0421','\u0441','\u0422','\u0442',
+ '\u0423','\u0443','\u0424','\u0444','\u0425','\u0445','\u0426','\u0446',
+ '\u0427','\u0447','\u0428','\u0448','\u0429','\u0449','\u042a','\u044a',
+ '\u042b','\u044b','\u042c','\u044c','\u042d','\u044d','\u042e','\u044e',
+ '\u042f','\u044f','\u0490','\u0491','\u0402','\u0452','\u0403','\u0453',
+ '\u0404','\u0454','\u0405','\u0455','\u0406','\u0456','\u0407','\u0457',
+ '\u0408','\u0458','\u0409','\u0459','\u040a','\u045a','\u040b','\u045b',
+ '\u040c','\u045c','\u040e','\u045e','\u040f','\u045f','\u0462','\u0463',
+ '\u0472','\u0473','\u0474','\u0475','\u046a','\u046b','\ua640','\ua641',
+ '\u0429','\u0449','\u04c0','\u04cf','\u0466','\u0467','\u0000','\u0000',
+ '\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000',
+ '\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000',
+ '\u0000','\u0000','\u0400','\u0450','\u0000','\u0000','\u040d','\u045d',
+ '\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000',
+ '\u0000','\u0000','\u0000','\u0000','\u0301','\u0300'
+ },
+ // WP Charset 11: Japanese (185 chars)
+ {
+ '\u3041','\u3043','\u3045','\u3047','\u3049','\u3053','\u3083','\u3085',
+ '\u3087','\u3094','\u3095','\u3096','\u3042','\u3044','\u3046','\u3048',
+ '\u304a','\u304b','\u304d','\u3047','\u3051','\u3053','\u304c','\u304e',
+ '\u3050','\u3052','\u3054','\u3055','\u3057','\u3059','\u305b','\u305d',
+ '\u3056','\u3058','\u305a','\u305c','\u305e','\u305f','\u3051','\u3064',
+ '\u3066','\u3068','\u3060','\u3062','\u3065','\u3067','\u3069','\u306a',
+ '\u306b','\u306c','\u306d','\u306e','\u306f','\u3072','\u3075','\u3078',
+ '\u307b','\u3070','\u3073','\u3076','\u3079','\u307c','\u3071','\u3074',
+ '\u3077','\u307a','\u307d','\u307e','\u307f','\u3080','\u3081','\u3082',
+ '\u3084','\u3086','\u3088','\u3089','\u308a','\u308b','\u308c','\u308d',
+ '\u308e','\u3092','\u3093','\u3014','\u3015','\uff3b','\uff3d','\u300c',
+ '\u300d','\u300c','\u300d','\u302a','\u3002','\u3001','\u309d','\u309e',
+ '\u3003','\u30fc','\u309b','\u309c','\u30a1','\u30a3','\u30a5','\u30a7',
+ '\u30a9','\u30c3','\u30e3','\u30e5','\u3057','\u30f4','\u30f5','\u30f6',
+ '\u30a2','\u30a4','\u30a6','\u30a8','\u30aa','\u30ab','\u30ad','\u30af',
+ '\u30b1','\u30b3','\u30ac','\u30ae','\u30b0','\u30b2','\u30b4','\u30b5',
+ '\u30c4','\u30b9','\u30bb','\u30bd','\u30b6','\u30b8','\u30ba','\u30bc',
+ '\u30be','\u30bf','\u30c1','\u30c4','\u30c6','\u30c8','\u30c0','\u30c2',
+ '\u30c5','\u30c7','\u30c9','\u30ca','\u30cb','\u30cc','\u30cd','\u30ce',
+ '\u30cf','\u30d2','\u30d5','\u30d8','\u03d0','\u30db','\u30d3','\u30d6',
+ '\u30d9','\u30dc','\u30d1','\u30d4','\u30d7','\u30da','\u30dd','\u30de',
+ '\u30df','\u30e0','\u30e1','\u30e2','\u30e4','\u30e6','\u30e8','\u30e9',
+ '\u30ea','\u30ab','\u30ec','\u30ed','\u30ef','\u30f2','\u30f3','\u30fd',
+ '\u30fe'
+ },
+ // WP Charset 12: User-defined (255 chars)
+ {
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' '
+ }
+ };
+
+ //TODO map multi-characters
+
+ /**
+ * Constructor.
+ */
+ private WP5Charsets() {
+ }
+
+ public static void append(StringBuilder out, int charset, int charval) {
+ if (charset >= WP5Charsets.EXTENDED_CHARSETS.length) {
+ LOG.debug("Unsupported WordPerfect 5.x charset: " + charset);
+ out.append(' ');
+ } else if (charval >= WP5Charsets.EXTENDED_CHARSETS[charset].length) {
+ LOG.debug("Unsupported WordPerfect 5.x charset (" + charset
+ + ") character value: " + charval);
+ out.append(' ');
+ } else {
+ out.append(WP5Charsets.EXTENDED_CHARSETS[charset][charval]);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
new file mode 100644
index 0000000..b95490e
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+import java.io.IOException;
+
+/**
+ * Extracts WordPerfect Document Area text from a WordPerfect document
+ * version 5.x.
+ * @author Pascal Essiembre
+ */
+class WP5DocumentAreaExtractor extends WPDocumentAreaExtractor {
+
+ protected void extract(int c, WPInputStream in, StringBuilder out)
+ throws IOException {
+
+ // 0-31: control characters
+ if (c == 10) {
+ out.append('\n'); // hard return ("Enter")
+ } else if (c == 11) {
+ out.append(' '); // soft page break
+ } else if (c == 12) {
+ out.append('\n'); // hard page break
+ } else if (c == 13) {
+ out.append(' '); // soft return (line wrap)
+
+ // 32-126: ASCII characters
+ } else if (c >= 32 && c <= 126) {
+ out.append((char) c); // ASCII character
+
+ // 128-191: single-byte functions
+ } else if (c == 140) {
+ out.append('\n'); // combination hard return/soft page (WP5.1)
+ } else if (c >= 144 && c <= 149) {
+ out.append(' '); // deletable/invisible soft return/page
+ } else if (c == 153) {
+ out.append('\4'); // Dormant Hard return (WP5.1)
+ } else if (c == 160) {
+ out.append('\u00A0'); // Hard space
+ } else if (c >= 169 && c <= 171) {
+ out.append('-'); // Hard hyphen
+
+ // 192-207: fixed-length multi-byte function
+ } else if (c == 192) {
+ // extended char
+ int charval = in.readWP();
+ int charset = in.readWP();
+ in.readWP(); // closing character
+ WP5Charsets.append(out, charset, charval);
+ } else if (c >= 193 && c <= 207) {
+ skipUntilChar(in, c); // opening/closing chars are same
+
+ // 208-255: variable-length multi-byte function
+ } else if (c >= 208 && c <= 255) {
+ // Variable-Length Multi-Byte Functions
+ in.readWP(); // subgroup (the function code)
+ int functionSize = in.readWPShort();
+ for (int i = 0; i < functionSize; i++) {
+ in.readWP();
+ }
+ //TODO Are there functions containing data? Like footnotes?
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Charsets.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Charsets.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Charsets.java
new file mode 100644
index 0000000..8c62983
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Charsets.java
@@ -0,0 +1,447 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+/**
+ * WordPerfect 6+ constant values used for mapping WordPerfect charsets to
+ * unicode equivalents when possible.
+ * @author Pascal Essiembre
+ */
+final class WP6Charsets {
+
+ private static final Logger LOG = LogManager.getLogger(WP6Charsets.class);
+
+ public static final char[] DEFAULT_EXTENDED_INTL_CHARS = new char[] {
+ '\0', '\u00E5', '\u00C5', '\u00E6', '\u00C6',
+ '\u00E4', '\u00C4', '\u00E1', '\u00E0', '\u00E2',
+ '\u00E3', '\u00C3', '\u00E7', '\u00C7', '\u00EB',
+ '\u00E9', '\u00C9', '\u00E8', '\u00EA', '\u00ED',
+ '\u00F1', '\u00D1', '\u00F8', '\u00D8', '\u00F5',
+ '\u00D5', '\u00F6', '\u00D6', '\u00FC', '\u00DC',
+ '\u00FA', '\u00F9', '\u00DF',
+ };
+
+ /**
+ * Extended character sets used when fixed-length multi-byte functions
+ * with a byte value of 240 (0xF0) are found in a WordPerfect document.
+ * Those character set codes may be specific to WordPerfect
+ * file specifications and may or may not be considered standard
+ * outside WordPerfect. Applies to version 6 and likely higher.
+ */
+ public static final char[][] EXTENDED_CHARSETS = new char[][] {
+ // WP Charset 0: ASCII (96 chars)
+ {
+ ' ', '!', '"', '#', '$', '%', '&', '\'', '(', ')',
+ '*', '+', ',', '-', '.', '/', '0', '1', '2', '3',
+ '4', '5', '6', '7', '8', '9', ':', ';', '<', '=',
+ '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
+ 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q',
+ 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[',
+ '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e',
+ 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+ 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y',
+ 'z', '{', '|', '}', '~', '\u00A0'
+ },
+ // WP Charset 1: Multinational (242 chars)
+ {
+ '\u0300','\u00b7','\u0303','\u0302','\u0335','\u0338','\u0301','\u0308',
+ '\u0304','\u0313','\u0315','\u02bc','\u0326','\u0315','\u00b0','\u0307',
+ '\u030b','\u0327','\u0328','\u030c','\u0337','\u0305','\u0306','\u00df',
+ '\u0138','\u006a','\u00c1','\u00e1','\u00c2','\u00e2','\u00c4','\u00e4',
+ '\u00c0','\u00e0','\u00c5','\u00e5','\u00c6','\u00e6','\u00c7','\u00e7',
+ '\u00c9','\u00e9','\u00ca','\u00ea','\u00cb','\u00eb','\u00c8','\u00e8',
+ '\u00cd','\u00ed','\u00ce','\u00ee','\u00cf','\u00ef','\u00cc','\u00ec',
+ '\u00d1','\u00f1','\u00d3','\u00f3','\u00d4','\u00f4','\u00d6','\u00f6',
+ '\u00d2','\u00f2','\u00da','\u00fa','\u00db','\u00fb','\u00dc','\u00fc',
+ '\u00d9','\u00f9','\u0178','\u00ff','\u00c3','\u00e3','\u0110','\u0111',
+ '\u00d8','\u00f8','\u00d5','\u00f5','\u00dd','\u00fd','\u00d0','\u00f0',
+ '\u00de','\u00fe','\u0102','\u0103','\u0100','\u0101','\u0104','\u0105',
+ '\u0106','\u0107','\u010c','\u010d','\u0108','\u0109','\u010a','\u010b',
+ '\u010e','\u010f','\u011a','\u011b','\u0116','\u0117','\u0112','\u0113',
+ '\u0118','\u0119','\u01f4','\u01f5','\u011e','\u011f','\u01e6','\u01e7',
+ '\u0122','\u0123','\u011c','\u011d','\u0120','\u0121','\u0124','\u0125',
+ '\u0126','\u0127','\u0130','\u0069','\u012a','\u012b','\u012e','\u012f',
+ '\u0128','\u0129','\u0132','\u0133','\u0134','\u0135','\u0136','\u0137',
+ '\u0139','\u013a','\u013d','\u013e','\u013b','\u013c','\u013f','\u0140',
+ '\u0141','\u0142','\u0143','\u0144','\u0000','\u0149','\u0147','\u0148',
+ '\u0145','\u0146','\u0150','\u0151','\u014c','\u014d','\u0152','\u0153',
+ '\u0154','\u0155','\u0158','\u0159','\u0156','\u0157','\u015a','\u015b',
+ '\u0160','\u0161','\u015e','\u015f','\u015c','\u015d','\u0164','\u0165',
+ '\u0162','\u0163','\u0166','\u0167','\u016c','\u016d','\u0170','\u0171',
+ '\u016a','\u016b','\u0172','\u0173','\u016e','\u016f','\u0168','\u0169',
+ '\u0174','\u0175','\u0176','\u0177','\u0179','\u017a','\u017d','\u017e',
+ '\u017b','\u017c','\u014a','\u014b','\u0000','\u0000','\u0000','\u0000',
+ '\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000',
+ '\u0000','\u0000','\u1ef2','\u1ef3','\u010e','\u010f','\u01a0','\u01a1',
+ '\u01af','\u01b0','\u0114','\u0115','\u012c','\u012d','\u0049','\u0131',
+ '\u014e','\u014f'
+ },
+ // WP Charset 2: Phonetic (145 chars)
+ {
+ '\u02b9','\u02ba','\u02bb','\u0020','\u02bd','\u02bc','\u0020','\u02be',
+ '\u02bf','\u0310','\u02d0','\u02d1','\u0306','\u032e','\u0329','\u02c8',
+ '\u02cc','\u02c9','\u02ca','\u02cb','\u02cd','\u02ce','\u02cf','\u02c6',
+ '\u02c7','\u02dc','\u0325','\u02da','\u032d','\u032c','\u0323','\u0308',
+ '\u0324','\u031c','\u031d','\u031e','\u031f','\u0320','\u0321','\u0322',
+ '\u032a','\u032b','\u02d2','\u02d3','\u0361','\u0356','\u005f','\u2017',
+ '\u033e','\u02db','\u0327','\u0233','\u030d','\u02b0','\u02b6','\u0250',
+ '\u0251','\u0252','\u0253','\u0299','\u0254','\u0255','\u0297','\u0256',
+ '\u0257','\u0258','\u0259','\u025a','\u025b','\u025c','\u025d','\u029a',
+ '\u025e','\u025f','\u0278','\u0261','\u0260','\u0262','\u029b','\u0263',
+ '\u0264','\u0265','\u0266','\u0267','\u029c','\u0268','\u026a','\u0269',
+ '\u029d','\u029e','\u026b','\u026c','\u026d','\u029f','\u026e','\u028e',
+ '\u026f','\u0270','\u0271','\u0272','\u0273','\u0274','\u0276','\u0277',
+ '\u02a0','\u0279','\u027a','\u027b','\u027c','\u027d','\u027e','\u027f',
+ '\u0280','\u0281','\u0282','\u0283','\u0284','\u0285','\u0286','\u0287',
+ '\u0288','\u0275','\u0289','\u028a','\u028c','\u028b','\u028d','\u03c7',
+ '\u028f','\u0290','\u0291','\u0292','\u0293','\u0294','\u0295','\u0296',
+ '\u02a1','\u02a2','\u0298','\u02a3','\u02a4','\u02a5','\u02a6','\u02a7',
+ '\u02a8'
+ },
+ // WP Charset 3: Box Drawing (88 chars)
+ {
+ '\u2591','\u2592','\u2593','\u2588','\u258c','\u2580','\u2590','\u2584',
+ '\u2500','\u2502','\u250c','\u2510','\u2518','\u2514','\u251c','\u252c',
+ '\u2524','\u2534','\u253c','\u2550','\u2551','\u2554','\u2557','\u255d',
+ '\u255a','\u2560','\u2566','\u2563','\u2569','\u256c','\u2552','\u2555',
+ '\u255b','\u2558','\u2553','\u2556','\u255c','\u2559','\u255e','\u2565',
+ '\u2561','\u2568','\u255f','\u2564','\u2562','\u2567','\u256b','\u256a',
+ '\u2574','\u2575','\u2576','\u2577','\u2578','\u2579','\u257a','\u257b',
+ '\u257c','\u257e','\u257d','\u257f','\u251f','\u2522','\u251e','\u2521',
+ '\u252e','\u2532','\u252d','\u2531','\u2527','\u2526','\u252a','\u2529',
+ '\u2536','\u253a','\u2535','\u2539','\u2541','\u2546','\u253e','\u2540',
+ '\u2544','\u254a','\u253d','\u2545','\u2548','\u2543','\u2549','\u2547'
+ },
+ // WP Charset 4: Typographic Symbols (102 chars)
+ {
+ '\u25cf','\u25cb','\u25a0','\u2022','\u002a','\u00b6','\u00a7','\u00a1',
+ '\u00bf','\u00ab','\u00bb','\u00a3','\u00a5','\u20a7','\u0192','\u00aa',
+ '\u00ba','\u00bd','\u00bc','\u00a2','\u00b2','\u207f','\u00ae','\u00a9',
+ '\u00a4','\u00be','\u00b3','\u201b','\u2019','\u2018','\u201f','\u201d',
+ '\u201c','\u2013','\u2014','\u2039','\u203a','\u25cb','\u25a1','\u2020',
+ '\u2021','\u2122','\u2120','\u211e','\u25cf','\u25e6','\u25a0','\u25aa',
+ '\u25a1','\u25ab','\u2012','\ufb00','\ufb03','\ufb04','\ufb01','\ufb02',
+ '\u2026','\u0024','\u20a3','\u20a2','\u20a0','\u20a4','\u201a','\u201e',
+ '\u2153','\u2154','\u215b','\u215c','\u215d','\u215e','\u24c2','\u24c5',
+ '\u20ac','\u2105','\u2106','\u2030','\u2116','\u2014','\u00b9','\u2409',
+ '\u240c','\u240d','\u240a','\u2424','\u240b','\u267c','\u20a9','\u20a6',
+ '\u20a8','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u0020','\u0020','\u0020','\u1d11','\u1d12'
+ // last two uncertain
+ },
+ // WP Charset 5: Iconic Symbol (255 chars)
+ {
+ '\u2661','\u2662','\u2667','\u2664','\u2642','\u2640','\u263c','\u263a',
+ '\u263b','\u266a','\u266c','\u25ac','\u2302','\u203c','\u221a','\u21a8',
+ '\u2310','\u2319','\u25d8','\u25d9','\u21b5','\u2104','\u261c','\u23b5',
+ '\u2610','\u2612','\u2639','\u266f','\u266d','\u266e','\u260e','\u231a',
+ '\u231b','\u2701','\u2702','\u2703','\u2704','\u260e','\u2706','\u2707',
+ '\u2708','\u2709','\u261b','\u261e','\u270c','\u270d','\u270e','\u270f',
+ '\u2710','\u2711','\u2712','\u2713','\u2714','\u2715','\u2716','\u2717',
+ '\u2718','\u2719','\u271a','\u271b','\u271c','\u271d','\u271e','\u271f',
+ '\u2720','\u2721','\u2722','\u2723','\u2724','\u2725','\u2726','\u2727',
+ '\u2605','\u2606','\u272a','\u272b','\u272c','\u272d','\u272e','\u272f',
+ '\u2730','\u2731','\u2732','\u2733','\u2734','\u2735','\u2736','\u2737',
+ '\u2738','\u2739','\u273a','\u273b','\u273c','\u273d','\u273e','\u273f',
+ '\u2740','\u2741','\u2742','\u2743','\u2744','\u2745','\u2746','\u2747',
+ '\u2748','\u2749','\u274a','\u274b','\u25cf','\u274d','\u25a0','\u274f',
+ '\u2750','\u2751','\u2752','\u25b2','\u25bc','\u25c6','\u2756','\u25d7',
+ '\u2758','\u2759','\u275a','\u275b','\u275c','\u275d','\u275e','\u2036',
+ '\u2033','\u0020','\u0020','\u0020','\u0020','\u2329','\u232a','\u005b',
+ '\u005d','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u2190','\u0020','\u0020','\u0020','\u0020','\u0020','\u21e8','\u21e6',
+ '\u2794','\u0020','\u0020','\u0020','\u0020','\u0020','\u25d6','\u0020',
+ '\u0020','\u2761','\u2762','\u2763','\u2764','\u2765','\u2766','\u2767',
+ '\u2663','\u2666','\u2665','\u2660','\u2780','\u2781','\u2782','\u2783',
+ '\u2784','\u2785','\u2786','\u2787','\u2788','\u2789','\u2776','\u2777',
+ '\u2778','\u2779','\u277a','\u277b','\u277c','\u277d','\u277e','\u277f',
+ '\u2780','\u2781','\u2782','\u2783','\u2784','\u2785','\u2786','\u2787',
+ '\u2788','\u2789','\u278a','\u278b','\u278c','\u278d','\u278e','\u278f',
+ '\u2790','\u2791','\u2792','\u2793','\u2794','\u2192','\u2194','\u2195',
+ '\u2798','\u2799','\u279a','\u279b','\u279c','\u279d','\u279e','\u279f',
+ '\u27a0','\u27a1','\u27a2','\u27a3','\u27a4','\u27a5','\u27a6','\u27a7',
+ '\u27a8','\u27a9','\u27aa','\u27ab','\u27ac','\u27ad','\u27ae','\u27af',
+ '\u0020','\u27b1','\u27b2','\u27b3','\u27b4','\u27b5','\u27b6','\u27b7',
+ '\u27b8','\u27b9','\u27ba','\u27bb','\u27bc','\u27bd','\u27be'
+ },
+ // WP Charset 6: Math/Scientific (238 chars)
+ {
+ '\u2212','\u00b1','\u2264','\u2265','\u221d','\u002f','\u2215','\u2216',
+ '\u00f7','\u2223','\u27e8','\u27e9','\u223c','\u2248','\u2261','\u2208',
+ '\u2229','\u2225','\u2211','\u221e','\u00ac','\u2192','\u2190','\u2191',
+ '\u2193','\u2194','\u2195','\u25b8','\u25c2','\u25b4','\u25be','\u22c5',
+ '\u00b7','\u2218','\u2219','\u212b','\u00b0','\u00b5','\u203e','\u00d7',
+ '\u222b','\u220f','\u2213','\u2207','\u2202','\u2032','\u2033','\u2192',
+ '\u212f','\u2113','\u210f','\u2111','\u211c','\u2118','\u21c4','\u21c6',
+ '\u21d2','\u21d0','\u21d1','\u21d3','\u21d4','\u21d5','\u2197','\u2198',
+ '\u2196','\u2199','\u222a','\u2282','\u2283','\u2286','\u2287','\u220d',
+ '\u2205','\u2308','\u2309','\u230a','\u230b','\u226a','\u226b','\u2220',
+ '\u2297','\u2295','\u2296','\u2a38','\u2299','\u2227','\u2228','\u22bb',
+ '\u22a4','\u22a5','\u2312','\u22a2','\u22a3','\u25a1','\u25a0','\u25ca',
+ '\u25c6','\u27e6','\u27e7','\u2260','\u2262','\u2235','\u2234','\u2237',
+ '\u222e','\u2112','\u212d','\u2128','\u2118','\u20dd','\u29cb','\u25c7',
+ '\u22c6','\u2034','\u2210','\u2243','\u2245','\u227a','\u227c','\u227b',
+ '\u227d','\u2203','\u2200','\u22d8','\u22d9','\u228e','\u228a','\u228b',
+ '\u2293','\u2294','\u228f','\u2291','\u22e4','\u2290','\u2292','\u22e5',
+ '\u25b3','\u25bd','\u25c3','\u25b9','\u22c8','\u2323','\u2322','\u25ef',
+ '\u219d','\u21a9','\u21aa','\u21a3','\u21bc','\u21bd','\u21c0','\u21c1',
+ '\u21cc','\u21cb','\u21bf','\u21be','\u21c3','\u21c2','\u21c9','\u21c7',
+ '\u22d3','\u22d2','\u22d0','\u22d1','\u229a','\u229b','\u229d','\u2127',
+ '\u2221','\u2222','\u25c3','\u25b9','\u25b5','\u25bf','\u2214','\u2250',
+ '\u2252','\u2253','\u224e','\u224d','\u22a8','\u2258','\u226c','\u0285',
+ '\u2605','\u226e','\u2270','\u226f','\u2271','\u2241','\u2244','\u2247',
+ '\u2249','\u2280','\u22e0','\u2281','\u22e1','\u2284','\u2285','\u2288',
+ '\u2289','\u0020','\u0020','\u22e2','\u22e3','\u2226','\u2224','\u226d',
+ '\u2204','\u2209','\u2247','\u2130','\u2131','\u2102','\u0020','\u2115',
+ '\u211d','\u225f','\u22be','\u220b','\u22ef','\u2026','\u22ee','\u22f1',
+ '\u0020','\u20e1','\u002b','\u002d','\u003d','\u002a','\u2032','\u2033',
+ '\u2034','\u210b','\u2118','\u2272','\u2273','\u0020'
+ },
+ // WP Charset 7 Math/Scientific Extended (229 chars)
+ {
+ '\u2320','\u2321','\u23a5','\u23bd','\u221a','\u0020','\u2211','\u220f',
+ '\u2210','\u222b','\u222e','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u23a7','\u23a8','\u23a9','\u23aa','\u0020','\u0020','\u0020',
+ '\u0020','\u23ab','\u23ac','\u23ad','\u23aa','\u0020','\u0020','\u0020',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u222a','\u222b','\u0020',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u239b','\u239d','\u239c','\u0020','\u0020','\u0020','\u0020','\u239e',
+ '\u23a8','\u239f','\u0020','\u0020','\u0020','\u0020','\u23a1','\u23a3',
+ '\u23a2','\u0020','\u20aa','\u0020','\u0020','\u23a4','\u23a6','\u23a5',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u22c3','\u22c2','\u228e','\u2a04','\u2294','\u2a06','\u2227','\u22c0',
+ '\u2228','\u22c1','\u2297','\u2a02','\u2295','\u2a01','\u2299','\u2a00',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u0020','\u0020','\u0020','\u229d','\u0020','\u2238','\u0020',
+ '\u27e6','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u27e7',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u21bc','\u21bd','\u0020','\u296c','\u296d','\u296a','\u296b','\u0020',
+ '\u21c9','\u21c7','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u21be','\u21bf','\u21c3','\u21c2','\u0020','\u2293','\u2a05','\u23a1',
+ '\u0020','\u0020','\u0020','\u0020','\u0020'
+ },
+ // WP Charset 8: Greek (219 chars)
+ {
+ '\u0391','\u03b1','\u0392','\u03b2','\u0392','\u03d0','\u0393','\u03b3',
+ '\u0394','\u03b4','\u0395','\u03b5','\u0396','\u03b6','\u0397','\u03b7',
+ '\u0398','\u03b8','\u0399','\u03b9','\u039a','\u03ba','\u039b','\u03bb',
+ '\u039c','\u03bc','\u039d','\u03bd','\u039e','\u03be','\u039f','\u03bf',
+ '\u03a0','\u03c0','\u03a1','\u03c1','\u03a3','\u03c3','\u03a3','\u03c2',
+ '\u03a4','\u03c4','\u03a5','\u03c5','\u03a6','\u03c6','\u03a7','\u03c7',
+ '\u03a8','\u03c8','\u03a9','\u03c9','\u0386','\u03ac','\u0388','\u03ad',
+ '\u0389','\u03ae','\u038a','\u03af','\u03aa','\u03ca','\u038c','\u03cc',
+ '\u038e','\u03cd','\u03ab','\u03cb','\u038f','\u03ce','\u03b5','\u03d1',
+ '\u03f0','\u03d6','\u03f1','\u03c2','\u03d2','\u03d5','\u03c9','\u037e',
+ '\u0387','\u0374','\u0375','\u0384','\u00a8','\u0385','\u1fed','\u1fef',
+ '\u1fc0','\u1fbd','\u1ffe','\u037a','\u1fce','\u1fde','\u1fcd','\u1fdd',
+ '\u1fcf','\u1fdf','\u0384','\u1fef','\u1fc0','\u1fbd','\u1ffe','\u1fce',
+ '\u1fde','\u1fcd','\u1fdd','\u1fcf','\u1fdf','\u1f70','\u1fb6','\u1fb3',
+ '\u1fb4','\u1fb2','\u1fb7','\u1f00','\u1f04','\u1f02','\u1f06','\u1f80',
+ '\u1f84','\u1f82','\u1f86','\u1f01','\u1f05','\u1f03','\u1f07','\u1f81',
+ '\u1f85','\u1f83','\u1f87','\u1f72','\u1f10','\u1f14','\u1f12','\u1f11',
+ '\u1f15','\u1f13','\u1f74','\u1fc6','\u1fc3','\u1fc4','\u1fc2','\u1fc7',
+ '\u1f20','\u1f24','\u1f22','\u1f26','\u1f90','\u1f94','\u1f92','\u1f96',
+ '\u1f21','\u1f25','\u1f23','\u1f27','\u1f91','\u1f95','\u1f93','\u1f97',
+ '\u1f76','\u1fd6','\u0390','\u1fd2','\u1f30','\u1f34','\u1f32','\u1f36',
+ '\u1f31','\u1f35','\u1f33','\u1f37','\u1f78','\u1f40','\u1f44','\u1f42',
+ '\u1f41','\u1f45','\u1f43','\u1fe5','\u1fe4','\u1f7a','\u1fe6','\u03b0',
+ '\u1fe2','\u1f50','\u1f54','\u1f52','\u1f56','\u1f51','\u1f55','\u1f53',
+ '\u1f57','\u1f7c','\u1ff6','\u1ff3','\u1ff4','\u1ff2','\u1ff7','\u1f60',
+ '\u1f64','\u1f62','\u1f66','\u1fa0','\u1fa4','\u1fa2','\u1fa6','\u1f61',
+ '\u1f65','\u1f63','\u1f67','\u1fa1','\u1fa5','\u1fa3','\u1fa7','\u03da',
+ '\u03dc','\u03de','\u03e0'
+ },
+ // WP Charset 9: Hebrew (123 chars)
+ {
+ '\u05d0','\u05d1','\u05d2','\u05d3','\u05d4','\u05d5','\u05d6','\u05d7',
+ '\u05d8','\u05d9','\u05da','\u05db','\u05dc','\u05dd','\u05de','\u05df',
+ '\u05e0','\u05e1','\u05e2','\u05e3','\u05e4','\u05e5','\u05e6','\u05e7',
+ '\u05e8','\u05e9','\u05ea','\u05be','\u05c0','\u05c3','\u05f3','\u05f4',
+ '\u05b0','\u05b1','\u05b2','\u05b3','\u05b4','\u05b5','\u05b6','\u05b7',
+ '\u05b8','\u05b9','\u05b9','\u05bb','\u05bc','\u05bd','\u05bf','\u05b7',
+ '\ufb1e','\u05f0','\u05f1','\u05f2','\ufb1f','\u0591','\u0596','\u0020',
+ '\u05a4','\u059a','\u059b','\u05a3','\u05a5','\u05a6','\u05a7','\u05a2',
+ '\u0592','\u0593','\u0594','\u0595','\u0597','\u0598','\u0599','\u05a8',
+ '\u05f3','\u05f3','\u05f4','\u0020','\u05a9','\u05a0','\u059f','\u05ab',
+ '\u05ac','\u05af','\u05c4','\u05aa','\ufb30','\ufb31','\u05d1','\ufb32',
+ '\ufb33','\ufb34','\ufb35','\ufb4b','\ufb36','\u05d7','\ufb38','\ufb39',
+ '\ufb1d','\ufb3b','\ufb3a','\u05da','\u05da','\u05da','\u05da','\u05da',
+ '\u05da','\ufb3c','\ufb3e','\ufb40','\u05d5','\ufb41','\ufb44','\u05e4',
+ '\ufb46','\ufb47','\ufb2b','\ufb2d','\ufb2a','\ufb2c','\ufb4a','\u05dc',
+ '\ufb3c','\ufb49','\u20aa'
+ },
+ // WP Charset 10: Cyrillic (250 chars)
+ {
+ '\u0410','\u0430','\u0411','\u0431','\u0412','\u0432','\u0413','\u0433',
+ '\u0414','\u0434','\u0415','\u0435','\u0401','\u0451','\u0416','\u0436',
+ '\u0417','\u0437','\u0418','\u0438','\u0419','\u0439','\u041a','\u043a',
+ '\u041b','\u043b','\u041c','\u043c','\u041d','\u043d','\u041e','\u043e',
+ '\u041f','\u043f','\u0420','\u0440','\u0421','\u0441','\u0422','\u0442',
+ '\u0423','\u0443','\u0424','\u0444','\u0425','\u0445','\u0426','\u0446',
+ '\u0427','\u0447','\u0428','\u0448','\u0429','\u0449','\u042a','\u044a',
+ '\u042b','\u044b','\u042c','\u044c','\u042d','\u044d','\u042e','\u044e',
+ '\u042f','\u044f','\u04d8','\u04d9','\u0403','\u0453','\u0490','\u0491',
+ '\u0492','\u0493','\u0402','\u0452','\u0404','\u0454','\u0404','\u0454',
+ '\u0496','\u0497','\u0405','\u0455','\u0020','\u0020','\u0418','\u0438',
+ '\u0406','\u0456','\u0407','\u0457','\u0020','\u0020','\u0408','\u0458',
+ '\u040c','\u045c','\u049a','\u049b','\u04c3','\u04c4','\u049c','\u049d',
+ '\u0409','\u0459','\u04a2','\u04a3','\u040a','\u045a','\u047a','\u047b',
+ '\u0460','\u0461','\u040b','\u045b','\u040e','\u045e','\u04ee','\u04ef',
+ '\u04ae','\u04af','\u04b0','\u04b1','\u0194','\u0263','\u04b2','\u04b3',
+ '\u0425','\u0445','\u04ba','\u04bb','\u047e','\u047f','\u040f','\u045f',
+ '\u04b6','\u04b7','\u04b8','\u04b9','\u0428','\u0448','\u0462','\u0463',
+ '\u0466','\u0467','\u046a','\u046b','\u046e','\u046f','\u0470','\u0471',
+ '\u0472','\u0473','\u0474','\u0475','\u0410','\u0430','\u0415','\u0435',
+ '\u0404','\u0454','\u0418','\u0438','\u0406','\u0456','\u0407','\u0457',
+ '\u041e','\u043e','\u0423','\u0443','\u042b','\u044b','\u042d','\u044d',
+ '\u042e','\u044e','\u042f','\u044f','\u0410','\u0430','\u0400','\u0450',
+ '\u0401','\u0451','\u040d','\u045d','\u041e','\u043e','\u0423','\u0443',
+ '\u042b','\u044b','\u042d','\u044d','\u042e','\u044e','\u042f','\u044f',
+ '\u0301','\u0300','\u0308','\u0306','\u0326','\u0328','\u0304','\u0020',
+ '\u201e','\u201c','\u10d0','\u10d1','\u10d2','\u10d3','\u10d4','\u10d5',
+ '\u10d6','\u10f1','\u10d7','\u10d8','\u10d9','\u10da','\u10db','\u10dc',
+ '\u10f2','\u10dd','\u10de','\u10df','\u10e0','\u10e1','\u10e2','\u10e3',
+ '\u10f3','\u10e4','\u10e5','\u10e6','\u10e7','\u10e8','\u10e9','\u10ea',
+ '\u10eb','\u10ec','\u10ed','\u10ee','\u10f4','\u10ef','\u10f0','\u10f5',
+ '\u10f6','\u10e3'
+ },
+ // WP Charset 11: Japanese (63 chars)
+ {
+ '\uff61','\uff62','\uff63','\uff64','\uff65','\uff66','\uff67','\uff68',
+ '\uff69','\uff6a','\uff6b','\uff6c','\uff6d','\uff6e','\uff6f','\uff70',
+ '\uff71','\uff72','\uff73','\uff74','\uff75','\uff76','\uff77','\uff78',
+ '\uff79','\uff7a','\uff7b','\uff7c','\uff7d','\uff7e','\uff7f','\uff80',
+ '\uff81','\uff82','\uff83','\uff84','\uff85','\uff86','\uff87','\uff88',
+ '\uff89','\uff8a','\uff8b','\uff8c','\uff8d','\uff8e','\uff8f','\uff90',
+ '\uff91','\uff92','\uff93','\uff94','\uff95','\uff96','\uff97','\uff98',
+ '\uff99','\uff9a','\uff9b','\uff9c','\uff9d','\uff9e','\uff9f'
+ },
+ // WP Charset 12: Current Font Symbols (256 chars)
+ { //TODO implement Current Font Symbols
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' '
+ },
+ // WP Charset 13: Arabic (196 chars)
+ {
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u0020','\u064e','\ufe77','\u064f','\ufe79','\u0650','\ufe7b',
+ '\u064b','\u064c','\u064c','\u064d','\u0652','\ufe7f','\u0651','\ufe7d',
+ '\ufc60','\ufcf2','\ufc61','\ufcf3','\ufc62','\ufcf4','\u064b','\ufc5e',
+ '\ufc5e','\ufc5f','\u0653','\u0670','\u0654','\u0020','\u060c','\u061b',
+ '\u061f','\u066d','\u066a','\u00bb','\u00ab','\u0029','\u0028','\u0661',
+ '\u0662','\u0663','\u0664','\u0665','\u0666','\u0667','\u0668','\u0669',
+ '\u0660','\u0662','\u0627','\ufe8e','\u0628','\ufe91','\ufe92','\ufe90',
+ '\u062a','\ufe97','\ufe98','\ufe96','\u062b','\ufe9b','\ufe9c','\ufe9a',
+ '\u062c','\ufe9f','\ufea0','\ufe9e','\u062d','\ufea3','\ufea4','\ufea2',
+ '\u062e','\ufea7','\ufea8','\ufea6','\u062f','\ufeaa','\u0630','\ufeac',
+ '\u0631','\ufeae','\u0632','\ufeaf','\u0633','\ufeb3','\ufeb4','\ufeb2',
+ '\u0634','\ufeb7','\ufeb8','\ufeb6','\u0635','\ufebb','\ufebc','\ufeba',
+ '\u0636','\ufebf','\ufec0','\ufebe','\u0637','\ufec3','\ufec4','\ufec2',
+ '\u0638','\ufec7','\ufec8','\ufec6','\u0639','\ufecb','\ufecc','\ufeca',
+ '\u063a','\ufecf','\ufed0','\ufece','\u0641','\ufed3','\ufed4','\ufed2',
+ '\u0642','\ufed7','\ufed8','\ufed6','\u0643','\ufedb','\ufedc','\ufeda',
+ '\u0644','\ufedf','\ufee0','\ufede','\u0645','\ufee3','\ufee4','\ufee2',
+ '\u0646','\ufee7','\ufee8','\ufee6','\u0647','\ufeeb','\ufeec','\ufeea',
+ '\u0629','\ufe94','\u0648','\ufeee','\u064a','\ufef3','\ufef4','\ufef2',
+ '\u0649','\ufef3','\ufef4','\ufef0','\u0621','\u0623','\ufe84','\u0625',
+ '\ufe88','\u0624','\ufe86','\u0626','\ufe8b','\ufe8c','\ufe8a','\ufd3d',
+ '\ufd3c','\u0622','\ufe82','\u0671','\ufb51','\ufefb','\ufefc','\ufef7',
+ '\ufef8','\ufef9','\ufefa','\u0020','\ufefc','\ufef5','\ufef6','\u0020',
+ '\u0020','\ufdf2','\u0640','\u0640'
+ },
+ // WP Charset 14: Arabic Script (220 chars)
+ {
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0615',
+ '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0615','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+ '\u0020','\u06d4','\u0020','\u0020','\u00b0','\u0020','\u065a','\u0020',
+ '\u065a','\u0659','\u0020','\u0020','\u0654','\u064c','\ufc5e','\u065a',
+ '\u065a','\u06f4','\u06f4','\u06f5','\u06f6','\u06f6','\u06f7','\u06f8',
+ '\u067b','\ufb54','\ufb55','\ufb53','\u0680','\ufb5c','\ufb5d','\ufb5b',
+ '\u067e','\ufb58','\ufb59','\ufb57','\u0679','\ufb68','\ufb69','\ufb67',
+ '\u067c','\u067c','\u067c','\u067c','\u067f','\ufb64','\ufb65','\ufb63',
+ '\u067d','\u067d','\u067d','\u067d','\u067a','\ufb60','\ufb61','\ufb5f',
+ '\u0684','\ufb74','\ufb75','\ufb73','\u0683','\ufb78','\ufb79','\ufb77',
+ '\u0686','\ufb7c','\ufb7d','\ufb7b','\u0687','\ufb80','\ufb81','\ufb7f',
+ '\u0685','\u0685','\u0685','\u0685','\u0681','\u0681','\u0681','\u0681',
+ '\u0688','\ufb89','\u0689','\u0689','\u068c','\ufb85','\u068e','\ufb87',
+ '\u068a','\u068a','\u068d','\ufb83','\u0693','\u0693','\u0691','\ufb8d',
+ '\u0699','\u0699','\u0695','\u0695','\u0692','\u0692','\u0698','\ufb8b',
+ '\u0696','\u0696','\u075b','\u075b','\u069a','\u069a','\u069a','\u069a',
+ '\u06a0','\u06a0','\u06a0','\u06a0','\u06a4','\ufb6c','\ufb6d','\ufb6b',
+ '\u06a6','\ufb70','\ufb71','\ufb6f','\u06a9','\ufb90','\ufb91','\ufb8f',
+ '\u0643','\ufedb','\ufedc','\ufeda','\u06aa','\u06aa','\u06aa','\u06aa',
+ '\u06af','\ufb94','\ufb95','\ufb93','\u06af','\ufb94','\ufb95','\ufb93',
+ '\u06ab','\u06ab','\u06ab','\u06ab','\u06b1','\ufb9c','\ufb9d','\ufb9b',
+ '\u06b3','\ufb98','\ufb99','\ufb97','\u06b5','\u06b5','\u06b5','\u06b5',
+ '\u0020','\u0020','\u06ba','\u0020','\u0020','\ufb9f','\u06bc','\u06bc',
+ '\u06bc','\u06bc','\u06bb','\ufba2','\ufba3','\ufba1','\u06c6','\ufbda',
+ '\u0020','\u0020','\u06ca','\u06ca','\u06c1','\ufba8','\ufba9','\ufba7',
+ '\u06ce','\u06ce','\u06ce','\u06ce','\u06d2','\ufbaf','\u06d1','\u06d1',
+ '\u06d1','\u06d1','\u06c0','\ufba5'
+ },
+ };
+
+ //TODO map multi-characters
+
+ /**
+ * Constructor.
+ */
+ private WP6Charsets() {
+ }
+
+ public static void append(StringBuilder out, int charset, int charval) {
+ if (charset >= WP6Charsets.EXTENDED_CHARSETS.length) {
+ LOG.debug("Unsupported WordPerfect 6+ charset: " + charset);
+ out.append(' ');
+ } else if (charval >= WP6Charsets.EXTENDED_CHARSETS[charset].length) {
+ LOG.debug("Unsupported WordPerfect 6+ charset (" + charset
+ + ") character value: " + charval);
+ out.append(' ');
+ } else {
+ out.append(WP6Charsets.EXTENDED_CHARSETS[charset][charval]);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Constants.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Constants.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Constants.java
deleted file mode 100644
index 194bad7..0000000
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Constants.java
+++ /dev/null
@@ -1,432 +0,0 @@
-/* Copyright 2015-2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.wordperfect;
-
-/**
- * WordPerfect constant values used for parsing and extracting text.
- * @author Pascal Essiembre
- */
-final class WP6Constants {
-
-
- public static final String WP6_FILE_ID = "�WPC";
- public static final int WP6_PRODUCT_TYPE = 1;
- public static final int WP6_FILE_TYPE_WP_DOCUMENT = 10;
- public static final int WP6_FILE_TYPE_WPD = 36;
-
- public static final char[] DEFAULT_EXTENDED_INTL_CHARS = new char[] {
- '\0', '\u00E5', '\u00C5', '\u00E6', '\u00C6',
- '\u00E4', '\u00C4', '\u00E1', '\u00E0', '\u00E2',
- '\u00E3', '\u00C3', '\u00E7', '\u00C7', '\u00EB',
- '\u00E9', '\u00C9', '\u00E8', '\u00EA', '\u00ED',
- '\u00F1', '\u00D1', '\u00F8', '\u00D8', '\u00F5',
- '\u00D5', '\u00F6', '\u00D6', '\u00FC', '\u00DC',
- '\u00FA', '\u00F9', '\u00DF',
- };
-
- /**
- * Extended character sets used when fixed-length multi-byte functions
- * with a byte value of 240 (0xF0) are found in a WordPerfect document.
- * Those character set codes may be specific to WordPerfect
- * file specifications and may or may not be considered standard
- * outside WordPerfect.
- */
- public static final char[][] EXTENDED_CHARSETS = new char[][] {
- // WP Charset 0: ASCII (95 chars)
- {
- ' ', '"', '#', '$', '%', '&', '\'', '(', ')', '*',
- '+', ',', '-', '.', '/', '0', '1', '2', '3', '4',
- '5', '6', '7', '8', '9', ':', ';', '<', '=', '>',
- '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
- 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
- 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', '\\',
- ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f',
- 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
- 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
- '{', '|', '}', '~', '\u00A0'
- },
- // WP Charset 1: Multinational (242 chars)
- {
- '\u0300','\u00b7','\u0303','\u0302','\u0335','\u0338','\u0301','\u0308',
- '\u0304','\u0313','\u0315','\u02bc','\u0326','\u0315','\u00b0','\u0307',
- '\u030b','\u0327','\u0328','\u030c','\u0337','\u0305','\u0306','\u00df',
- '\u0138','\u006a','\u00c1','\u00e1','\u00c2','\u00e2','\u00c4','\u00e4',
- '\u00c0','\u00e0','\u00c5','\u00e5','\u00c6','\u00e6','\u00c7','\u00e7',
- '\u00c9','\u00e9','\u00ca','\u00ea','\u00cb','\u00eb','\u00c8','\u00e8',
- '\u00cd','\u00ed','\u00ce','\u00ee','\u00cf','\u00ef','\u00cc','\u00ec',
- '\u00d1','\u00f1','\u00d3','\u00f3','\u00d4','\u00f4','\u00d6','\u00f6',
- '\u00d2','\u00f2','\u00da','\u00fa','\u00db','\u00fb','\u00dc','\u00fc',
- '\u00d9','\u00f9','\u0178','\u00ff','\u00c3','\u00e3','\u0110','\u0111',
- '\u00d8','\u00f8','\u00d5','\u00f5','\u00dd','\u00fd','\u00d0','\u00f0',
- '\u00de','\u00fe','\u0102','\u0103','\u0100','\u0101','\u0104','\u0105',
- '\u0106','\u0107','\u010c','\u010d','\u0108','\u0109','\u010a','\u010b',
- '\u010e','\u010f','\u011a','\u011b','\u0116','\u0117','\u0112','\u0113',
- '\u0118','\u0119','\u01f4','\u01f5','\u011e','\u011f','\u01e6','\u01e7',
- '\u0122','\u0123','\u011c','\u011d','\u0120','\u0121','\u0124','\u0125',
- '\u0126','\u0127','\u0130','\u0069','\u012a','\u012b','\u012e','\u012f',
- '\u0128','\u0129','\u0132','\u0133','\u0134','\u0135','\u0136','\u0137',
- '\u0139','\u013a','\u013d','\u013e','\u013b','\u013c','\u013f','\u0140',
- '\u0141','\u0142','\u0143','\u0144','\u0000','\u0149','\u0147','\u0148',
- '\u0145','\u0146','\u0150','\u0151','\u014c','\u014d','\u0152','\u0153',
- '\u0154','\u0155','\u0158','\u0159','\u0156','\u0157','\u015a','\u015b',
- '\u0160','\u0161','\u015e','\u015f','\u015c','\u015d','\u0164','\u0165',
- '\u0162','\u0163','\u0166','\u0167','\u016c','\u016d','\u0170','\u0171',
- '\u016a','\u016b','\u0172','\u0173','\u016e','\u016f','\u0168','\u0169',
- '\u0174','\u0175','\u0176','\u0177','\u0179','\u017a','\u017d','\u017e',
- '\u017b','\u017c','\u014a','\u014b','\u0000','\u0000','\u0000','\u0000',
- '\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000',
- '\u0000','\u0000','\u1ef2','\u1ef3','\u010e','\u010f','\u01a0','\u01a1',
- '\u01af','\u01b0','\u0114','\u0115','\u012c','\u012d','\u0049','\u0131',
- '\u014e','\u014f'
- },
- // WP Charset 2: Phonetic (145 chars)
- {
- '\u02b9','\u02ba','\u02bb','\u0020','\u02bd','\u02bc','\u0020','\u02be',
- '\u02bf','\u0310','\u02d0','\u02d1','\u0306','\u032e','\u0329','\u02c8',
- '\u02cc','\u02c9','\u02ca','\u02cb','\u02cd','\u02ce','\u02cf','\u02c6',
- '\u02c7','\u02dc','\u0325','\u02da','\u032d','\u032c','\u0323','\u0308',
- '\u0324','\u031c','\u031d','\u031e','\u031f','\u0320','\u0321','\u0322',
- '\u032a','\u032b','\u02d2','\u02d3','\u0361','\u0356','\u005f','\u2017',
- '\u033e','\u02db','\u0327','\u0233','\u030d','\u02b0','\u02b6','\u0250',
- '\u0251','\u0252','\u0253','\u0299','\u0254','\u0255','\u0297','\u0256',
- '\u0257','\u0258','\u0259','\u025a','\u025b','\u025c','\u025d','\u029a',
- '\u025e','\u025f','\u0278','\u0261','\u0260','\u0262','\u029b','\u0263',
- '\u0264','\u0265','\u0266','\u0267','\u029c','\u0268','\u026a','\u0269',
- '\u029d','\u029e','\u026b','\u026c','\u026d','\u029f','\u026e','\u028e',
- '\u026f','\u0270','\u0271','\u0272','\u0273','\u0274','\u0276','\u0277',
- '\u02a0','\u0279','\u027a','\u027b','\u027c','\u027d','\u027e','\u027f',
- '\u0280','\u0281','\u0282','\u0283','\u0284','\u0285','\u0286','\u0287',
- '\u0288','\u0275','\u0289','\u028a','\u028c','\u028b','\u028d','\u03c7',
- '\u028f','\u0290','\u0291','\u0292','\u0293','\u0294','\u0295','\u0296',
- '\u02a1','\u02a2','\u0298','\u02a3','\u02a4','\u02a5','\u02a6','\u02a7',
- '\u02a8'
- },
- // WP Charset 3: Box Drawing (88 chars)
- {
- '\u2591','\u2592','\u2593','\u2588','\u258c','\u2580','\u2590','\u2584',
- '\u2500','\u2502','\u250c','\u2510','\u2518','\u2514','\u251c','\u252c',
- '\u2524','\u2534','\u253c','\u2550','\u2551','\u2554','\u2557','\u255d',
- '\u255a','\u2560','\u2566','\u2563','\u2569','\u256c','\u2552','\u2555',
- '\u255b','\u2558','\u2553','\u2556','\u255c','\u2559','\u255e','\u2565',
- '\u2561','\u2568','\u255f','\u2564','\u2562','\u2567','\u256b','\u256a',
- '\u2574','\u2575','\u2576','\u2577','\u2578','\u2579','\u257a','\u257b',
- '\u257c','\u257e','\u257d','\u257f','\u251f','\u2522','\u251e','\u2521',
- '\u252e','\u2532','\u252d','\u2531','\u2527','\u2526','\u252a','\u2529',
- '\u2536','\u253a','\u2535','\u2539','\u2541','\u2546','\u253e','\u2540',
- '\u2544','\u254a','\u253d','\u2545','\u2548','\u2543','\u2549','\u2547'
- },
- // WP Charset 4: Typographic Symbols (102 chars)
- {
- '\u25cf','\u25cb','\u25a0','\u2022','\u002a','\u00b6','\u00a7','\u00a1',
- '\u00bf','\u00ab','\u00bb','\u00a3','\u00a5','\u20a7','\u0192','\u00aa',
- '\u00ba','\u00bd','\u00bc','\u00a2','\u00b2','\u207f','\u00ae','\u00a9',
- '\u00a4','\u00be','\u00b3','\u201b','\u2019','\u2018','\u201f','\u201d',
- '\u201c','\u2013','\u2014','\u2039','\u203a','\u25cb','\u25a1','\u2020',
- '\u2021','\u2122','\u2120','\u211e','\u25cf','\u25e6','\u25a0','\u25aa',
- '\u25a1','\u25ab','\u2012','\ufb00','\ufb03','\ufb04','\ufb01','\ufb02',
- '\u2026','\u0024','\u20a3','\u20a2','\u20a0','\u20a4','\u201a','\u201e',
- '\u2153','\u2154','\u215b','\u215c','\u215d','\u215e','\u24c2','\u24c5',
- '\u20ac','\u2105','\u2106','\u2030','\u2116','\u2014','\u00b9','\u2409',
- '\u240c','\u240d','\u240a','\u2424','\u240b','\u267c','\u20a9','\u20a6',
- '\u20a8','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u0020','\u0020','\u0020','\u1d11','\u1d12'
- // last two uncertain
- },
- // WP Charset 5: Iconic Symbol (255 chars)
- {
- '\u2661','\u2662','\u2667','\u2664','\u2642','\u2640','\u263c','\u263a',
- '\u263b','\u266a','\u266c','\u25ac','\u2302','\u203c','\u221a','\u21a8',
- '\u2310','\u2319','\u25d8','\u25d9','\u21b5','\u2104','\u261c','\u23b5',
- '\u2610','\u2612','\u2639','\u266f','\u266d','\u266e','\u260e','\u231a',
- '\u231b','\u2701','\u2702','\u2703','\u2704','\u260e','\u2706','\u2707',
- '\u2708','\u2709','\u261b','\u261e','\u270c','\u270d','\u270e','\u270f',
- '\u2710','\u2711','\u2712','\u2713','\u2714','\u2715','\u2716','\u2717',
- '\u2718','\u2719','\u271a','\u271b','\u271c','\u271d','\u271e','\u271f',
- '\u2720','\u2721','\u2722','\u2723','\u2724','\u2725','\u2726','\u2727',
- '\u2605','\u2606','\u272a','\u272b','\u272c','\u272d','\u272e','\u272f',
- '\u2730','\u2731','\u2732','\u2733','\u2734','\u2735','\u2736','\u2737',
- '\u2738','\u2739','\u273a','\u273b','\u273c','\u273d','\u273e','\u273f',
- '\u2740','\u2741','\u2742','\u2743','\u2744','\u2745','\u2746','\u2747',
- '\u2748','\u2749','\u274a','\u274b','\u25cf','\u274d','\u25a0','\u274f',
- '\u2750','\u2751','\u2752','\u25b2','\u25bc','\u25c6','\u2756','\u25d7',
- '\u2758','\u2759','\u275a','\u275b','\u275c','\u275d','\u275e','\u2036',
- '\u2033','\u0020','\u0020','\u0020','\u0020','\u2329','\u232a','\u005b',
- '\u005d','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u2190','\u0020','\u0020','\u0020','\u0020','\u0020','\u21e8','\u21e6',
- '\u2794','\u0020','\u0020','\u0020','\u0020','\u0020','\u25d6','\u0020',
- '\u0020','\u2761','\u2762','\u2763','\u2764','\u2765','\u2766','\u2767',
- '\u2663','\u2666','\u2665','\u2660','\u2780','\u2781','\u2782','\u2783',
- '\u2784','\u2785','\u2786','\u2787','\u2788','\u2789','\u2776','\u2777',
- '\u2778','\u2779','\u277a','\u277b','\u277c','\u277d','\u277e','\u277f',
- '\u2780','\u2781','\u2782','\u2783','\u2784','\u2785','\u2786','\u2787',
- '\u2788','\u2789','\u278a','\u278b','\u278c','\u278d','\u278e','\u278f',
- '\u2790','\u2791','\u2792','\u2793','\u2794','\u2192','\u2194','\u2195',
- '\u2798','\u2799','\u279a','\u279b','\u279c','\u279d','\u279e','\u279f',
- '\u27a0','\u27a1','\u27a2','\u27a3','\u27a4','\u27a5','\u27a6','\u27a7',
- '\u27a8','\u27a9','\u27aa','\u27ab','\u27ac','\u27ad','\u27ae','\u27af',
- '\u0020','\u27b1','\u27b2','\u27b3','\u27b4','\u27b5','\u27b6','\u27b7',
- '\u27b8','\u27b9','\u27ba','\u27bb','\u27bc','\u27bd','\u27be'
- },
- // WP Charset 6: Math/Scientific (238 chars)
- {
- '\u2212','\u00b1','\u2264','\u2265','\u221d','\u002f','\u2215','\u2216',
- '\u00f7','\u2223','\u27e8','\u27e9','\u223c','\u2248','\u2261','\u2208',
- '\u2229','\u2225','\u2211','\u221e','\u00ac','\u2192','\u2190','\u2191',
- '\u2193','\u2194','\u2195','\u25b8','\u25c2','\u25b4','\u25be','\u22c5',
- '\u00b7','\u2218','\u2219','\u212b','\u00b0','\u00b5','\u203e','\u00d7',
- '\u222b','\u220f','\u2213','\u2207','\u2202','\u2032','\u2033','\u2192',
- '\u212f','\u2113','\u210f','\u2111','\u211c','\u2118','\u21c4','\u21c6',
- '\u21d2','\u21d0','\u21d1','\u21d3','\u21d4','\u21d5','\u2197','\u2198',
- '\u2196','\u2199','\u222a','\u2282','\u2283','\u2286','\u2287','\u220d',
- '\u2205','\u2308','\u2309','\u230a','\u230b','\u226a','\u226b','\u2220',
- '\u2297','\u2295','\u2296','\u2a38','\u2299','\u2227','\u2228','\u22bb',
- '\u22a4','\u22a5','\u2312','\u22a2','\u22a3','\u25a1','\u25a0','\u25ca',
- '\u25c6','\u27e6','\u27e7','\u2260','\u2262','\u2235','\u2234','\u2237',
- '\u222e','\u2112','\u212d','\u2128','\u2118','\u20dd','\u29cb','\u25c7',
- '\u22c6','\u2034','\u2210','\u2243','\u2245','\u227a','\u227c','\u227b',
- '\u227d','\u2203','\u2200','\u22d8','\u22d9','\u228e','\u228a','\u228b',
- '\u2293','\u2294','\u228f','\u2291','\u22e4','\u2290','\u2292','\u22e5',
- '\u25b3','\u25bd','\u25c3','\u25b9','\u22c8','\u2323','\u2322','\u25ef',
- '\u219d','\u21a9','\u21aa','\u21a3','\u21bc','\u21bd','\u21c0','\u21c1',
- '\u21cc','\u21cb','\u21bf','\u21be','\u21c3','\u21c2','\u21c9','\u21c7',
- '\u22d3','\u22d2','\u22d0','\u22d1','\u229a','\u229b','\u229d','\u2127',
- '\u2221','\u2222','\u25c3','\u25b9','\u25b5','\u25bf','\u2214','\u2250',
- '\u2252','\u2253','\u224e','\u224d','\u22a8','\u2258','\u226c','\u0285',
- '\u2605','\u226e','\u2270','\u226f','\u2271','\u2241','\u2244','\u2247',
- '\u2249','\u2280','\u22e0','\u2281','\u22e1','\u2284','\u2285','\u2288',
- '\u2289','\u0020','\u0020','\u22e2','\u22e3','\u2226','\u2224','\u226d',
- '\u2204','\u2209','\u2247','\u2130','\u2131','\u2102','\u0020','\u2115',
- '\u211d','\u225f','\u22be','\u220b','\u22ef','\u2026','\u22ee','\u22f1',
- '\u0020','\u20e1','\u002b','\u002d','\u003d','\u002a','\u2032','\u2033',
- '\u2034','\u210b','\u2118','\u2272','\u2273','\u0020'
- },
- // WP Charset 7 Math/Scientific Extended (229 chars)
- {
- '\u2320','\u2321','\u23a5','\u23bd','\u221a','\u0020','\u2211','\u220f',
- '\u2210','\u222b','\u222e','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u23a7','\u23a8','\u23a9','\u23aa','\u0020','\u0020','\u0020',
- '\u0020','\u23ab','\u23ac','\u23ad','\u23aa','\u0020','\u0020','\u0020',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u222a','\u222b','\u0020',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u239b','\u239d','\u239c','\u0020','\u0020','\u0020','\u0020','\u239e',
- '\u23a8','\u239f','\u0020','\u0020','\u0020','\u0020','\u23a1','\u23a3',
- '\u23a2','\u0020','\u20aa','\u0020','\u0020','\u23a4','\u23a6','\u23a5',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u22c3','\u22c2','\u228e','\u2a04','\u2294','\u2a06','\u2227','\u22c0',
- '\u2228','\u22c1','\u2297','\u2a02','\u2295','\u2a01','\u2299','\u2a00',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u0020','\u0020','\u0020','\u229d','\u0020','\u2238','\u0020',
- '\u27e6','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u27e7',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u21bc','\u21bd','\u0020','\u296c','\u296d','\u296a','\u296b','\u0020',
- '\u21c9','\u21c7','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u21be','\u21bf','\u21c3','\u21c2','\u0020','\u2293','\u2a05','\u23a1',
- '\u0020','\u0020','\u0020','\u0020','\u0020'
- },
- // WP Charset 8: Greek (219 chars)
- {
- '\u0391','\u03b1','\u0392','\u03b2','\u0392','\u03d0','\u0393','\u03b3',
- '\u0394','\u03b4','\u0395','\u03b5','\u0396','\u03b6','\u0397','\u03b7',
- '\u0398','\u03b8','\u0399','\u03b9','\u039a','\u03ba','\u039b','\u03bb',
- '\u039c','\u03bc','\u039d','\u03bd','\u039e','\u03be','\u039f','\u03bf',
- '\u03a0','\u03c0','\u03a1','\u03c1','\u03a3','\u03c3','\u03a3','\u03c2',
- '\u03a4','\u03c4','\u03a5','\u03c5','\u03a6','\u03c6','\u03a7','\u03c7',
- '\u03a8','\u03c8','\u03a9','\u03c9','\u0386','\u03ac','\u0388','\u03ad',
- '\u0389','\u03ae','\u038a','\u03af','\u03aa','\u03ca','\u038c','\u03cc',
- '\u038e','\u03cd','\u03ab','\u03cb','\u038f','\u03ce','\u03b5','\u03d1',
- '\u03f0','\u03d6','\u03f1','\u03c2','\u03d2','\u03d5','\u03c9','\u037e',
- '\u0387','\u0374','\u0375','\u0384','\u00a8','\u0385','\u1fed','\u1fef',
- '\u1fc0','\u1fbd','\u1ffe','\u037a','\u1fce','\u1fde','\u1fcd','\u1fdd',
- '\u1fcf','\u1fdf','\u0384','\u1fef','\u1fc0','\u1fbd','\u1ffe','\u1fce',
- '\u1fde','\u1fcd','\u1fdd','\u1fcf','\u1fdf','\u1f70','\u1fb6','\u1fb3',
- '\u1fb4','\u1fb2','\u1fb7','\u1f00','\u1f04','\u1f02','\u1f06','\u1f80',
- '\u1f84','\u1f82','\u1f86','\u1f01','\u1f05','\u1f03','\u1f07','\u1f81',
- '\u1f85','\u1f83','\u1f87','\u1f72','\u1f10','\u1f14','\u1f12','\u1f11',
- '\u1f15','\u1f13','\u1f74','\u1fc6','\u1fc3','\u1fc4','\u1fc2','\u1fc7',
- '\u1f20','\u1f24','\u1f22','\u1f26','\u1f90','\u1f94','\u1f92','\u1f96',
- '\u1f21','\u1f25','\u1f23','\u1f27','\u1f91','\u1f95','\u1f93','\u1f97',
- '\u1f76','\u1fd6','\u0390','\u1fd2','\u1f30','\u1f34','\u1f32','\u1f36',
- '\u1f31','\u1f35','\u1f33','\u1f37','\u1f78','\u1f40','\u1f44','\u1f42',
- '\u1f41','\u1f45','\u1f43','\u1fe5','\u1fe4','\u1f7a','\u1fe6','\u03b0',
- '\u1fe2','\u1f50','\u1f54','\u1f52','\u1f56','\u1f51','\u1f55','\u1f53',
- '\u1f57','\u1f7c','\u1ff6','\u1ff3','\u1ff4','\u1ff2','\u1ff7','\u1f60',
- '\u1f64','\u1f62','\u1f66','\u1fa0','\u1fa4','\u1fa2','\u1fa6','\u1f61',
- '\u1f65','\u1f63','\u1f67','\u1fa1','\u1fa5','\u1fa3','\u1fa7','\u03da',
- '\u03dc','\u03de','\u03e0'
- },
- // WP Charset 9: Hebrew (123 chars)
- {
- '\u05d0','\u05d1','\u05d2','\u05d3','\u05d4','\u05d5','\u05d6','\u05d7',
- '\u05d8','\u05d9','\u05da','\u05db','\u05dc','\u05dd','\u05de','\u05df',
- '\u05e0','\u05e1','\u05e2','\u05e3','\u05e4','\u05e5','\u05e6','\u05e7',
- '\u05e8','\u05e9','\u05ea','\u05be','\u05c0','\u05c3','\u05f3','\u05f4',
- '\u05b0','\u05b1','\u05b2','\u05b3','\u05b4','\u05b5','\u05b6','\u05b7',
- '\u05b8','\u05b9','\u05b9','\u05bb','\u05bc','\u05bd','\u05bf','\u05b7',
- '\ufb1e','\u05f0','\u05f1','\u05f2','\ufb1f','\u0591','\u0596','\u0020',
- '\u05a4','\u059a','\u059b','\u05a3','\u05a5','\u05a6','\u05a7','\u05a2',
- '\u0592','\u0593','\u0594','\u0595','\u0597','\u0598','\u0599','\u05a8',
- '\u05f3','\u05f3','\u05f4','\u0020','\u05a9','\u05a0','\u059f','\u05ab',
- '\u05ac','\u05af','\u05c4','\u05aa','\ufb30','\ufb31','\u05d1','\ufb32',
- '\ufb33','\ufb34','\ufb35','\ufb4b','\ufb36','\u05d7','\ufb38','\ufb39',
- '\ufb1d','\ufb3b','\ufb3a','\u05da','\u05da','\u05da','\u05da','\u05da',
- '\u05da','\ufb3c','\ufb3e','\ufb40','\u05d5','\ufb41','\ufb44','\u05e4',
- '\ufb46','\ufb47','\ufb2b','\ufb2d','\ufb2a','\ufb2c','\ufb4a','\u05dc',
- '\ufb3c','\ufb49','\u20aa'
- },
- // WP Charset 10: Cyrillic (250 chars)
- {
- '\u0410','\u0430','\u0411','\u0431','\u0412','\u0432','\u0413','\u0433',
- '\u0414','\u0434','\u0415','\u0435','\u0401','\u0451','\u0416','\u0436',
- '\u0417','\u0437','\u0418','\u0438','\u0419','\u0439','\u041a','\u043a',
- '\u041b','\u043b','\u041c','\u043c','\u041d','\u043d','\u041e','\u043e',
- '\u041f','\u043f','\u0420','\u0440','\u0421','\u0441','\u0422','\u0442',
- '\u0423','\u0443','\u0424','\u0444','\u0425','\u0445','\u0426','\u0446',
- '\u0427','\u0447','\u0428','\u0448','\u0429','\u0449','\u042a','\u044a',
- '\u042b','\u044b','\u042c','\u044c','\u042d','\u044d','\u042e','\u044e',
- '\u042f','\u044f','\u04d8','\u04d9','\u0403','\u0453','\u0490','\u0491',
- '\u0492','\u0493','\u0402','\u0452','\u0404','\u0454','\u0404','\u0454',
- '\u0496','\u0497','\u0405','\u0455','\u0020','\u0020','\u0418','\u0438',
- '\u0406','\u0456','\u0407','\u0457','\u0020','\u0020','\u0408','\u0458',
- '\u040c','\u045c','\u049a','\u049b','\u04c3','\u04c4','\u049c','\u049d',
- '\u0409','\u0459','\u04a2','\u04a3','\u040a','\u045a','\u047a','\u047b',
- '\u0460','\u0461','\u040b','\u045b','\u040e','\u045e','\u04ee','\u04ef',
- '\u04ae','\u04af','\u04b0','\u04b1','\u0194','\u0263','\u04b2','\u04b3',
- '\u0425','\u0445','\u04ba','\u04bb','\u047e','\u047f','\u040f','\u045f',
- '\u04b6','\u04b7','\u04b8','\u04b9','\u0428','\u0448','\u0462','\u0463',
- '\u0466','\u0467','\u046a','\u046b','\u046e','\u046f','\u0470','\u0471',
- '\u0472','\u0473','\u0474','\u0475','\u0410','\u0430','\u0415','\u0435',
- '\u0404','\u0454','\u0418','\u0438','\u0406','\u0456','\u0407','\u0457',
- '\u041e','\u043e','\u0423','\u0443','\u042b','\u044b','\u042d','\u044d',
- '\u042e','\u044e','\u042f','\u044f','\u0410','\u0430','\u0400','\u0450',
- '\u0401','\u0451','\u040d','\u045d','\u041e','\u043e','\u0423','\u0443',
- '\u042b','\u044b','\u042d','\u044d','\u042e','\u044e','\u042f','\u044f',
- '\u0301','\u0300','\u0308','\u0306','\u0326','\u0328','\u0304','\u0020',
- '\u201e','\u201c','\u10d0','\u10d1','\u10d2','\u10d3','\u10d4','\u10d5',
- '\u10d6','\u10f1','\u10d7','\u10d8','\u10d9','\u10da','\u10db','\u10dc',
- '\u10f2','\u10dd','\u10de','\u10df','\u10e0','\u10e1','\u10e2','\u10e3',
- '\u10f3','\u10e4','\u10e5','\u10e6','\u10e7','\u10e8','\u10e9','\u10ea',
- '\u10eb','\u10ec','\u10ed','\u10ee','\u10f4','\u10ef','\u10f0','\u10f5',
- '\u10f6','\u10e3'
- },
- // WP Charset 11: Japanese (63 chars)
- {
- '\uff61','\uff62','\uff63','\uff64','\uff65','\uff66','\uff67','\uff68',
- '\uff69','\uff6a','\uff6b','\uff6c','\uff6d','\uff6e','\uff6f','\uff70',
- '\uff71','\uff72','\uff73','\uff74','\uff75','\uff76','\uff77','\uff78',
- '\uff79','\uff7a','\uff7b','\uff7c','\uff7d','\uff7e','\uff7f','\uff80',
- '\uff81','\uff82','\uff83','\uff84','\uff85','\uff86','\uff87','\uff88',
- '\uff89','\uff8a','\uff8b','\uff8c','\uff8d','\uff8e','\uff8f','\uff90',
- '\uff91','\uff92','\uff93','\uff94','\uff95','\uff96','\uff97','\uff98',
- '\uff99','\uff9a','\uff9b','\uff9c','\uff9d','\uff9e','\uff9f'
- },
- // WP Charset 12: Current Font Symbols (256 chars)
- { //TODO implement Current Font Symbols
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
- ' '
- },
- // WP Charset 13: Arabic (196 chars)
- {
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u0020','\u064e','\ufe77','\u064f','\ufe79','\u0650','\ufe7b',
- '\u064b','\u064c','\u064c','\u064d','\u0652','\ufe7f','\u0651','\ufe7d',
- '\ufc60','\ufcf2','\ufc61','\ufcf3','\ufc62','\ufcf4','\u064b','\ufc5e',
- '\ufc5e','\ufc5f','\u0653','\u0670','\u0654','\u0020','\u060c','\u061b',
- '\u061f','\u066d','\u066a','\u00bb','\u00ab','\u0029','\u0028','\u0661',
- '\u0662','\u0663','\u0664','\u0665','\u0666','\u0667','\u0668','\u0669',
- '\u0660','\u0662','\u0627','\ufe8e','\u0628','\ufe91','\ufe92','\ufe90',
- '\u062a','\ufe97','\ufe98','\ufe96','\u062b','\ufe9b','\ufe9c','\ufe9a',
- '\u062c','\ufe9f','\ufea0','\ufe9e','\u062d','\ufea3','\ufea4','\ufea2',
- '\u062e','\ufea7','\ufea8','\ufea6','\u062f','\ufeaa','\u0630','\ufeac',
- '\u0631','\ufeae','\u0632','\ufeaf','\u0633','\ufeb3','\ufeb4','\ufeb2',
- '\u0634','\ufeb7','\ufeb8','\ufeb6','\u0635','\ufebb','\ufebc','\ufeba',
- '\u0636','\ufebf','\ufec0','\ufebe','\u0637','\ufec3','\ufec4','\ufec2',
- '\u0638','\ufec7','\ufec8','\ufec6','\u0639','\ufecb','\ufecc','\ufeca',
- '\u063a','\ufecf','\ufed0','\ufece','\u0641','\ufed3','\ufed4','\ufed2',
- '\u0642','\ufed7','\ufed8','\ufed6','\u0643','\ufedb','\ufedc','\ufeda',
- '\u0644','\ufedf','\ufee0','\ufede','\u0645','\ufee3','\ufee4','\ufee2',
- '\u0646','\ufee7','\ufee8','\ufee6','\u0647','\ufeeb','\ufeec','\ufeea',
- '\u0629','\ufe94','\u0648','\ufeee','\u064a','\ufef3','\ufef4','\ufef2',
- '\u0649','\ufef3','\ufef4','\ufef0','\u0621','\u0623','\ufe84','\u0625',
- '\ufe88','\u0624','\ufe86','\u0626','\ufe8b','\ufe8c','\ufe8a','\ufd3d',
- '\ufd3c','\u0622','\ufe82','\u0671','\ufb51','\ufefb','\ufefc','\ufef7',
- '\ufef8','\ufef9','\ufefa','\u0020','\ufefc','\ufef5','\ufef6','\u0020',
- '\u0020','\ufdf2','\u0640','\u0640'
- },
- // WP Charset 14: Arabic Script (220 chars)
- {
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0615',
- '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0615','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
- '\u0020','\u06d4','\u0020','\u0020','\u00b0','\u0020','\u065a','\u0020',
- '\u065a','\u0659','\u0020','\u0020','\u0654','\u064c','\ufc5e','\u065a',
- '\u065a','\u06f4','\u06f4','\u06f5','\u06f6','\u06f6','\u06f7','\u06f8',
- '\u067b','\ufb54','\ufb55','\ufb53','\u0680','\ufb5c','\ufb5d','\ufb5b',
- '\u067e','\ufb58','\ufb59','\ufb57','\u0679','\ufb68','\ufb69','\ufb67',
- '\u067c','\u067c','\u067c','\u067c','\u067f','\ufb64','\ufb65','\ufb63',
- '\u067d','\u067d','\u067d','\u067d','\u067a','\ufb60','\ufb61','\ufb5f',
- '\u0684','\ufb74','\ufb75','\ufb73','\u0683','\ufb78','\ufb79','\ufb77',
- '\u0686','\ufb7c','\ufb7d','\ufb7b','\u0687','\ufb80','\ufb81','\ufb7f',
- '\u0685','\u0685','\u0685','\u0685','\u0681','\u0681','\u0681','\u0681',
- '\u0688','\ufb89','\u0689','\u0689','\u068c','\ufb85','\u068e','\ufb87',
- '\u068a','\u068a','\u068d','\ufb83','\u0693','\u0693','\u0691','\ufb8d',
- '\u0699','\u0699','\u0695','\u0695','\u0692','\u0692','\u0698','\ufb8b',
- '\u0696','\u0696','\u075b','\u075b','\u069a','\u069a','\u069a','\u069a',
- '\u06a0','\u06a0','\u06a0','\u06a0','\u06a4','\ufb6c','\ufb6d','\ufb6b',
- '\u06a6','\ufb70','\ufb71','\ufb6f','\u06a9','\ufb90','\ufb91','\ufb8f',
- '\u0643','\ufedb','\ufedc','\ufeda','\u06aa','\u06aa','\u06aa','\u06aa',
- '\u06af','\ufb94','\ufb95','\ufb93','\u06af','\ufb94','\ufb95','\ufb93',
- '\u06ab','\u06ab','\u06ab','\u06ab','\u06b1','\ufb9c','\ufb9d','\ufb9b',
- '\u06b3','\ufb98','\ufb99','\ufb97','\u06b5','\u06b5','\u06b5','\u06b5',
- '\u0020','\u0020','\u06ba','\u0020','\u0020','\ufb9f','\u06bc','\u06bc',
- '\u06bc','\u06bc','\u06bb','\ufba2','\ufba3','\ufba1','\u06c6','\ufbda',
- '\u0020','\u0020','\u06ca','\u06ca','\u06c1','\ufba8','\ufba9','\ufba7',
- '\u06ce','\u06ce','\u06ce','\u06ce','\u06d2','\ufbaf','\u06d1','\u06d1',
- '\u06d1','\u06d1','\u06c0','\ufba5'
- },
- };
-
-
- /**
- * Constructor.
- */
- private WP6Constants() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
new file mode 100644
index 0000000..659ad17
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+import java.io.IOException;
+
+/**
+ * Extracts WordPerfect Document Area text from a WordPerfect document
+ * version 6+.
+ * @author Pascal Essiembre
+ */
+class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
+
+ protected void extract(int c, WPInputStream in, StringBuilder out)
+ throws IOException {
+ if (c > 0 && c <= 32) {
+ out.append(WP6Charsets.DEFAULT_EXTENDED_INTL_CHARS[c]);
+ } else if (c >= 33 && c <= 126) {
+ out.append((char) c);
+ } else if (c == 128) {
+ out.append(' '); // Soft space
+ } else if (c == 129) {
+ out.append('\u00A0'); // Hard space
+ } else if (c == 129) {
+ out.append('-'); // Hard hyphen
+ } else if (c == 135 || c == 137) {
+ out.append('\n'); // Dormant Hard return
+ } else if (c == 138) {
+ // skip to closing pair surrounding page number
+ skipUntilChar(in, 139);
+ } else if (c == 198) {
+ // end of cell
+ out.append('\t');
+ } else if (c >= 180 && c <= 207) {
+ out.append('\n');
+ } else if (c >= 208 && c <= 239) {
+ // Variable-Length Multi-Byte Functions
+ int subgroup = in.readWP();
+ int functionSize = in.readWPShort();
+ for (int i = 0; i < functionSize - 4; i++) {
+ in.readWP();
+ }
+
+ // End-of-Line group
+ if (c == 208) {
+ if (subgroup >= 1 && subgroup <= 3) {
+ out.append(' ');
+ } else if (subgroup == 10) {
+ // end of cell
+ out.append('\t');
+ } else if (subgroup >= 4 && subgroup <= 19) {
+ out.append('\n');
+ } else if (subgroup >= 20 && subgroup <= 22) {
+ out.append(' ');
+ } else if (subgroup >= 23 && subgroup <= 28) {
+ out.append('\n');
+ }
+ } else if (c == 213) {
+ out.append(' ');
+ } else if (c == 224) {
+ out.append('\t');
+ }
+ //TODO Are there functions containing data? Like footnotes?
+
+ } else if (c == 240) {
+ // extended char
+ int charval = in.readWP();
+ int charset = in.readWP();
+ in.readWP(); // closing character
+ WP6Charsets.append(out, charset, charval);
+ } else if (c >= 241 && c <= 254) {
+ skipUntilChar(in, c);
+ } else if (c == 255) {
+ skipUntilChar(in, c);
+ }
+
+ // Ignored codes above 127:
+
+ // 130,131,133: soft hyphens
+ // 134: invisible return in line
+ // 136: soft end of center/align
+ // 140: style separator mark
+ // 141,142: start/end of text to skip
+ // 143: exited hyphenation
+ // 144: cancel hyphenation
+ // 145-151: match functions
+ // 152-179: unknown/ignored
+ // 255: reserved, cannot be used
+ }
+}