You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/01/04 17:32:06 UTC

[1/3] tika git commit: TIKA-2228 - WordPerfect parser update to handle 5.x from Pascal Essiembre. This closes #142.

Repository: tika
Updated Branches:
  refs/heads/master ef1d9077b -> 940e6f483


http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6FileHeader.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6FileHeader.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6FileHeader.java
deleted file mode 100644
index 6a95335..0000000
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6FileHeader.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/* Copyright 2015-2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.wordperfect;
-
-
-/**
- * WordPerfect file header.
- * @author Pascal Essiembre
- */
-class WP6FileHeader {
-
-    // Normal header
-    private String fileId;
-    private long docAreaPointer;
-    private int productType;
-    private int fileType;
-    private int majorVersion;
-    private int minorVersion;
-    private boolean encrypted;
-    private int indexAreaPointer;
-
-    // Extended header
-    private long fileSize;
-    
-    public WP6FileHeader() {
-        super();
-    }
-
-    public String getFileId() {
-        return fileId;
-    }
-
-    public void setFileId(String fileId) {
-        this.fileId = fileId;
-    }
-
-    public long getDocAreaPointer() {
-        return docAreaPointer;
-    }
-
-    public void setDocAreaPointer(long docAreaPointer) {
-        this.docAreaPointer = docAreaPointer;
-    }
-
-    public int getProductType() {
-        return productType;
-    }
-
-    public void setProductType(int productType) {
-        this.productType = productType;
-    }
-
-    public int getFileType() {
-        return fileType;
-    }
-
-    public void setFileType(int fileType) {
-        this.fileType = fileType;
-    }
-    
-    public int getMajorVersion() {
-        return majorVersion;
-    }
-
-    public void setMajorVersion(int majorVersion) {
-        this.majorVersion = majorVersion;
-    }
-
-    public int getMinorVersion() {
-        return minorVersion;
-    }
-
-    public void setMinorVersion(int minorVersion) {
-        this.minorVersion = minorVersion;
-    }
-    
-    public boolean isEncrypted() {
-        return encrypted;
-    }
-
-    public void setEncrypted(boolean encrypted) {
-        this.encrypted = encrypted;
-    }
-
-    public int getIndexAreaPointer() {
-        return indexAreaPointer;
-    }
-
-    public void setIndexAreaPointer(int indexAreaPointer) {
-        this.indexAreaPointer = indexAreaPointer;
-    }
-    
-    public long getFileSize() {
-        return fileSize;
-    }
-    public void setFileSize(long fileSize) {
-        this.fileSize = fileSize;
-    }
-
-    @Override
-    public String toString() {
-        return "WP6FileHeader{" +
-                "fileId='" + fileId + '\'' +
-                ", docAreaPointer=" + docAreaPointer +
-                ", productType=" + productType +
-                ", fileType=" + fileType +
-                ", majorVersion=" + majorVersion +
-                ", minorVersion=" + minorVersion +
-                ", encrypted=" + encrypted +
-                ", indexAreaPointer=" + indexAreaPointer +
-                ", fileSize=" + fileSize +
-                '}';
-    }
-
-    @Override
-    public int hashCode() {
-        final int prime = 31;
-        int result = 1;
-        result = prime * result
-                + (int) (docAreaPointer ^ (docAreaPointer >>> 32));
-        result = prime * result + (encrypted ? 1231 : 1237);
-        result = prime * result + ((fileId == null) ? 0 : fileId.hashCode());
-        result = prime * result + (int) (fileSize ^ (fileSize >>> 32));
-        result = prime * result + fileType;
-        result = prime * result + indexAreaPointer;
-        result = prime * result + majorVersion;
-        result = prime * result + minorVersion;
-        result = prime * result + productType;
-        return result;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-        if (obj == null) {
-            return false;
-        }
-        if (!(obj instanceof WP6FileHeader)) {
-            return false;
-        }
-        WP6FileHeader other = (WP6FileHeader) obj;
-        if (docAreaPointer != other.docAreaPointer) {
-            return false;
-        }
-        if (encrypted != other.encrypted) {
-            return false;
-        }
-        if (fileId == null) {
-            if (other.fileId != null) {
-                return false;
-            }
-        } else if (!fileId.equals(other.fileId)) {
-            return false;
-        }
-        if (fileSize != other.fileSize) {
-            return false;
-        }
-        if (fileType != other.fileType) {
-            return false;
-        }
-        if (indexAreaPointer != other.indexAreaPointer) {
-            return false;
-        }
-        if (majorVersion != other.majorVersion) {
-            return false;
-        }
-        if (minorVersion != other.minorVersion) {
-            return false;
-        }
-        if (productType != other.productType) {
-            return false;
-        }
-        return true;
-    }
-
-    
-}

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6TextExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6TextExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6TextExtractor.java
deleted file mode 100644
index 8c03d81..0000000
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6TextExtractor.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/* Copyright 2015-2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.wordperfect;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.tika.exception.EncryptedDocumentException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.exception.UnsupportedFormatException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.WordPerfect;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.sax.XHTMLContentHandler;
-import org.xml.sax.SAXException;
-
-/**
- * Extracts text from a WordPerfect document according to WP6 File Format.
- * This format appears to be compatible with more recent versions too.
- * @author Pascal Essiembre
- */
-class WP6TextExtractor {
-
-    public void extract(
-            InputStream input, XHTMLContentHandler xhtml, Metadata metadata) 
-            throws IOException, SAXException, TikaException {
-        WPInputStream in = new WPInputStream(input);
-        
-        WP6FileHeader header = parseFileHeader(in);
-
-        applyMetadata(header, metadata);
-
-        if (header.getMajorVersion() == 0) {
-            MediaType version = WordPerfectParser.WP_UNK;
-            if (header.getMinorVersion() == 0) {
-                version = WordPerfectParser.WP_5_0;
-            } else if (header.getMinorVersion() == 1) {
-                version = WordPerfectParser.WP_5_1;
-            }
-            metadata.set(Metadata.CONTENT_TYPE, version.toString());
-            throw new UnsupportedFormatException("Parser doesn't support this version:"+version.toString());
-        }
-
-        if (header.getMajorVersion() != 2) {
-            metadata.set(Metadata.CONTENT_TYPE, WordPerfectParser.WP_UNK.toString());
-            throw new UnsupportedFormatException("Parser doesn't recognize this version");
-        }
-
-        if (header.isEncrypted()) {
-            throw new EncryptedDocumentException();
-        }
-
-        // For text extraction we can safely ignore WP Index Area and
-        // Packet Data Area and jump right away to Document Area.
-        extractDocumentText(in, header.getDocAreaPointer(), xhtml);
-        
-    }
-
-    private void applyMetadata(WP6FileHeader header, Metadata metadata) {
-        metadata.set(WordPerfect.FILE_SIZE,
-                Long.toString(header.getFileSize()));
-        metadata.set(WordPerfect.FILE_ID, header.getFileId());
-        metadata.set(WordPerfect.PRODUCT_TYPE, header.getProductType());
-        metadata.set(WordPerfect.FILE_TYPE, header.getFileType());
-        metadata.set(WordPerfect.MAJOR_VERSION, header.getMajorVersion());
-        metadata.set(WordPerfect.MINOR_VERSION, header.getMinorVersion());
-        metadata.set(WordPerfect.ENCRYPTED, 
-                Boolean.toString(header.isEncrypted()));
-    }
-        
-    private void extractDocumentText(
-            WPInputStream in, long offset, XHTMLContentHandler xhtml) 
-                    throws IOException, SAXException {
-        xhtml.startElement("p");
-        
-        // Move to offset (for some reason skip() did not work).
-        for (int i = 0; i < offset; i++) {
-            in.readWPByte();
-        }
-
-        int chunk = 4096;
-        StringBuilder out = new StringBuilder(chunk);
-        
-        int c;
-        while ((c = in.read()) != -1) {
-            if (c > 0 && c <= 32) {
-                out.append(WP6Constants.DEFAULT_EXTENDED_INTL_CHARS[c]);
-            } else if (c >= 33 && c <= 126) {
-                out.append((char) c);
-            } else if (c == 128) {
-                out.append(' ');      // Soft space
-            } else if (c == 129) {
-                out.append('\u00A0'); // Hard space
-            } else if (c == 129) {
-                out.append('-');      // Hard hyphen
-            } else if (c == 135 || c == 137) {
-                out.append('\n');      // Dormant Hard return
-            } else if (c == 138) {
-                // skip to closing pair surrounding page number
-                skipUntilChar(in, 139);
-            } else if (c == 198) {
-                // end of cell
-                out.append('\t');
-            } else if (c >= 180 && c <= 207) {
-                out.append('\n');
-            } else if (c >= 208 && c <= 239) {
-                // Variable-Length Multi-Byte Functions
-                int subgroup = in.readWP();
-                int functionSize = in.readWPShort();
-                for (int i = 0; i < functionSize - 4; i++) {
-                    in.readWP();
-                }
-                
-                // End-of-Line group
-                if (c == 208) {
-                    if (subgroup >= 1 && subgroup <= 3) {
-                        out.append(' ');
-                    } else if (subgroup == 10) {
-                        // end of cell
-                        out.append('\t');
-                    } else if (subgroup >= 4 && subgroup <= 19) {
-                        out.append('\n');
-                    } else if (subgroup >= 20 && subgroup <= 22) {
-                        out.append(' ');
-                    } else if (subgroup >= 23 && subgroup <= 28) {
-                        out.append('\n');
-                    }
-                } else if (c == 213) {
-                    out.append(' ');
-                } else if (c == 224) {
-                    out.append('\t');
-                }
-                //TODO Are there functions containing data? Like footnotes?
-                
-            } else if (c == 240) {
-                // extended char
-                int charval = in.readWP();
-                int charset = in.readWP();
-                in.readWP(); // closing character
-  
-                //TODO implement all charsets
-                if (charset == 4 || charset == 5) {
-                    out.append(
-                            WP6Constants.EXTENDED_CHARSETS[charset][charval]);
-                } else {
-                    out.append("[TODO:charset" + charset + "]");
-                }
-            } else if (c >= 241 && c <= 254) {
-                skipUntilChar(in, c);
-            } else if (c == 255) {
-                skipUntilChar(in, c);
-            }
-            
-            if (out.length() >= chunk) {
-                xhtml.characters(out.toString());
-                out.setLength(0);
-            }
-        }
-        
-        // Ignored codes above 127:
-        
-        // 130,131,133: soft hyphens
-        // 134: invisible return in line
-        // 136: soft end of center/align
-        // 140: style separator mark
-        // 141,142: start/end of text to skip
-        // 143: exited hyphenation
-        // 144: cancel hyphenation
-        // 145-151: match functions
-        // 152-179: unknown/ignored
-        // 255: reserved, cannot be used
-        
-        xhtml.characters(out.toString());
-        out.setLength(0);
-        xhtml.endElement("p");
-    }
-
-    // Skips until the given character is encountered.
-    private int skipUntilChar(WPInputStream in, int targetChar)
-            throws IOException {
-        int count = 0;
-        int c;
-        while ((c = in.read()) != -1) {
-            count++;
-            if (c == targetChar) {
-                return count;
-            }
-        }
-        return count;
-    }
-    
-    private WP6FileHeader parseFileHeader(WPInputStream in) 
-            throws IOException {
-        WP6FileHeader header = new WP6FileHeader();
-
-        // File header
-        in.mark(30);
-        header.setFileId(in.readWPString(4));         // 1-4
-        header.setDocAreaPointer(in.readWPLong());    // 5-8
-        header.setProductType(in.readWP());             // 9
-        header.setFileType(in.readWPChar());          // 10
-        header.setMajorVersion(in.readWP());            // 11
-        header.setMinorVersion(in.readWP());            // 12
-        header.setEncrypted(in.readWPShort() != 0);   // 13-14
-        header.setIndexAreaPointer(in.readWPShort()); // 15-16
-        try {
-            in.skip(4); // 4 reserved bytes: skip     // 17-20
-            header.setFileSize(in.readWPLong());      // 21-24
-        } catch (IOException e) {
-            // May fail if not extended error, which is fine.
-        }
-        in.reset();
-
-        //TODO header may be shared between corel products, so move validation
-        //specific to each product elsewhere?
-        //TODO convert to logs only, and let it fail elsewhere?
-//        if (!WP6Constants.WP6_FILE_ID.equals(header.getFileId())) {
-//            throw new IOException("Not a WordPerfect file. File must start "
-//                    + "with " + WP6Constants.WP6_FILE_ID + " but was "
-//                    + header.getFileId());
-//        }
-//        if (WP6Constants.WP6_PRODUCT_TYPE != header.getProductType()) {
-//            throw new IOException("Not a WordPerfect file. Product type "
-//                    + "must be " + WP6Constants.WP6_PRODUCT_TYPE + " but was "
-//                    + header.getProductType());
-//        }
-        //TODO perform file type validation?
-        return header;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
new file mode 100644
index 0000000..002ecee
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+import java.io.IOException;
+
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.SAXException;
+
+/**
+ * Extracts WordPerfect Document Area text from a WordPerfect document.
+ * @author Pascal Essiembre
+ */
+abstract class WPDocumentAreaExtractor {
+
+    public void extract(WPInputStream in, XHTMLContentHandler xhtml) 
+            throws IOException, SAXException {
+        int chunk = 4096;
+        StringBuilder out = new StringBuilder(chunk);
+        int c;
+        while ((c = in.read()) != -1) {
+            extract(c, in, out);
+            if (out.length() >= chunk) {
+                xhtml.characters(out.toString());
+                out.setLength(0);
+            }
+        }
+        xhtml.characters(out.toString());
+        out.setLength(0);
+    }
+ 
+    protected abstract void extract(
+            int c, WPInputStream in, StringBuilder out) throws IOException;
+    
+    
+    // Skips until the given character is encountered.
+    protected int skipUntilChar(WPInputStream in, int targetChar)
+            throws IOException {
+        int count = 0;
+        int c;
+        while ((c = in.read()) != -1) {
+            count++;
+            if (c == targetChar) {
+                return count;
+            }
+        }
+        return count;
+    }
+}

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPInputStream.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPInputStream.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPInputStream.java
index 67c3200..9b01aa5 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPInputStream.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPInputStream.java
@@ -1,8 +1,10 @@
-/* Copyright 2015-2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
@@ -12,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.tika.parser.wordperfect;
 
 import java.io.BufferedInputStream;
@@ -24,6 +27,7 @@ import org.apache.commons.lang.StringUtils;
 
 /**
  * {@link InputStream} wrapper adding WordPerfect-specific byte-reading methods.
+ * Applies to both 5.x and 6+ documents.
  * @author Pascal Essiembre
  */
 class WPInputStream extends InputStream {

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixArea.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixArea.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixArea.java
new file mode 100644
index 0000000..5c08528
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixArea.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+
+/**
+ * Section of a WordPerfect file called "Prefix Area" containing  
+ * information about the specific file format/version.
+ * @author Pascal Essiembre
+ */
+class WPPrefixArea {
+
+    public static final String WP_FILE_ID = "�WPC";
+    public static final int WP_PRODUCT_TYPE = 1;
+
+    // WordPerfect 5.x header values
+    public static final int WP5_FILE_TYPE = 10;
+    public static final int WP5_MAJOR_VERSION = 0;
+    public static final int WP5_0_MINOR_VERSION = 0;
+    public static final int WP5_1_MINOR_VERSION = 1; // also for WPWin 5.2
+
+    // WordPerfect 6+ header values
+    public static final int WP6_FILE_TYPE = 36;
+    public static final int WP6_MAJOR_VERSION = 2;
+    public static final int WP6_MINOR_VERSION = 0;
+    
+    
+    // Normal header
+    private String fileId;
+    private long docAreaPointer;
+    private int productType;
+    private int fileType;
+    private int majorVersion;
+    private int minorVersion;
+    private boolean encrypted;
+    private int indexAreaPointer;
+
+    // Extended header
+    private long fileSize;
+    
+    public WPPrefixArea() {
+        super();
+    }
+
+    public String getFileId() {
+        return fileId;
+    }
+
+    public void setFileId(String fileId) {
+        this.fileId = fileId;
+    }
+
+    public long getDocAreaPointer() {
+        return docAreaPointer;
+    }
+
+    public void setDocAreaPointer(long docAreaPointer) {
+        this.docAreaPointer = docAreaPointer;
+    }
+
+    public int getProductType() {
+        return productType;
+    }
+
+    public void setProductType(int productType) {
+        this.productType = productType;
+    }
+
+    public int getFileType() {
+        return fileType;
+    }
+
+    public void setFileType(int fileType) {
+        this.fileType = fileType;
+    }
+    
+    public int getMajorVersion() {
+        return majorVersion;
+    }
+
+    public void setMajorVersion(int majorVersion) {
+        this.majorVersion = majorVersion;
+    }
+
+    public int getMinorVersion() {
+        return minorVersion;
+    }
+
+    public void setMinorVersion(int minorVersion) {
+        this.minorVersion = minorVersion;
+    }
+    
+    public boolean isEncrypted() {
+        return encrypted;
+    }
+
+    public void setEncrypted(boolean encrypted) {
+        this.encrypted = encrypted;
+    }
+
+    public int getIndexAreaPointer() {
+        return indexAreaPointer;
+    }
+
+    public void setIndexAreaPointer(int indexAreaPointer) {
+        this.indexAreaPointer = indexAreaPointer;
+    }
+    
+    public long getFileSize() {
+        return fileSize;
+    }
+    public void setFileSize(long fileSize) {
+        this.fileSize = fileSize;
+    }
+
+    @Override
+    public String toString() {
+        return "WP6FileHeader{" +
+                "fileId='" + fileId + '\'' +
+                ", docAreaPointer=" + docAreaPointer +
+                ", productType=" + productType +
+                ", fileType=" + fileType +
+                ", majorVersion=" + majorVersion +
+                ", minorVersion=" + minorVersion +
+                ", encrypted=" + encrypted +
+                ", indexAreaPointer=" + indexAreaPointer +
+                ", fileSize=" + fileSize +
+                '}';
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result
+                + (int) (docAreaPointer ^ (docAreaPointer >>> 32));
+        result = prime * result + (encrypted ? 1231 : 1237);
+        result = prime * result + ((fileId == null) ? 0 : fileId.hashCode());
+        result = prime * result + (int) (fileSize ^ (fileSize >>> 32));
+        result = prime * result + fileType;
+        result = prime * result + indexAreaPointer;
+        result = prime * result + majorVersion;
+        result = prime * result + minorVersion;
+        result = prime * result + productType;
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (!(obj instanceof WPPrefixArea)) {
+            return false;
+        }
+        WPPrefixArea other = (WPPrefixArea) obj;
+        if (docAreaPointer != other.docAreaPointer) {
+            return false;
+        }
+        if (encrypted != other.encrypted) {
+            return false;
+        }
+        if (fileId == null) {
+            if (other.fileId != null) {
+                return false;
+            }
+        } else if (!fileId.equals(other.fileId)) {
+            return false;
+        }
+        if (fileSize != other.fileSize) {
+            return false;
+        }
+        if (fileType != other.fileType) {
+            return false;
+        }
+        if (indexAreaPointer != other.indexAreaPointer) {
+            return false;
+        }
+        if (majorVersion != other.majorVersion) {
+            return false;
+        }
+        if (minorVersion != other.minorVersion) {
+            return false;
+        }
+        if (productType != other.productType) {
+            return false;
+        }
+        return true;
+    }
+
+    
+}

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixAreaExtractor.java
new file mode 100644
index 0000000..1f587f2
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPPrefixAreaExtractor.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+import java.io.IOException;
+
+/**
+ * Extracts WordPerfect Prefix Area data from a WordPerfect document.
+ * Applies to both 5.x and 6+ documents.
+ * @author Pascal Essiembre
+ */
+final class WPPrefixAreaExtractor {
+
+    private WPPrefixAreaExtractor() {
+        super();
+    }
+
+    //WP5.x:
+    //  Prefix Area:            16 bytes (standard header)
+    
+    //WP6.x:
+    //  Prefix Area:            30 bytes (16 standard header + 14 index header)
+    
+    public static WPPrefixArea extract(WPInputStream in) 
+            throws IOException {
+        WPPrefixArea prefixArea = new WPPrefixArea();
+
+        in.mark(30);
+        prefixArea.setFileId(in.readWPString(4));         // 1-4
+        prefixArea.setDocAreaPointer(in.readWPLong());    // 5-8
+        prefixArea.setProductType(in.readWP());           // 9
+        prefixArea.setFileType(in.readWPChar());          // 10
+        prefixArea.setMajorVersion(in.readWP());          // 11
+        prefixArea.setMinorVersion(in.readWP());          // 12
+        prefixArea.setEncrypted(in.readWPShort() != 0);   // 13-14
+        prefixArea.setIndexAreaPointer(in.readWPShort()); // 15-16
+
+        // only applies to 6.x:
+        prefixArea.setFileSize(-1);
+        if (prefixArea.getMajorVersion() == WPPrefixArea.WP6_MAJOR_VERSION) {
+            try {
+                in.skip(4); // 4 reserved bytes: skip     // 17-20
+                prefixArea.setFileSize(in.readWPLong());      // 21-24
+            } catch (IOException e) {
+                // May fail if no index header, which is fine.
+            }
+        }
+        
+        in.reset();
+
+        return prefixArea;
+    }
+}

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
index 5cd81e2..4c6d445 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
@@ -1,8 +1,10 @@
-/* Copyright 2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
@@ -16,11 +18,16 @@ package org.apache.tika.parser.wordperfect;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.Set;
 
+import org.apache.tika.exception.EncryptedDocumentException;
 import org.apache.tika.exception.TikaException;
+import org.apache.tika.exception.UnsupportedFormatException;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.WordPerfect;
 import org.apache.tika.mime.MediaType;
 import org.apache.tika.parser.AbstractParser;
 import org.apache.tika.parser.ParseContext;
@@ -49,7 +56,8 @@ public class WordPerfectParser extends AbstractParser {
     final static MediaType WP_6_x = new MediaType(WP_BASE, "version", "6.x");
 
     private static final Set<MediaType> SUPPORTED_TYPES =
-            Collections.unmodifiableSet(Collections.singleton(WP_6_x));
+            Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
+                    WP_5_0, WP_5_1, WP_6_x)));
     
     @Override
     public Set<MediaType> getSupportedTypes(ParseContext context) {
@@ -60,17 +68,87 @@ public class WordPerfectParser extends AbstractParser {
     public void parse(InputStream stream, ContentHandler handler, 
             Metadata metadata, ParseContext context)
             throws IOException, SAXException, TikaException {
-        if (metadata.get(Metadata.CONTENT_TYPE) == null) {
-            metadata.set(Metadata.CONTENT_TYPE, "application/wordperfect");
+        
+        WPInputStream wpStream = new WPInputStream(stream);
+        WPPrefixArea prefixArea = WPPrefixAreaExtractor.extract(wpStream);
+        
+        ensureFileSupport(prefixArea, metadata);
+        
+        applyMetadata(prefixArea, metadata);
+        
+        extractDocumentArea(prefixArea, wpStream, 
+                new XHTMLContentHandler(handler, metadata));
+    }
+    
+    private void extractDocumentArea(WPPrefixArea prefixArea,
+            WPInputStream in, XHTMLContentHandler xhtml)
+                    throws SAXException, IOException {
+
+        // Move to offset (for some reason skip() did not work).
+        for (int i = 0; i < prefixArea.getDocAreaPointer(); i++) {
+            in.readWPByte();
         }
         
-        XHTMLContentHandler xhtml =
-                new XHTMLContentHandler(handler, metadata);
         xhtml.startDocument();
+        xhtml.startElement("p");
 
-        WP6TextExtractor extractor = new WP6TextExtractor();
-        extractor.extract(stream, xhtml, metadata);
-
+        getDocumentAreaExtractor(prefixArea).extract(in, xhtml);
+        
+        xhtml.endElement("p");
         xhtml.endDocument();
     }
+    
+    private void ensureFileSupport(WPPrefixArea pa, Metadata metadata)
+            throws UnsupportedFormatException, EncryptedDocumentException {
+        if (pa.getMajorVersion() != WPPrefixArea.WP5_MAJOR_VERSION
+                && pa.getMajorVersion() != WPPrefixArea.WP6_MAJOR_VERSION) {
+            metadata.set(Metadata.CONTENT_TYPE, WP_UNK.toString());
+            throw new UnsupportedFormatException(
+                    "Parser doesn't recognize this major version: "
+                            + pa.getMajorVersion());
+        }
+        if (pa.isEncrypted()) {
+            throw new EncryptedDocumentException();
+        }
+    }
+    
+    private void applyMetadata(WPPrefixArea pa, Metadata metadata) {
+        // Should we force the more precise media type if only the base
+        // form is found?  Or shall we store a friendly representation
+        // of the version in a new field?
+        if (metadata.get(Metadata.CONTENT_TYPE) == null) {
+            if (pa.getMajorVersion() == WPPrefixArea.WP6_MAJOR_VERSION) {
+                metadata.set(Metadata.CONTENT_TYPE, WP_6_x.toString());
+            } else if (pa.getMinorVersion() == 
+                    WPPrefixArea.WP5_0_MINOR_VERSION) {
+                metadata.set(Metadata.CONTENT_TYPE, WP_5_0.toString());
+            } else if (pa.getMinorVersion() == 
+                    WPPrefixArea.WP5_1_MINOR_VERSION) {
+                metadata.set(Metadata.CONTENT_TYPE, WP_5_1.toString());
+            } else {
+                metadata.set(Metadata.CONTENT_TYPE, WP_BASE.toString());
+            }
+        }
+        
+        metadata.set(WordPerfect.FILE_ID, pa.getFileId());
+        metadata.set(WordPerfect.PRODUCT_TYPE, pa.getProductType());
+        metadata.set(WordPerfect.FILE_TYPE, pa.getFileType());
+        metadata.set(WordPerfect.MAJOR_VERSION, pa.getMajorVersion());
+        metadata.set(WordPerfect.MINOR_VERSION, pa.getMinorVersion());
+        metadata.set(WordPerfect.ENCRYPTED, Boolean.toString(pa.isEncrypted()));
+        
+        if (pa.getFileSize() > -1) {
+            metadata.set(
+                    WordPerfect.FILE_SIZE, Long.toString(pa.getFileSize()));
+        }
+    }
+    
+    private WPDocumentAreaExtractor getDocumentAreaExtractor(
+            WPPrefixArea prefixArea) {
+        if (prefixArea.getMajorVersion() == WPPrefixArea.WP6_MAJOR_VERSION) {
+            return new WP6DocumentAreaExtractor();
+        }
+        // we can safely assume v5 as exception would have been thrown
+        return new WP5DocumentAreaExtractor();
+    }
 }

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/QuattroProTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/QuattroProTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/QuattroProTest.java
index cd43a84..687caf1 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/QuattroProTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/QuattroProTest.java
@@ -1,8 +1,10 @@
-/* Copyright 2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WPInputStreamTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WPInputStreamTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WPInputStreamTest.java
index d204e0c..878388e 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WPInputStreamTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WPInputStreamTest.java
@@ -30,8 +30,7 @@ public class WPInputStreamTest {
 
     @Test
     public void testReadByte() throws Exception {
-        WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
-        try {
+        try (WPInputStream wpInputStream = emptyWPStream()) {
             wpInputStream.readWPByte();
             fail("should have thrown EOF");
         } catch (EOFException e) {
@@ -42,8 +41,7 @@ public class WPInputStreamTest {
 
     @Test
     public void testReadShort() throws Exception {
-        WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
-        try {
+        try (WPInputStream wpInputStream = emptyWPStream()) {
             wpInputStream.readWPShort();
             fail("should have thrown EOF");
         } catch (EOFException e) {
@@ -54,8 +52,7 @@ public class WPInputStreamTest {
 
     @Test
     public void testReadChar() throws Exception {
-        WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
-        try {
+        try (WPInputStream wpInputStream = emptyWPStream()) {
             wpInputStream.readWPChar();
             fail("should have thrown EOF");
         } catch (EOFException e) {
@@ -65,8 +62,7 @@ public class WPInputStreamTest {
 
     @Test
     public void testReadHex() throws Exception {
-        WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
-        try {
+        try (WPInputStream wpInputStream = emptyWPStream()) {
             wpInputStream.readWPHex();
             fail("should have thrown EOF");
         } catch (EOFException e) {
@@ -76,8 +72,7 @@ public class WPInputStreamTest {
 
     @Test
     public void testReadHexString() throws Exception {
-        WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
-        try {
+        try (WPInputStream wpInputStream = emptyWPStream()) {
             wpInputStream.readWPHexString(10);
             fail("should have thrown EOF");
         } catch (EOFException e) {
@@ -87,8 +82,7 @@ public class WPInputStreamTest {
 
     @Test
     public void testReadLong() throws Exception {
-        WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
-        try {
+        try (WPInputStream wpInputStream = emptyWPStream()) {
             wpInputStream.readWPLong();
             fail("should have thrown EOF");
         } catch (EOFException e) {
@@ -99,8 +93,7 @@ public class WPInputStreamTest {
 
     @Test
     public void testReadString() throws Exception {
-        WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
-        try {
+        try (WPInputStream wpInputStream = emptyWPStream()) {
             wpInputStream.readWPString(10);
             fail("should have thrown EOF");
         } catch (EOFException e) {
@@ -110,8 +103,7 @@ public class WPInputStreamTest {
 
     @Test
     public void testReadArr() throws Exception {
-        WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
-        try {
+        try (WPInputStream wpInputStream = emptyWPStream()) {
             byte[] buffer = new byte[10];
             wpInputStream.read(buffer);
         } catch (EOFException e) {
@@ -121,8 +113,7 @@ public class WPInputStreamTest {
 
     @Test
     public void testReadArrOffset() throws Exception {
-        WPInputStream wpInputStream = new WPInputStream(new ByteArrayInputStream(new byte[0]));
-        try {
+        try (WPInputStream wpInputStream = emptyWPStream()) {
             byte[] buffer = new byte[10];
             wpInputStream.read(buffer, 0, 2);
         } catch (EOFException e) {
@@ -130,5 +121,7 @@ public class WPInputStreamTest {
         }
     }
 
-
+    private WPInputStream emptyWPStream() {
+        return new WPInputStream(new ByteArrayInputStream(new byte[0]));
+    }
 }

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
index 8eafc18..89abca0 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
@@ -1,8 +1,10 @@
-/* Copyright 2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
@@ -26,31 +28,32 @@ import org.junit.Test;
  */
 public class WordPerfectTest extends TikaTest {
 
-
     @Test
     public void testWordPerfectParser() throws Exception {
-
         XMLResult r = getXML("testWordPerfect.wpd");
-        assertEquals("application/vnd.wordperfect; version=6.x",
+        assertEquals(WordPerfectParser.WP_6_x.toString(),
                 r.metadata.get(Metadata.CONTENT_TYPE));
         assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
         assertContains("test test", r.xml);
     }
 
-
     @Test
     public void testVersion50() throws Exception {
         //test file "testWordPerfect_5_0.wp" is from govdocs1: 126546.wp
         XMLResult r = getXML("testWordPerfect_5_0.wp");
-        assertEquals(WordPerfectParser.WP_5_0.toString(), r.metadata.get(Metadata.CONTENT_TYPE));
-        assertEquals("org.apache.tika.parser.EmptyParser", r.metadata.get("X-Parsed-By"));
+        assertEquals(WordPerfectParser.WP_5_0.toString(),
+                r.metadata.get(Metadata.CONTENT_TYPE));
+        assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
+        assertContains("Surrounded by her family", r.xml);
     }
 
     @Test
     public void testVersion51() throws Exception {
         //testfile "testWordperfect_5_1.wp is from govdocs1: 758750.wp
         XMLResult r = getXML("testWordPerfect_5_1.wp");
-        assertEquals(WordPerfectParser.WP_5_1.toString(), r.metadata.get(Metadata.CONTENT_TYPE));
-        assertEquals("org.apache.tika.parser.EmptyParser", r.metadata.get("X-Parsed-By"));
+        assertEquals(WordPerfectParser.WP_5_1.toString(),
+                r.metadata.get(Metadata.CONTENT_TYPE));
+        assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
+        assertContains("Summer is the peak travel time", r.xml);
     }
 }


[3/3] tika git commit: TIKA-2230 -- add paragraph markup to WordPerfect parsers

Posted by ta...@apache.org.
TIKA-2230 -- add paragraph markup to WordPerfect parsers


Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/940e6f48
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/940e6f48
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/940e6f48

Branch: refs/heads/master
Commit: 940e6f483ec87725870b3c9d62ed0b2ab9a5811e
Parents: 6dc442d
Author: tballison <ta...@mitre.org>
Authored: Wed Jan 4 12:31:47 2017 -0500
Committer: tballison <ta...@mitre.org>
Committed: Wed Jan 4 12:31:47 2017 -0500

----------------------------------------------------------------------
 .../wordperfect/WP5DocumentAreaExtractor.java   | 15 ++++----
 .../wordperfect/WP6DocumentAreaExtractor.java   | 15 ++++----
 .../wordperfect/WPDocumentAreaExtractor.java    | 37 ++++++++++++++++----
 .../parser/wordperfect/WordPerfectParser.java   |  2 --
 .../parser/wordperfect/WordPerfectTest.java     |  7 ++--
 5 files changed, 53 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
index b95490e..bd59725 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
@@ -18,6 +18,9 @@ package org.apache.tika.parser.wordperfect;
 
 import java.io.IOException;
 
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.SAXException;
+
 /**
  * Extracts WordPerfect Document Area text from a WordPerfect document
  * version 5.x.
@@ -25,16 +28,16 @@ import java.io.IOException;
  */
 class WP5DocumentAreaExtractor extends WPDocumentAreaExtractor {
     
-    protected void extract(int c, WPInputStream in, StringBuilder out)
-            throws IOException {
+    protected void extract(int c, WPInputStream in, StringBuilder out, XHTMLContentHandler xhtml)
+            throws IOException, SAXException {
 
         // 0-31: control characters
         if (c == 10) {
-            out.append('\n');     // hard return ("Enter")
+            endParagraph(out, xhtml);// hard return ("Enter")
         } else if (c == 11) {
             out.append(' ');      // soft page break
         } else if (c == 12) {
-            out.append('\n');     // hard page break
+            endParagraph(out, xhtml);// hard page break
         } else if (c == 13) {
             out.append(' ');      // soft return (line wrap)
             
@@ -44,11 +47,11 @@ class WP5DocumentAreaExtractor extends WPDocumentAreaExtractor {
 
         // 128-191: single-byte functions
         } else if (c == 140) {
-            out.append('\n');     // combination hard return/soft page (WP5.1)
+            endParagraph(out, xhtml);// combination hard return/soft page (WP5.1)
         } else if (c >= 144 && c <= 149) {
             out.append(' ');      // deletable/invisible soft return/page
         } else if (c == 153) {
-            out.append('\4');     // Dormant Hard return (WP5.1)
+            endParagraph(out, xhtml);// Dormant Hard return (WP5.1)
         } else if (c == 160) {
             out.append('\u00A0'); // Hard space
         } else if (c >= 169 && c <= 171) {

http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
index 659ad17..862c858 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
@@ -18,6 +18,9 @@ package org.apache.tika.parser.wordperfect;
 
 import java.io.IOException;
 
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.SAXException;
+
 /**
  * Extracts WordPerfect Document Area text from a WordPerfect document
  * version 6+.
@@ -25,8 +28,8 @@ import java.io.IOException;
  */
 class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
     
-    protected void extract(int c, WPInputStream in, StringBuilder out)
-            throws IOException {
+    protected void extract(int c, WPInputStream in, StringBuilder out, XHTMLContentHandler xhtml)
+            throws IOException, SAXException {
         if (c > 0 && c <= 32) {
             out.append(WP6Charsets.DEFAULT_EXTENDED_INTL_CHARS[c]);
         } else if (c >= 33 && c <= 126) {
@@ -38,7 +41,7 @@ class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
         } else if (c == 129) {
             out.append('-');      // Hard hyphen
         } else if (c == 135 || c == 137) {
-            out.append('\n');      // Dormant Hard return
+            endParagraph(out, xhtml); // Dormant Hard return
         } else if (c == 138) {
             // skip to closing pair surrounding page number
             skipUntilChar(in, 139);
@@ -46,7 +49,7 @@ class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
             // end of cell
             out.append('\t');
         } else if (c >= 180 && c <= 207) {
-            out.append('\n');
+            endParagraph(out, xhtml);
         } else if (c >= 208 && c <= 239) {
             // Variable-Length Multi-Byte Functions
             int subgroup = in.readWP();
@@ -63,11 +66,11 @@ class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
                     // end of cell
                     out.append('\t');
                 } else if (subgroup >= 4 && subgroup <= 19) {
-                    out.append('\n');
+                    endParagraph(out, xhtml);
                 } else if (subgroup >= 20 && subgroup <= 22) {
                     out.append(' ');
                 } else if (subgroup >= 23 && subgroup <= 28) {
-                    out.append('\n');
+                    endParagraph(out, xhtml);
                 }
             } else if (c == 213) {
                 out.append(' ');

http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
index 002ecee..ee3980e 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WPDocumentAreaExtractor.java
@@ -27,26 +27,51 @@ import org.xml.sax.SAXException;
  */
 abstract class WPDocumentAreaExtractor {
 
+    boolean startedP = false;
     public void extract(WPInputStream in, XHTMLContentHandler xhtml) 
             throws IOException, SAXException {
         int chunk = 4096;
         StringBuilder out = new StringBuilder(chunk);
         int c;
         while ((c = in.read()) != -1) {
-            extract(c, in, out);
+            extract(c, in, out, xhtml);
             if (out.length() >= chunk) {
                 xhtml.characters(out.toString());
                 out.setLength(0);
             }
         }
-        xhtml.characters(out.toString());
-        out.setLength(0);
+        endParagraph(out, xhtml);
     }
  
     protected abstract void extract(
-            int c, WPInputStream in, StringBuilder out) throws IOException;
-    
-    
+            int c, WPInputStream in, StringBuilder out, XHTMLContentHandler xhtml) throws IOException, SAXException;
+
+
+    protected void lazilyStartParagraph(XHTMLContentHandler xhtml) throws SAXException {
+        if (! startedP) {
+            xhtml.startElement("p");
+        }
+        startedP = true;
+    }
+
+    /**
+     * This assumes that the &lt;p&gt; was started before you get here.
+     * And the user is required to close the &lt;p&gt; at the end of the document.
+     *
+     * These are currently handled by {@link #extract(WPInputStream, XHTMLContentHandler)}.
+     *
+     * @param xhtml
+     * @throws SAXException
+     */
+    protected void endParagraph(StringBuilder buffer, XHTMLContentHandler xhtml) throws SAXException {
+        lazilyStartParagraph(xhtml);
+
+        xhtml.characters(buffer.toString());
+        buffer.setLength(0);
+        xhtml.endElement("p");
+        startedP = false;
+    }
+
     // Skips until the given character is encountered.
     protected int skipUntilChar(WPInputStream in, int targetChar)
             throws IOException {

http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
index 4c6d445..a18fee1 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WordPerfectParser.java
@@ -90,11 +90,9 @@ public class WordPerfectParser extends AbstractParser {
         }
         
         xhtml.startDocument();
-        xhtml.startElement("p");
 
         getDocumentAreaExtractor(prefixArea).extract(in, xhtml);
         
-        xhtml.endElement("p");
         xhtml.endDocument();
     }
     

http://git-wip-us.apache.org/repos/asf/tika/blob/940e6f48/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
index 89abca0..0da8959 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/wordperfect/WordPerfectTest.java
@@ -34,7 +34,7 @@ public class WordPerfectTest extends TikaTest {
         assertEquals(WordPerfectParser.WP_6_x.toString(),
                 r.metadata.get(Metadata.CONTENT_TYPE));
         assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
-        assertContains("test test", r.xml);
+        assertContains("<p>AND FURTHER</p>", r.xml);
     }
 
     @Test
@@ -44,7 +44,7 @@ public class WordPerfectTest extends TikaTest {
         assertEquals(WordPerfectParser.WP_5_0.toString(),
                 r.metadata.get(Metadata.CONTENT_TYPE));
         assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
-        assertContains("Surrounded by her family", r.xml);
+        assertContains("<p>Surrounded by her family", r.xml);
     }
 
     @Test
@@ -54,6 +54,7 @@ public class WordPerfectTest extends TikaTest {
         assertEquals(WordPerfectParser.WP_5_1.toString(),
                 r.metadata.get(Metadata.CONTENT_TYPE));
         assertEquals(1, r.metadata.getValues(Metadata.CONTENT_TYPE).length);
-        assertContains("Summer is the peak travel time", r.xml);
+        assertContains("<p>STUDY RESULTS: Existing condition", r.xml);
+        assertContains("Seattle nonstop flights.</p>", r.xml);
     }
 }


[2/3] tika git commit: TIKA-2228 - WordPerfect parser update to handle 5.x from Pascal Essiembre. This closes #142.

Posted by ta...@apache.org.
TIKA-2228 - WordPerfect parser update to handle 5.x from Pascal Essiembre. This closes #142.


Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/6dc442da
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/6dc442da
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/6dc442da

Branch: refs/heads/master
Commit: 6dc442da50dbb22bbf2b73076d5080f32af03067
Parents: ef1d907
Author: tballison <ta...@mitre.org>
Authored: Wed Jan 4 11:54:26 2017 -0500
Committer: tballison <ta...@mitre.org>
Committed: Wed Jan 4 11:54:26 2017 -0500

----------------------------------------------------------------------
 .../parser/wordperfect/QPWTextExtractor.java    |  12 +-
 .../parser/wordperfect/QuattroProParser.java    |   3 +-
 .../tika/parser/wordperfect/WP5Charsets.java    | 205 +++++++++
 .../wordperfect/WP5DocumentAreaExtractor.java   |  78 ++++
 .../tika/parser/wordperfect/WP6Charsets.java    | 447 +++++++++++++++++++
 .../tika/parser/wordperfect/WP6Constants.java   | 432 ------------------
 .../wordperfect/WP6DocumentAreaExtractor.java   | 104 +++++
 .../tika/parser/wordperfect/WP6FileHeader.java  | 191 --------
 .../parser/wordperfect/WP6TextExtractor.java    | 243 ----------
 .../wordperfect/WPDocumentAreaExtractor.java    |  63 +++
 .../tika/parser/wordperfect/WPInputStream.java  |  14 +-
 .../tika/parser/wordperfect/WPPrefixArea.java   | 209 +++++++++
 .../wordperfect/WPPrefixAreaExtractor.java      |  67 +++
 .../parser/wordperfect/WordPerfectParser.java   | 104 ++++-
 .../tika/parser/wordperfect/QuattroProTest.java |  12 +-
 .../parser/wordperfect/WPInputStreamTest.java   |  31 +-
 .../parser/wordperfect/WordPerfectTest.java     |  29 +-
 17 files changed, 1316 insertions(+), 928 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QPWTextExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QPWTextExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QPWTextExtractor.java
index 1f5f68f..0bf9731 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QPWTextExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QPWTextExtractor.java
@@ -1,8 +1,10 @@
-/* Copyright 2015-2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QuattroProParser.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QuattroProParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QuattroProParser.java
index 8b29b4d..64895a4 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QuattroProParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/QuattroProParser.java
@@ -1,5 +1,4 @@
-/* Copyright 2016 Norconex Inc.
- *
+/*
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5Charsets.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5Charsets.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5Charsets.java
new file mode 100644
index 0000000..6db8afa
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5Charsets.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+/**
+ * WordPerfect 5.x constant values used for mapping WordPerfect charsets to
+ * unicode equivalents when possible.
+ * @author Pascal Essiembre
+ */
+final class WP5Charsets {
+
+    private static final Logger LOG = LogManager.getLogger(WP5Charsets.class);
+
+    /**
+     * Extended character sets used when fixed-length multi-byte functions
+     * with a byte value of 192 (0xC0) are found in a WordPerfect document.
+     * Those character set codes may be specific to WordPerfect 
+     * file specifications and may or may not be considered standard 
+     * outside WordPerfect. Applies to version 5.x.
+     */
+    public static final char[][] EXTENDED_CHARSETS = new char[][] {
+        // WP Charset 0: ASCII (same as WP6)
+        WP6Charsets.EXTENDED_CHARSETS[0],
+        // WP Charset 1: Multinational 1 (same as WP6)
+        WP6Charsets.EXTENDED_CHARSETS[1],
+        // WP Charset 2: Multinational 2 (28 chars)
+        {
+        '\u0323','\u0324','\u02da','\u0325','\u02bc','\u032d','\u2017','\u005f',
+        '\u0138','\u032e','\u033e','\u2018','\u0020','\u02bd','\u02db','\u0327',
+        '\u0321','\u0322','\u030d','\u2019','\u0329','\u0020','\u0621','\u02be',
+        '\u0306','\u0310','\u2032','\u2034'            
+        },
+        // WP Charset 3: Box Drawing (same as WP6)
+        WP6Charsets.EXTENDED_CHARSETS[3],
+        // WP Charset 4: Typographic Symbols (same as WP6)
+        WP6Charsets.EXTENDED_CHARSETS[4],
+        // WP Charset 5: Iconic Symbol (35 chars)
+        {
+        '\u2665','\u2666','\u2663','\u2660','\u2642','\u2640','\u263c','\u263a',
+        '\u263b','\u266a','\u266c','\u25ac','\u2302','\u203c','\u221a','\u21a8',
+        '\u2310','\u2319','\u25d8','\u25d9','\u21b5','\u261e','\u261c','\u2713',
+        '\u2610','\u2612','\u2639','\u266f','\u266d','\u266e','\u260e','\u231a',
+        '\u231b','\u2104','\u23b5'            
+        },
+        // WP Charset 6: Math/Scientific (same as WP6)
+        WP6Charsets.EXTENDED_CHARSETS[6],
+        // WP Charset 7 Math/Scientific Extended (same as WP6)
+        WP6Charsets.EXTENDED_CHARSETS[7],
+        // WP Charset 8: Greek (210 chars)
+        {
+        '\u0391','\u03b1','\u0392','\u03b2','\u0392','\u03d0','\u0393','\u03b3',
+        '\u0394','\u03b4','\u0395','\u03b5','\u0396','\u03b6','\u0397','\u03b7',
+        '\u0398','\u03b8','\u0399','\u03b9','\u039a','\u03ba','\u039b','\u03bb',
+        '\u039c','\u03bc','\u039d','\u03bd','\u039e','\u03be','\u039f','\u03bf',
+        '\u03a0','\u03c0','\u03a1','\u03c1','\u03a3','\u03c3','\u03f9','\u03db',
+        '\u03a4','\u03c4','\u03a5','\u03c5','\u03a6','\u03d5','\u03a7','\u03c7',
+        '\u03a8','\u03c8','\u03a9','\u03c9','\u03ac','\u03ad','\u03ae','\u03af',
+        '\u03ca','\u03cc','\u03cd','\u03cb','\u03ce','\u03b5','\u03d1','\u03f0',
+        '\u03d6','\u1fe5','\u03d2','\u03c6','\u03c9','\u037e','\u0387','\u0384',
+        '\u00a8','\u0385','\u1fed','\u1fef','\u1fc0','\u1fbd','\u1fbf','\u1fbe',
+        '\u1fce','\u1fde','\u1fcd','\u1fdd','\u1fcf','\u1fdf','\u0384','\u1fef',
+        '\u1fc0','\u1fbd','\u1fbf','\u1fce','\u1fde','\u1fcd','\u1fdd','\u1fcf',
+        '\u1fdf','\u1f70','\u1fb6','\u1fb3','\u1fb4','\u1fb7','\u1f00','\u1f04',
+        '\u1f02','\u1f06','\u1f80','\u1f84','\u1f86','\u1f01','\u1f05','\u1f03',
+        '\u1f07','\u1f81','\u1f85','\u1f87','\u1f72','\u1f10','\u1f14','\u1f13',
+        '\u1f11','\u1f15','\u1f13','\u1f74','\u1fc6','\u1fc3','\u1fc4','\u1fc2',
+        '\u1fc7','\u1f20','\u1f24','\u1f22','\u1f26','\u1f90','\u1f94','\u1f96',
+        '\u1f21','\u1f25','\u1f23','\u1f27','\u1f91','\u1f95','\u1f97','\u1f76',
+        '\u1fd6','\u0390','\u1fd2','\u1f30','\u1f34','\u1f32','\u1f36','\u1f31',
+        '\u1f35','\u1f33','\u1f37','\u1f78','\u1f40','\u1f44','\u1f42','\u1f41',
+        '\u1f45','\u1f43','\u1f7a','\u1fe6','\u03b0','\u1fe3','\u1f50','\u1f54',
+        '\u1f52','\u1f56','\u1f51','\u1f55','\u1f53','\u1f57','\u1f7c','\u1ff6',
+        '\u1ff3','\u1ff4','\u1ff2','\u1ff7','\u1f60','\u1f64','\u1f62','\u1f66',
+        '\u1fa0','\u1fa4','\u1fa6','\u1f61','\u1f65','\u1f63','\u1f67','\u1fa1',
+        '\u1fa5','\u1fa7','\u0374','\u0375','\u03db','\u03dd','\u03d9','\u03e1',
+        '\u0386','\u0388','\u0389','\u038a','\u038c','\u038e','\u038f','\u03aa',
+        '\u03ab','\u1fe5'
+        },
+        // WP Charset 9: Hebrew (119 chars)
+        {
+        '\u05d0','\u05d1','\u05d2','\u05d3','\u05d4','\u05d5','\u05d6','\u05d7',
+        '\u05d8','\u05d9','\u05da','\u05db','\u05dc','\u05dd','\u05de','\u05df',
+        '\u05e0','\u05e1','\u05e2','\u05e3','\u05e4','\u05e5','\u05e6','\u05e7',
+        '\u05e8','\u05e9','\u05ea','\u05be','\u05c0','\u05c3','\u05f3','\u05f4',
+        '\u05b0','\u05b1','\u05b2','\u05b3','\u05b4','\u05b5','\u05b6','\u05b7',
+        '\u05b8','\u05b9','\u05ba','\u05bb','\u05bc','\u05bd','\u05bf','\u05b7',
+        '\ufbe1','\u05f0','\u05f1','\u05f2','\u0591','\u0596','\u05ad','\u05a4',
+        '\u059a','\u059b','\u05a3','\u05a5','\u05a6','\u05a7','\u09aa','\u0592',
+        '\u0593','\u0594','\u0595','\u0597','\u0598','\u0599','\u05a8','\u059c',
+        '\u059d','\u059e','\u05a1','\u05a9','\u05a0','\u059f','\u05ab','\u05ac',
+        '\u05af','\u05c4','\u0544','\u05d0','\ufb31','\ufb32','\ufb33','\ufb34',
+        '\ufb35','\ufb4b','\ufb36','\u05d7','\ufb38','\ufb39','\ufb3b','\ufb3a',
+        '\u05da','\u05da','\u05da','\u05da','\u05da','\u05da','\ufb3c','\ufb3e',
+        '\ufb40','\u05df','\ufb41','\ufb44','\ufb46','\ufb47','\ufb2b','\ufb2d',
+        '\ufb2a','\ufb2c','\ufb4a','\ufb4c','\ufb4e','\ufb1f','\ufb1d'
+        },
+        // WP Charset 10: Cyrillic (150 chars)
+        {
+        '\u0410','\u0430','\u0411','\u0431','\u0412','\u0432','\u0413','\u0433',
+        '\u0414','\u0434','\u0415','\u0435','\u0401','\u0451','\u0416','\u0436',
+        '\u0417','\u0437','\u0418','\u0438','\u0419','\u0439','\u041a','\u043a',
+        '\u041b','\u043b','\u041c','\u043c','\u041d','\u043d','\u041e','\u043e',
+        '\u041f','\u043f','\u0420','\u0440','\u0421','\u0441','\u0422','\u0442',
+        '\u0423','\u0443','\u0424','\u0444','\u0425','\u0445','\u0426','\u0446',
+        '\u0427','\u0447','\u0428','\u0448','\u0429','\u0449','\u042a','\u044a',
+        '\u042b','\u044b','\u042c','\u044c','\u042d','\u044d','\u042e','\u044e',
+        '\u042f','\u044f','\u0490','\u0491','\u0402','\u0452','\u0403','\u0453',
+        '\u0404','\u0454','\u0405','\u0455','\u0406','\u0456','\u0407','\u0457',
+        '\u0408','\u0458','\u0409','\u0459','\u040a','\u045a','\u040b','\u045b',
+        '\u040c','\u045c','\u040e','\u045e','\u040f','\u045f','\u0462','\u0463',
+        '\u0472','\u0473','\u0474','\u0475','\u046a','\u046b','\ua640','\ua641',
+        '\u0429','\u0449','\u04c0','\u04cf','\u0466','\u0467','\u0000','\u0000',
+        '\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000',
+        '\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000',
+        '\u0000','\u0000','\u0400','\u0450','\u0000','\u0000','\u040d','\u045d',
+        '\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000',
+        '\u0000','\u0000','\u0000','\u0000','\u0301','\u0300'
+        },
+        // WP Charset 11: Japanese (185 chars)
+        {
+        '\u3041','\u3043','\u3045','\u3047','\u3049','\u3053','\u3083','\u3085',
+        '\u3087','\u3094','\u3095','\u3096','\u3042','\u3044','\u3046','\u3048',
+        '\u304a','\u304b','\u304d','\u3047','\u3051','\u3053','\u304c','\u304e',
+        '\u3050','\u3052','\u3054','\u3055','\u3057','\u3059','\u305b','\u305d',
+        '\u3056','\u3058','\u305a','\u305c','\u305e','\u305f','\u3051','\u3064',
+        '\u3066','\u3068','\u3060','\u3062','\u3065','\u3067','\u3069','\u306a',
+        '\u306b','\u306c','\u306d','\u306e','\u306f','\u3072','\u3075','\u3078',
+        '\u307b','\u3070','\u3073','\u3076','\u3079','\u307c','\u3071','\u3074',
+        '\u3077','\u307a','\u307d','\u307e','\u307f','\u3080','\u3081','\u3082',
+        '\u3084','\u3086','\u3088','\u3089','\u308a','\u308b','\u308c','\u308d',
+        '\u308e','\u3092','\u3093','\u3014','\u3015','\uff3b','\uff3d','\u300c',
+        '\u300d','\u300c','\u300d','\u302a','\u3002','\u3001','\u309d','\u309e',
+        '\u3003','\u30fc','\u309b','\u309c','\u30a1','\u30a3','\u30a5','\u30a7',
+        '\u30a9','\u30c3','\u30e3','\u30e5','\u3057','\u30f4','\u30f5','\u30f6',
+        '\u30a2','\u30a4','\u30a6','\u30a8','\u30aa','\u30ab','\u30ad','\u30af',
+        '\u30b1','\u30b3','\u30ac','\u30ae','\u30b0','\u30b2','\u30b4','\u30b5',
+        '\u30c4','\u30b9','\u30bb','\u30bd','\u30b6','\u30b8','\u30ba','\u30bc',
+        '\u30be','\u30bf','\u30c1','\u30c4','\u30c6','\u30c8','\u30c0','\u30c2',
+        '\u30c5','\u30c7','\u30c9','\u30ca','\u30cb','\u30cc','\u30cd','\u30ce',
+        '\u30cf','\u30d2','\u30d5','\u30d8','\u03d0','\u30db','\u30d3','\u30d6',
+        '\u30d9','\u30dc','\u30d1','\u30d4','\u30d7','\u30da','\u30dd','\u30de',
+        '\u30df','\u30e0','\u30e1','\u30e2','\u30e4','\u30e6','\u30e8','\u30e9',
+        '\u30ea','\u30ab','\u30ec','\u30ed','\u30ef','\u30f2','\u30f3','\u30fd',
+        '\u30fe'
+        },
+        // WP Charset 12: User-defined (255 chars)
+        {  
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' '
+        }
+    }; 
+
+    //TODO map multi-characters
+    
+    /**
+     * Constructor.
+     */
+    private WP5Charsets() {
+    }
+
+    public static void append(StringBuilder out, int charset, int charval) {
+        if (charset >= WP5Charsets.EXTENDED_CHARSETS.length) {
+            LOG.debug("Unsupported WordPerfect 5.x charset: " + charset);
+            out.append(' ');
+        } else if (charval >= WP5Charsets.EXTENDED_CHARSETS[charset].length) {
+            LOG.debug("Unsupported WordPerfect 5.x charset (" + charset 
+                    + ") character value: " + charval);
+            out.append(' ');
+        } else {
+            out.append(WP5Charsets.EXTENDED_CHARSETS[charset][charval]);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
new file mode 100644
index 0000000..b95490e
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP5DocumentAreaExtractor.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+import java.io.IOException;
+
+/**
+ * Extracts WordPerfect Document Area text from a WordPerfect document
+ * version 5.x.
+ * @author Pascal Essiembre
+ */
+class WP5DocumentAreaExtractor extends WPDocumentAreaExtractor {
+    
+    protected void extract(int c, WPInputStream in, StringBuilder out)
+            throws IOException {
+
+        // 0-31: control characters
+        if (c == 10) {
+            out.append('\n');     // hard return ("Enter")
+        } else if (c == 11) {
+            out.append(' ');      // soft page break
+        } else if (c == 12) {
+            out.append('\n');     // hard page break
+        } else if (c == 13) {
+            out.append(' ');      // soft return (line wrap)
+            
+        // 32-126: ASCII characters
+        } else if (c >= 32 && c <= 126) {
+            out.append((char) c); // ASCII character
+
+        // 128-191: single-byte functions
+        } else if (c == 140) {
+            out.append('\n');     // combination hard return/soft page (WP5.1)
+        } else if (c >= 144 && c <= 149) {
+            out.append(' ');      // deletable/invisible soft return/page
+        } else if (c == 153) {
+            out.append('\4');     // Dormant Hard return (WP5.1)
+        } else if (c == 160) {
+            out.append('\u00A0'); // Hard space
+        } else if (c >= 169 && c <= 171) {
+            out.append('-');      // Hard hyphen
+            
+        // 192-207: fixed-length multi-byte function
+        } else if (c == 192) {
+            // extended char
+            int charval = in.readWP();
+            int charset = in.readWP();
+            in.readWP(); // closing character
+            WP5Charsets.append(out, charset, charval);
+        } else if (c >= 193 && c <= 207) {
+            skipUntilChar(in, c); // opening/closing chars are same
+
+        // 208-255: variable-length multi-byte function
+        } else if (c >= 208 && c <= 255) {
+            // Variable-Length Multi-Byte Functions
+            in.readWP(); // subgroup (the function code)
+            int functionSize = in.readWPShort();
+            for (int i = 0; i < functionSize; i++) {
+                in.readWP();
+            }
+            //TODO Are there functions containing data? Like footnotes?
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Charsets.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Charsets.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Charsets.java
new file mode 100644
index 0000000..8c62983
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Charsets.java
@@ -0,0 +1,447 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+/**
+ * WordPerfect 6+ constant values used for mapping WordPerfect charsets to
+ * unicode equivalents when possible.
+ * @author Pascal Essiembre
+ */
+final class WP6Charsets {
+
+    private static final Logger LOG = LogManager.getLogger(WP6Charsets.class);
+
+    public static final char[] DEFAULT_EXTENDED_INTL_CHARS = new char[] {
+        '\0',     '\u00E5', '\u00C5', '\u00E6', '\u00C6',
+        '\u00E4', '\u00C4', '\u00E1', '\u00E0', '\u00E2',
+        '\u00E3', '\u00C3', '\u00E7', '\u00C7', '\u00EB', 
+        '\u00E9', '\u00C9', '\u00E8', '\u00EA', '\u00ED', 
+        '\u00F1', '\u00D1', '\u00F8', '\u00D8', '\u00F5', 
+        '\u00D5', '\u00F6', '\u00D6', '\u00FC', '\u00DC', 
+        '\u00FA', '\u00F9', '\u00DF',
+    };
+    
+    /**
+     * Extended character sets used when fixed-length multi-byte functions
+     * with a byte value of 240 (0xF0) are found in a WordPerfect document.
+     * Those character set codes may be specific to WordPerfect 
+     * file specifications and may or may not be considered standard 
+     * outside WordPerfect. Applies to version 6 and likely higher.
+     */
+    public static final char[][] EXTENDED_CHARSETS = new char[][] {
+        // WP Charset 0: ASCII (96 chars)
+        {
+        ' ', '!', '"', '#', '$', '%', '&', '\'', '(', ')',
+        '*', '+', ',', '-', '.', '/', '0', '1', '2', '3',
+        '4', '5', '6', '7', '8', '9', ':', ';', '<', '=',
+        '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
+        'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q',
+        'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[',
+        '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e',
+        'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+        'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y',
+        'z', '{', '|', '}', '~', '\u00A0'
+        },
+        // WP Charset 1: Multinational (242 chars)
+        {
+        '\u0300','\u00b7','\u0303','\u0302','\u0335','\u0338','\u0301','\u0308',
+        '\u0304','\u0313','\u0315','\u02bc','\u0326','\u0315','\u00b0','\u0307',
+        '\u030b','\u0327','\u0328','\u030c','\u0337','\u0305','\u0306','\u00df',
+        '\u0138','\u006a','\u00c1','\u00e1','\u00c2','\u00e2','\u00c4','\u00e4',
+        '\u00c0','\u00e0','\u00c5','\u00e5','\u00c6','\u00e6','\u00c7','\u00e7',
+        '\u00c9','\u00e9','\u00ca','\u00ea','\u00cb','\u00eb','\u00c8','\u00e8',
+        '\u00cd','\u00ed','\u00ce','\u00ee','\u00cf','\u00ef','\u00cc','\u00ec',
+        '\u00d1','\u00f1','\u00d3','\u00f3','\u00d4','\u00f4','\u00d6','\u00f6',
+        '\u00d2','\u00f2','\u00da','\u00fa','\u00db','\u00fb','\u00dc','\u00fc',
+        '\u00d9','\u00f9','\u0178','\u00ff','\u00c3','\u00e3','\u0110','\u0111',
+        '\u00d8','\u00f8','\u00d5','\u00f5','\u00dd','\u00fd','\u00d0','\u00f0',
+        '\u00de','\u00fe','\u0102','\u0103','\u0100','\u0101','\u0104','\u0105',
+        '\u0106','\u0107','\u010c','\u010d','\u0108','\u0109','\u010a','\u010b',
+        '\u010e','\u010f','\u011a','\u011b','\u0116','\u0117','\u0112','\u0113',
+        '\u0118','\u0119','\u01f4','\u01f5','\u011e','\u011f','\u01e6','\u01e7',
+        '\u0122','\u0123','\u011c','\u011d','\u0120','\u0121','\u0124','\u0125',
+        '\u0126','\u0127','\u0130','\u0069','\u012a','\u012b','\u012e','\u012f',
+        '\u0128','\u0129','\u0132','\u0133','\u0134','\u0135','\u0136','\u0137',
+        '\u0139','\u013a','\u013d','\u013e','\u013b','\u013c','\u013f','\u0140',
+        '\u0141','\u0142','\u0143','\u0144','\u0000','\u0149','\u0147','\u0148',
+        '\u0145','\u0146','\u0150','\u0151','\u014c','\u014d','\u0152','\u0153',
+        '\u0154','\u0155','\u0158','\u0159','\u0156','\u0157','\u015a','\u015b',
+        '\u0160','\u0161','\u015e','\u015f','\u015c','\u015d','\u0164','\u0165',
+        '\u0162','\u0163','\u0166','\u0167','\u016c','\u016d','\u0170','\u0171',
+        '\u016a','\u016b','\u0172','\u0173','\u016e','\u016f','\u0168','\u0169',
+        '\u0174','\u0175','\u0176','\u0177','\u0179','\u017a','\u017d','\u017e',
+        '\u017b','\u017c','\u014a','\u014b','\u0000','\u0000','\u0000','\u0000',
+        '\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000',
+        '\u0000','\u0000','\u1ef2','\u1ef3','\u010e','\u010f','\u01a0','\u01a1',
+        '\u01af','\u01b0','\u0114','\u0115','\u012c','\u012d','\u0049','\u0131',
+        '\u014e','\u014f'
+        },
+        // WP Charset 2: Phonetic (145 chars)
+        {
+        '\u02b9','\u02ba','\u02bb','\u0020','\u02bd','\u02bc','\u0020','\u02be',
+        '\u02bf','\u0310','\u02d0','\u02d1','\u0306','\u032e','\u0329','\u02c8',
+        '\u02cc','\u02c9','\u02ca','\u02cb','\u02cd','\u02ce','\u02cf','\u02c6',
+        '\u02c7','\u02dc','\u0325','\u02da','\u032d','\u032c','\u0323','\u0308',
+        '\u0324','\u031c','\u031d','\u031e','\u031f','\u0320','\u0321','\u0322',
+        '\u032a','\u032b','\u02d2','\u02d3','\u0361','\u0356','\u005f','\u2017',
+        '\u033e','\u02db','\u0327','\u0233','\u030d','\u02b0','\u02b6','\u0250',
+        '\u0251','\u0252','\u0253','\u0299','\u0254','\u0255','\u0297','\u0256',
+        '\u0257','\u0258','\u0259','\u025a','\u025b','\u025c','\u025d','\u029a',
+        '\u025e','\u025f','\u0278','\u0261','\u0260','\u0262','\u029b','\u0263',
+        '\u0264','\u0265','\u0266','\u0267','\u029c','\u0268','\u026a','\u0269',
+        '\u029d','\u029e','\u026b','\u026c','\u026d','\u029f','\u026e','\u028e',
+        '\u026f','\u0270','\u0271','\u0272','\u0273','\u0274','\u0276','\u0277',
+        '\u02a0','\u0279','\u027a','\u027b','\u027c','\u027d','\u027e','\u027f',
+        '\u0280','\u0281','\u0282','\u0283','\u0284','\u0285','\u0286','\u0287',
+        '\u0288','\u0275','\u0289','\u028a','\u028c','\u028b','\u028d','\u03c7',
+        '\u028f','\u0290','\u0291','\u0292','\u0293','\u0294','\u0295','\u0296',
+        '\u02a1','\u02a2','\u0298','\u02a3','\u02a4','\u02a5','\u02a6','\u02a7',
+        '\u02a8'
+        },
+        // WP Charset 3: Box Drawing (88 chars)
+        {
+        '\u2591','\u2592','\u2593','\u2588','\u258c','\u2580','\u2590','\u2584',
+        '\u2500','\u2502','\u250c','\u2510','\u2518','\u2514','\u251c','\u252c',
+        '\u2524','\u2534','\u253c','\u2550','\u2551','\u2554','\u2557','\u255d',
+        '\u255a','\u2560','\u2566','\u2563','\u2569','\u256c','\u2552','\u2555',
+        '\u255b','\u2558','\u2553','\u2556','\u255c','\u2559','\u255e','\u2565',
+        '\u2561','\u2568','\u255f','\u2564','\u2562','\u2567','\u256b','\u256a',
+        '\u2574','\u2575','\u2576','\u2577','\u2578','\u2579','\u257a','\u257b',
+        '\u257c','\u257e','\u257d','\u257f','\u251f','\u2522','\u251e','\u2521',
+        '\u252e','\u2532','\u252d','\u2531','\u2527','\u2526','\u252a','\u2529',
+        '\u2536','\u253a','\u2535','\u2539','\u2541','\u2546','\u253e','\u2540',
+        '\u2544','\u254a','\u253d','\u2545','\u2548','\u2543','\u2549','\u2547'
+        },
+        // WP Charset 4: Typographic Symbols (102 chars)
+        {
+        '\u25cf','\u25cb','\u25a0','\u2022','\u002a','\u00b6','\u00a7','\u00a1',
+        '\u00bf','\u00ab','\u00bb','\u00a3','\u00a5','\u20a7','\u0192','\u00aa',
+        '\u00ba','\u00bd','\u00bc','\u00a2','\u00b2','\u207f','\u00ae','\u00a9',
+        '\u00a4','\u00be','\u00b3','\u201b','\u2019','\u2018','\u201f','\u201d',
+        '\u201c','\u2013','\u2014','\u2039','\u203a','\u25cb','\u25a1','\u2020',
+        '\u2021','\u2122','\u2120','\u211e','\u25cf','\u25e6','\u25a0','\u25aa',
+        '\u25a1','\u25ab','\u2012','\ufb00','\ufb03','\ufb04','\ufb01','\ufb02',
+        '\u2026','\u0024','\u20a3','\u20a2','\u20a0','\u20a4','\u201a','\u201e',
+        '\u2153','\u2154','\u215b','\u215c','\u215d','\u215e','\u24c2','\u24c5',
+        '\u20ac','\u2105','\u2106','\u2030','\u2116','\u2014','\u00b9','\u2409',
+        '\u240c','\u240d','\u240a','\u2424','\u240b','\u267c','\u20a9','\u20a6',
+        '\u20a8','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u0020','\u0020','\u0020','\u1d11','\u1d12'
+                                            // last two uncertain
+        },
+        // WP Charset 5: Iconic Symbol (255 chars)
+        {
+        '\u2661','\u2662','\u2667','\u2664','\u2642','\u2640','\u263c','\u263a',
+        '\u263b','\u266a','\u266c','\u25ac','\u2302','\u203c','\u221a','\u21a8',
+        '\u2310','\u2319','\u25d8','\u25d9','\u21b5','\u2104','\u261c','\u23b5',
+        '\u2610','\u2612','\u2639','\u266f','\u266d','\u266e','\u260e','\u231a',
+        '\u231b','\u2701','\u2702','\u2703','\u2704','\u260e','\u2706','\u2707',
+        '\u2708','\u2709','\u261b','\u261e','\u270c','\u270d','\u270e','\u270f',
+        '\u2710','\u2711','\u2712','\u2713','\u2714','\u2715','\u2716','\u2717',
+        '\u2718','\u2719','\u271a','\u271b','\u271c','\u271d','\u271e','\u271f',
+        '\u2720','\u2721','\u2722','\u2723','\u2724','\u2725','\u2726','\u2727',
+        '\u2605','\u2606','\u272a','\u272b','\u272c','\u272d','\u272e','\u272f',
+        '\u2730','\u2731','\u2732','\u2733','\u2734','\u2735','\u2736','\u2737',
+        '\u2738','\u2739','\u273a','\u273b','\u273c','\u273d','\u273e','\u273f',
+        '\u2740','\u2741','\u2742','\u2743','\u2744','\u2745','\u2746','\u2747',
+        '\u2748','\u2749','\u274a','\u274b','\u25cf','\u274d','\u25a0','\u274f',
+        '\u2750','\u2751','\u2752','\u25b2','\u25bc','\u25c6','\u2756','\u25d7',
+        '\u2758','\u2759','\u275a','\u275b','\u275c','\u275d','\u275e','\u2036',
+        '\u2033','\u0020','\u0020','\u0020','\u0020','\u2329','\u232a','\u005b',
+        '\u005d','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u2190','\u0020','\u0020','\u0020','\u0020','\u0020','\u21e8','\u21e6',
+        '\u2794','\u0020','\u0020','\u0020','\u0020','\u0020','\u25d6','\u0020',
+        '\u0020','\u2761','\u2762','\u2763','\u2764','\u2765','\u2766','\u2767',
+        '\u2663','\u2666','\u2665','\u2660','\u2780','\u2781','\u2782','\u2783',
+        '\u2784','\u2785','\u2786','\u2787','\u2788','\u2789','\u2776','\u2777',
+        '\u2778','\u2779','\u277a','\u277b','\u277c','\u277d','\u277e','\u277f',
+        '\u2780','\u2781','\u2782','\u2783','\u2784','\u2785','\u2786','\u2787',
+        '\u2788','\u2789','\u278a','\u278b','\u278c','\u278d','\u278e','\u278f',
+        '\u2790','\u2791','\u2792','\u2793','\u2794','\u2192','\u2194','\u2195',
+        '\u2798','\u2799','\u279a','\u279b','\u279c','\u279d','\u279e','\u279f',
+        '\u27a0','\u27a1','\u27a2','\u27a3','\u27a4','\u27a5','\u27a6','\u27a7',
+        '\u27a8','\u27a9','\u27aa','\u27ab','\u27ac','\u27ad','\u27ae','\u27af',
+        '\u0020','\u27b1','\u27b2','\u27b3','\u27b4','\u27b5','\u27b6','\u27b7',
+        '\u27b8','\u27b9','\u27ba','\u27bb','\u27bc','\u27bd','\u27be'
+        },
+        // WP Charset 6: Math/Scientific (238 chars)
+        {
+        '\u2212','\u00b1','\u2264','\u2265','\u221d','\u002f','\u2215','\u2216',
+        '\u00f7','\u2223','\u27e8','\u27e9','\u223c','\u2248','\u2261','\u2208',
+        '\u2229','\u2225','\u2211','\u221e','\u00ac','\u2192','\u2190','\u2191',
+        '\u2193','\u2194','\u2195','\u25b8','\u25c2','\u25b4','\u25be','\u22c5',
+        '\u00b7','\u2218','\u2219','\u212b','\u00b0','\u00b5','\u203e','\u00d7',
+        '\u222b','\u220f','\u2213','\u2207','\u2202','\u2032','\u2033','\u2192',
+        '\u212f','\u2113','\u210f','\u2111','\u211c','\u2118','\u21c4','\u21c6',
+        '\u21d2','\u21d0','\u21d1','\u21d3','\u21d4','\u21d5','\u2197','\u2198',
+        '\u2196','\u2199','\u222a','\u2282','\u2283','\u2286','\u2287','\u220d',
+        '\u2205','\u2308','\u2309','\u230a','\u230b','\u226a','\u226b','\u2220',
+        '\u2297','\u2295','\u2296','\u2a38','\u2299','\u2227','\u2228','\u22bb',
+        '\u22a4','\u22a5','\u2312','\u22a2','\u22a3','\u25a1','\u25a0','\u25ca',
+        '\u25c6','\u27e6','\u27e7','\u2260','\u2262','\u2235','\u2234','\u2237',
+        '\u222e','\u2112','\u212d','\u2128','\u2118','\u20dd','\u29cb','\u25c7',
+        '\u22c6','\u2034','\u2210','\u2243','\u2245','\u227a','\u227c','\u227b',
+        '\u227d','\u2203','\u2200','\u22d8','\u22d9','\u228e','\u228a','\u228b',
+        '\u2293','\u2294','\u228f','\u2291','\u22e4','\u2290','\u2292','\u22e5',
+        '\u25b3','\u25bd','\u25c3','\u25b9','\u22c8','\u2323','\u2322','\u25ef',
+        '\u219d','\u21a9','\u21aa','\u21a3','\u21bc','\u21bd','\u21c0','\u21c1',
+        '\u21cc','\u21cb','\u21bf','\u21be','\u21c3','\u21c2','\u21c9','\u21c7',
+        '\u22d3','\u22d2','\u22d0','\u22d1','\u229a','\u229b','\u229d','\u2127',
+        '\u2221','\u2222','\u25c3','\u25b9','\u25b5','\u25bf','\u2214','\u2250',
+        '\u2252','\u2253','\u224e','\u224d','\u22a8','\u2258','\u226c','\u0285',
+        '\u2605','\u226e','\u2270','\u226f','\u2271','\u2241','\u2244','\u2247',
+        '\u2249','\u2280','\u22e0','\u2281','\u22e1','\u2284','\u2285','\u2288',
+        '\u2289','\u0020','\u0020','\u22e2','\u22e3','\u2226','\u2224','\u226d',
+        '\u2204','\u2209','\u2247','\u2130','\u2131','\u2102','\u0020','\u2115',
+        '\u211d','\u225f','\u22be','\u220b','\u22ef','\u2026','\u22ee','\u22f1',
+        '\u0020','\u20e1','\u002b','\u002d','\u003d','\u002a','\u2032','\u2033',
+        '\u2034','\u210b','\u2118','\u2272','\u2273','\u0020'
+        },
+        // WP Charset 7 Math/Scientific Extended (229 chars)
+        {
+        '\u2320','\u2321','\u23a5','\u23bd','\u221a','\u0020','\u2211','\u220f',
+        '\u2210','\u222b','\u222e','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u23a7','\u23a8','\u23a9','\u23aa','\u0020','\u0020','\u0020',
+        '\u0020','\u23ab','\u23ac','\u23ad','\u23aa','\u0020','\u0020','\u0020',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u222a','\u222b','\u0020',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u239b','\u239d','\u239c','\u0020','\u0020','\u0020','\u0020','\u239e',
+        '\u23a8','\u239f','\u0020','\u0020','\u0020','\u0020','\u23a1','\u23a3',
+        '\u23a2','\u0020','\u20aa','\u0020','\u0020','\u23a4','\u23a6','\u23a5',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u22c3','\u22c2','\u228e','\u2a04','\u2294','\u2a06','\u2227','\u22c0',
+        '\u2228','\u22c1','\u2297','\u2a02','\u2295','\u2a01','\u2299','\u2a00',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u0020','\u0020','\u0020','\u229d','\u0020','\u2238','\u0020',
+        '\u27e6','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u27e7',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u21bc','\u21bd','\u0020','\u296c','\u296d','\u296a','\u296b','\u0020',
+        '\u21c9','\u21c7','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u21be','\u21bf','\u21c3','\u21c2','\u0020','\u2293','\u2a05','\u23a1',
+        '\u0020','\u0020','\u0020','\u0020','\u0020'
+        },
+        // WP Charset 8: Greek (219 chars)
+        {
+        '\u0391','\u03b1','\u0392','\u03b2','\u0392','\u03d0','\u0393','\u03b3',
+        '\u0394','\u03b4','\u0395','\u03b5','\u0396','\u03b6','\u0397','\u03b7',
+        '\u0398','\u03b8','\u0399','\u03b9','\u039a','\u03ba','\u039b','\u03bb',
+        '\u039c','\u03bc','\u039d','\u03bd','\u039e','\u03be','\u039f','\u03bf',
+        '\u03a0','\u03c0','\u03a1','\u03c1','\u03a3','\u03c3','\u03a3','\u03c2',
+        '\u03a4','\u03c4','\u03a5','\u03c5','\u03a6','\u03c6','\u03a7','\u03c7',
+        '\u03a8','\u03c8','\u03a9','\u03c9','\u0386','\u03ac','\u0388','\u03ad',
+        '\u0389','\u03ae','\u038a','\u03af','\u03aa','\u03ca','\u038c','\u03cc',
+        '\u038e','\u03cd','\u03ab','\u03cb','\u038f','\u03ce','\u03b5','\u03d1',
+        '\u03f0','\u03d6','\u03f1','\u03c2','\u03d2','\u03d5','\u03c9','\u037e',
+        '\u0387','\u0374','\u0375','\u0384','\u00a8','\u0385','\u1fed','\u1fef',
+        '\u1fc0','\u1fbd','\u1ffe','\u037a','\u1fce','\u1fde','\u1fcd','\u1fdd',
+        '\u1fcf','\u1fdf','\u0384','\u1fef','\u1fc0','\u1fbd','\u1ffe','\u1fce',
+        '\u1fde','\u1fcd','\u1fdd','\u1fcf','\u1fdf','\u1f70','\u1fb6','\u1fb3',
+        '\u1fb4','\u1fb2','\u1fb7','\u1f00','\u1f04','\u1f02','\u1f06','\u1f80',
+        '\u1f84','\u1f82','\u1f86','\u1f01','\u1f05','\u1f03','\u1f07','\u1f81',
+        '\u1f85','\u1f83','\u1f87','\u1f72','\u1f10','\u1f14','\u1f12','\u1f11',
+        '\u1f15','\u1f13','\u1f74','\u1fc6','\u1fc3','\u1fc4','\u1fc2','\u1fc7',
+        '\u1f20','\u1f24','\u1f22','\u1f26','\u1f90','\u1f94','\u1f92','\u1f96',
+        '\u1f21','\u1f25','\u1f23','\u1f27','\u1f91','\u1f95','\u1f93','\u1f97',
+        '\u1f76','\u1fd6','\u0390','\u1fd2','\u1f30','\u1f34','\u1f32','\u1f36',
+        '\u1f31','\u1f35','\u1f33','\u1f37','\u1f78','\u1f40','\u1f44','\u1f42',
+        '\u1f41','\u1f45','\u1f43','\u1fe5','\u1fe4','\u1f7a','\u1fe6','\u03b0',
+        '\u1fe2','\u1f50','\u1f54','\u1f52','\u1f56','\u1f51','\u1f55','\u1f53',
+        '\u1f57','\u1f7c','\u1ff6','\u1ff3','\u1ff4','\u1ff2','\u1ff7','\u1f60',
+        '\u1f64','\u1f62','\u1f66','\u1fa0','\u1fa4','\u1fa2','\u1fa6','\u1f61',
+        '\u1f65','\u1f63','\u1f67','\u1fa1','\u1fa5','\u1fa3','\u1fa7','\u03da',
+        '\u03dc','\u03de','\u03e0'
+        },
+        // WP Charset 9: Hebrew (123 chars)
+        {
+        '\u05d0','\u05d1','\u05d2','\u05d3','\u05d4','\u05d5','\u05d6','\u05d7',
+        '\u05d8','\u05d9','\u05da','\u05db','\u05dc','\u05dd','\u05de','\u05df',
+        '\u05e0','\u05e1','\u05e2','\u05e3','\u05e4','\u05e5','\u05e6','\u05e7',
+        '\u05e8','\u05e9','\u05ea','\u05be','\u05c0','\u05c3','\u05f3','\u05f4',
+        '\u05b0','\u05b1','\u05b2','\u05b3','\u05b4','\u05b5','\u05b6','\u05b7',
+        '\u05b8','\u05b9','\u05b9','\u05bb','\u05bc','\u05bd','\u05bf','\u05b7',
+        '\ufb1e','\u05f0','\u05f1','\u05f2','\ufb1f','\u0591','\u0596','\u0020',
+        '\u05a4','\u059a','\u059b','\u05a3','\u05a5','\u05a6','\u05a7','\u05a2',
+        '\u0592','\u0593','\u0594','\u0595','\u0597','\u0598','\u0599','\u05a8',
+        '\u05f3','\u05f3','\u05f4','\u0020','\u05a9','\u05a0','\u059f','\u05ab',
+        '\u05ac','\u05af','\u05c4','\u05aa','\ufb30','\ufb31','\u05d1','\ufb32',
+        '\ufb33','\ufb34','\ufb35','\ufb4b','\ufb36','\u05d7','\ufb38','\ufb39',
+        '\ufb1d','\ufb3b','\ufb3a','\u05da','\u05da','\u05da','\u05da','\u05da',
+        '\u05da','\ufb3c','\ufb3e','\ufb40','\u05d5','\ufb41','\ufb44','\u05e4',
+        '\ufb46','\ufb47','\ufb2b','\ufb2d','\ufb2a','\ufb2c','\ufb4a','\u05dc',
+        '\ufb3c','\ufb49','\u20aa'
+        },
+        // WP Charset 10: Cyrillic (250 chars)
+        {
+        '\u0410','\u0430','\u0411','\u0431','\u0412','\u0432','\u0413','\u0433',
+        '\u0414','\u0434','\u0415','\u0435','\u0401','\u0451','\u0416','\u0436',
+        '\u0417','\u0437','\u0418','\u0438','\u0419','\u0439','\u041a','\u043a',
+        '\u041b','\u043b','\u041c','\u043c','\u041d','\u043d','\u041e','\u043e',
+        '\u041f','\u043f','\u0420','\u0440','\u0421','\u0441','\u0422','\u0442',
+        '\u0423','\u0443','\u0424','\u0444','\u0425','\u0445','\u0426','\u0446',
+        '\u0427','\u0447','\u0428','\u0448','\u0429','\u0449','\u042a','\u044a',
+        '\u042b','\u044b','\u042c','\u044c','\u042d','\u044d','\u042e','\u044e',
+        '\u042f','\u044f','\u04d8','\u04d9','\u0403','\u0453','\u0490','\u0491',
+        '\u0492','\u0493','\u0402','\u0452','\u0404','\u0454','\u0404','\u0454',
+        '\u0496','\u0497','\u0405','\u0455','\u0020','\u0020','\u0418','\u0438',
+        '\u0406','\u0456','\u0407','\u0457','\u0020','\u0020','\u0408','\u0458',
+        '\u040c','\u045c','\u049a','\u049b','\u04c3','\u04c4','\u049c','\u049d',
+        '\u0409','\u0459','\u04a2','\u04a3','\u040a','\u045a','\u047a','\u047b',
+        '\u0460','\u0461','\u040b','\u045b','\u040e','\u045e','\u04ee','\u04ef',
+        '\u04ae','\u04af','\u04b0','\u04b1','\u0194','\u0263','\u04b2','\u04b3',
+        '\u0425','\u0445','\u04ba','\u04bb','\u047e','\u047f','\u040f','\u045f',
+        '\u04b6','\u04b7','\u04b8','\u04b9','\u0428','\u0448','\u0462','\u0463',
+        '\u0466','\u0467','\u046a','\u046b','\u046e','\u046f','\u0470','\u0471',
+        '\u0472','\u0473','\u0474','\u0475','\u0410','\u0430','\u0415','\u0435',
+        '\u0404','\u0454','\u0418','\u0438','\u0406','\u0456','\u0407','\u0457',
+        '\u041e','\u043e','\u0423','\u0443','\u042b','\u044b','\u042d','\u044d',
+        '\u042e','\u044e','\u042f','\u044f','\u0410','\u0430','\u0400','\u0450',
+        '\u0401','\u0451','\u040d','\u045d','\u041e','\u043e','\u0423','\u0443',
+        '\u042b','\u044b','\u042d','\u044d','\u042e','\u044e','\u042f','\u044f',
+        '\u0301','\u0300','\u0308','\u0306','\u0326','\u0328','\u0304','\u0020',
+        '\u201e','\u201c','\u10d0','\u10d1','\u10d2','\u10d3','\u10d4','\u10d5',
+        '\u10d6','\u10f1','\u10d7','\u10d8','\u10d9','\u10da','\u10db','\u10dc',
+        '\u10f2','\u10dd','\u10de','\u10df','\u10e0','\u10e1','\u10e2','\u10e3',
+        '\u10f3','\u10e4','\u10e5','\u10e6','\u10e7','\u10e8','\u10e9','\u10ea',
+        '\u10eb','\u10ec','\u10ed','\u10ee','\u10f4','\u10ef','\u10f0','\u10f5',
+        '\u10f6','\u10e3'
+        },
+        // WP Charset 11: Japanese (63 chars)
+        {
+        '\uff61','\uff62','\uff63','\uff64','\uff65','\uff66','\uff67','\uff68',
+        '\uff69','\uff6a','\uff6b','\uff6c','\uff6d','\uff6e','\uff6f','\uff70',
+        '\uff71','\uff72','\uff73','\uff74','\uff75','\uff76','\uff77','\uff78',
+        '\uff79','\uff7a','\uff7b','\uff7c','\uff7d','\uff7e','\uff7f','\uff80',
+        '\uff81','\uff82','\uff83','\uff84','\uff85','\uff86','\uff87','\uff88',
+        '\uff89','\uff8a','\uff8b','\uff8c','\uff8d','\uff8e','\uff8f','\uff90',
+        '\uff91','\uff92','\uff93','\uff94','\uff95','\uff96','\uff97','\uff98',
+        '\uff99','\uff9a','\uff9b','\uff9c','\uff9d','\uff9e','\uff9f'
+        },
+        // WP Charset 12: Current Font Symbols (256 chars)
+        {  //TODO implement Current Font Symbols
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+        ' '
+        },
+        // WP Charset 13: Arabic (196 chars)
+        {
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u0020','\u064e','\ufe77','\u064f','\ufe79','\u0650','\ufe7b',
+        '\u064b','\u064c','\u064c','\u064d','\u0652','\ufe7f','\u0651','\ufe7d',
+        '\ufc60','\ufcf2','\ufc61','\ufcf3','\ufc62','\ufcf4','\u064b','\ufc5e',
+        '\ufc5e','\ufc5f','\u0653','\u0670','\u0654','\u0020','\u060c','\u061b',
+        '\u061f','\u066d','\u066a','\u00bb','\u00ab','\u0029','\u0028','\u0661',
+        '\u0662','\u0663','\u0664','\u0665','\u0666','\u0667','\u0668','\u0669',
+        '\u0660','\u0662','\u0627','\ufe8e','\u0628','\ufe91','\ufe92','\ufe90',
+        '\u062a','\ufe97','\ufe98','\ufe96','\u062b','\ufe9b','\ufe9c','\ufe9a',
+        '\u062c','\ufe9f','\ufea0','\ufe9e','\u062d','\ufea3','\ufea4','\ufea2',
+        '\u062e','\ufea7','\ufea8','\ufea6','\u062f','\ufeaa','\u0630','\ufeac',
+        '\u0631','\ufeae','\u0632','\ufeaf','\u0633','\ufeb3','\ufeb4','\ufeb2',
+        '\u0634','\ufeb7','\ufeb8','\ufeb6','\u0635','\ufebb','\ufebc','\ufeba',
+        '\u0636','\ufebf','\ufec0','\ufebe','\u0637','\ufec3','\ufec4','\ufec2',
+        '\u0638','\ufec7','\ufec8','\ufec6','\u0639','\ufecb','\ufecc','\ufeca',
+        '\u063a','\ufecf','\ufed0','\ufece','\u0641','\ufed3','\ufed4','\ufed2',
+        '\u0642','\ufed7','\ufed8','\ufed6','\u0643','\ufedb','\ufedc','\ufeda',
+        '\u0644','\ufedf','\ufee0','\ufede','\u0645','\ufee3','\ufee4','\ufee2',
+        '\u0646','\ufee7','\ufee8','\ufee6','\u0647','\ufeeb','\ufeec','\ufeea',
+        '\u0629','\ufe94','\u0648','\ufeee','\u064a','\ufef3','\ufef4','\ufef2',
+        '\u0649','\ufef3','\ufef4','\ufef0','\u0621','\u0623','\ufe84','\u0625',
+        '\ufe88','\u0624','\ufe86','\u0626','\ufe8b','\ufe8c','\ufe8a','\ufd3d',
+        '\ufd3c','\u0622','\ufe82','\u0671','\ufb51','\ufefb','\ufefc','\ufef7',
+        '\ufef8','\ufef9','\ufefa','\u0020','\ufefc','\ufef5','\ufef6','\u0020',
+        '\u0020','\ufdf2','\u0640','\u0640'
+        },
+        // WP Charset 14: Arabic Script (220 chars)
+        {
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0615',
+        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0615','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
+        '\u0020','\u06d4','\u0020','\u0020','\u00b0','\u0020','\u065a','\u0020',
+        '\u065a','\u0659','\u0020','\u0020','\u0654','\u064c','\ufc5e','\u065a',
+        '\u065a','\u06f4','\u06f4','\u06f5','\u06f6','\u06f6','\u06f7','\u06f8',
+        '\u067b','\ufb54','\ufb55','\ufb53','\u0680','\ufb5c','\ufb5d','\ufb5b',
+        '\u067e','\ufb58','\ufb59','\ufb57','\u0679','\ufb68','\ufb69','\ufb67',
+        '\u067c','\u067c','\u067c','\u067c','\u067f','\ufb64','\ufb65','\ufb63',
+        '\u067d','\u067d','\u067d','\u067d','\u067a','\ufb60','\ufb61','\ufb5f',
+        '\u0684','\ufb74','\ufb75','\ufb73','\u0683','\ufb78','\ufb79','\ufb77',
+        '\u0686','\ufb7c','\ufb7d','\ufb7b','\u0687','\ufb80','\ufb81','\ufb7f',
+        '\u0685','\u0685','\u0685','\u0685','\u0681','\u0681','\u0681','\u0681',
+        '\u0688','\ufb89','\u0689','\u0689','\u068c','\ufb85','\u068e','\ufb87',
+        '\u068a','\u068a','\u068d','\ufb83','\u0693','\u0693','\u0691','\ufb8d',
+        '\u0699','\u0699','\u0695','\u0695','\u0692','\u0692','\u0698','\ufb8b',
+        '\u0696','\u0696','\u075b','\u075b','\u069a','\u069a','\u069a','\u069a',
+        '\u06a0','\u06a0','\u06a0','\u06a0','\u06a4','\ufb6c','\ufb6d','\ufb6b',
+        '\u06a6','\ufb70','\ufb71','\ufb6f','\u06a9','\ufb90','\ufb91','\ufb8f',
+        '\u0643','\ufedb','\ufedc','\ufeda','\u06aa','\u06aa','\u06aa','\u06aa',
+        '\u06af','\ufb94','\ufb95','\ufb93','\u06af','\ufb94','\ufb95','\ufb93',
+        '\u06ab','\u06ab','\u06ab','\u06ab','\u06b1','\ufb9c','\ufb9d','\ufb9b',
+        '\u06b3','\ufb98','\ufb99','\ufb97','\u06b5','\u06b5','\u06b5','\u06b5',
+        '\u0020','\u0020','\u06ba','\u0020','\u0020','\ufb9f','\u06bc','\u06bc',
+        '\u06bc','\u06bc','\u06bb','\ufba2','\ufba3','\ufba1','\u06c6','\ufbda',
+        '\u0020','\u0020','\u06ca','\u06ca','\u06c1','\ufba8','\ufba9','\ufba7',
+        '\u06ce','\u06ce','\u06ce','\u06ce','\u06d2','\ufbaf','\u06d1','\u06d1',
+        '\u06d1','\u06d1','\u06c0','\ufba5'
+        },
+    }; 
+
+    //TODO map multi-characters
+    
+    /**
+     * Constructor.
+     */
+    private WP6Charsets() {
+    }
+
+    public static void append(StringBuilder out, int charset, int charval) {
+        if (charset >= WP6Charsets.EXTENDED_CHARSETS.length) {
+            LOG.debug("Unsupported WordPerfect 6+ charset: " + charset);
+            out.append(' ');
+        } else if (charval >= WP6Charsets.EXTENDED_CHARSETS[charset].length) {
+            LOG.debug("Unsupported WordPerfect 6+ charset (" + charset 
+                    + ") character value: " + charval);
+            out.append(' ');
+        } else {
+            out.append(WP6Charsets.EXTENDED_CHARSETS[charset][charval]);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Constants.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Constants.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Constants.java
deleted file mode 100644
index 194bad7..0000000
--- a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6Constants.java
+++ /dev/null
@@ -1,432 +0,0 @@
-/* Copyright 2015-2016 Norconex Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.wordperfect;
-
-/**
- * WordPerfect constant values used for parsing and extracting text.
- * @author Pascal Essiembre
- */
-final class WP6Constants {
-
-
-    public static final String WP6_FILE_ID = "�WPC";
-    public static final int WP6_PRODUCT_TYPE = 1;
-    public static final int WP6_FILE_TYPE_WP_DOCUMENT = 10;
-    public static final int WP6_FILE_TYPE_WPD = 36;
-    
-    public static final char[] DEFAULT_EXTENDED_INTL_CHARS = new char[] {
-        '\0',     '\u00E5', '\u00C5', '\u00E6', '\u00C6',
-        '\u00E4', '\u00C4', '\u00E1', '\u00E0', '\u00E2',
-        '\u00E3', '\u00C3', '\u00E7', '\u00C7', '\u00EB', 
-        '\u00E9', '\u00C9', '\u00E8', '\u00EA', '\u00ED', 
-        '\u00F1', '\u00D1', '\u00F8', '\u00D8', '\u00F5', 
-        '\u00D5', '\u00F6', '\u00D6', '\u00FC', '\u00DC', 
-        '\u00FA', '\u00F9', '\u00DF',
-    };
-    
-    /**
-     * Extended character sets used when fixed-length multi-byte functions
-     * with a byte value of 240 (0xF0) are found in a WordPerfect document.
-     * Those character set codes may be specific to WordPerfect 
-     * file specifications and may or may not be considered standard 
-     * outside WordPerfect.
-     */
-    public static final char[][] EXTENDED_CHARSETS = new char[][] {
-        // WP Charset 0: ASCII (95 chars)
-        {
-        ' ', '"', '#', '$', '%', '&', '\'', '(', ')', '*',
-        '+', ',', '-', '.', '/', '0', '1', '2', '3', '4',
-        '5', '6', '7', '8', '9', ':', ';', '<', '=', '>',
-        '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
-        'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
-        'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', '\\',
-        ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f',
-        'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
-        'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
-        '{', '|', '}', '~', '\u00A0'
-        },
-        // WP Charset 1: Multinational (242 chars)
-        {
-        '\u0300','\u00b7','\u0303','\u0302','\u0335','\u0338','\u0301','\u0308',
-        '\u0304','\u0313','\u0315','\u02bc','\u0326','\u0315','\u00b0','\u0307',
-        '\u030b','\u0327','\u0328','\u030c','\u0337','\u0305','\u0306','\u00df',
-        '\u0138','\u006a','\u00c1','\u00e1','\u00c2','\u00e2','\u00c4','\u00e4',
-        '\u00c0','\u00e0','\u00c5','\u00e5','\u00c6','\u00e6','\u00c7','\u00e7',
-        '\u00c9','\u00e9','\u00ca','\u00ea','\u00cb','\u00eb','\u00c8','\u00e8',
-        '\u00cd','\u00ed','\u00ce','\u00ee','\u00cf','\u00ef','\u00cc','\u00ec',
-        '\u00d1','\u00f1','\u00d3','\u00f3','\u00d4','\u00f4','\u00d6','\u00f6',
-        '\u00d2','\u00f2','\u00da','\u00fa','\u00db','\u00fb','\u00dc','\u00fc',
-        '\u00d9','\u00f9','\u0178','\u00ff','\u00c3','\u00e3','\u0110','\u0111',
-        '\u00d8','\u00f8','\u00d5','\u00f5','\u00dd','\u00fd','\u00d0','\u00f0',
-        '\u00de','\u00fe','\u0102','\u0103','\u0100','\u0101','\u0104','\u0105',
-        '\u0106','\u0107','\u010c','\u010d','\u0108','\u0109','\u010a','\u010b',
-        '\u010e','\u010f','\u011a','\u011b','\u0116','\u0117','\u0112','\u0113',
-        '\u0118','\u0119','\u01f4','\u01f5','\u011e','\u011f','\u01e6','\u01e7',
-        '\u0122','\u0123','\u011c','\u011d','\u0120','\u0121','\u0124','\u0125',
-        '\u0126','\u0127','\u0130','\u0069','\u012a','\u012b','\u012e','\u012f',
-        '\u0128','\u0129','\u0132','\u0133','\u0134','\u0135','\u0136','\u0137',
-        '\u0139','\u013a','\u013d','\u013e','\u013b','\u013c','\u013f','\u0140',
-        '\u0141','\u0142','\u0143','\u0144','\u0000','\u0149','\u0147','\u0148',
-        '\u0145','\u0146','\u0150','\u0151','\u014c','\u014d','\u0152','\u0153',
-        '\u0154','\u0155','\u0158','\u0159','\u0156','\u0157','\u015a','\u015b',
-        '\u0160','\u0161','\u015e','\u015f','\u015c','\u015d','\u0164','\u0165',
-        '\u0162','\u0163','\u0166','\u0167','\u016c','\u016d','\u0170','\u0171',
-        '\u016a','\u016b','\u0172','\u0173','\u016e','\u016f','\u0168','\u0169',
-        '\u0174','\u0175','\u0176','\u0177','\u0179','\u017a','\u017d','\u017e',
-        '\u017b','\u017c','\u014a','\u014b','\u0000','\u0000','\u0000','\u0000',
-        '\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000','\u0000',
-        '\u0000','\u0000','\u1ef2','\u1ef3','\u010e','\u010f','\u01a0','\u01a1',
-        '\u01af','\u01b0','\u0114','\u0115','\u012c','\u012d','\u0049','\u0131',
-        '\u014e','\u014f'
-        },
-        // WP Charset 2: Phonetic (145 chars)
-        {
-        '\u02b9','\u02ba','\u02bb','\u0020','\u02bd','\u02bc','\u0020','\u02be',
-        '\u02bf','\u0310','\u02d0','\u02d1','\u0306','\u032e','\u0329','\u02c8',
-        '\u02cc','\u02c9','\u02ca','\u02cb','\u02cd','\u02ce','\u02cf','\u02c6',
-        '\u02c7','\u02dc','\u0325','\u02da','\u032d','\u032c','\u0323','\u0308',
-        '\u0324','\u031c','\u031d','\u031e','\u031f','\u0320','\u0321','\u0322',
-        '\u032a','\u032b','\u02d2','\u02d3','\u0361','\u0356','\u005f','\u2017',
-        '\u033e','\u02db','\u0327','\u0233','\u030d','\u02b0','\u02b6','\u0250',
-        '\u0251','\u0252','\u0253','\u0299','\u0254','\u0255','\u0297','\u0256',
-        '\u0257','\u0258','\u0259','\u025a','\u025b','\u025c','\u025d','\u029a',
-        '\u025e','\u025f','\u0278','\u0261','\u0260','\u0262','\u029b','\u0263',
-        '\u0264','\u0265','\u0266','\u0267','\u029c','\u0268','\u026a','\u0269',
-        '\u029d','\u029e','\u026b','\u026c','\u026d','\u029f','\u026e','\u028e',
-        '\u026f','\u0270','\u0271','\u0272','\u0273','\u0274','\u0276','\u0277',
-        '\u02a0','\u0279','\u027a','\u027b','\u027c','\u027d','\u027e','\u027f',
-        '\u0280','\u0281','\u0282','\u0283','\u0284','\u0285','\u0286','\u0287',
-        '\u0288','\u0275','\u0289','\u028a','\u028c','\u028b','\u028d','\u03c7',
-        '\u028f','\u0290','\u0291','\u0292','\u0293','\u0294','\u0295','\u0296',
-        '\u02a1','\u02a2','\u0298','\u02a3','\u02a4','\u02a5','\u02a6','\u02a7',
-        '\u02a8'
-        },
-        // WP Charset 3: Box Drawing (88 chars)
-        {
-        '\u2591','\u2592','\u2593','\u2588','\u258c','\u2580','\u2590','\u2584',
-        '\u2500','\u2502','\u250c','\u2510','\u2518','\u2514','\u251c','\u252c',
-        '\u2524','\u2534','\u253c','\u2550','\u2551','\u2554','\u2557','\u255d',
-        '\u255a','\u2560','\u2566','\u2563','\u2569','\u256c','\u2552','\u2555',
-        '\u255b','\u2558','\u2553','\u2556','\u255c','\u2559','\u255e','\u2565',
-        '\u2561','\u2568','\u255f','\u2564','\u2562','\u2567','\u256b','\u256a',
-        '\u2574','\u2575','\u2576','\u2577','\u2578','\u2579','\u257a','\u257b',
-        '\u257c','\u257e','\u257d','\u257f','\u251f','\u2522','\u251e','\u2521',
-        '\u252e','\u2532','\u252d','\u2531','\u2527','\u2526','\u252a','\u2529',
-        '\u2536','\u253a','\u2535','\u2539','\u2541','\u2546','\u253e','\u2540',
-        '\u2544','\u254a','\u253d','\u2545','\u2548','\u2543','\u2549','\u2547'
-        },
-        // WP Charset 4: Typographic Symbols (102 chars)
-        {
-        '\u25cf','\u25cb','\u25a0','\u2022','\u002a','\u00b6','\u00a7','\u00a1',
-        '\u00bf','\u00ab','\u00bb','\u00a3','\u00a5','\u20a7','\u0192','\u00aa',
-        '\u00ba','\u00bd','\u00bc','\u00a2','\u00b2','\u207f','\u00ae','\u00a9',
-        '\u00a4','\u00be','\u00b3','\u201b','\u2019','\u2018','\u201f','\u201d',
-        '\u201c','\u2013','\u2014','\u2039','\u203a','\u25cb','\u25a1','\u2020',
-        '\u2021','\u2122','\u2120','\u211e','\u25cf','\u25e6','\u25a0','\u25aa',
-        '\u25a1','\u25ab','\u2012','\ufb00','\ufb03','\ufb04','\ufb01','\ufb02',
-        '\u2026','\u0024','\u20a3','\u20a2','\u20a0','\u20a4','\u201a','\u201e',
-        '\u2153','\u2154','\u215b','\u215c','\u215d','\u215e','\u24c2','\u24c5',
-        '\u20ac','\u2105','\u2106','\u2030','\u2116','\u2014','\u00b9','\u2409',
-        '\u240c','\u240d','\u240a','\u2424','\u240b','\u267c','\u20a9','\u20a6',
-        '\u20a8','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u0020','\u0020','\u0020','\u1d11','\u1d12'
-                                            // last two uncertain
-        },
-        // WP Charset 5: Iconic Symbol (255 chars)
-        {
-        '\u2661','\u2662','\u2667','\u2664','\u2642','\u2640','\u263c','\u263a',
-        '\u263b','\u266a','\u266c','\u25ac','\u2302','\u203c','\u221a','\u21a8',
-        '\u2310','\u2319','\u25d8','\u25d9','\u21b5','\u2104','\u261c','\u23b5',
-        '\u2610','\u2612','\u2639','\u266f','\u266d','\u266e','\u260e','\u231a',
-        '\u231b','\u2701','\u2702','\u2703','\u2704','\u260e','\u2706','\u2707',
-        '\u2708','\u2709','\u261b','\u261e','\u270c','\u270d','\u270e','\u270f',
-        '\u2710','\u2711','\u2712','\u2713','\u2714','\u2715','\u2716','\u2717',
-        '\u2718','\u2719','\u271a','\u271b','\u271c','\u271d','\u271e','\u271f',
-        '\u2720','\u2721','\u2722','\u2723','\u2724','\u2725','\u2726','\u2727',
-        '\u2605','\u2606','\u272a','\u272b','\u272c','\u272d','\u272e','\u272f',
-        '\u2730','\u2731','\u2732','\u2733','\u2734','\u2735','\u2736','\u2737',
-        '\u2738','\u2739','\u273a','\u273b','\u273c','\u273d','\u273e','\u273f',
-        '\u2740','\u2741','\u2742','\u2743','\u2744','\u2745','\u2746','\u2747',
-        '\u2748','\u2749','\u274a','\u274b','\u25cf','\u274d','\u25a0','\u274f',
-        '\u2750','\u2751','\u2752','\u25b2','\u25bc','\u25c6','\u2756','\u25d7',
-        '\u2758','\u2759','\u275a','\u275b','\u275c','\u275d','\u275e','\u2036',
-        '\u2033','\u0020','\u0020','\u0020','\u0020','\u2329','\u232a','\u005b',
-        '\u005d','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u2190','\u0020','\u0020','\u0020','\u0020','\u0020','\u21e8','\u21e6',
-        '\u2794','\u0020','\u0020','\u0020','\u0020','\u0020','\u25d6','\u0020',
-        '\u0020','\u2761','\u2762','\u2763','\u2764','\u2765','\u2766','\u2767',
-        '\u2663','\u2666','\u2665','\u2660','\u2780','\u2781','\u2782','\u2783',
-        '\u2784','\u2785','\u2786','\u2787','\u2788','\u2789','\u2776','\u2777',
-        '\u2778','\u2779','\u277a','\u277b','\u277c','\u277d','\u277e','\u277f',
-        '\u2780','\u2781','\u2782','\u2783','\u2784','\u2785','\u2786','\u2787',
-        '\u2788','\u2789','\u278a','\u278b','\u278c','\u278d','\u278e','\u278f',
-        '\u2790','\u2791','\u2792','\u2793','\u2794','\u2192','\u2194','\u2195',
-        '\u2798','\u2799','\u279a','\u279b','\u279c','\u279d','\u279e','\u279f',
-        '\u27a0','\u27a1','\u27a2','\u27a3','\u27a4','\u27a5','\u27a6','\u27a7',
-        '\u27a8','\u27a9','\u27aa','\u27ab','\u27ac','\u27ad','\u27ae','\u27af',
-        '\u0020','\u27b1','\u27b2','\u27b3','\u27b4','\u27b5','\u27b6','\u27b7',
-        '\u27b8','\u27b9','\u27ba','\u27bb','\u27bc','\u27bd','\u27be'
-        },
-        // WP Charset 6: Math/Scientific (238 chars)
-        {
-        '\u2212','\u00b1','\u2264','\u2265','\u221d','\u002f','\u2215','\u2216',
-        '\u00f7','\u2223','\u27e8','\u27e9','\u223c','\u2248','\u2261','\u2208',
-        '\u2229','\u2225','\u2211','\u221e','\u00ac','\u2192','\u2190','\u2191',
-        '\u2193','\u2194','\u2195','\u25b8','\u25c2','\u25b4','\u25be','\u22c5',
-        '\u00b7','\u2218','\u2219','\u212b','\u00b0','\u00b5','\u203e','\u00d7',
-        '\u222b','\u220f','\u2213','\u2207','\u2202','\u2032','\u2033','\u2192',
-        '\u212f','\u2113','\u210f','\u2111','\u211c','\u2118','\u21c4','\u21c6',
-        '\u21d2','\u21d0','\u21d1','\u21d3','\u21d4','\u21d5','\u2197','\u2198',
-        '\u2196','\u2199','\u222a','\u2282','\u2283','\u2286','\u2287','\u220d',
-        '\u2205','\u2308','\u2309','\u230a','\u230b','\u226a','\u226b','\u2220',
-        '\u2297','\u2295','\u2296','\u2a38','\u2299','\u2227','\u2228','\u22bb',
-        '\u22a4','\u22a5','\u2312','\u22a2','\u22a3','\u25a1','\u25a0','\u25ca',
-        '\u25c6','\u27e6','\u27e7','\u2260','\u2262','\u2235','\u2234','\u2237',
-        '\u222e','\u2112','\u212d','\u2128','\u2118','\u20dd','\u29cb','\u25c7',
-        '\u22c6','\u2034','\u2210','\u2243','\u2245','\u227a','\u227c','\u227b',
-        '\u227d','\u2203','\u2200','\u22d8','\u22d9','\u228e','\u228a','\u228b',
-        '\u2293','\u2294','\u228f','\u2291','\u22e4','\u2290','\u2292','\u22e5',
-        '\u25b3','\u25bd','\u25c3','\u25b9','\u22c8','\u2323','\u2322','\u25ef',
-        '\u219d','\u21a9','\u21aa','\u21a3','\u21bc','\u21bd','\u21c0','\u21c1',
-        '\u21cc','\u21cb','\u21bf','\u21be','\u21c3','\u21c2','\u21c9','\u21c7',
-        '\u22d3','\u22d2','\u22d0','\u22d1','\u229a','\u229b','\u229d','\u2127',
-        '\u2221','\u2222','\u25c3','\u25b9','\u25b5','\u25bf','\u2214','\u2250',
-        '\u2252','\u2253','\u224e','\u224d','\u22a8','\u2258','\u226c','\u0285',
-        '\u2605','\u226e','\u2270','\u226f','\u2271','\u2241','\u2244','\u2247',
-        '\u2249','\u2280','\u22e0','\u2281','\u22e1','\u2284','\u2285','\u2288',
-        '\u2289','\u0020','\u0020','\u22e2','\u22e3','\u2226','\u2224','\u226d',
-        '\u2204','\u2209','\u2247','\u2130','\u2131','\u2102','\u0020','\u2115',
-        '\u211d','\u225f','\u22be','\u220b','\u22ef','\u2026','\u22ee','\u22f1',
-        '\u0020','\u20e1','\u002b','\u002d','\u003d','\u002a','\u2032','\u2033',
-        '\u2034','\u210b','\u2118','\u2272','\u2273','\u0020'
-        },
-        // WP Charset 7 Math/Scientific Extended (229 chars)
-        {
-        '\u2320','\u2321','\u23a5','\u23bd','\u221a','\u0020','\u2211','\u220f',
-        '\u2210','\u222b','\u222e','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u23a7','\u23a8','\u23a9','\u23aa','\u0020','\u0020','\u0020',
-        '\u0020','\u23ab','\u23ac','\u23ad','\u23aa','\u0020','\u0020','\u0020',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u222a','\u222b','\u0020',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u239b','\u239d','\u239c','\u0020','\u0020','\u0020','\u0020','\u239e',
-        '\u23a8','\u239f','\u0020','\u0020','\u0020','\u0020','\u23a1','\u23a3',
-        '\u23a2','\u0020','\u20aa','\u0020','\u0020','\u23a4','\u23a6','\u23a5',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u22c3','\u22c2','\u228e','\u2a04','\u2294','\u2a06','\u2227','\u22c0',
-        '\u2228','\u22c1','\u2297','\u2a02','\u2295','\u2a01','\u2299','\u2a00',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u0020','\u0020','\u0020','\u229d','\u0020','\u2238','\u0020',
-        '\u27e6','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u27e7',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u21bc','\u21bd','\u0020','\u296c','\u296d','\u296a','\u296b','\u0020',
-        '\u21c9','\u21c7','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u21be','\u21bf','\u21c3','\u21c2','\u0020','\u2293','\u2a05','\u23a1',
-        '\u0020','\u0020','\u0020','\u0020','\u0020'
-        },
-        // WP Charset 8: Greek (219 chars)
-        {
-        '\u0391','\u03b1','\u0392','\u03b2','\u0392','\u03d0','\u0393','\u03b3',
-        '\u0394','\u03b4','\u0395','\u03b5','\u0396','\u03b6','\u0397','\u03b7',
-        '\u0398','\u03b8','\u0399','\u03b9','\u039a','\u03ba','\u039b','\u03bb',
-        '\u039c','\u03bc','\u039d','\u03bd','\u039e','\u03be','\u039f','\u03bf',
-        '\u03a0','\u03c0','\u03a1','\u03c1','\u03a3','\u03c3','\u03a3','\u03c2',
-        '\u03a4','\u03c4','\u03a5','\u03c5','\u03a6','\u03c6','\u03a7','\u03c7',
-        '\u03a8','\u03c8','\u03a9','\u03c9','\u0386','\u03ac','\u0388','\u03ad',
-        '\u0389','\u03ae','\u038a','\u03af','\u03aa','\u03ca','\u038c','\u03cc',
-        '\u038e','\u03cd','\u03ab','\u03cb','\u038f','\u03ce','\u03b5','\u03d1',
-        '\u03f0','\u03d6','\u03f1','\u03c2','\u03d2','\u03d5','\u03c9','\u037e',
-        '\u0387','\u0374','\u0375','\u0384','\u00a8','\u0385','\u1fed','\u1fef',
-        '\u1fc0','\u1fbd','\u1ffe','\u037a','\u1fce','\u1fde','\u1fcd','\u1fdd',
-        '\u1fcf','\u1fdf','\u0384','\u1fef','\u1fc0','\u1fbd','\u1ffe','\u1fce',
-        '\u1fde','\u1fcd','\u1fdd','\u1fcf','\u1fdf','\u1f70','\u1fb6','\u1fb3',
-        '\u1fb4','\u1fb2','\u1fb7','\u1f00','\u1f04','\u1f02','\u1f06','\u1f80',
-        '\u1f84','\u1f82','\u1f86','\u1f01','\u1f05','\u1f03','\u1f07','\u1f81',
-        '\u1f85','\u1f83','\u1f87','\u1f72','\u1f10','\u1f14','\u1f12','\u1f11',
-        '\u1f15','\u1f13','\u1f74','\u1fc6','\u1fc3','\u1fc4','\u1fc2','\u1fc7',
-        '\u1f20','\u1f24','\u1f22','\u1f26','\u1f90','\u1f94','\u1f92','\u1f96',
-        '\u1f21','\u1f25','\u1f23','\u1f27','\u1f91','\u1f95','\u1f93','\u1f97',
-        '\u1f76','\u1fd6','\u0390','\u1fd2','\u1f30','\u1f34','\u1f32','\u1f36',
-        '\u1f31','\u1f35','\u1f33','\u1f37','\u1f78','\u1f40','\u1f44','\u1f42',
-        '\u1f41','\u1f45','\u1f43','\u1fe5','\u1fe4','\u1f7a','\u1fe6','\u03b0',
-        '\u1fe2','\u1f50','\u1f54','\u1f52','\u1f56','\u1f51','\u1f55','\u1f53',
-        '\u1f57','\u1f7c','\u1ff6','\u1ff3','\u1ff4','\u1ff2','\u1ff7','\u1f60',
-        '\u1f64','\u1f62','\u1f66','\u1fa0','\u1fa4','\u1fa2','\u1fa6','\u1f61',
-        '\u1f65','\u1f63','\u1f67','\u1fa1','\u1fa5','\u1fa3','\u1fa7','\u03da',
-        '\u03dc','\u03de','\u03e0'
-        },
-        // WP Charset 9: Hebrew (123 chars)
-        {
-        '\u05d0','\u05d1','\u05d2','\u05d3','\u05d4','\u05d5','\u05d6','\u05d7',
-        '\u05d8','\u05d9','\u05da','\u05db','\u05dc','\u05dd','\u05de','\u05df',
-        '\u05e0','\u05e1','\u05e2','\u05e3','\u05e4','\u05e5','\u05e6','\u05e7',
-        '\u05e8','\u05e9','\u05ea','\u05be','\u05c0','\u05c3','\u05f3','\u05f4',
-        '\u05b0','\u05b1','\u05b2','\u05b3','\u05b4','\u05b5','\u05b6','\u05b7',
-        '\u05b8','\u05b9','\u05b9','\u05bb','\u05bc','\u05bd','\u05bf','\u05b7',
-        '\ufb1e','\u05f0','\u05f1','\u05f2','\ufb1f','\u0591','\u0596','\u0020',
-        '\u05a4','\u059a','\u059b','\u05a3','\u05a5','\u05a6','\u05a7','\u05a2',
-        '\u0592','\u0593','\u0594','\u0595','\u0597','\u0598','\u0599','\u05a8',
-        '\u05f3','\u05f3','\u05f4','\u0020','\u05a9','\u05a0','\u059f','\u05ab',
-        '\u05ac','\u05af','\u05c4','\u05aa','\ufb30','\ufb31','\u05d1','\ufb32',
-        '\ufb33','\ufb34','\ufb35','\ufb4b','\ufb36','\u05d7','\ufb38','\ufb39',
-        '\ufb1d','\ufb3b','\ufb3a','\u05da','\u05da','\u05da','\u05da','\u05da',
-        '\u05da','\ufb3c','\ufb3e','\ufb40','\u05d5','\ufb41','\ufb44','\u05e4',
-        '\ufb46','\ufb47','\ufb2b','\ufb2d','\ufb2a','\ufb2c','\ufb4a','\u05dc',
-        '\ufb3c','\ufb49','\u20aa'
-        },
-        // WP Charset 10: Cyrillic (250 chars)
-        {
-        '\u0410','\u0430','\u0411','\u0431','\u0412','\u0432','\u0413','\u0433',
-        '\u0414','\u0434','\u0415','\u0435','\u0401','\u0451','\u0416','\u0436',
-        '\u0417','\u0437','\u0418','\u0438','\u0419','\u0439','\u041a','\u043a',
-        '\u041b','\u043b','\u041c','\u043c','\u041d','\u043d','\u041e','\u043e',
-        '\u041f','\u043f','\u0420','\u0440','\u0421','\u0441','\u0422','\u0442',
-        '\u0423','\u0443','\u0424','\u0444','\u0425','\u0445','\u0426','\u0446',
-        '\u0427','\u0447','\u0428','\u0448','\u0429','\u0449','\u042a','\u044a',
-        '\u042b','\u044b','\u042c','\u044c','\u042d','\u044d','\u042e','\u044e',
-        '\u042f','\u044f','\u04d8','\u04d9','\u0403','\u0453','\u0490','\u0491',
-        '\u0492','\u0493','\u0402','\u0452','\u0404','\u0454','\u0404','\u0454',
-        '\u0496','\u0497','\u0405','\u0455','\u0020','\u0020','\u0418','\u0438',
-        '\u0406','\u0456','\u0407','\u0457','\u0020','\u0020','\u0408','\u0458',
-        '\u040c','\u045c','\u049a','\u049b','\u04c3','\u04c4','\u049c','\u049d',
-        '\u0409','\u0459','\u04a2','\u04a3','\u040a','\u045a','\u047a','\u047b',
-        '\u0460','\u0461','\u040b','\u045b','\u040e','\u045e','\u04ee','\u04ef',
-        '\u04ae','\u04af','\u04b0','\u04b1','\u0194','\u0263','\u04b2','\u04b3',
-        '\u0425','\u0445','\u04ba','\u04bb','\u047e','\u047f','\u040f','\u045f',
-        '\u04b6','\u04b7','\u04b8','\u04b9','\u0428','\u0448','\u0462','\u0463',
-        '\u0466','\u0467','\u046a','\u046b','\u046e','\u046f','\u0470','\u0471',
-        '\u0472','\u0473','\u0474','\u0475','\u0410','\u0430','\u0415','\u0435',
-        '\u0404','\u0454','\u0418','\u0438','\u0406','\u0456','\u0407','\u0457',
-        '\u041e','\u043e','\u0423','\u0443','\u042b','\u044b','\u042d','\u044d',
-        '\u042e','\u044e','\u042f','\u044f','\u0410','\u0430','\u0400','\u0450',
-        '\u0401','\u0451','\u040d','\u045d','\u041e','\u043e','\u0423','\u0443',
-        '\u042b','\u044b','\u042d','\u044d','\u042e','\u044e','\u042f','\u044f',
-        '\u0301','\u0300','\u0308','\u0306','\u0326','\u0328','\u0304','\u0020',
-        '\u201e','\u201c','\u10d0','\u10d1','\u10d2','\u10d3','\u10d4','\u10d5',
-        '\u10d6','\u10f1','\u10d7','\u10d8','\u10d9','\u10da','\u10db','\u10dc',
-        '\u10f2','\u10dd','\u10de','\u10df','\u10e0','\u10e1','\u10e2','\u10e3',
-        '\u10f3','\u10e4','\u10e5','\u10e6','\u10e7','\u10e8','\u10e9','\u10ea',
-        '\u10eb','\u10ec','\u10ed','\u10ee','\u10f4','\u10ef','\u10f0','\u10f5',
-        '\u10f6','\u10e3'
-        },
-        // WP Charset 11: Japanese (63 chars)
-        {
-        '\uff61','\uff62','\uff63','\uff64','\uff65','\uff66','\uff67','\uff68',
-        '\uff69','\uff6a','\uff6b','\uff6c','\uff6d','\uff6e','\uff6f','\uff70',
-        '\uff71','\uff72','\uff73','\uff74','\uff75','\uff76','\uff77','\uff78',
-        '\uff79','\uff7a','\uff7b','\uff7c','\uff7d','\uff7e','\uff7f','\uff80',
-        '\uff81','\uff82','\uff83','\uff84','\uff85','\uff86','\uff87','\uff88',
-        '\uff89','\uff8a','\uff8b','\uff8c','\uff8d','\uff8e','\uff8f','\uff90',
-        '\uff91','\uff92','\uff93','\uff94','\uff95','\uff96','\uff97','\uff98',
-        '\uff99','\uff9a','\uff9b','\uff9c','\uff9d','\uff9e','\uff9f'
-        },
-        // WP Charset 12: Current Font Symbols (256 chars)
-        {  //TODO implement Current Font Symbols
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
-        ' '
-        },
-        // WP Charset 13: Arabic (196 chars)
-        {
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u0020','\u064e','\ufe77','\u064f','\ufe79','\u0650','\ufe7b',
-        '\u064b','\u064c','\u064c','\u064d','\u0652','\ufe7f','\u0651','\ufe7d',
-        '\ufc60','\ufcf2','\ufc61','\ufcf3','\ufc62','\ufcf4','\u064b','\ufc5e',
-        '\ufc5e','\ufc5f','\u0653','\u0670','\u0654','\u0020','\u060c','\u061b',
-        '\u061f','\u066d','\u066a','\u00bb','\u00ab','\u0029','\u0028','\u0661',
-        '\u0662','\u0663','\u0664','\u0665','\u0666','\u0667','\u0668','\u0669',
-        '\u0660','\u0662','\u0627','\ufe8e','\u0628','\ufe91','\ufe92','\ufe90',
-        '\u062a','\ufe97','\ufe98','\ufe96','\u062b','\ufe9b','\ufe9c','\ufe9a',
-        '\u062c','\ufe9f','\ufea0','\ufe9e','\u062d','\ufea3','\ufea4','\ufea2',
-        '\u062e','\ufea7','\ufea8','\ufea6','\u062f','\ufeaa','\u0630','\ufeac',
-        '\u0631','\ufeae','\u0632','\ufeaf','\u0633','\ufeb3','\ufeb4','\ufeb2',
-        '\u0634','\ufeb7','\ufeb8','\ufeb6','\u0635','\ufebb','\ufebc','\ufeba',
-        '\u0636','\ufebf','\ufec0','\ufebe','\u0637','\ufec3','\ufec4','\ufec2',
-        '\u0638','\ufec7','\ufec8','\ufec6','\u0639','\ufecb','\ufecc','\ufeca',
-        '\u063a','\ufecf','\ufed0','\ufece','\u0641','\ufed3','\ufed4','\ufed2',
-        '\u0642','\ufed7','\ufed8','\ufed6','\u0643','\ufedb','\ufedc','\ufeda',
-        '\u0644','\ufedf','\ufee0','\ufede','\u0645','\ufee3','\ufee4','\ufee2',
-        '\u0646','\ufee7','\ufee8','\ufee6','\u0647','\ufeeb','\ufeec','\ufeea',
-        '\u0629','\ufe94','\u0648','\ufeee','\u064a','\ufef3','\ufef4','\ufef2',
-        '\u0649','\ufef3','\ufef4','\ufef0','\u0621','\u0623','\ufe84','\u0625',
-        '\ufe88','\u0624','\ufe86','\u0626','\ufe8b','\ufe8c','\ufe8a','\ufd3d',
-        '\ufd3c','\u0622','\ufe82','\u0671','\ufb51','\ufefb','\ufefc','\ufef7',
-        '\ufef8','\ufef9','\ufefa','\u0020','\ufefc','\ufef5','\ufef6','\u0020',
-        '\u0020','\ufdf2','\u0640','\u0640'
-        },
-        // WP Charset 14: Arabic Script (220 chars)
-        {
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0615',
-        '\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0615','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020','\u0020',
-        '\u0020','\u06d4','\u0020','\u0020','\u00b0','\u0020','\u065a','\u0020',
-        '\u065a','\u0659','\u0020','\u0020','\u0654','\u064c','\ufc5e','\u065a',
-        '\u065a','\u06f4','\u06f4','\u06f5','\u06f6','\u06f6','\u06f7','\u06f8',
-        '\u067b','\ufb54','\ufb55','\ufb53','\u0680','\ufb5c','\ufb5d','\ufb5b',
-        '\u067e','\ufb58','\ufb59','\ufb57','\u0679','\ufb68','\ufb69','\ufb67',
-        '\u067c','\u067c','\u067c','\u067c','\u067f','\ufb64','\ufb65','\ufb63',
-        '\u067d','\u067d','\u067d','\u067d','\u067a','\ufb60','\ufb61','\ufb5f',
-        '\u0684','\ufb74','\ufb75','\ufb73','\u0683','\ufb78','\ufb79','\ufb77',
-        '\u0686','\ufb7c','\ufb7d','\ufb7b','\u0687','\ufb80','\ufb81','\ufb7f',
-        '\u0685','\u0685','\u0685','\u0685','\u0681','\u0681','\u0681','\u0681',
-        '\u0688','\ufb89','\u0689','\u0689','\u068c','\ufb85','\u068e','\ufb87',
-        '\u068a','\u068a','\u068d','\ufb83','\u0693','\u0693','\u0691','\ufb8d',
-        '\u0699','\u0699','\u0695','\u0695','\u0692','\u0692','\u0698','\ufb8b',
-        '\u0696','\u0696','\u075b','\u075b','\u069a','\u069a','\u069a','\u069a',
-        '\u06a0','\u06a0','\u06a0','\u06a0','\u06a4','\ufb6c','\ufb6d','\ufb6b',
-        '\u06a6','\ufb70','\ufb71','\ufb6f','\u06a9','\ufb90','\ufb91','\ufb8f',
-        '\u0643','\ufedb','\ufedc','\ufeda','\u06aa','\u06aa','\u06aa','\u06aa',
-        '\u06af','\ufb94','\ufb95','\ufb93','\u06af','\ufb94','\ufb95','\ufb93',
-        '\u06ab','\u06ab','\u06ab','\u06ab','\u06b1','\ufb9c','\ufb9d','\ufb9b',
-        '\u06b3','\ufb98','\ufb99','\ufb97','\u06b5','\u06b5','\u06b5','\u06b5',
-        '\u0020','\u0020','\u06ba','\u0020','\u0020','\ufb9f','\u06bc','\u06bc',
-        '\u06bc','\u06bc','\u06bb','\ufba2','\ufba3','\ufba1','\u06c6','\ufbda',
-        '\u0020','\u0020','\u06ca','\u06ca','\u06c1','\ufba8','\ufba9','\ufba7',
-        '\u06ce','\u06ce','\u06ce','\u06ce','\u06d2','\ufbaf','\u06d1','\u06d1',
-        '\u06d1','\u06d1','\u06c0','\ufba5'
-        },
-    }; 
-
-    
-    /**
-     * Constructor.
-     */
-    private WP6Constants() {
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/tika/blob/6dc442da/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
new file mode 100644
index 0000000..659ad17
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/wordperfect/WP6DocumentAreaExtractor.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.wordperfect;
+
+import java.io.IOException;
+
+/**
+ * Extracts WordPerfect Document Area text from a WordPerfect document
+ * version 6+.
+ * @author Pascal Essiembre
+ */
+class WP6DocumentAreaExtractor extends WPDocumentAreaExtractor {
+    
+    protected void extract(int c, WPInputStream in, StringBuilder out)
+            throws IOException {
+        if (c > 0 && c <= 32) {
+            out.append(WP6Charsets.DEFAULT_EXTENDED_INTL_CHARS[c]);
+        } else if (c >= 33 && c <= 126) {
+            out.append((char) c);
+        } else if (c == 128) {
+            out.append(' ');      // Soft space
+        } else if (c == 129) {
+            out.append('\u00A0'); // Hard space
+        } else if (c == 129) {
+            out.append('-');      // Hard hyphen
+        } else if (c == 135 || c == 137) {
+            out.append('\n');      // Dormant Hard return
+        } else if (c == 138) {
+            // skip to closing pair surrounding page number
+            skipUntilChar(in, 139);
+        } else if (c == 198) {
+            // end of cell
+            out.append('\t');
+        } else if (c >= 180 && c <= 207) {
+            out.append('\n');
+        } else if (c >= 208 && c <= 239) {
+            // Variable-Length Multi-Byte Functions
+            int subgroup = in.readWP();
+            int functionSize = in.readWPShort();
+            for (int i = 0; i < functionSize - 4; i++) {
+                in.readWP();
+            }
+            
+            // End-of-Line group
+            if (c == 208) {
+                if (subgroup >= 1 && subgroup <= 3) {
+                    out.append(' ');
+                } else if (subgroup == 10) {
+                    // end of cell
+                    out.append('\t');
+                } else if (subgroup >= 4 && subgroup <= 19) {
+                    out.append('\n');
+                } else if (subgroup >= 20 && subgroup <= 22) {
+                    out.append(' ');
+                } else if (subgroup >= 23 && subgroup <= 28) {
+                    out.append('\n');
+                }
+            } else if (c == 213) {
+                out.append(' ');
+            } else if (c == 224) {
+                out.append('\t');
+            }
+            //TODO Are there functions containing data? Like footnotes?
+            
+        } else if (c == 240) {
+            // extended char
+            int charval = in.readWP();
+            int charset = in.readWP();
+            in.readWP(); // closing character
+            WP6Charsets.append(out, charset, charval);
+        } else if (c >= 241 && c <= 254) {
+            skipUntilChar(in, c);
+        } else if (c == 255) {
+            skipUntilChar(in, c);
+        }
+        
+        // Ignored codes above 127:
+        
+        // 130,131,133: soft hyphens
+        // 134: invisible return in line
+        // 136: soft end of center/align
+        // 140: style separator mark
+        // 141,142: start/end of text to skip
+        // 143: exited hyphenation
+        // 144: cancel hyphenation
+        // 145-151: match functions
+        // 152-179: unknown/ignored
+        // 255: reserved, cannot be used
+    }
+}