You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2016/05/26 14:21:58 UTC
[2/2] tika git commit: TIKA-1985 - fixes for reading headers with
extra padding, applying encoding detection to headers, add datetime field
TIKA-1985 - fixes for reading headers with extra padding, applying encoding detection to headers, add datetime field
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/e855648a
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/e855648a
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/e855648a
Branch: refs/heads/2.x
Commit: e855648af835a586d438d61751573b49cf622649
Parents: 15ec358
Author: tballison <ta...@mitre.org>
Authored: Thu May 26 10:21:48 2016 -0400
Committer: tballison <ta...@mitre.org>
Committed: Thu May 26 10:21:48 2016 -0400
----------------------------------------------------------------------
.../org/apache/tika/parser/dbf/DBFCell.java | 32 +++++++++++++++
.../apache/tika/parser/dbf/DBFColumnHeader.java | 20 ++++++++--
.../apache/tika/parser/dbf/DBFFileHeader.java | 41 +++++++++++++-------
.../org/apache/tika/parser/dbf/DBFParser.java | 10 +++--
.../apache/tika/parser/dbf/DBFParserTest.java | 11 ++++++
5 files changed, 92 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/e855648a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFCell.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFCell.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFCell.java
index 2a7dc26..ee1d8db 100644
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFCell.java
+++ b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFCell.java
@@ -17,13 +17,20 @@
package org.apache.tika.parser.dbf;
import org.apache.commons.io.IOUtils;
+import org.apache.tika.io.EndianUtils;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.Arrays;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
import java.util.Locale;
+import java.util.TimeZone;
class DBFCell {
@@ -48,6 +55,8 @@ class DBFCell {
return new String(getBytes(), StandardCharsets.US_ASCII).trim();
case L:
return new String(getBytes(), StandardCharsets.US_ASCII).trim();
+ case T:
+ return getFormattedDateTime();
default:
//TODO: find examples of other cell types for testing
return new String(getBytes(), StandardCharsets.US_ASCII).trim();
@@ -113,4 +122,27 @@ class DBFCell {
return String.format(Locale.ROOT,
"%s/%s/%s", month, day, year);
}
+
+ String getFormattedDateTime() {
+ //sometimes 12/31/1899 instead of 01/01/4713 BC.
+ //http://stackoverflow.com/questions/20026154/convert-dbase-timestamp
+ //TODO: add heuristic for deciding;
+ //TODO: find example of file with time != 0
+ Calendar baseCalendar = GregorianCalendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT);
+// baseCalendar.set(1899, 11, 31, 0, 0, 0);
+ baseCalendar.set(-4712, 0, 1, 0, 0, 0);
+ byte[] bytes = getBytes();
+ try (InputStream is = new ByteArrayInputStream(getBytes())) {
+
+ int date = EndianUtils.readIntLE(is);
+ int time = EndianUtils.readIntLE(is);
+ baseCalendar.add(Calendar.DATE, date);
+ DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
+ return df.format(baseCalendar.getTime());
+ } catch (IOException|EndianUtils.BufferUnderrunException e) {
+
+ }
+ return "";
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tika/blob/e855648a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFColumnHeader.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFColumnHeader.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFColumnHeader.java
index ff6353a..a85f184 100644
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFColumnHeader.java
+++ b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFColumnHeader.java
@@ -16,10 +16,13 @@
*/
package org.apache.tika.parser.dbf;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static org.apache.tika.parser.dbf.DBFColumnHeader.ColType.AT;
+import static org.apache.tika.parser.dbf.DBFColumnHeader.ColType.NULL;
import static org.apache.tika.parser.dbf.DBFColumnHeader.ColType.PLUS;
class DBFColumnHeader {
@@ -41,6 +44,8 @@ class DBFColumnHeader {
PLUS, //autoincrement
AT, //timestamp dbase level 7
O, //double
+ NULL //null
+
}
private final static Map<Integer, ColType> COL_TYPE_MAP =
new ConcurrentHashMap<>();
@@ -48,16 +53,18 @@ class DBFColumnHeader {
static {
for (ColType type : ColType.values()) {
if (type.equals(PLUS)) {
- COL_TYPE_MAP.put((int)'+', PLUS);
+ COL_TYPE_MAP.put((int) '+', PLUS);
} else if (type.equals(AT)) {
- COL_TYPE_MAP.put((int)'@', AT);
+ COL_TYPE_MAP.put((int) '@', AT);
+ } else if (type.equals(NULL)) {
+ COL_TYPE_MAP.put((int)'0', NULL);
} else {
COL_TYPE_MAP.put((int) type.toString().charAt(0), type);
}
}
}
- String name;
+ byte[] name;
private ColType colType = null;
int fieldLength = -1;
int decimalCount = -1;
@@ -65,7 +72,8 @@ class DBFColumnHeader {
public void setType(int type) {
colType = COL_TYPE_MAP.get(type);
if (colType == null) {
- throw new IllegalArgumentException("Unrecognized column type for column: " + name +
+ throw new IllegalArgumentException("Unrecognized column type for column: " +
+ getName(StandardCharsets.US_ASCII) +
". I regret I don't recognize: " + (char) type);
}
}
@@ -74,6 +82,10 @@ class DBFColumnHeader {
return colType;
}
+ String getName(Charset charset) {
+ return new String(name, charset).trim();
+ }
+
@Override
public String toString() {
return "DBFColumnHeader{" +
http://git-wip-us.apache.org/repos/asf/tika/blob/e855648a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFFileHeader.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFFileHeader.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFFileHeader.java
index 7132c36..2a3bee6 100644
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFFileHeader.java
+++ b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFFileHeader.java
@@ -26,6 +26,8 @@ import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Calendar;
import java.util.GregorianCalendar;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Locale;
import java.util.TimeZone;
@@ -69,12 +71,21 @@ class DBFFileHeader {
header.numBytesInRecord = EndianUtils.readShortLE(is);
IOUtils.skipFully(is, 20);//TODO: can get useful info out of here
- int numCols = (header.numBytesInHeader - 32) / 32;
-
- header.cols = new DBFColumnHeader[numCols];
- for (int i = 0; i < numCols; i++) {
- header.cols[i] = readCol(is);
+ int numCols = 0;//(header.numBytesInHeader - 32) / 32;
+ List<DBFColumnHeader> headers = new LinkedList<>();
+ int bytesAccountedFor = 0;
+ while (true) {
+ DBFColumnHeader colHeader = readCol(is);
+ bytesAccountedFor += colHeader.fieldLength;
+ numCols++;
+ headers.add(colHeader);
+ if (bytesAccountedFor >= header.numBytesInRecord-1) {
+ break;
+ }
}
+
+ header.cols = headers.toArray(new DBFColumnHeader[headers.size()]);
+
int endOfHeader = is.read();
if (endOfHeader != 13) {
throw new TikaException("Expected new line at end of header");
@@ -87,26 +98,26 @@ class DBFFileHeader {
}
private static DBFColumnHeader readCol(InputStream is) throws IOException, TikaException {
- byte[] headerName = new byte[11];
- IOUtils.readFully(is, headerName);
+ byte[] fieldRecord = new byte[32];
+ IOUtils.readFully(is, fieldRecord);
+
DBFColumnHeader col = new DBFColumnHeader();
- headerName = DBFReader.trim(headerName);
- col.name = new String(headerName, StandardCharsets.US_ASCII);
- int colType = is.read();
+ col.name = new byte[11];
+ System.arraycopy(fieldRecord, 0, col.name, 0, 10);
+
+ int colType = fieldRecord[11] & 0xFF;
if (colType < 0) {
throw new IOException("File truncated before coltype in header");
}
col.setType(colType);
- IOUtils.skipFully(is, 4);//field data address
- col.fieldLength = is.read();
+ col.fieldLength = fieldRecord[16] & 0xFF;
if (col.fieldLength < 0) {
- throw new TikaException("Field length for column "+headerName+"is < 0");
+ throw new TikaException("Field length for column "+col.getName(StandardCharsets.US_ASCII)+" is < 0");
} else if (col.fieldLength > DBFReader.MAX_FIELD_LENGTH) {
throw new TikaException("Field length ("+col.fieldLength+") is greater than DBReader.MAX_FIELD_LENGTH ("+
DBFReader.MAX_FIELD_LENGTH+")");
}
- col.decimalCount = is.read();
- IOUtils.skipFully(is, 14); //TODO: might have useful info in some versions
+ col.decimalCount = fieldRecord[17] & 0xFF;
return col;
}
http://git-wip-us.apache.org/repos/asf/tika/blob/e855648a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java
index f8fa1a2..d4f2bda 100644
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java
+++ b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java
@@ -16,7 +16,7 @@
*/
package org.apache.tika.parser.dbf;
-import org.apache.tika.detect.AutoDetectReader;
+import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
@@ -24,10 +24,12 @@ import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AbstractParser;
import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.txt.Icu4jEncodingDetector;
import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -92,7 +94,7 @@ public class DBFParser extends AbstractParser {
xhtml.startElement("thead");
for (DBFColumnHeader col : header.getCols()) {
xhtml.startElement("th");
- xhtml.characters(col.name);
+ xhtml.characters(col.getName(charset));
xhtml.endElement("th");
}
xhtml.endElement("thead");
@@ -133,7 +135,9 @@ public class DBFParser extends AbstractParser {
}
byte[] bytes = bos.toByteArray();
if (bytes.length > 20) {
- charset = new AutoDetectReader(TikaInputStream.get(bytes)).getCharset();
+ EncodingDetector detector = new Icu4jEncodingDetector();
+ detector.detect(TikaInputStream.get(bytes), new Metadata());
+ charset = detector.detect(new ByteArrayInputStream(bytes), new Metadata());
}
return charset;
}
http://git-wip-us.apache.org/repos/asf/tika/blob/e855648a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/dbf/DBFParserTest.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/dbf/DBFParserTest.java b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/dbf/DBFParserTest.java
index 202c8c8..25180d2 100644
--- a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/dbf/DBFParserTest.java
+++ b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/dbf/DBFParserTest.java
@@ -135,6 +135,17 @@ public class DBFParserTest extends TikaTest {
}
}
+ @Test
+ public void testEncodingInHeaderAndDateTime() throws Exception {
+ XMLResult r = getXML("prem2007_2.dbf");
+ String xml = r.xml.replaceAll("[\\r\\n\\t]", " ");
+ assertEquals(DBFReader.Version.VISUAL_FOXPRO.getFullMimeString(),
+ r.metadata.get(Metadata.CONTENT_TYPE));
+ assertContains("<th>\u8389\u3053\u6664\u93a2</th>", xml);//header
+ assertContains("<td>\u9f60\u8915</td>", xml);//content
+ assertContains("<td>2010-04-20T00:00:00Z</td>", xml);
+ }
+
InputStream truncate(String testFileName, int length) throws IOException {
byte[] bytes = new byte[length];
try (InputStream is = getResourceAsStream("/test-documents/"+testFileName)) {