You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2019/02/28 19:45:36 UTC
[tika] branch master updated: TIKA-2833 -- initial commit with csv
detection and swapping out the TXTParser in favor of the CSVParser
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/master by this push:
new baf0bb9 TIKA-2833 -- initial commit with csv detection and swapping out the TXTParser in favor of the CSVParser
baf0bb9 is described below
commit baf0bb9c9c9fcc433c84b6047f78611af6675283
Author: TALLISON <ta...@apache.org>
AuthorDate: Wed Feb 27 21:52:46 2019 -0500
TIKA-2833 -- initial commit with csv detection and swapping
out the TXTParser in favor of the CSVParser
# Conflicts:
# tika-parsers/src/main/java/org/apache/tika/parser/csv/CSVParser.java
# tika-parsers/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java
# tika-parsers/src/test/java/org/apache/tika/parser/csv/TextAndCSVParserTest.java
---
CHANGES.txt | 4 +
.../tika/extractor/TestEmbeddedDocumentUtil.java | 8 +-
.../java/org/apache/tika/parser/csv/CSVParams.java | 72 ++++
.../java/org/apache/tika/parser/csv/CSVParser.java | 313 --------------
.../java/org/apache/tika/parser/csv/CSVResult.java | 73 ++++
.../org/apache/tika/parser/csv/CSVSniffer.java | 375 +++++++++++++++++
.../apache/tika/parser/csv/TextAndCSVParser.java | 460 +++++++++++++++++++++
.../tika/parser/mail/MailContentHandler.java | 12 +-
.../apache/tika/parser/mbox/OutlookPSTParser.java | 15 +-
.../parser/microsoft/AbstractPOIFSExtractor.java | 25 +-
.../tika/parser/microsoft/OutlookExtractor.java | 5 +-
.../services/org.apache.tika.parser.Parser | 3 +-
.../tika/config/TikaEncodingDetectorTest.java | 2 +-
.../org/apache/tika/parser/csv/CSVSnifferTest.java | 115 ++++++
...SVParserTest.java => TextAndCSVParserTest.java} | 101 +++--
15 files changed, 1216 insertions(+), 367 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index ad55bce..14107a3 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -8,6 +8,10 @@ Release 2.0.0 - ???
Release 1.21 - ????
+ * Add CSV detection and replace TXTParser with TextAndCSVParser;
+ users can turn off CSV detection by excluding the TextAndCSVParser
+ and adding back the TXTParser via tika-config (TIKA-2833).
+
* Add a CSVParser. CSV detection is currently based solely on filename
and/or information conveyed via Metadata (TIKA-2826).
diff --git a/tika-app/src/test/java/org/apache/tika/extractor/TestEmbeddedDocumentUtil.java b/tika-app/src/test/java/org/apache/tika/extractor/TestEmbeddedDocumentUtil.java
index 2262998..ac969f2 100644
--- a/tika-app/src/test/java/org/apache/tika/extractor/TestEmbeddedDocumentUtil.java
+++ b/tika-app/src/test/java/org/apache/tika/extractor/TestEmbeddedDocumentUtil.java
@@ -38,9 +38,9 @@ public class TestEmbeddedDocumentUtil {
Parser p = new AutoDetectParser();
ParseContext parseContext = new ParseContext();
parseContext.set(Parser.class, p);
- Parser txtParser = EmbeddedDocumentUtil.tryToFindExistingLeafParser(org.apache.tika.parser.txt.TXTParser.class, parseContext);
+ Parser txtParser = EmbeddedDocumentUtil.tryToFindExistingLeafParser(org.apache.tika.parser.csv.TextAndCSVParser.class, parseContext);
assertNotNull(txtParser);
- assertEquals(org.apache.tika.parser.txt.TXTParser.class, txtParser.getClass());
+ assertEquals(org.apache.tika.parser.csv.TextAndCSVParser.class, txtParser.getClass());
}
@@ -51,8 +51,8 @@ public class TestEmbeddedDocumentUtil {
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, -1));
ParseContext parseContext = new ParseContext();
parseContext.set(Parser.class, wrapper);
- Parser txtParser = EmbeddedDocumentUtil.tryToFindExistingLeafParser(org.apache.tika.parser.txt.TXTParser.class, parseContext);
+ Parser txtParser = EmbeddedDocumentUtil.tryToFindExistingLeafParser(org.apache.tika.parser.csv.TextAndCSVParser.class, parseContext);
assertNotNull(txtParser);
- assertEquals(org.apache.tika.parser.txt.TXTParser.class, txtParser.getClass());
+ assertEquals(org.apache.tika.parser.csv.TextAndCSVParser.class, txtParser.getClass());
}
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/csv/CSVParams.java b/tika-parsers/src/main/java/org/apache/tika/parser/csv/CSVParams.java
new file mode 100644
index 0000000..7e0ffb1
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/csv/CSVParams.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.csv;
+
+import java.nio.charset.Charset;
+
+import org.apache.tika.mime.MediaType;
+
+public class CSVParams {
+
+ private MediaType mediaType = null;
+ private Character delimiter = null;
+ private Charset charset = null;
+
+ CSVParams() {}
+ CSVParams(MediaType mediaType, Charset charset) {
+ this.mediaType = mediaType;
+ this.charset = charset;
+ }
+
+ CSVParams(MediaType mediaType, Charset charset, Character delimiter) {
+ this.mediaType = mediaType;
+ this.charset = charset;
+ this.delimiter = delimiter;
+ }
+
+ public boolean isEmpty() {
+ return mediaType == null && delimiter == null && charset == null;
+ }
+
+ public boolean isComplete() {
+ return mediaType != null && delimiter != null && charset != null;
+ }
+
+ public MediaType getMediaType() {
+ return mediaType;
+ }
+
+ public void setMediaType(MediaType mediaType) {
+ this.mediaType = mediaType;
+ }
+
+ public Character getDelimiter() {
+ return delimiter;
+ }
+
+ public void setDelimiter(Character delimiter) {
+ this.delimiter = delimiter;
+ }
+
+ public Charset getCharset() {
+ return charset;
+ }
+
+ public void setCharset(Charset charset) {
+ this.charset = charset;
+ }
+}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/csv/CSVParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/csv/CSVParser.java
deleted file mode 100644
index 7fb57f6..0000000
--- a/tika-parsers/src/main/java/org/apache/tika/parser/csv/CSVParser.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.csv;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.nio.charset.Charset;
-import java.nio.charset.UnsupportedCharsetException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.csv.CSVFormat;
-import org.apache.commons.csv.CSVRecord;
-import org.apache.tika.Tika;
-import org.apache.tika.config.Field;
-import org.apache.tika.detect.AutoDetectReader;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.IOUtils;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.Property;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.parser.AbstractParser;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.sax.XHTMLContentHandler;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-
-public class CSVParser extends AbstractParser {
- private static final String CSV_PREFIX = "csv";
- public static final Property DELIMITER = Property.externalText(
- CSV_PREFIX+ TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER+"delimiter");
-
- private static final String TD = "td";
- private static final String TR = "tr";
- private static final String TABLE = "table";
-
- private static final MediaType CSV = MediaType.text("csv");
- private static final MediaType TSV = MediaType.text("tsv");
-
- private static final int DEFAULT_MARK_LIMIT = 20000;
-
- //TODO: add | or make this configurable?
- private static final char[] CANDIDATE_DELIMITERS = new char[]{',', '\t'};
-
- private static final Map<Character, String> DELIMITERS = new HashMap<>();
-
- static {
- DELIMITERS.put(',', "comma");
- DELIMITERS.put('\t', "tab");
- }
-
-
-
- @Field
- private int markLimit = DEFAULT_MARK_LIMIT;
-
- private static final Set<MediaType> SUPPORTED_TYPES =
- Collections.unmodifiableSet(new HashSet<MediaType>(Arrays.asList(
- CSV, TSV)));
-
- @Override
- public Set<MediaType> getSupportedTypes(ParseContext context) {
- return SUPPORTED_TYPES;
- }
-
- @Override
- public void parse(InputStream stream, ContentHandler handler,
- Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException {
-
- String override = metadata.get(TikaCoreProperties.CONTENT_TYPE_OVERRIDE);
- Character overrideDelimiter = null;
- Charset overrideCharset = null;
- if (override != null) {
- MediaType mediaType = MediaType.parse(override);
- String charset = mediaType.getParameters().get("charset");
- overrideDelimiter = mediaType.getBaseType().toString().endsWith("tsv") ? '\t' : ',';
- if (charset != null) {
- try {
- overrideCharset = Charset.forName(charset);
- } catch (UnsupportedCharsetException e) {
- //swallow
- }
- }
- }
- if (overrideDelimiter == null || overrideCharset == null) {
- if (!stream.markSupported()) {
- stream = new BufferedInputStream(stream);
- }
- }
- //buffer the firstx bytes to detect delimiter
- byte[] firstX = null;
- if (overrideDelimiter == null) {
- firstX = readFirstX(stream, markLimit);
- }
- Charset charset = null;
- Reader reader = null;
- org.apache.commons.csv.CSVParser commonsParser = null;
- try {
- //need to detect if nothing has been sent in via override
- if (overrideCharset == null) {
- reader = new AutoDetectReader(stream);
- charset = ((AutoDetectReader) reader).getCharset();
- } else {
- reader = new BufferedReader(new InputStreamReader(stream, overrideCharset));
- charset = overrideCharset;
- }
- CSVFormat csvFormat = null;
- if (overrideDelimiter == null) {
- csvFormat = guessFormat(firstX, charset, metadata);
- } else {
- csvFormat = CSVFormat.EXCEL.withDelimiter(overrideDelimiter);
- }
- metadata.set(DELIMITER, DELIMITERS.get(csvFormat.getDelimiter()));
-
- if (overrideCharset == null || overrideDelimiter == null) {
- MediaType mediaType = (csvFormat.getDelimiter() == '\t') ? TSV : CSV;
- MediaType type = new MediaType(mediaType, charset);
- metadata.set(Metadata.CONTENT_TYPE, type.toString());
- // deprecated, see TIKA-431
- metadata.set(Metadata.CONTENT_ENCODING, charset.name());
- }
-
- XHTMLContentHandler xhtmlContentHandler = new XHTMLContentHandler(handler, metadata);
- commonsParser = new org.apache.commons.csv.CSVParser(reader, csvFormat);
- xhtmlContentHandler.startDocument();
- xhtmlContentHandler.startElement(TABLE);
- try {
- for (CSVRecord row : commonsParser) {
- xhtmlContentHandler.startElement(TR);
- for (String cell : row) {
- xhtmlContentHandler.startElement(TD);
- xhtmlContentHandler.characters(cell);
- xhtmlContentHandler.endElement(TD);
- }
- xhtmlContentHandler.endElement(TR);
- }
- } catch (IllegalStateException e) {
- throw new TikaException("exception parsing the csv", e);
- }
-
- xhtmlContentHandler.endElement(TABLE);
- xhtmlContentHandler.endDocument();
- } finally {
- if (commonsParser != null) {
- try {
- commonsParser.close();
- } catch (IOException e) {
- //swallow
- }
- }
- IOUtils.closeQuietly(reader);
- }
- }
-
- private byte[] readFirstX(InputStream stream, int markLimit) throws IOException {
- byte[] bytes = new byte[markLimit];
-
- try {
- stream.mark(markLimit);
- int numRead = IOUtils.read(stream, bytes, 0, bytes.length);
- if (numRead < markLimit) {
- byte[] dest = new byte[numRead];
- System.arraycopy(bytes, 0, dest, 0, numRead);
- bytes = dest;
- }
- } finally {
- stream.reset();
- }
- return bytes;
- }
-
- private CSVFormat guessFormat(byte[] bytes, Charset charset, Metadata metadata) throws IOException {
-
- String mediaTypeString = metadata.get(Metadata.CONTENT_TYPE);
- char bestDelimiter = (mediaTypeString.endsWith("csv")) ? ',' : '\t';
- CSVReadTestResult bestResult = null;
-
- for (char c : CANDIDATE_DELIMITERS) {
-
- try (Reader reader = new InputStreamReader(new ByteArrayInputStream(bytes), charset)) {
-
- CSVReadTestResult testResult = attemptCSVRead(c, bytes.length, reader);
- if (bestResult == null || testResult.isBetterThan(bestResult)) {
- bestResult = testResult;
- bestDelimiter = c;
- }
- }
- }
- return CSVFormat.EXCEL.withDelimiter(bestDelimiter);
- }
-
- private CSVReadTestResult attemptCSVRead(char delimiter, int bytesTotal, Reader reader) throws IOException {
-
- //maps <rowLength, numberOfRows>
- Map<Integer, Integer> colCounts = new HashMap<>();
- long lastCharacterPosition = -1L;
- int rowCount = 0;
- boolean illegalStateException = false;
- try {
- org.apache.commons.csv.CSVParser p = new org.apache.commons.csv.CSVParser(reader, CSVFormat.EXCEL.withDelimiter(delimiter));
-
- for (CSVRecord row : p) {
- int colCount = row.size();
- lastCharacterPosition = row.getCharacterPosition();
- Integer cnt = colCounts.get(colCount);
- if (cnt == null) {
- cnt = 1;
- } else {
- cnt++;
- }
- colCounts.put(colCount, cnt);
- rowCount++;
- }
- } catch (IllegalStateException e) {
- //this could be bad encapsulation -- invalid char between encapsulated token
- //swallow while guessing
- illegalStateException = true;
- }
-
- int mostCommonColCount = -1;
- int totalCount = 0;
- for (Integer count : colCounts.values()) {
- if (count > mostCommonColCount) {
- mostCommonColCount = count;
- }
- totalCount += count;
- }
- double percentMostCommonRowLength = -1.0f;
- if (totalCount > 0) {
- percentMostCommonRowLength = (double) mostCommonColCount / (double) totalCount;
- }
- return new CSVReadTestResult(bytesTotal, lastCharacterPosition, rowCount, percentMostCommonRowLength, illegalStateException);
-
- }
-
- private static class CSVReadTestResult {
- private final int bytesTotal;
- private final long bytesParsed;
- private final int rowCount;
- //the percentage of the rows that have the
- //the most common row length -- maybe use stdev?
- private final double percentMostCommonRowLength;
- private final boolean illegalStateException;
-
- public CSVReadTestResult(int bytesTotal, long bytesParsed, int rowCount,
- double percentMostCommonRowLength, boolean illegalStateException) {
- this.bytesTotal = bytesTotal;
- this.bytesParsed = bytesParsed;
- this.rowCount = rowCount;
- this.percentMostCommonRowLength = percentMostCommonRowLength;
- this.illegalStateException = illegalStateException;
- }
-
- public boolean isBetterThan(CSVReadTestResult bestResult) {
- if (bestResult == null) {
- return true;
- }
- if (illegalStateException && ! bestResult.illegalStateException) {
- return false;
- } else if (! illegalStateException && bestResult.illegalStateException) {
- return true;
- }
- //if there are >= 3 rows in both, select the one with the better
- //percentMostCommonRowLength
- if (this.rowCount >= 3 && bestResult.rowCount >= 3) {
- if (percentMostCommonRowLength > bestResult.percentMostCommonRowLength) {
- return true;
- } else {
- return false;
- }
- }
-
- //if there's a big difference between the number of bytes parsed,
- //pick the one that allowed more parsed bytes
- if (bytesTotal > 0 && Math.abs((bestResult.bytesParsed - bytesParsed) / bytesTotal) > 0.1f) {
- if (bytesParsed > bestResult.bytesParsed) {
- return true;
- } else {
- return false;
- }
- }
- //add other heuristics as necessary
-
- //if there's no other information,
- //default to not better = default
- return false;
- }
- }
-}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/csv/CSVResult.java b/tika-parsers/src/main/java/org/apache/tika/parser/csv/CSVResult.java
new file mode 100644
index 0000000..13dad8e
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/csv/CSVResult.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.csv;
+
+import java.util.Objects;
+
+import org.apache.tika.mime.MediaType;
+
+public class CSVResult implements Comparable<CSVResult> {
+
+ static CSVResult TEXT = new CSVResult(1.0, MediaType.TEXT_PLAIN, '\n');
+
+ private final double confidence;
+ private final MediaType mediaType;
+ private final char delimiter;
+
+ public CSVResult(double confidence, MediaType mediaType, char delimiter) {
+ this.confidence = confidence;
+ this.mediaType = mediaType;
+ this.delimiter = delimiter;
+ }
+
+ public MediaType getMediaType() {
+ return mediaType;
+ }
+
+ public Character getDelimiter() {
+ return delimiter;
+ }
+
+ /**
+ * Sorts in descending order of confidence
+ * @param o
+ * @return
+ */
+ @Override
+ public int compareTo(CSVResult o) {
+ return Double.compare(o.confidence, this.confidence);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ CSVResult csvResult = (CSVResult) o;
+ return Double.compare(csvResult.confidence, confidence) == 0 &&
+ delimiter == csvResult.delimiter &&
+ mediaType.equals(csvResult.mediaType);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(confidence, mediaType, delimiter);
+ }
+
+ public double getConfidence() {
+ return confidence;
+ }
+}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java b/tika-parsers/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
new file mode 100644
index 0000000..b463272
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
@@ -0,0 +1,375 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.parser.csv;
+
+import java.io.BufferedReader;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.PushbackReader;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.io.input.ProxyReader;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.tika.mime.MediaType;
+
+class CSVSniffer {
+ private static final int DEFAULT_MARK_LIMIT = 10000;
+ private static final int PUSH_BACK = 2;
+ static final int EOF = -1;
+ static final int NEW_LINE = '\n';
+ static final int CARRIAGE_RETURN = '\r';
+ private static final int SPACE = ' ';
+
+ private final char[] delimiters;
+ private final int markLimit;
+
+ CSVSniffer(char[] delimiters) {
+ this(DEFAULT_MARK_LIMIT, delimiters);
+ }
+
+ CSVSniffer(int markLimit, char[] delimiters) {
+ this.markLimit = markLimit;
+ this.delimiters = delimiters;
+ }
+
+ List<CSVResult> sniff(Reader reader) throws IOException {
+ if (! reader.markSupported()) {
+ reader = new BufferedReader(reader);
+ }
+ List<CSVResult> ret = new ArrayList<>();
+ for (char delimiter : delimiters) {
+ reader.mark(markLimit);
+ try {
+ CSVResult result = new Snifflet(delimiter).sniff(reader);
+ ret.add(result);
+ } finally {
+ reader.reset();
+ }
+ }
+ Collections.sort(ret);
+ return ret;
+ }
+
+ //gets the best result with confidence > 0
+ //otherwise, returns CSVResult
+
+ /**
+ *
+ * @param reader
+ * @return the best result with confidence > 0; if none exist, it returns {@link CSVResult#TEXT}
+ * @throws IOException
+ */
+ CSVResult getBest(Reader reader) throws IOException {
+ List<CSVResult> results = sniff(reader);
+
+ if (results.size() > 0) {
+ CSVResult result = results.get(0);
+ if (result.getConfidence() > 0.0) {
+ return result;
+ }
+ }
+ return CSVResult.TEXT;
+ }
+
+
+ //inner class that tests a single hypothesis/combination
+ //of parameters for delimiter and quote character
+ //this will throw an EOF before reading beyond the
+ //markLimit number of characters (not bytes!)
+ private class Snifflet {
+
+ private final char delimiter;
+
+ //hardcode this for now
+ private final char quoteCharacter = '"';
+
+ Map<Integer, MutableInt> rowLengthCounts = new HashMap<>();
+ int charsRead = 0;
+ int colCount = 0;
+ int encapsulated = 0; //number of cells that are encapsulated in dquotes (for now)
+ boolean parseException = false;
+
+ public Snifflet(char delimiter) {
+ this.delimiter = delimiter;
+ }
+
+ CSVResult sniff(Reader r) throws IOException {
+ boolean eof = false;
+ boolean hitMarkLimit = false;
+ int lastC = -1;
+ StringBuilder unquoted = new StringBuilder();
+ try (PushbackReader reader = new PushbackReader(new CloseShieldReader(r), PUSH_BACK)) {
+ int c = read(reader);
+ while (c != EOF) {
+ if (c == quoteCharacter) {
+ handleUnquoted(unquoted);
+ //test to make sure there isn't an unencapsulated quote character in the middle of a cell
+ if (lastC > -1 && lastC != delimiter && lastC != NEW_LINE && lastC != CARRIAGE_RETURN) {
+ parseException = true;
+ return calcResult();
+ }
+ //TODO: test to make sure cell doesn't start with escaped ""the quick brown cat"
+ boolean correctlyEncapsulated = consumeQuoted(reader, quoteCharacter);
+ if (! correctlyEncapsulated) {
+ parseException = true;
+ return calcResult();
+ }
+ } else if (c == delimiter) {
+ handleUnquoted(unquoted);
+ endColumn();
+ consumeSpaceCharacters(reader);
+ } else if (c == NEW_LINE || c == CARRIAGE_RETURN) {
+ if (unquoted.length() > 0) {
+ endColumn();
+ }
+ handleUnquoted(unquoted);
+ endRow();
+ consumeNewLines(reader);
+ } else {
+ unquoted.append((char) c);
+ }
+ lastC = c;
+ c = read(reader);
+ }
+ } catch (HitMarkLimitException e) {
+ hitMarkLimit = true;
+ } catch (UnsurprisingEOF e) {
+ //totally ignore
+ } catch (EOFException e) {
+ //the consume* throw this to avoid
+ //having to check -1 every time and
+ //having to rely on potentially wonky
+ //inputstreams not consistently returning -1
+ //after hitting EOF and returning the first -1.
+ //Yes. That's a thing.
+ eof = true;
+ } finally {
+ r.reset();
+ }
+ //if you've hit the marklimit or an eof on a truncated file
+ //don't add the last row's info
+ if (!hitMarkLimit && !eof && lastC != NEW_LINE && lastC != CARRIAGE_RETURN) {
+ handleUnquoted(unquoted);
+ endColumn();
+ endRow();
+ }
+ return calcResult();
+ }
+
+ private CSVResult calcResult() {
+ double confidence = getConfidence();
+ MediaType mediaType = TextAndCSVParser.CSV;
+ if (delimiter == '\t') {
+ mediaType = TextAndCSVParser.TSV;
+ }
+ return new CSVResult(confidence, mediaType, delimiter);
+ }
+
+ private void handleUnquoted(StringBuilder unquoted) {
+ if (unquoted.length() > 0) {
+ unquoted(unquoted.toString());
+ unquoted.setLength(0);
+ }
+ }
+
+ void consumeSpaceCharacters(PushbackReader reader) throws IOException {
+ int c = read(reader);
+ while (c == SPACE) {
+ c = read(reader);
+ }
+ if (c == EOF) {
+ throw new UnsurprisingEOF();
+ }
+ unread(reader, c);
+ }
+
+
+ /**
+ *
+ * @param reader
+ * @param quoteCharacter
+ * @return whether or not this was a correctly encapsulated cell
+ * @throws UnsurprisingEOF if the file ended immediately after the close quote
+ * @throws EOFException if the file ended in the middle of the encapsulated section
+ * @throws IOException on other IOExceptions
+ */
+ boolean consumeQuoted(PushbackReader reader, int quoteCharacter) throws IOException {
+ //this currently assumes excel "escaping" of double quotes:
+ //'the " quick' -> "the "" quick"
+ //we can make this more interesting later with other
+ //escaping options
+ int c = read(reader);
+ while (c != -1) {
+ if (c == quoteCharacter) {
+ int nextC = read(reader);
+ if (nextC == EOF) {
+ encapsulated++;
+ endColumn();
+ throw new UnsurprisingEOF();
+ } else if (nextC != quoteCharacter) {
+ encapsulated++;
+ endColumn();
+ unread(reader, nextC);
+ consumeSpaceCharacters(reader);
+ //now make sure that the next character is eof, \r\n
+ //or a delimiter
+ nextC = read(reader);
+ if (nextC == EOF) {
+ throw new UnsurprisingEOF();
+ } else if (nextC == NEW_LINE || nextC == CARRIAGE_RETURN) {
+ unread(reader, nextC);
+ return true;
+ } else if (nextC != delimiter) {
+ unread(reader, nextC);
+ return false;
+ }
+ unread(reader, nextC);
+ return true;
+ }
+ }
+ c = read(reader);
+ }
+ throw new EOFException();
+ }
+
+ private int read(PushbackReader reader) throws IOException {
+ if (charsRead >= markLimit -1) {
+ throw new HitMarkLimitException();
+ }
+ int c = reader.read();
+ if (c == EOF) {
+ return EOF;
+ }
+ charsRead++;
+ return c;
+ }
+
+ private void unread(PushbackReader reader, int c) throws IOException {
+ if (c != EOF) {
+ reader.unread(c);
+ charsRead--;
+ }
+ }
+ //consume all consecutive '\r\n' in any order
+ void consumeNewLines(PushbackReader reader) throws IOException {
+ int c = read(reader);
+ while (c == NEW_LINE || c == CARRIAGE_RETURN) {
+ c = read(reader);
+ }
+ if (c == EOF) {
+ throw new EOFException();
+ }
+ unread(reader, c);
+ return;
+ }
+
+
+ void endColumn() {
+ colCount++;
+ }
+
+ void endRow() {
+ MutableInt cnt = rowLengthCounts.get(colCount);
+ if (cnt == null) {
+ cnt = new MutableInt(1);
+ rowLengthCounts.put(colCount, cnt);
+ } else {
+ cnt.increment();
+ }
+ colCount = 0;
+ }
+
+ void unquoted(String string) {
+ //TODO -- do some analysis to make sure you don't have
+ //large tokens like 2,3,2,3,2,3,
+ }
+
+ double getConfidence() {
+ double confidence = 0.0f;
+
+ if (parseException) {
+ return -1.0f;
+ }
+ //TODO -- add tests for long tokens containing
+ //other delimiters, e.g. the,quick,brown,fox as a token
+ //when testing '\t'
+ double colCountConsistencyConf = calculateColumnCountConsistency();
+ if (colCountConsistencyConf > -1.0) {
+ confidence = colCountConsistencyConf;
+ }
+ //the idea is that if there are a bunch of encapsulated
+ //cells, then that should outweigh column length inconsistency
+ //this particular formula offers a small initial increase
+ //that eventually approaches 1.0
+ double encapsulatedBonus = 0;
+ if (encapsulated > 0) {
+ encapsulatedBonus = 1.0-(1.0d/Math.pow(encapsulated, 0.2));
+ }
+ return Math.min(confidence+encapsulatedBonus, 1.0);
+ }
+
+ private double calculateColumnCountConsistency() {
+ int max = -1;
+ int totalRows = 0;
+ //find the most common row
+ for (Map.Entry<Integer, MutableInt> e : rowLengthCounts.entrySet()) {
+ int numCols = e.getKey();
+ int count = e.getValue().intValue();
+ //require that numCols > 1 so that you had at least
+ //one delimiter in that row
+ if (numCols > 1 && count > max) {
+ max = count;
+ }
+ totalRows += count;
+ }
+ //if there's not enough info
+ if (max < 0 || totalRows < 3) {
+ return 0.0;
+ }
+
+ //TODO: convert this to continuous vs vague heuristic step function
+ double consistency = (double)max/(double)totalRows;
+ return ((1d-(1d/Math.pow(totalRows,0.3)))*consistency);
+ }
+
+ }
+
+ private static class UnsurprisingEOF extends EOFException {
+
+ }
+
+ private static class HitMarkLimitException extends EOFException {
+
+ }
+
+ private class CloseShieldReader extends ProxyReader {
+ public CloseShieldReader(Reader r) {
+ super(r);
+ }
+
+ @Override
+ public void close() throws IOException {
+ //do nothing
+ }
+ }
+}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/csv/TextAndCSVParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/csv/TextAndCSVParser.java
new file mode 100644
index 0000000..40d2b7b
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/csv/TextAndCSVParser.java
@@ -0,0 +1,460 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.csv;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVRecord;
+import org.apache.commons.io.input.CloseShieldInputStream;
+import org.apache.tika.config.Field;
+import org.apache.tika.detect.AutoDetectReader;
+import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.IOUtils;
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AbstractEncodingDetectorParser;
+import org.apache.tika.parser.AbstractParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+public class TextAndCSVParser extends AbstractEncodingDetectorParser {
+
+ private static final String CSV_PREFIX = "csv";
+ private static final String CHARSET = "charset";
+ private static final String DELIMITER = "delimiter";
+ public static final Property DELIMITER_PROPERTY = Property.externalText(
+ CSV_PREFIX + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER+DELIMITER);
+
+ private static final String TD = "td";
+ private static final String TR = "tr";
+ private static final String TABLE = "table";
+
+ static final MediaType CSV = MediaType.text("csv");
+ static final MediaType TSV = MediaType.text("tsv");
+
+ private static final int DEFAULT_MARK_LIMIT = 20000;
+
+ private static final char[] DEFAULT_DELIMITERS = new char[]{',', '\t'};
+
+ private static final Map<Character, String> CHAR_TO_STRING_DELIMITER_MAP = new HashMap<>();
+ private static final Map<String, Character> STRING_TO_CHAR_DELIMITER_MAP = new HashMap<>();
+
+ static {
+ CHAR_TO_STRING_DELIMITER_MAP.put(',', "comma");
+ CHAR_TO_STRING_DELIMITER_MAP.put('\t', "tab");
+ CHAR_TO_STRING_DELIMITER_MAP.put('|', "pipe");
+ CHAR_TO_STRING_DELIMITER_MAP.put(';', "semicolon");
+ CHAR_TO_STRING_DELIMITER_MAP.put(':', "colon");
+ }
+
+ static {
+ for (Map.Entry<Character, String> e : CHAR_TO_STRING_DELIMITER_MAP.entrySet()) {
+ STRING_TO_CHAR_DELIMITER_MAP.put(e.getValue(), e.getKey());
+ }
+ }
+ public TextAndCSVParser() {
+ super();
+ }
+
+ public TextAndCSVParser(EncodingDetector encodingDetector) {
+ super(encodingDetector);
+ }
+ private char[] delimiters = DEFAULT_DELIMITERS;
+
+ @Field
+ private int markLimit = DEFAULT_MARK_LIMIT;
+
+ private static final Set<MediaType> SUPPORTED_TYPES =
+ Collections.unmodifiableSet(new HashSet<MediaType>(Arrays.asList(
+ CSV, TSV, MediaType.TEXT_PLAIN)));
+
+ @Override
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return SUPPORTED_TYPES;
+ }
+
+ @Override
+ public void parse(InputStream stream, ContentHandler handler,
+ Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException {
+
+ CSVParams params = getOverride(metadata);
+ Reader reader = null;
+ Charset charset = null;
+ if (! params.isComplete()) {
+ reader = detect(params, stream, metadata, context);
+ if (params.getCharset() != null) {
+ charset = params.getCharset();
+ } else {
+ charset = ((AutoDetectReader) reader).getCharset();
+ }
+ } else {
+ reader = new BufferedReader(new InputStreamReader(stream, params.getCharset()));
+ charset = params.getCharset();
+ }
+ //if text or a non-csv/tsv category of text
+ //treat this as text and be done
+ //TODO -- if it was detected already as a non-csv subtype of text
+ if (! params.getMediaType().getBaseType().equals(CSV) &&
+ ! params.getMediaType().getBaseType().equals(TSV)) {
+ handleText(reader, charset, handler, metadata);
+ return;
+ }
+
+ updateMetadata(params, metadata);
+
+ CSVFormat csvFormat = CSVFormat.EXCEL.withDelimiter(params.getDelimiter());
+ metadata.set(DELIMITER_PROPERTY, CHAR_TO_STRING_DELIMITER_MAP.get(csvFormat.getDelimiter()));
+
+ XHTMLContentHandler xhtmlContentHandler = new XHTMLContentHandler(handler, metadata);
+ try (org.apache.commons.csv.CSVParser commonsParser =
+ new org.apache.commons.csv.CSVParser(reader, csvFormat)) {
+ xhtmlContentHandler.startDocument();
+ xhtmlContentHandler.startElement(TABLE);
+ try {
+ for (CSVRecord row : commonsParser) {
+ xhtmlContentHandler.startElement(TR);
+ for (String cell : row) {
+ xhtmlContentHandler.startElement(TD);
+ xhtmlContentHandler.characters(cell);
+ xhtmlContentHandler.endElement(TD);
+ }
+ xhtmlContentHandler.endElement(TR);
+ }
+ } catch (IllegalStateException e) {
+ //if there's a parse exception
+ //try to get the rest of the content...treat it as text for now
+ //There will be some content lost because of buffering.
+ //TODO -- figure out how to improve this
+ xhtmlContentHandler.endElement(TABLE);
+ xhtmlContentHandler.startElement("div", "name", "after exception");
+ handleText(reader, xhtmlContentHandler);
+ xhtmlContentHandler.endElement("div");
+ xhtmlContentHandler.endDocument();
+ //TODO -- consider dumping what's left in the reader as text
+ throw new TikaException("exception parsing the csv", e);
+ }
+
+ xhtmlContentHandler.endElement(TABLE);
+ xhtmlContentHandler.endDocument();
+ }
+ }
+
+ private void handleText(Reader reader, Charset charset,
+ ContentHandler handler, Metadata metadata)
+ throws SAXException, IOException, TikaException {
+ // Automatically detect the character encoding
+ //try to get detected content type; could be a subclass of text/plain
+ //such as vcal, etc.
+ String incomingMime = metadata.get(Metadata.CONTENT_TYPE);
+ MediaType mediaType = MediaType.TEXT_PLAIN;
+ if (incomingMime != null) {
+ MediaType tmpMediaType = MediaType.parse(incomingMime);
+ if (tmpMediaType != null) {
+ mediaType = tmpMediaType;
+ }
+ }
+ MediaType type = new MediaType(mediaType, charset);
+ metadata.set(Metadata.CONTENT_TYPE, type.toString());
+ // deprecated, see TIKA-431
+ metadata.set(Metadata.CONTENT_ENCODING, charset.name());
+
+ XHTMLContentHandler xhtml =
+ new XHTMLContentHandler(handler, metadata);
+ xhtml.startDocument();
+ handleText(reader, xhtml);
+ xhtml.endDocument();
+ }
+
+ private static void handleText(Reader reader, XHTMLContentHandler xhtml)
+ throws SAXException, IOException {
+ xhtml.startElement("p");
+ char[] buffer = new char[4096];
+ int n = reader.read(buffer);
+ while (n != -1) {
+ xhtml.characters(buffer, 0, n);
+ n = reader.read(buffer);
+ }
+ xhtml.endElement("p");
+
+ }
+
+ private void updateMetadata(CSVParams params, Metadata metadata) {
+ MediaType mediaType = (params.getDelimiter() == '\t') ? TSV : CSV;
+ Map<String, String> attrs = new HashMap<>();
+ attrs.put(CHARSET, params.getCharset().name());
+ if (params.getDelimiter() != null) {
+ if (CHAR_TO_STRING_DELIMITER_MAP.containsKey(params.getDelimiter())) {
+ attrs.put(DELIMITER, CHAR_TO_STRING_DELIMITER_MAP.get(params.getDelimiter()));
+ } else {
+ attrs.put(DELIMITER, Integer.toString((int)params.getDelimiter().charValue()));
+ }
+ }
+ MediaType type = new MediaType(mediaType, attrs);
+ metadata.set(Metadata.CONTENT_TYPE, type.toString());
+ // deprecated, see TIKA-431
+ metadata.set(Metadata.CONTENT_ENCODING, params.getCharset().name());
+
+ }
+
+ private Reader detect(CSVParams params, InputStream stream,
+ Metadata metadata, ParseContext context) throws IOException, TikaException {
+ //if the file was already identified as not .txt, .csv or .tsv
+ //don't even try to csv or not
+ String mediaString = metadata.get(Metadata.CONTENT_TYPE);
+ if (mediaString != null) {
+ MediaType mediaType = MediaType.parse(mediaString);
+ if (! SUPPORTED_TYPES.contains(mediaType.getBaseType())) {
+ params.setMediaType(mediaType);
+ return new AutoDetectReader(
+ new CloseShieldInputStream(stream),
+ metadata, getEncodingDetector(context));
+ }
+ }
+ Reader reader = null;
+ if (params.getCharset() == null) {
+ reader = new AutoDetectReader(
+ new CloseShieldInputStream(stream),
+ metadata, getEncodingDetector(context));
+ params.setCharset(((AutoDetectReader)reader).getCharset());
+ if (params.isComplete()) {
+ return reader;
+ }
+ } else {
+ reader = new BufferedReader(new InputStreamReader(
+ new CloseShieldInputStream(stream), params.getCharset()));
+ }
+
+ if (params.getDelimiter() == null &&
+ (params.getMediaType() == null ||
+ isCSVOrTSV(params.getMediaType()))) {
+
+ CSVSniffer sniffer = new CSVSniffer(delimiters);
+ CSVResult result = sniffer.getBest(reader);
+ //we should require a higher confidence if the content-type
+ //is text/plain -- e.g. if the file name ends in .txt or
+ //the parent parser has an indication that this is txt
+ //(as in mail attachment headers)
+ params.setMediaType(result.getMediaType());
+ params.setDelimiter(result.getDelimiter());
+ }
+ return reader;
+ }
+
+ private CSVParams getOverride(Metadata metadata) {
+ String override = metadata.get(TikaCoreProperties.CONTENT_TYPE_OVERRIDE);
+ if (override == null) {
+ return new CSVParams();
+ }
+ MediaType mediaType = MediaType.parse(override);
+ if (mediaType == null) {
+ return new CSVParams();
+ }
+ String charsetString = mediaType.getParameters().get(CHARSET);
+ Charset charset = null;
+ if (charsetString != null) {
+ try {
+ charset = Charset.forName(charsetString);
+ } catch (UnsupportedCharsetException e) {
+
+ }
+ }
+ if (! isCSVOrTSV(mediaType)) {
+ return new CSVParams(mediaType, charset);
+ }
+
+ String delimiterString = mediaType.getParameters().get(DELIMITER);
+ if (delimiterString == null) {
+ return new CSVParams(mediaType, charset);
+ }
+ if (STRING_TO_CHAR_DELIMITER_MAP.containsKey(delimiterString)) {
+ return new CSVParams(mediaType, charset, (char) STRING_TO_CHAR_DELIMITER_MAP.get(delimiterString));
+ }
+ if (delimiterString.length() == 1) {
+ return new CSVParams(mediaType, charset, delimiterString.charAt(0));
+ }
+ //TODO: log bad/unrecognized delimiter string
+ return new CSVParams(mediaType, charset);
+ }
+
+ static boolean isCSVOrTSV(MediaType mediaType) {
+ if (mediaType == null) {
+ return false;
+ }
+ if (mediaType.getBaseType().equals(TSV) ||
+ mediaType.getBaseType().equals(CSV)) {
+ return true;
+ }
+ return false;
+ }
+ private byte[] readFirstX(InputStream stream, int markLimit) throws IOException {
+ byte[] bytes = new byte[markLimit];
+
+ try {
+ stream.mark(markLimit);
+ int numRead = IOUtils.read(stream, bytes, 0, bytes.length);
+ if (numRead < markLimit) {
+ byte[] dest = new byte[numRead];
+ System.arraycopy(bytes, 0, dest, 0, numRead);
+ bytes = dest;
+ }
+ } finally {
+ stream.reset();
+ }
+ return bytes;
+ }
+
+ private CSVFormat guessFormat(byte[] bytes, Charset charset, Metadata metadata) throws IOException {
+
+ String mediaTypeString = metadata.get(Metadata.CONTENT_TYPE);
+ char bestDelimiter = (mediaTypeString.endsWith("csv")) ? ',' : '\t';
+ CSVReadTestResult bestResult = null;
+
+ for (char c : DEFAULT_DELIMITERS) {
+
+ try (Reader reader = new InputStreamReader(new ByteArrayInputStream(bytes), charset)) {
+
+ CSVReadTestResult testResult = attemptCSVRead(c, bytes.length, reader);
+ if (bestResult == null || testResult.isBetterThan(bestResult)) {
+ bestResult = testResult;
+ bestDelimiter = c;
+ }
+ }
+ }
+ return CSVFormat.EXCEL.withDelimiter(bestDelimiter);
+ }
+
+ private CSVReadTestResult attemptCSVRead(char delimiter, int bytesTotal, Reader reader) throws IOException {
+
+ //maps <rowLength, numberOfRows>
+ Map<Integer, Integer> colCounts = new HashMap<>();
+ long lastCharacterPosition = -1L;
+ int rowCount = 0;
+ boolean illegalStateException = false;
+ try {
+ org.apache.commons.csv.CSVParser p = new org.apache.commons.csv.CSVParser(reader, CSVFormat.EXCEL.withDelimiter(delimiter));
+
+ for (CSVRecord row : p) {
+ int colCount = row.size();
+ lastCharacterPosition = row.getCharacterPosition();
+ Integer cnt = colCounts.get(colCount);
+ if (cnt == null) {
+ cnt = 1;
+ } else {
+ cnt++;
+ }
+ colCounts.put(colCount, cnt);
+ rowCount++;
+ }
+ } catch (IllegalStateException e) {
+ //this could be bad encapsulation -- invalid char between encapsulated token
+ //swallow while guessing
+ illegalStateException = true;
+ }
+
+ int mostCommonColCount = -1;
+ int totalCount = 0;
+ for (Integer count : colCounts.values()) {
+ if (count > mostCommonColCount) {
+ mostCommonColCount = count;
+ }
+ totalCount += count;
+ }
+ double percentMostCommonRowLength = -1.0f;
+ if (totalCount > 0) {
+ percentMostCommonRowLength = (double) mostCommonColCount / (double) totalCount;
+ }
+ return new CSVReadTestResult(bytesTotal, lastCharacterPosition, rowCount, percentMostCommonRowLength, illegalStateException);
+
+ }
+
+ private static class CSVReadTestResult {
+ private final int bytesTotal;
+ private final long bytesParsed;
+ private final int rowCount;
+ //the percentage of the rows that have the
+ //the most common row length -- maybe use stdev?
+ private final double percentMostCommonRowLength;
+ private final boolean illegalStateException;
+
+ public CSVReadTestResult(int bytesTotal, long bytesParsed, int rowCount,
+ double percentMostCommonRowLength, boolean illegalStateException) {
+ this.bytesTotal = bytesTotal;
+ this.bytesParsed = bytesParsed;
+ this.rowCount = rowCount;
+ this.percentMostCommonRowLength = percentMostCommonRowLength;
+ this.illegalStateException = illegalStateException;
+ }
+
+ public boolean isBetterThan(CSVReadTestResult bestResult) {
+ if (bestResult == null) {
+ return true;
+ }
+ if (illegalStateException && ! bestResult.illegalStateException) {
+ return false;
+ } else if (! illegalStateException && bestResult.illegalStateException) {
+ return true;
+ }
+ //if there are >= 3 rows in both, select the one with the better
+ //percentMostCommonRowLength
+ if (this.rowCount >= 3 && bestResult.rowCount >= 3) {
+ if (percentMostCommonRowLength > bestResult.percentMostCommonRowLength) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ //if there's a big difference between the number of bytes parsed,
+ //pick the one that allowed more parsed bytes
+ if (bytesTotal > 0 && Math.abs((bestResult.bytesParsed - bytesParsed) / bytesTotal) > 0.1f) {
+ if (bytesParsed > bestResult.bytesParsed) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ //add other heuristics as necessary
+
+ //if there's no other information,
+ //default to not better = default
+ return false;
+ }
+ }
+
+
+}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java b/tika-parsers/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
index fa30ee0..f837384 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
@@ -45,6 +45,7 @@ import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.csv.TextAndCSVParser;
import org.apache.tika.parser.html.HtmlParser;
import org.apache.tika.parser.rtf.RTFParser;
import org.apache.tika.parser.txt.TXTParser;
@@ -586,6 +587,7 @@ class MailContentHandler implements ContentHandler {
private void handleInlineBodyPart(BodyContents part) throws MimeException, IOException {
String contentType = part.metadata.get(Metadata.CONTENT_TYPE);
Parser parser = null;
+ boolean inlineText = false;
if (MediaType.TEXT_HTML.toString().equalsIgnoreCase(contentType)) {
parser =
EmbeddedDocumentUtil.tryToFindExistingLeafParser(HtmlParser.class, parseContext);
@@ -595,6 +597,10 @@ class MailContentHandler implements ContentHandler {
} else if (MediaType.TEXT_PLAIN.toString().equalsIgnoreCase(contentType)) {
parser =
EmbeddedDocumentUtil.tryToFindExistingLeafParser(TXTParser.class, parseContext);
+ if (parser == null) {
+ parser = EmbeddedDocumentUtil.tryToFindExistingLeafParser(TextAndCSVParser.class, parseContext);
+ inlineText = true;
+ }
}
@@ -607,10 +613,14 @@ class MailContentHandler implements ContentHandler {
//parse inline
try {
+ Metadata inlineMetadata = new Metadata();
+ if (inlineText) {
+ inlineMetadata.set(TikaCoreProperties.CONTENT_TYPE_OVERRIDE, MediaType.TEXT_PLAIN.toString());
+ }
parser.parse(
new ByteArrayInputStream(part.bytes),
new EmbeddedContentHandler(new BodyContentHandler(handler)),
- new Metadata(), parseContext
+ inlineMetadata, parseContext
);
} catch (SAXException | TikaException e) {
throw new MimeException(e);
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java
index 05b67e5..56290aa 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java
@@ -21,7 +21,6 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.singleton;
import java.io.ByteArrayInputStream;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Set;
@@ -32,12 +31,9 @@ import com.pff.PSTFile;
import com.pff.PSTFolder;
import com.pff.PSTMessage;
import com.pff.PSTRecipient;
-import org.apache.poi.ss.formula.functions.T;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
-import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor;
-import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Message;
import org.apache.tika.metadata.Metadata;
@@ -46,12 +42,7 @@ import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AbstractParser;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
import org.apache.tika.parser.microsoft.OutlookExtractor;
-import org.apache.tika.parser.rtf.RTFParser;
-import org.apache.tika.parser.txt.TXTParser;
-import org.apache.tika.sax.BodyContentHandler;
-import org.apache.tika.sax.EmbeddedContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
@@ -219,8 +210,10 @@ public class OutlookPSTParser extends AbstractParser {
//the underlying bytes from the pstMail object...
byte[] mailContent = pstMail.getBody().getBytes(UTF_8);
- mailMetadata.set(TikaCoreProperties.CONTENT_TYPE_OVERRIDE, MediaType.TEXT_PLAIN.toString());
- embeddedExtractor.parseEmbedded(new ByteArrayInputStream(mailContent), handler, mailMetadata, true);
+ mailMetadata.set(TikaCoreProperties.CONTENT_TYPE_OVERRIDE,
+ MediaType.TEXT_PLAIN.toString());
+ embeddedExtractor.parseEmbedded(new ByteArrayInputStream(mailContent),
+ handler, mailMetadata, true);
}
private void parseMailAttachments(XHTMLContentHandler xhtml, PSTMessage email,
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java
index 5c7d445..b889c1e 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java
@@ -104,24 +104,33 @@ abstract class AbstractPOIFSExtractor {
String relationshipID, ClassID storageClassID, String mediaType, XHTMLContentHandler xhtml,
boolean outputHtml)
throws IOException, SAXException, TikaException {
+ handleEmbeddedResource(resource, new Metadata(), filename,
+ relationshipID, storageClassID, mediaType, xhtml, outputHtml);
+ }
+
+ protected void handleEmbeddedResource(TikaInputStream resource, Metadata embeddedMetadata, String filename,
+ String relationshipID, ClassID storageClassID, String mediaType, XHTMLContentHandler xhtml,
+ boolean outputHtml)
+ throws IOException, SAXException, TikaException {
+
try {
- Metadata metadata = new Metadata();
+
if (filename != null) {
- metadata.set(Metadata.TIKA_MIME_FILE, filename);
- metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, filename);
+ embeddedMetadata.set(Metadata.TIKA_MIME_FILE, filename);
+ embeddedMetadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, filename);
}
if (relationshipID != null) {
- metadata.set(TikaCoreProperties.EMBEDDED_RELATIONSHIP_ID, relationshipID);
+ embeddedMetadata.set(TikaCoreProperties.EMBEDDED_RELATIONSHIP_ID, relationshipID);
}
if (storageClassID != null) {
- metadata.set(TikaCoreProperties.EMBEDDED_STORAGE_CLASS_ID, storageClassID.toString());
+ embeddedMetadata.set(TikaCoreProperties.EMBEDDED_STORAGE_CLASS_ID, storageClassID.toString());
}
if (mediaType != null) {
- metadata.set(Metadata.CONTENT_TYPE, mediaType);
+ embeddedMetadata.set(Metadata.CONTENT_TYPE, mediaType);
}
- if (embeddedDocumentUtil.shouldParseEmbedded(metadata)) {
- embeddedDocumentUtil.parseEmbedded(resource, xhtml, metadata, outputHtml);
+ if (embeddedDocumentUtil.shouldParseEmbedded(embeddedMetadata)) {
+ embeddedDocumentUtil.parseEmbedded(resource, xhtml, embeddedMetadata, outputHtml);
}
} finally {
resource.close();
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
index 136bbf2..a33f89b 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
@@ -376,8 +376,11 @@ public class OutlookExtractor extends AbstractPOIFSExtractor {
if (textChunk != null) {
byte[] data = getValue(textChunk);
if (data != null) {
+ Metadata chunkMetadata = new Metadata();
+ chunkMetadata.set(TikaCoreProperties.CONTENT_TYPE_OVERRIDE,
+ MediaType.TEXT_PLAIN.toString());
handleEmbeddedResource(
- TikaInputStream.get(data),
+ TikaInputStream.get(data), chunkMetadata, null,
"text-body", null,
MediaType.TEXT_PLAIN.toString(), xhtml, true
);
diff --git a/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser b/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser
index cfabd0f..330d473 100644
--- a/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser
+++ b/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser
@@ -19,7 +19,7 @@ org.apache.tika.parser.audio.AudioParser
org.apache.tika.parser.audio.MidiParser
org.apache.tika.parser.crypto.Pkcs7Parser
org.apache.tika.parser.crypto.TSDParser
-org.apache.tika.parser.csv.CSVParser
+org.apache.tika.parser.csv.TextAndCSVParser
org.apache.tika.parser.dwg.DWGParser
org.apache.tika.parser.envi.EnviHeaderParser
org.apache.tika.parser.epub.EpubParser
@@ -61,7 +61,6 @@ org.apache.tika.parser.pkg.PackageParser
org.apache.tika.parser.pkg.RarParser
org.apache.tika.parser.rtf.RTFParser
org.apache.tika.parser.sas.SAS7BDATParser
-org.apache.tika.parser.txt.TXTParser
org.apache.tika.parser.video.FLVParser
org.apache.tika.parser.wordperfect.QuattroProParser
org.apache.tika.parser.wordperfect.WordPerfectParser
diff --git a/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java b/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
index f6278e0..859b709 100644
--- a/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
@@ -158,7 +158,7 @@ public class TikaEncodingDetectorTest extends AbstractTikaConfigTest {
List<Parser> parsers = new ArrayList<>();
findEncodingDetectionParsers(p, parsers);
- assertEquals(4, parsers.size());
+ assertEquals(5, parsers.size());
for (Parser encodingDetectingParser : parsers) {
EncodingDetector encodingDetector = ((AbstractEncodingDetectorParser) encodingDetectingParser).getEncodingDetector();
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/csv/CSVSnifferTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/csv/CSVSnifferTest.java
new file mode 100644
index 0000000..998473c
--- /dev/null
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/csv/CSVSnifferTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.csv;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.io.ByteOrderMark;
+import org.apache.tika.TikaTest;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.Parser;
+import org.junit.Test;
+
+public class CSVSnifferTest extends TikaTest {
+
+ private static char[] DELIMITERS = new char[]{ ',', '\t'};
+
+ private static byte[] CSV_BASIC =
+ ("the,quick,brown\tfox\n" +
+ "jumped \tover,the\tlazy,\tdog\n"+
+ "and then,ran,down\tthe\tstreet").getBytes(StandardCharsets.UTF_8);
+
+ private static byte[] TSV_BASIC =
+ ("the\tquick\tbrown,fox\n" +
+ "jumped ,over\tthe,lazy\t,dog\n"+
+ "and then\tran\tdown,the,street").getBytes(StandardCharsets.UTF_8);
+
+ private static byte[] CSV_MID_CELL_QUOTE_EXCEPTION =
+ ("the,quick,brown\"fox\n" +
+ "jumped over,the lazy,dog\n"+
+ "and then,ran,down the street").getBytes(StandardCharsets.UTF_8);
+
+
+ private static byte[] ALLOW_SPACES_BEFORE_QUOTE =
+ ("the,quick, \"brown\"\"fox\"\n" +
+ "jumped over,the lazy,dog\n"+
+ "and then,ran,down the street").getBytes(StandardCharsets.UTF_8);
+
+ private static byte[] ALLOW_SPACES_AFTER_QUOTE =
+ ("the,\"quick\" ,brown fox\n" +
+ "jumped over,the lazy,dog\n"+
+ "and then,ran,down the street").getBytes(StandardCharsets.UTF_8);
+
+ @Test
+ public void testCSVBasic() throws Exception {
+ List<CSVResult> results = sniff(DELIMITERS, CSV_BASIC, StandardCharsets.UTF_8);
+ assertEquals(2, results.size());
+ assertEquals(new Character(','), results.get(0).getDelimiter());
+
+ results = sniff(DELIMITERS, TSV_BASIC, StandardCharsets.UTF_8);
+ assertEquals(2, results.size());
+ assertEquals(new Character('\t'), results.get(0).getDelimiter());
+ }
+
+ private static List<CSVResult> sniff(char[] delimiters, byte[] bytes, Charset charset) throws IOException {
+ CSVSniffer sniffer = new CSVSniffer(delimiters);
+ try (BufferedReader reader = new BufferedReader(
+ new InputStreamReader(new ByteArrayInputStream(bytes), charset))) {
+ return sniffer.sniff(reader);
+ }
+ }
+
+ @Test
+ public void testCSVMidCellQuoteException() throws Exception {
+ List<CSVResult> results = sniff(DELIMITERS, CSV_MID_CELL_QUOTE_EXCEPTION, StandardCharsets.UTF_8);
+ assertEquals(2, results.size());
+ }
+
+ @Test
+ public void testAllowWhiteSpacesAroundAQuote() throws Exception {
+ List<CSVResult> results = sniff(DELIMITERS,
+ ALLOW_SPACES_BEFORE_QUOTE, StandardCharsets.UTF_8);
+ assertEquals(2, results.size());
+ assertEquals(new Character(','), results.get(0).getDelimiter());
+
+ results = sniff(DELIMITERS, ALLOW_SPACES_AFTER_QUOTE, StandardCharsets.UTF_8);
+ assertEquals(2, results.size());
+ assertEquals(new Character(','), results.get(0).getDelimiter());
+ }
+
+ @Test
+ public void testSort() {
+ List<CSVResult> list = new ArrayList<>();
+ list.add(new CSVResult(0.1, MediaType.TEXT_HTML, '-'));
+ list.add(new CSVResult(0.2, MediaType.TEXT_PLAIN, ','));
+ Collections.sort(list);
+ assertEquals(0.2, list.get(0).getConfidence(), 0.00001);
+ }
+}
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/csv/CSVParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/csv/TextAndCSVParserTest.java
similarity index 65%
rename from tika-parsers/src/test/java/org/apache/tika/parser/csv/CSVParserTest.java
rename to tika-parsers/src/test/java/org/apache/tika/parser/csv/TextAndCSVParserTest.java
index de6ec29..44f74e9 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/csv/CSVParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/csv/TextAndCSVParserTest.java
@@ -17,19 +17,25 @@
package org.apache.tika.parser.csv;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
+import java.io.InputStream;
import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.commons.io.ByteOrderMark;
import org.apache.tika.TikaTest;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.Parser;
import org.junit.Test;
-public class CSVParserTest extends TikaTest {
+public class TextAndCSVParserTest extends TikaTest {
private static byte[] CSV_UTF8 =
("the,quick,brown\tfox\n" +
@@ -67,8 +73,9 @@ public class CSVParserTest extends TikaTest {
Metadata metadata = new Metadata();
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.csv");
XMLResult xmlResult = getXML(new ByteArrayInputStream(CSV_UTF8), PARSER, metadata);
- assertEquals("comma", xmlResult.metadata.get(CSVParser.DELIMITER));
- assertEquals("text/csv; charset=ISO-8859-1", xmlResult.metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("comma", xmlResult.metadata.get(TextAndCSVParser.DELIMITER_PROPERTY));
+ assertMediaTypeEquals("csv", "ISO-8859-1","comma",
+ xmlResult.metadata.get(Metadata.CONTENT_TYPE));
assertContainsIgnoreWhiteSpaceDiffs(EXPECTED_CSV, xmlResult.xml);
}
@@ -77,8 +84,10 @@ public class CSVParserTest extends TikaTest {
Metadata metadata = new Metadata();
metadata.set(TikaCoreProperties.CONTENT_TYPE_OVERRIDE, "text/csv; charset=UTF-8");
XMLResult xmlResult = getXML(new ByteArrayInputStream(CSV_UTF8), PARSER, metadata);
- assertEquals("comma", xmlResult.metadata.get(CSVParser.DELIMITER));
- assertEquals("text/csv; charset=UTF-8", xmlResult.metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("comma", xmlResult.metadata.get(TextAndCSVParser.DELIMITER_PROPERTY));
+ assertMediaTypeEquals("csv", "UTF-8","comma",
+ xmlResult.metadata.get(Metadata.CONTENT_TYPE));
+
assertContainsIgnoreWhiteSpaceDiffs(EXPECTED_CSV, xmlResult.xml);
}
@@ -87,8 +96,9 @@ public class CSVParserTest extends TikaTest {
Metadata metadata = new Metadata();
metadata.set(Metadata.CONTENT_TYPE, "text/csv");
XMLResult xmlResult = getXML(new ByteArrayInputStream(CSV_UTF8), PARSER, metadata);
- assertEquals("comma", xmlResult.metadata.get(CSVParser.DELIMITER));
- assertEquals("text/csv; charset=ISO-8859-1", xmlResult.metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("comma", xmlResult.metadata.get(TextAndCSVParser.DELIMITER_PROPERTY));
+ assertMediaTypeEquals("csv", "ISO-8859-1","comma",
+ xmlResult.metadata.get(Metadata.CONTENT_TYPE));
assertContainsIgnoreWhiteSpaceDiffs(EXPECTED_CSV, xmlResult.xml);
}
@@ -97,8 +107,9 @@ public class CSVParserTest extends TikaTest {
Metadata metadata = new Metadata();
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.csv");
XMLResult xmlResult = getXML(new ByteArrayInputStream(CSV_UTF_16LE), PARSER, metadata);
- assertEquals("comma", xmlResult.metadata.get(CSVParser.DELIMITER));
- assertEquals("text/csv; charset=UTF-16LE", xmlResult.metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("comma", xmlResult.metadata.get(TextAndCSVParser.DELIMITER_PROPERTY));
+ assertMediaTypeEquals("csv", "UTF-16LE","comma",
+ xmlResult.metadata.get(Metadata.CONTENT_TYPE));
assertContainsIgnoreWhiteSpaceDiffs(EXPECTED_CSV, xmlResult.xml);
}
@@ -108,25 +119,20 @@ public class CSVParserTest extends TikaTest {
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.csv");
XMLResult xmlResult = getXML(new ByteArrayInputStream(
concat(ByteOrderMark.UTF_16LE.getBytes(), CSV_UTF_16LE)), PARSER, metadata);
- assertEquals("comma", xmlResult.metadata.get(CSVParser.DELIMITER));
- assertEquals("text/csv; charset=UTF-16LE", xmlResult.metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("comma", xmlResult.metadata.get(TextAndCSVParser.DELIMITER_PROPERTY));
+ assertMediaTypeEquals("csv", "UTF-16LE","comma",
+ xmlResult.metadata.get(Metadata.CONTENT_TYPE));
assertContainsIgnoreWhiteSpaceDiffs(EXPECTED_CSV, xmlResult.xml);
}
- private static byte[] concat(byte[] bytesA, byte[] bytesB) {
- byte[] ret = new byte[bytesA.length+bytesB.length];
- System.arraycopy(bytesA, 0, ret, 0, bytesA.length);
- System.arraycopy(bytesB, 0, ret, bytesA.length, bytesB.length);
- return ret;
- }
-
@Test
public void testTSV_UTF8() throws Exception {
Metadata metadata = new Metadata();
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.csv");
XMLResult xmlResult = getXML(new ByteArrayInputStream(TSV_UTF8), PARSER, metadata);
- assertEquals("tab", xmlResult.metadata.get(CSVParser.DELIMITER));
- assertEquals("text/tsv; charset=ISO-8859-1", xmlResult.metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("tab", xmlResult.metadata.get(TextAndCSVParser.DELIMITER_PROPERTY));
+ assertMediaTypeEquals("tsv", "ISO-8859-1","tab",
+ xmlResult.metadata.get(Metadata.CONTENT_TYPE));
assertContainsIgnoreWhiteSpaceDiffs(EXPECTED_TSV, xmlResult.xml);
}
@@ -135,16 +141,18 @@ public class CSVParserTest extends TikaTest {
Metadata metadata = new Metadata();
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.csv");
XMLResult xmlResult = getXML(new ByteArrayInputStream(TSV_UTF_16LE), PARSER, metadata);
- assertEquals("tab", xmlResult.metadata.get(CSVParser.DELIMITER));
- assertEquals("text/tsv; charset=UTF-16LE", xmlResult.metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("tab", xmlResult.metadata.get(TextAndCSVParser.DELIMITER_PROPERTY));
+ assertMediaTypeEquals("tsv", "UTF-16LE","tab",
+ xmlResult.metadata.get(Metadata.CONTENT_TYPE));
assertContainsIgnoreWhiteSpaceDiffs(EXPECTED_TSV, xmlResult.xml);
}
@Test
public void testBadCsv() throws Exception {
//this causes an IllegalStateException during delimiter detection
- //when trying to parse with ','; therefore, the parser backs off to '\t'.
- //this isn't necessarily the best outcome, but we want to make sure
+ //when trying to parse with ','; therefore, the parser backs off to
+ //treating this as straight text.
+ //This isn't necessarily the best outcome, but we want to make sure
//that an IllegalStateException during delimiter guessing doesn't
//make the parse fail.
@@ -154,11 +162,52 @@ public class CSVParserTest extends TikaTest {
Metadata metadata = new Metadata();
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.csv");
XMLResult xmlResult = getXML(new ByteArrayInputStream(csv), PARSER, metadata);
- assertEquals("tab", xmlResult.metadata.get(CSVParser.DELIMITER));
- assertEquals("text/tsv; charset=ISO-8859-1", xmlResult.metadata.get(Metadata.CONTENT_TYPE));
+ assertNull(xmlResult.metadata.get(TextAndCSVParser.DELIMITER_PROPERTY));
+ assertEquals("text/csv; charset=ISO-8859-1", xmlResult.metadata.get(Metadata.CONTENT_TYPE));
+ assertContains("the,quick", xmlResult.xml);
+ }
+
+ @Test
+ public void testLong() throws Exception {
+ //test mark/reset worked on the sniffers
+ StringBuilder sb = new StringBuilder();
+ for (int rows = 0; rows < 1000; rows++) {
+ for (int cols = 0; cols < 10; cols++) {
+ sb.append("2").append(",");
+ }
+ sb.append("\n");
+ }
+ Metadata metadata = new Metadata();
+ XMLResult xmlResult = getXML(new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8)), PARSER, metadata);
+ assertMediaTypeEquals("csv", "ISO-8859-1","comma",
+ xmlResult.metadata.get(Metadata.CONTENT_TYPE));
}
private void assertContainsIgnoreWhiteSpaceDiffs(String expected, String xml) {
assertContains(expected, xml.replaceAll("[\r\n\t ]", " "));
}
+
+ private static void assertMediaTypeEquals(String csv, String charset, String delimiter, String mediaTypeString) {
+ if (mediaTypeString == null) {
+ fail("media type string must not be null");
+ }
+ MediaType expected = mediaType(csv, charset, delimiter);
+ MediaType observed = MediaType.parse(mediaTypeString);
+ assertEquals(expected, observed);
+ }
+
+ private static MediaType mediaType(String csv, String charset, String delimiter) {
+ Map<String, String> attrs = new HashMap<>();
+ attrs.put("charset", charset);
+ attrs.put("delimiter", delimiter);
+ return new MediaType(MediaType.text(csv), attrs);
+ }
+
+ private static byte[] concat(byte[] bytesA, byte[] bytesB) {
+ byte[] ret = new byte[bytesA.length+bytesB.length];
+ System.arraycopy(bytesA, 0, ret, 0, bytesA.length);
+ System.arraycopy(bytesB, 0, ret, bytesA.length, bytesB.length);
+ return ret;
+ }
+
}