You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/02/27 15:32:22 UTC
[2/2] tika git commit: TIKA-2278 clean up extract exception handling
TIKA-2278 clean up extract exception handling
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/a279d039
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/a279d039
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/a279d039
Branch: refs/heads/2.x
Commit: a279d039de234093828e0602468688060e425791
Parents: 35756b1
Author: tballison <ta...@mitre.org>
Authored: Mon Feb 27 10:32:09 2017 -0500
Committer: tballison <ta...@mitre.org>
Committed: Mon Feb 27 10:32:09 2017 -0500
----------------------------------------------------------------------
.../org/apache/tika/eval/AbstractProfiler.java | 72 ++++++++-------
.../org/apache/tika/eval/ExtractComparer.java | 97 ++++++++++----------
.../org/apache/tika/eval/ExtractProfiler.java | 46 +++++++---
.../apache/tika/eval/XMLErrorLogUpdater.java | 4 +-
.../tika/eval/batch/DBConsumersManager.java | 17 +++-
.../tika/eval/batch/EvalConsumerBuilder.java | 37 +++++---
.../tika/eval/batch/EvalConsumersBuilder.java | 14 ++-
.../tika/eval/batch/FileComparerBuilder.java | 39 +++-----
.../eval/batch/SingleFileConsumerBuilder.java | 34 ++-----
.../main/java/org/apache/tika/eval/db/Cols.java | 4 +-
.../java/org/apache/tika/eval/db/DBUtil.java | 3 +-
.../org/apache/tika/eval/db/MimeBuffer.java | 5 +
.../java/org/apache/tika/eval/io/DBWriter.java | 24 ++---
.../org/apache/tika/eval/io/ExtractReader.java | 64 +++++++++----
.../tika/eval/io/ExtractReaderException.java | 48 ++++++++++
.../src/main/resources/comparison-reports.xml | 24 ++---
.../resources/tika-eval-comparison-config.xml | 10 +-
.../resources/tika-eval-profiler-config.xml | 7 ++
.../test/java/org/apache/tika/MockDBWriter.java | 13 ++-
.../org/apache/tika/eval/ProfilerBatchTest.java | 23 ++---
.../apache/tika/eval/SimpleComparerTest.java | 15 +--
.../apache/tika/eval/io/ExtractReaderTest.java | 12 ++-
...ingle-file-profiler-crawl-extract-config.xml | 4 +
.../single-file-profiler-crawl-input-config.xml | 4 +
24 files changed, 378 insertions(+), 242 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java b/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
index daa964a..f792e77 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
@@ -47,6 +47,7 @@ import org.apache.tika.config.TikaConfig;
import org.apache.tika.eval.db.ColInfo;
import org.apache.tika.eval.db.Cols;
import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.ExtractReaderException;
import org.apache.tika.eval.io.IDBWriter;
import org.apache.tika.eval.tokens.AnalyzerManager;
import org.apache.tika.eval.tokens.CommonTokenCountManager;
@@ -77,9 +78,9 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
};
static final long NON_EXISTENT_FILE_LENGTH = -1l;
- public static TableInfo REF_EXTRACT_ERROR_TYPES = new TableInfo("ref_extract_error_types",
- new ColInfo(Cols.EXTRACT_ERROR_TYPE_ID, Types.INTEGER),
- new ColInfo(Cols.EXTRACT_ERROR_DESCRIPTION, Types.VARCHAR, 128)
+ public static TableInfo REF_EXTRACT_EXCEPTION_TYPES = new TableInfo("ref_extract_exception_types",
+ new ColInfo(Cols.EXTRACT_EXCEPTION_TYPE_ID, Types.INTEGER),
+ new ColInfo(Cols.EXTRACT_EXCEPTION_DESCRIPTION, Types.VARCHAR, 128)
);
@@ -97,10 +98,8 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
public static final String FALSE = Boolean.toString(false);
- protected static final AtomicInteger CONTAINER_ID = new AtomicInteger();
protected static final AtomicInteger ID = new AtomicInteger();
-
private final static String UNKNOWN_EXTENSION = "unk";
//make this configurable
private final static String DIGEST_KEY = "X-TIKA:digest:MD5";
@@ -111,13 +110,6 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
final AnalyzerManager analyzerManager;
final TokenCounter tokenCounter;
- public enum EXTRACT_ERROR_TYPE {
- //what do you see when you look at the extract file
- NO_EXTRACT_FILE,
- ZERO_BYTE_EXTRACT_FILE,
- EXTRACT_PARSE_EXCEPTION
- }
-
public enum EXCEPTION_TYPE {
RUNTIME,
ENCRYPTION,
@@ -125,8 +117,11 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
UNSUPPORTED_VERSION,
}
+ /**
+ * If information was gathered from the log file about
+ * a parse error
+ */
public enum PARSE_ERROR_TYPE {
- //what was gathered from the log file during the batch run
OOM,
TIMEOUT
}
@@ -181,29 +176,15 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
}
}
- protected void writeError(TableInfo extractErrorTable, String containerId,
- String filePath, Path extractsA) throws IOException {
+
+
+ protected void writeExtractException(TableInfo extractExceptionTable, String containerId,
+ String filePath, ExtractReaderException.TYPE type) throws IOException {
Map<Cols, String> data = new HashMap<>();
data.put(Cols.CONTAINER_ID, containerId);
data.put(Cols.FILE_PATH, filePath);
- int errorCode = -1;
- long len = -1;
- if (extractsA != null) {
- try {
- len = Files.size(extractsA);
- } catch (IOException e) {
- //swallow
- }
- }
- if (extractsA == null) {
- errorCode = EXTRACT_ERROR_TYPE.NO_EXTRACT_FILE.ordinal();
- } else if (len == 0) {
- errorCode = EXTRACT_ERROR_TYPE.ZERO_BYTE_EXTRACT_FILE.ordinal();
- } else {
- errorCode = EXTRACT_ERROR_TYPE.EXTRACT_PARSE_EXCEPTION.ordinal();
- }
- data.put(Cols.EXTRACT_ERROR_TYPE_ID, Integer.toString(errorCode));
- writer.writeRow(extractErrorTable, data);
+ data.put(Cols.EXTRACT_EXCEPTION_TYPE_ID, Integer.toString(type.ordinal()));
+ writer.writeRow(extractExceptionTable, data);
}
@@ -235,7 +216,7 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
data.put(Cols.FILE_NAME, fps.getRelativeSourceFilePath().getFileName().toString());
} else {
data.put(Cols.IS_EMBEDDED, TRUE);
- data.put(Cols.FILE_NAME, FilenameUtils.getName(m.get(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH)));
+ data.put(Cols.FILE_NAME, getFileName(m.get(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH)));
}
data.put(Cols.FILE_EXTENSION,
FilenameUtils.getExtension(fps.getRelativeSourceFilePath().getFileName().toString()));
@@ -266,6 +247,29 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
}
}
+ private static String getFileName(String path) {
+ if (path == null) {
+ return "";
+ }
+ //filenameUtils checks for a null byte in the path.
+ //it will throw an IllegalArgumentException if there is a null byte.
+ //given that we're recording names and not using them on a file path
+ //we should ignore this.
+ try {
+ return FilenameUtils.getName(path);
+ } catch (IllegalArgumentException e) {
+ logger.warn(e.getMessage() + " in "+path);
+ }
+ path = path.replaceAll("\u0000", " ");
+ try {
+ return FilenameUtils.getName(path);
+ } catch (IllegalArgumentException e) {
+ logger.warn("Again: " + e.getMessage() + " in "+path);
+ }
+ //give up
+ return "";
+ }
+
protected void writeExceptionData(String fileId, Metadata m, TableInfo exceptionTable) {
Map<Cols, String> data = new HashMap<>();
getExceptionStrings(m, data);
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java b/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
index 8b3d266..8e8cb97 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
@@ -37,6 +37,7 @@ import org.apache.tika.eval.db.ColInfo;
import org.apache.tika.eval.db.Cols;
import org.apache.tika.eval.db.TableInfo;
import org.apache.tika.eval.io.ExtractReader;
+import org.apache.tika.eval.io.ExtractReaderException;
import org.apache.tika.eval.io.IDBWriter;
import org.apache.tika.eval.tokens.ContrastStatistics;
import org.apache.tika.eval.tokens.TokenContraster;
@@ -74,8 +75,9 @@ public class ExtractComparer extends AbstractProfiler {
"for json-formatted extract files, " +
"process full metadata list ('as_is'=default), " +
"take just the first/container document ('first_only'), " +
- "concatenate all content into the first metadata item ('concatenate_content')")
- );
+ "concatenate all content into the first metadata item ('concatenate_content')"))
+ .addOption("minExtractLength", true, "minimum extract length to process (in bytes)")
+ .addOption("maxExtractLength", true, "maximum extract length to process (in bytes)");
}
public static void USAGE() {
@@ -85,7 +87,7 @@ public class ExtractComparer extends AbstractProfiler {
"java -jar tika-eval-x.y.jar Compare -extractsA extractsA -extractsB extractsB -db mydb",
"Tool: Compare",
ExtractComparer.OPTIONS,
- "Note: for h2 db, do not include the .mv.db at the end of the db name.");
+ "Note: do not include the .mv.db at the end of the db name.");
}
private final static String FIELD_A = "fa";
@@ -140,10 +142,10 @@ public class ExtractComparer extends AbstractProfiler {
public static TableInfo EXCEPTION_TABLE_B = new TableInfo ("exceptions_b",
ExtractProfiler.EXCEPTION_TABLE.getColInfos());
- public static TableInfo ERROR_TABLE_A = new TableInfo("extract_errors_a",
- ExtractProfiler.ERROR_TABLE.getColInfos());
- public static TableInfo ERROR_TABLE_B = new TableInfo("extract_errors_b",
- ExtractProfiler.ERROR_TABLE.getColInfos());
+ public static TableInfo EXTRACT_EXCEPTION_TABLE_A = new TableInfo("extract_exceptions_a",
+ ExtractProfiler.EXTRACT_EXCEPTION_TABLE.getColInfos());
+ public static TableInfo EXTRACT_EXCEPTION_TABLE_B = new TableInfo("extract_exceptions_b",
+ ExtractProfiler.EXTRACT_EXCEPTION_TABLE.getColInfos());
//need to parameterize?
@@ -153,8 +155,8 @@ public class ExtractComparer extends AbstractProfiler {
private final Path extractsA;
private final Path extractsB;
- private final long minJsonLength;
- private final long maxJsonLength;
+ private final long minExtractLength;
+ private final long maxExtractLength;
private final ExtractReader.ALTER_METADATA_LIST alterExtractList;
private final TokenContraster tokenContraster = new TokenContraster();
@@ -165,8 +167,8 @@ public class ExtractComparer extends AbstractProfiler {
IDBWriter writer, long minJsonLength,
long maxJsonLength, ExtractReader.ALTER_METADATA_LIST alterExtractList) {
super(queue, writer);
- this.minJsonLength = minJsonLength;
- this.maxJsonLength = maxJsonLength;
+ this.minExtractLength = minJsonLength;
+ this.maxExtractLength = maxJsonLength;
this.inputDir = inputDir;
this.extractsA = extractsA;
this.extractsB = extractsB;
@@ -190,23 +192,6 @@ public class ExtractComparer extends AbstractProfiler {
fpsB = getPathsFromSrcCrawl(metadata, inputDir, extractsB);
}
- if (minJsonLength > -1) {
- //if both files exist and are < minJsonLength, skip em
- if (fpsA.getExtractFileLength() > NON_EXISTENT_FILE_LENGTH
- && fpsA.getExtractFileLength() < minJsonLength
- && fpsB.getExtractFileLength() > NON_EXISTENT_FILE_LENGTH
- && fpsB.getExtractFileLength() < minJsonLength) {
- return false;
- }
- }
- if (maxJsonLength > -1) {
- if ((fpsA.getExtractFileLength() > maxJsonLength) ||
- (fpsB.getExtractFileLength() > maxJsonLength)) {
- return false;
- }
- }
-
-
try {
compareFiles(fpsA, fpsB);
} catch (Throwable e) {
@@ -219,17 +204,33 @@ public class ExtractComparer extends AbstractProfiler {
}
//protected for testing, should find better way so that this can be private!
- protected void compareFiles(EvalFilePaths fpsA, EvalFilePaths fpsB) throws IOException {
+ protected void compareFiles(EvalFilePaths fpsA,
+ EvalFilePaths fpsB) throws IOException {
+
+ ExtractReaderException.TYPE extractExceptionA = null;
+ ExtractReaderException.TYPE extractExceptionB = null;
- List<Metadata> metadataListA =
- extractReader.loadExtract(fpsA.getExtractFile(), alterExtractList);
- List<Metadata> metadataListB =
- extractReader.loadExtract(fpsB.getExtractFile(), alterExtractList);
+ List<Metadata> metadataListA = null;
+ if (extractExceptionA == null) {
+ try {
+ metadataListA = extractReader.loadExtract(fpsA.getExtractFile(),
+ alterExtractList, minExtractLength, maxExtractLength);
+ } catch (ExtractReaderException e) {
+ extractExceptionA = e.getType();
+ }
+ }
+
+ List<Metadata> metadataListB = null;
+ try {
+ metadataListB = extractReader.loadExtract(fpsB.getExtractFile(),
+ alterExtractList, minExtractLength, maxExtractLength);
+ } catch (ExtractReaderException e) {
+ extractExceptionB = e.getType();
+ }
- //array indices for those metadata items handled in
- //"that"
+ //array indices for those metadata items handled in B
Set<Integer> handledB = new HashSet<>();
- String containerID = Integer.toString(CONTAINER_ID.getAndIncrement());
+ String containerID = Integer.toString(ID.getAndIncrement());
//container table
Map<Cols, String> contData = new HashMap<>();
contData.put(Cols.CONTAINER_ID, containerID);
@@ -245,19 +246,19 @@ public class ExtractComparer extends AbstractProfiler {
contData.put(Cols.EXTRACT_FILE_LENGTH_A, extractFileLengthA > NON_EXISTENT_FILE_LENGTH ?
Long.toString(extractFileLengthA) : "");
- long extractFileLengthB = getFileLength(fpsA.getExtractFile());
+ long extractFileLengthB = getFileLength(fpsB.getExtractFile());
contData.put(Cols.EXTRACT_FILE_LENGTH_B, extractFileLengthB > NON_EXISTENT_FILE_LENGTH ?
Long.toString(extractFileLengthB) : "");
writer.writeRow(COMPARISON_CONTAINERS, contData);
- if (metadataListA == null) {
- writeError(ERROR_TABLE_A, containerID, fpsA.getRelativeSourceFilePath().toString(),
- fpsA.getExtractFile());
+ if (extractExceptionA != null) {
+ writeExtractException(EXTRACT_EXCEPTION_TABLE_A, containerID, fpsA.getRelativeSourceFilePath().toString(),
+ extractExceptionA);
}
- if (metadataListB == null) {
- writeError(ERROR_TABLE_B, containerID, fpsB.getRelativeSourceFilePath().toString(),
- fpsB.getExtractFile());
+ if (extractExceptionB != null) {
+ writeExtractException(EXTRACT_EXCEPTION_TABLE_B, containerID, fpsB.getRelativeSourceFilePath().toString(),
+ extractExceptionB);
}
if (metadataListA == null && metadataListB == null) {
@@ -269,7 +270,8 @@ public class ExtractComparer extends AbstractProfiler {
//now get that metadata
if (metadataListA != null) {
for (int i = 0; i < metadataListA.size(); i++) {
- String fileId = Integer.toString(ID.getAndIncrement());
+ //the first file should have the same id as the container id
+ String fileId = (i == 0) ? containerID : Integer.toString(ID.getAndIncrement());
Metadata metadataA = metadataListA.get(i);
Metadata metadataB = null;
//TODO: shouldn't be fileA!!!!
@@ -277,7 +279,7 @@ public class ExtractComparer extends AbstractProfiler {
writeExceptionData(fileId, metadataA, EXCEPTION_TABLE_A);
int matchIndex = getMatch(i, metadataListA, metadataListB);
- if (matchIndex > -1) {
+ if (matchIndex > -1 && ! handledB.contains(matchIndex)) {
metadataB = metadataListB.get(matchIndex);
handledB.add(matchIndex);
}
@@ -315,7 +317,7 @@ public class ExtractComparer extends AbstractProfiler {
}
}
}
- //now try to get any Metadata objects in "that"
+ //now try to get any Metadata objects in B
//that haven't yet been handled.
if (metadataListB != null) {
for (int i = 0; i < metadataListB.size(); i++) {
@@ -323,7 +325,8 @@ public class ExtractComparer extends AbstractProfiler {
continue;
}
Metadata metadataB = metadataListB.get(i);
- String fileId = Integer.toString(ID.getAndIncrement());
+ //the first file should have the same id as the container id
+ String fileId = (i == 0) ? containerID : Integer.toString(ID.getAndIncrement());
writeProfileData(fpsB, i, metadataB, fileId, containerID, numAttachmentsB, PROFILES_B);
writeEmbeddedFilePathData(i, fileId, null, metadataB);
writeExceptionData(fileId, metadataB, EXCEPTION_TABLE_B);
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java b/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
index f3b10ca..d3f58ef 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
@@ -33,6 +33,7 @@ import org.apache.tika.eval.db.ColInfo;
import org.apache.tika.eval.db.Cols;
import org.apache.tika.eval.db.TableInfo;
import org.apache.tika.eval.io.ExtractReader;
+import org.apache.tika.eval.io.ExtractReaderException;
import org.apache.tika.eval.io.IDBWriter;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.RecursiveParserWrapper;
@@ -63,7 +64,9 @@ public class ExtractProfiler extends AbstractProfiler {
"for json-formatted extract files, " +
"process full metadata list ('as_is'=default), " +
"take just the first/container document ('first_only'), " +
- "concatenate all content into the first metadata item ('concatenate_content')"));
+ "concatenate all content into the first metadata item ('concatenate_content')"))
+ .addOption("minExtractLength", true, "minimum extract length to process (in bytes)")
+ .addOption("maxExtractLength", true, "maximum extract length to process (in bytes)");
}
@@ -79,10 +82,10 @@ public class ExtractProfiler extends AbstractProfiler {
private final static String FIELD = "f";
- public static TableInfo ERROR_TABLE = new TableInfo("errors",
+ public static TableInfo EXTRACT_EXCEPTION_TABLE = new TableInfo("extract_exceptions",
new ColInfo(Cols.CONTAINER_ID, Types.INTEGER),
new ColInfo(Cols.FILE_PATH, Types.VARCHAR, FILE_PATH_MAX_LEN),
- new ColInfo(Cols.EXTRACT_ERROR_TYPE_ID, Types.INTEGER),
+ new ColInfo(Cols.EXTRACT_EXCEPTION_TYPE_ID, Types.INTEGER),
new ColInfo(Cols.PARSE_ERROR_TYPE_ID, Types.INTEGER)
);
@@ -144,15 +147,20 @@ public class ExtractProfiler extends AbstractProfiler {
private final Path inputDir;
private final Path extracts;
+ private final long minExtractLength;
+ private final long maxExtractLength;
private final ExtractReader.ALTER_METADATA_LIST alterExtractList;
private final ExtractReader extractReader = new ExtractReader();
public ExtractProfiler(ArrayBlockingQueue<FileResource> queue,
Path inputDir, Path extracts,
- IDBWriter dbWriter, ExtractReader.ALTER_METADATA_LIST alterExtractList) {
+ IDBWriter dbWriter, long minExtractLength,
+ long maxExtractLength, ExtractReader.ALTER_METADATA_LIST alterExtractList) {
super(queue, dbWriter);
this.inputDir = inputDir;
this.extracts = extracts;
+ this.minExtractLength = minExtractLength;
+ this.maxExtractLength = maxExtractLength;
this.alterExtractList = alterExtractList;
}
@@ -167,15 +175,25 @@ public class ExtractProfiler extends AbstractProfiler {
} else {
fps = getPathsFromSrcCrawl(metadata, inputDir, extracts);
}
- List<Metadata> metadataList = extractReader.loadExtract(fps.getExtractFile(), alterExtractList);
+ int containerId = ID.incrementAndGet();
+ String containerIdString = Integer.toString(containerId);
+
+ ExtractReaderException.TYPE extractExceptionType = null;
+
+ List<Metadata> metadataList = null;
+ try {
+ metadataList = extractReader.loadExtract(fps.getExtractFile(),
+ alterExtractList, minExtractLength, maxExtractLength);
+ } catch (ExtractReaderException e) {
+ extractExceptionType = e.getType();
+ }
Map<Cols, String> contOutput = new HashMap<>();
- String containerId = Integer.toString(CONTAINER_ID.incrementAndGet());
Long srcFileLen = getSourceFileLength(fps, metadataList);
contOutput.put(Cols.LENGTH,
srcFileLen > NON_EXISTENT_FILE_LENGTH ?
Long.toString(srcFileLen): "");
- contOutput.put(Cols.CONTAINER_ID, containerId);
+ contOutput.put(Cols.CONTAINER_ID, containerIdString);
contOutput.put(Cols.FILE_PATH, fps.getRelativeSourceFilePath().toString());
if (fps.getExtractFileLength() > 0) {
@@ -191,23 +209,22 @@ public class ExtractProfiler extends AbstractProfiler {
}
- if (metadataList == null) {
+ if (extractExceptionType != null) {
try {
- writeError(ERROR_TABLE, containerId,
- fps.getRelativeSourceFilePath().toString(), fps.getExtractFile());
+ writeExtractException(EXTRACT_EXCEPTION_TABLE, containerIdString,
+ fps.getRelativeSourceFilePath().toString(), extractExceptionType);
} catch (IOException e) {
throw new RuntimeException(e);
}
return true;
}
- //TODO: calculate num_attachments, add to profile table
-
List<Integer> numAttachments = countAttachments(metadataList);
int i = 0;
for (Metadata m : metadataList) {
- String fileId = Integer.toString(ID.incrementAndGet());
- writeProfileData(fps, i, m, fileId, containerId, numAttachments, PROFILE_TABLE);
+ //the first file should have the same id as the container id
+ String fileId = (i == 0) ? containerIdString : Integer.toString(ID.incrementAndGet());
+ writeProfileData(fps, i, m, fileId, containerIdString, numAttachments, PROFILE_TABLE);
writeEmbeddedPathData(i, fileId, m, EMBEDDED_FILE_PATH_TABLE);
writeExceptionData(fileId, m, EXCEPTION_TABLE);
try {
@@ -220,6 +237,7 @@ public class ExtractProfiler extends AbstractProfiler {
return true;
}
+
private void writeEmbeddedPathData(int i, String fileId, Metadata m,
TableInfo embeddedFilePathTable) {
if (i == 0) {
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java b/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java
index eaaf228..9f9bed5 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java
@@ -62,8 +62,8 @@ public class XMLErrorLogUpdater {
Path db = Paths.get(args[2]);
DBUtil dbUtil = new H2Util(db);
Connection connection = dbUtil.getConnection(true);
- writer.update(connection, ExtractComparer.ERROR_TABLE_A, xmlLogFileA);
- writer.update(connection, ExtractComparer.ERROR_TABLE_B, xmlLogFileB);
+ writer.update(connection, ExtractComparer.EXTRACT_EXCEPTION_TABLE_A, xmlLogFileA);
+ writer.update(connection, ExtractComparer.EXTRACT_EXCEPTION_TABLE_B, xmlLogFileB);
connection.commit();
connection.close();
}
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java
index 2c655cc..f353662 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java
@@ -28,18 +28,21 @@ import org.apache.tika.batch.FileResourceConsumer;
import org.apache.tika.eval.AbstractProfiler;
import org.apache.tika.eval.XMLErrorLogUpdater;
import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.MimeBuffer;
import org.apache.tika.eval.db.TableInfo;
public class DBConsumersManager extends ConsumersManager {
- private Connection conn;
- List<LogTablePair> errorLogs = new ArrayList<>();
+ private final Connection conn;
+ private final MimeBuffer mimeBuffer;
+ private final List<LogTablePair> errorLogs = new ArrayList<>();
- public DBConsumersManager(DBUtil dbUtil, List<FileResourceConsumer> consumers)
+ public DBConsumersManager(DBUtil dbUtil, MimeBuffer mimeBuffer, List<FileResourceConsumer> consumers)
throws IOException {
super(consumers);
this.conn = dbUtil.getConnection(true);
+ this.mimeBuffer = mimeBuffer;
}
@@ -55,6 +58,13 @@ public class DBConsumersManager extends ConsumersManager {
}
}
}
+
+ try {
+ mimeBuffer.close();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+
//MUST HAPPEN AFTER consumers have closed and
//committed container information!!!
XMLErrorLogUpdater up = new XMLErrorLogUpdater();
@@ -66,6 +76,7 @@ public class DBConsumersManager extends ConsumersManager {
}
}
+
try {
conn.commit();
} catch (SQLException e) {
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
index 8135887..d21d1de 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
@@ -18,6 +18,7 @@
package org.apache.tika.eval.batch;
import java.io.IOException;
+import java.sql.Connection;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.List;
@@ -30,8 +31,11 @@ import org.apache.tika.batch.FileResourceConsumer;
import org.apache.tika.eval.AbstractProfiler;
import org.apache.tika.eval.db.Cols;
import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.MimeBuffer;
import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.DBWriter;
import org.apache.tika.eval.io.ExtractReader;
+import org.apache.tika.eval.io.ExtractReaderException;
import org.apache.tika.eval.io.IDBWriter;
public abstract class EvalConsumerBuilder {
@@ -39,27 +43,29 @@ public abstract class EvalConsumerBuilder {
protected ArrayBlockingQueue<FileResource> queue;
Map<String, String> localAttrs;
DBUtil dbUtil;
+ private MimeBuffer mimeBuffer;
+ AtomicInteger initialized = new AtomicInteger(0);
public void init(ArrayBlockingQueue<FileResource> queue, Map<String, String> localAttrs,
- DBUtil dbUtil) {
+ DBUtil dbUtil, MimeBuffer mimeBuffer) throws IOException, SQLException {
this.queue = queue;
this.localAttrs = localAttrs;
this.dbUtil = dbUtil;
+ this.mimeBuffer = mimeBuffer;
+ populateRefTables();
+ if (initialized.getAndIncrement() > 0) {
+ throw new RuntimeException("Can only init a consumer builder once!");
+ }
}
public abstract FileResourceConsumer build() throws IOException, SQLException;
protected abstract List<TableInfo> getTableInfo();
- protected abstract IDBWriter getDBWriter() throws IOException, SQLException;
-
protected abstract void addErrorLogTablePairs(DBConsumersManager manager);
- public void populateRefTables(IDBWriter writer) throws IOException, SQLException {
- //figure out cleaner way of doing this!
- if (count.getAndIncrement() > 0) {
- return;
- }
+ public void populateRefTables() throws IOException, SQLException {
+ IDBWriter writer = getDBWriter();
Map<Cols, String> m = new HashMap<>();
for (AbstractProfiler.PARSE_ERROR_TYPE t : AbstractProfiler.PARSE_ERROR_TYPE.values()) {
m.clear();
@@ -75,14 +81,19 @@ public abstract class EvalConsumerBuilder {
writer.writeRow(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES, m);
}
- for (AbstractProfiler.EXTRACT_ERROR_TYPE t :
- AbstractProfiler.EXTRACT_ERROR_TYPE.values()) {
+ for (ExtractReaderException.TYPE t :
+ ExtractReaderException.TYPE.values()) {
m.clear();
- m.put(Cols.EXTRACT_ERROR_TYPE_ID, Integer.toString(t.ordinal()));
- m.put(Cols.EXTRACT_ERROR_DESCRIPTION, t.name());
- writer.writeRow(AbstractProfiler.REF_EXTRACT_ERROR_TYPES, m);
+ m.put(Cols.EXTRACT_EXCEPTION_TYPE_ID, Integer.toString(t.ordinal()));
+ m.put(Cols.EXTRACT_EXCEPTION_DESCRIPTION, t.name());
+ writer.writeRow(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES, m);
}
+ writer.close();
+ }
+ protected IDBWriter getDBWriter() throws IOException, SQLException {
+ Connection conn = dbUtil.getConnection(true);
+ return new DBWriter(conn, getTableInfo(), dbUtil, mimeBuffer);
}
ExtractReader.ALTER_METADATA_LIST getAlterMetadata(Map<String, String> localAttrs) {
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
index 6caad63..883f908 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
@@ -31,9 +31,11 @@ import org.apache.tika.batch.FileResource;
import org.apache.tika.batch.FileResourceConsumer;
import org.apache.tika.batch.builders.AbstractConsumersBuilder;
import org.apache.tika.batch.builders.BatchProcessBuilder;
+import org.apache.tika.config.TikaConfig;
import org.apache.tika.eval.AbstractProfiler;
import org.apache.tika.eval.db.DBUtil;
import org.apache.tika.eval.db.H2Util;
+import org.apache.tika.eval.db.MimeBuffer;
import org.apache.tika.eval.util.LanguageIDWrapper;
import org.apache.tika.util.ClassLoaderUtil;
import org.apache.tika.util.PropsUtil;
@@ -84,15 +86,17 @@ public class EvalConsumersBuilder extends AbstractConsumersBuilder {
if (consumerBuilder == null) {
throw new RuntimeException("Must specify consumerBuilderClass in config file");
}
- consumerBuilder.init(queue, localAttrs, util);
+
+ MimeBuffer mimeBuffer = null;
try {
util.createDB(consumerBuilder.getTableInfo(), append);
- } catch (SQLException e) {
- throw new RuntimeException(e);
- } catch (IOException e) {
+ mimeBuffer = new MimeBuffer(util.getConnection(true), TikaConfig.getDefaultConfig());
+ consumerBuilder.init(queue, localAttrs, util, mimeBuffer);
+ } catch (IOException | SQLException e) {
throw new RuntimeException(e);
}
+
for (int i = 0; i < numConsumers; i++) {
try {
consumers.add(consumerBuilder.build());
@@ -103,7 +107,7 @@ public class EvalConsumersBuilder extends AbstractConsumersBuilder {
DBConsumersManager manager;
try {
- manager = new DBConsumersManager(util, consumers);
+ manager = new DBConsumersManager(util, mimeBuffer, consumers);
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java
index cface16..0b35cb4 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java
@@ -24,18 +24,13 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.tika.batch.FileResourceConsumer;
-import org.apache.tika.config.TikaConfig;
import org.apache.tika.eval.AbstractProfiler;
import org.apache.tika.eval.ExtractComparer;
import org.apache.tika.eval.db.TableInfo;
-import org.apache.tika.eval.io.DBWriter;
import org.apache.tika.eval.io.ExtractReader;
-import org.apache.tika.eval.io.IDBWriter;
import org.apache.tika.util.PropsUtil;
public class FileComparerBuilder extends EvalConsumerBuilder {
- private final static String WHICH_DB = "h2";//TODO: allow flexibility
-
@Override
public FileResourceConsumer build() throws IOException, SQLException {
@@ -50,27 +45,19 @@ public class FileComparerBuilder extends EvalConsumerBuilder {
Path inputRootDir = PropsUtil.getPath(localAttrs.get("inputDir"), null);
- long minJsonLength = PropsUtil.getLong(localAttrs.get("minJsonFileSizeBytes"), -1L);
- long maxJsonLength = PropsUtil.getLong(localAttrs.get("maxJsonFileSizeBytes"), -1L);
+ long minExtractLength = PropsUtil.getLong(localAttrs.get("minExtractLength"), -1L);
+ long maxExtractLength = PropsUtil.getLong(localAttrs.get("maxExtractLength"), -1L);
ExtractReader.ALTER_METADATA_LIST alterExtractList = getAlterMetadata(localAttrs);
- IDBWriter writer = getDBWriter();
- //TODO: clean up the writing of the ref tables!!!
- try {
- populateRefTables(writer);
- } catch (SQLException e) {
- throw new RuntimeException("Can't populate ref tables", e);
- }
-
if (inputRootDir == null) {
//this is for the sake of the crawler
throw new RuntimeException("Must specify an -inputDir");
}
- return new ExtractComparer(queue, inputRootDir, thisRootDir, thatRootDir, writer,
- minJsonLength, maxJsonLength, alterExtractList);
+ return new ExtractComparer(queue, inputRootDir, thisRootDir, thatRootDir, getDBWriter(),
+ minExtractLength, maxExtractLength, alterExtractList);
}
@Override
@@ -79,12 +66,12 @@ public class FileComparerBuilder extends EvalConsumerBuilder {
tableInfos.add(ExtractComparer.COMPARISON_CONTAINERS);
tableInfos.add(ExtractComparer.PROFILES_A);
tableInfos.add(ExtractComparer.PROFILES_B);
- tableInfos.add(ExtractComparer.ERROR_TABLE_A);
- tableInfos.add(ExtractComparer.ERROR_TABLE_B);
+ tableInfos.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
+ tableInfos.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
tableInfos.add(ExtractComparer.EXCEPTION_TABLE_A);
tableInfos.add(ExtractComparer.EXCEPTION_TABLE_B);
- tableInfos.add(ExtractComparer.ERROR_TABLE_A);
- tableInfos.add(ExtractComparer.ERROR_TABLE_B);
+ tableInfos.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
+ tableInfos.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
tableInfos.add(ExtractComparer.CONTENTS_TABLE_A);
tableInfos.add(ExtractComparer.CONTENTS_TABLE_B);
tableInfos.add(ExtractComparer.EMBEDDED_FILE_PATH_TABLE_A);
@@ -95,14 +82,10 @@ public class FileComparerBuilder extends EvalConsumerBuilder {
tableInfos.add(ExtractComparer.REF_PAIR_NAMES);
tableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
tableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
- tableInfos.add(AbstractProfiler.REF_EXTRACT_ERROR_TYPES);
+ tableInfos.add(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES);
return tableInfos;
}
- @Override
- protected IDBWriter getDBWriter() throws IOException, SQLException {
- return new DBWriter(getTableInfo(), TikaConfig.getDefaultConfig(), dbUtil);
- }
@Override
protected void addErrorLogTablePairs(DBConsumersManager manager) {
@@ -110,12 +93,12 @@ public class FileComparerBuilder extends EvalConsumerBuilder {
if (errorLogA == null) {
return;
}
- manager.addErrorLogTablePair(errorLogA, ExtractComparer.ERROR_TABLE_A);
+ manager.addErrorLogTablePair(errorLogA, ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
Path errorLogB = PropsUtil.getPath(localAttrs.get("errorLogFileB"), null);
if (errorLogB == null) {
return;
}
- manager.addErrorLogTablePair(errorLogB, ExtractComparer.ERROR_TABLE_B);
+ manager.addErrorLogTablePair(errorLogB, ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
}
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java
index ec45f32..c1c25d3 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java
@@ -24,20 +24,17 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.tika.batch.FileResourceConsumer;
-import org.apache.tika.config.TikaConfig;
import org.apache.tika.eval.AbstractProfiler;
import org.apache.tika.eval.ExtractProfiler;
import org.apache.tika.eval.db.TableInfo;
-import org.apache.tika.eval.io.DBWriter;
import org.apache.tika.eval.io.ExtractReader;
-import org.apache.tika.eval.io.IDBWriter;
import org.apache.tika.util.PropsUtil;
public class SingleFileConsumerBuilder extends EvalConsumerBuilder {
@Override
- public FileResourceConsumer build() throws IOException {
+ public FileResourceConsumer build() throws IOException, SQLException {
Path extracts = PropsUtil.getPath(localAttrs.get("extracts"), null);
if (extracts == null) {
throw new RuntimeException("Must specify \"extracts\" -- directory to crawl");
@@ -49,21 +46,11 @@ public class SingleFileConsumerBuilder extends EvalConsumerBuilder {
Path inputDir = PropsUtil.getPath(localAttrs.get("inputDir"), null);
- ExtractReader.ALTER_METADATA_LIST alterExtractList = getAlterMetadata(localAttrs);
+ long minExtractLength = PropsUtil.getLong(localAttrs.get("minExtractLength"), -1L);
+ long maxExtractLength = PropsUtil.getLong(localAttrs.get("maxExtractLength"), -1L);
- IDBWriter writer = null;
- try {
- writer = getDBWriter();
- } catch (SQLException ex) {
- throw new IOException(ex);
- }
+ ExtractReader.ALTER_METADATA_LIST alterExtractList = getAlterMetadata(localAttrs);
- //TODO: clean up the writing of the ref tables!!!
- try {
- populateRefTables(writer);
- } catch (SQLException e) {
- throw new RuntimeException("Can't populate ref tables", e);
- }
//we _could_ set this to extracts (if not null)
//here, but the Crawler defaults to "input" if nothing is passed
//so this won't work
@@ -73,7 +60,8 @@ public class SingleFileConsumerBuilder extends EvalConsumerBuilder {
if (extracts == null && inputDir != null) {
extracts = inputDir;
}
- return new ExtractProfiler(queue, inputDir, extracts, writer, alterExtractList);
+ return new ExtractProfiler(queue, inputDir, extracts, getDBWriter(),
+ minExtractLength, maxExtractLength, alterExtractList);
}
@Override
@@ -82,20 +70,16 @@ public class SingleFileConsumerBuilder extends EvalConsumerBuilder {
tableInfos.add(AbstractProfiler.MIME_TABLE);
tableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
tableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
- tableInfos.add(AbstractProfiler.REF_EXTRACT_ERROR_TYPES);
+ tableInfos.add(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES);
tableInfos.add(ExtractProfiler.CONTAINER_TABLE);
tableInfos.add(ExtractProfiler.PROFILE_TABLE);
- tableInfos.add(ExtractProfiler.ERROR_TABLE);
+ tableInfos.add(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
tableInfos.add(ExtractProfiler.EXCEPTION_TABLE);
tableInfos.add(ExtractProfiler.CONTENTS_TABLE);
tableInfos.add(ExtractProfiler.EMBEDDED_FILE_PATH_TABLE);
return tableInfos;
}
- @Override
- protected IDBWriter getDBWriter() throws IOException, SQLException {
- return new DBWriter(getTableInfo(), TikaConfig.getDefaultConfig(), dbUtil);
- }
@Override
protected void addErrorLogTablePairs(DBConsumersManager manager) {
@@ -103,6 +87,6 @@ public class SingleFileConsumerBuilder extends EvalConsumerBuilder {
if (errorLog == null) {
return;
}
- manager.addErrorLogTablePair(errorLog, ExtractProfiler.ERROR_TABLE);
+ manager.addErrorLogTablePair(errorLog, ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
}
}
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/db/Cols.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/Cols.java b/tika-eval/src/main/java/org/apache/tika/eval/db/Cols.java
index 0dff9f7..bf8784b 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/db/Cols.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/Cols.java
@@ -72,8 +72,8 @@ public enum Cols {
PARSE_ERROR_DESCRIPTION,
PARSE_EXCEPTION_DESCRIPTION,
- EXTRACT_ERROR_TYPE_ID,
- EXTRACT_ERROR_DESCRIPTION,
+ EXTRACT_EXCEPTION_TYPE_ID,
+ EXTRACT_EXCEPTION_DESCRIPTION,
//exceptions
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java b/tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java
index d99798d..1735735 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java
@@ -101,6 +101,7 @@ public abstract class DBUtil {
return insertStatement.executeUpdate();
} catch (SQLException e) {
logger.warn("couldn't insert data for this row: "+e.getMessage());
+ e.printStackTrace();
return -1;
}
}
@@ -130,7 +131,7 @@ public abstract class DBUtil {
st.setDouble(dbColOffset, Float.parseFloat(value));
break;
case Types.INTEGER:
- st.setDouble(dbColOffset, Integer.parseInt(value));
+ st.setInt(dbColOffset, Integer.parseInt(value));
break;
case Types.BIGINT:
st.setLong(dbColOffset, Long.parseLong(value));
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java b/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java
index 11a72f8..56f26b7 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java
@@ -33,6 +33,8 @@ public class MimeBuffer extends AbstractDBBuffer {
private final PreparedStatement st;
private final TikaConfig config;
+ private final Connection connection;
+
public MimeBuffer(Connection connection, TikaConfig config) throws SQLException {
st = connection.prepareStatement("insert into " + AbstractProfiler.MIME_TABLE.getName() + "( " +
@@ -40,6 +42,7 @@ public class MimeBuffer extends AbstractDBBuffer {
Cols.MIME_STRING + ", " +
Cols.FILE_EXTENSION + ") values (?,?,?);");
this.config = config;
+ this.connection = connection;
}
@Override
@@ -68,6 +71,8 @@ public class MimeBuffer extends AbstractDBBuffer {
@Override
public void close() throws SQLException {
st.close();
+ connection.commit();
+ connection.close();
}
private static class MimeUtil {
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java b/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java
index 383f25c..14a43fd 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java
@@ -23,11 +23,11 @@ import java.sql.SQLException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.io.IOExceptionWithCause;
import org.apache.log4j.Logger;
-import org.apache.tika.config.TikaConfig;
import org.apache.tika.eval.db.ColInfo;
import org.apache.tika.eval.db.Cols;
import org.apache.tika.eval.db.DBUtil;
@@ -40,29 +40,30 @@ import org.apache.tika.eval.db.TableInfo;
* as necessary.
*
* Beware, this deletes the db file with each initialization.
+ *
+ * Each thread must construct its own DBWriter because each
+ * DBWriter creates its own PreparedStatements at initialization.
*/
public class DBWriter implements IDBWriter {
-
+ private static final AtomicInteger WRITER_ID = new AtomicInteger();
+
private static Logger logger = Logger.getLogger(DBWriter.class);
private final AtomicLong insertedRows = new AtomicLong();
private final Long commitEveryX = 1000L;
- private final List<TableInfo> tableInfos;
private final Connection conn;
private final DBUtil dbUtil;
- private static MimeBuffer mimeBuffer;
+ private final MimeBuffer mimeBuffer;
+ private final int myId = WRITER_ID.getAndIncrement();
//<tableName, preparedStatement>
private final Map<String, PreparedStatement> inserts = new HashMap<>();
- public DBWriter(List<TableInfo> tableInfos, TikaConfig tikaConfig, DBUtil dbUtil)
+ public DBWriter(Connection connection, List<TableInfo> tableInfos, DBUtil dbUtil, MimeBuffer mimeBuffer)
throws IOException, SQLException {
- this.conn = dbUtil.getConnection(true);
- if (mimeBuffer == null) {
- mimeBuffer = new MimeBuffer(conn, tikaConfig);
- }
- this.tableInfos = tableInfos;
+ this.conn = connection;
+ this.mimeBuffer = mimeBuffer;
this.dbUtil = dbUtil;
for (TableInfo tableInfo : tableInfos) {
try {
@@ -115,7 +116,7 @@ public class DBWriter implements IDBWriter {
dbUtil.insert(p, table, data);
long rows = insertedRows.incrementAndGet();
if (rows % commitEveryX == 0) {
- logger.info("writer is committing after "+ rows + " rows");
+ logger.debug("writer ("+myId+") is committing after "+ rows + " rows");
conn.commit();
}
} catch (SQLException e) {
@@ -125,7 +126,6 @@ public class DBWriter implements IDBWriter {
public void close() throws IOException {
try {
- mimeBuffer.close();
conn.commit();
} catch (SQLException e){
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java b/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java
index f703408..fad933f 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java
@@ -5,6 +5,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -44,6 +45,8 @@ import org.slf4j.LoggerFactory;
public class ExtractReader {
+ public static final long IGNORE_LENGTH = -1L;
+
public enum ALTER_METADATA_LIST {
AS_IS, //leave the metadata list as is
FIRST_ONLY, //take only the metadata list for the "container" document
@@ -52,34 +55,63 @@ public class ExtractReader {
private final static Logger LOGGER = LoggerFactory.getLogger(ExtractReader.class);
TikaConfig tikaConfig = TikaConfig.getDefaultConfig();
- public List<Metadata> loadExtract(Path thisFile, ALTER_METADATA_LIST alterExtractList) {
+ public List<Metadata> loadExtract(Path extractFile, ALTER_METADATA_LIST alterExtractList,
+ long minExtractLength, long maxExtractLength) throws ExtractReaderException {
+
List<Metadata> metadataList = null;
- if (thisFile == null || !Files.isRegularFile(thisFile)) {
- return metadataList;
+ if (extractFile == null || !Files.isRegularFile(extractFile)) {
+ throw new ExtractReaderException(ExtractReaderException.TYPE.NO_EXTRACT_FILE);
}
- Reader reader = null;
- InputStream is = null;
- FileSuffixes fileSuffixes = parseSuffixes(thisFile.getFileName().toString());
+
+ FileSuffixes fileSuffixes = parseSuffixes(extractFile.getFileName().toString());
if (fileSuffixes.txtOrJson == null) {
- LOGGER.warn("file must end with .txt or .json: "+thisFile.getFileName().toString());
- return metadataList;
+ throw new ExtractReaderException(ExtractReaderException.TYPE.INCORRECT_EXTRACT_FILE_SUFFIX);
+ }
+ if (! Files.isRegularFile(extractFile)) {
+ throw new ExtractReaderException(ExtractReaderException.TYPE.NO_EXTRACT_FILE);
+ }
+
+ long length = -1L;
+ try {
+ length = Files.size(extractFile);
+ } catch (IOException e) {
+ throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION);
+ }
+
+ if (length == 0L) {
+ throw new ExtractReaderException(ExtractReaderException.TYPE.ZERO_BYTE_EXTRACT_FILE);
+ }
+
+ if (minExtractLength > IGNORE_LENGTH && length < minExtractLength) {
+ throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_FILE_TOO_SHORT);
+ }
+ if (maxExtractLength > IGNORE_LENGTH && length > maxExtractLength) {
+ throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_FILE_TOO_LONG);
}
+ Reader reader = null;
+ InputStream is = null;
try {
- is = Files.newInputStream(thisFile);
+ is = Files.newInputStream(extractFile);
if (fileSuffixes.compression != null) {
if (fileSuffixes.compression.equals("bz2")) {
is = new BZip2CompressorInputStream(is);
- } else if (fileSuffixes.compression.equals("gz")) {
+ } else if (fileSuffixes.compression.equals("gz")
+ || fileSuffixes.compression.equals("gzip")) {
is = new GzipCompressorInputStream(is);
} else if (fileSuffixes.compression.equals("zip")) {
is = new ZCompressorInputStream(is);
} else {
- LOGGER.warn("Can't yet process compression of type: "+fileSuffixes.compression);
+ LOGGER.warn("Can't yet process compression of type: " + fileSuffixes.compression);
+ return metadataList;
}
}
- reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+ reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
+ } catch (IOException e) {
+ throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION);
+ }
+ try {
if (fileSuffixes.txtOrJson.equals("json")) {
metadataList = JsonMetadataList.fromJson(reader);
if (alterExtractList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) {
@@ -103,15 +135,13 @@ public class ExtractReader {
metadataList.remove(metadataList.size()-1);
}
}
-
-
} else {
metadataList = generateListFromTextFile(reader, fileSuffixes);
}
} catch (IOException e) {
- LOGGER.warn("couldn't open:" + thisFile.toAbsolutePath(), e);
+ throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION);
} catch (TikaException e) {
- LOGGER.warn("couldn't open:" + thisFile.toAbsolutePath(), e);
+ throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_PARSE_EXCEPTION);
} finally {
IOUtils.closeQuietly(reader);
IOUtils.closeQuietly(is);
@@ -120,7 +150,7 @@ public class ExtractReader {
}
private List<Metadata> generateListFromTextFile(Reader reader,
- FileSuffixes fileSuffixes) throws IOException {
+ FileSuffixes fileSuffixes) throws IOException {
List<Metadata> metadataList = new ArrayList<>();
String content = IOUtils.toString(reader);
Metadata m = new Metadata();
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReaderException.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReaderException.java b/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReaderException.java
new file mode 100644
index 0000000..5aa2f3f
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReaderException.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.io;
+
+import java.io.IOException;
+
+/**
+ * Exception when trying to read extract
+ */
+public class ExtractReaderException extends IOException {
+
+ public enum TYPE {
+ //what do you see when you look at the extract file
+ NO_EXTRACT_FILE,
+ ZERO_BYTE_EXTRACT_FILE,
+ IO_EXCEPTION,
+ EXTRACT_PARSE_EXCEPTION,
+ EXTRACT_FILE_TOO_SHORT,
+ EXTRACT_FILE_TOO_LONG,
+ INCORRECT_EXTRACT_FILE_SUFFIX;//extract file must have suffix of .json or .txt,
+ // optionally followed by gzip, zip or bz2
+ }
+
+ private final TYPE type;
+
+ public ExtractReaderException(TYPE exceptionType) {
+ this.type = exceptionType;
+ }
+
+ public TYPE getType() {
+ return type;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/resources/comparison-reports.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/comparison-reports.xml b/tika-eval/src/main/resources/comparison-reports.xml
index d69cb2a..b980d69 100644
--- a/tika-eval/src/main/resources/comparison-reports.xml
+++ b/tika-eval/src/main/resources/comparison-reports.xml
@@ -536,26 +536,26 @@
order by MIME_TYPE asc, COUNT desc
</sql>
</report>
- <report reportName="extractErrorsA"
- reportFilename="exceptions/extract_errors_a.xlsx"
+ <report reportName="extractExceptionsA"
+ reportFilename="exceptions/extract_exceptions_a.xlsx"
format="xlsx"
includeSql="true">
<sql>
- select file_path, extract_error_description
- from extract_errors_a e
- join ref_extract_error_types t
- on e.extract_error_type_id=t.extract_error_type_id
+ select file_path, extract_exception_description
+ from extract_exceptions_a e
+ join ref_extract_exception_types t
+ on e.extract_exception_type_id=t.extract_exception_type_id
</sql>
</report>
- <report reportName="extractErrorsB"
- reportFilename="exceptions/extract_errors_b.xlsx"
+ <report reportName="extractExceptionsB"
+ reportFilename="exceptions/extract_exceptions_b.xlsx"
format="xlsx"
includeSql="true">
<sql>
- select file_path, extract_error_description
- from extract_errors_b e
- join ref_extract_error_types t
- on e.extract_error_type_id=t.extract_error_type_id
+ select file_path, extract_exception_description
+ from extract_exceptions_b e
+ join ref_extract_exception_types t
+ on e.extract_exception_type_id=t.extract_exception_type_id
</sql>
</report>
<report reportName="parseExceptionTypesA"
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/resources/tika-eval-comparison-config.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/tika-eval-comparison-config.xml b/tika-eval/src/main/resources/tika-eval-comparison-config.xml
index 8070672..7f13414 100644
--- a/tika-eval/src/main/resources/tika-eval-comparison-config.xml
+++ b/tika-eval/src/main/resources/tika-eval-comparison-config.xml
@@ -45,6 +45,11 @@
process full metadata list ('as_is'=default),
take just the first/container document ('first_only'),
concatenate all content into the first metadata item ('concatenate_content')"/>
+ <option opt="minExtractLength" hasArg="true"
+ description="minimum extract length to process"/>
+ <option opt="maxExtractLength" hasArg="true"
+ description="maximum extract length to process"/>
+
</commandline>
@@ -59,15 +64,14 @@
maxFilesToConsider="-1"
includeFilePat=""
excludeFilePat=""
- maxFileSizeBytes="10000000"
/>
<consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
consumerBuilderClass="org.apache.tika.eval.batch.FileComparerBuilder"
dbAppend="false"
crawlingInputDir="false"
- minJsonFileSizeBytes="-1"
- maxJsonFileSizeBytes="2000000"
+ minExtractLength="-1"
+ maxExtractLength="2000000"
commonTokens="resources/common_tokens"
/>
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/main/resources/tika-eval-profiler-config.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/tika-eval-profiler-config.xml b/tika-eval/src/main/resources/tika-eval-profiler-config.xml
index be7adf4..1425d24 100644
--- a/tika-eval/src/main/resources/tika-eval-profiler-config.xml
+++ b/tika-eval/src/main/resources/tika-eval-profiler-config.xml
@@ -42,6 +42,11 @@
process full metadata list ('as_is'=default),
take just the first/container document ('first_only'),
concatenate all content into the first metadata item ('concatenate_content')"/>
+ <option opt="minExtractLength" hasArg="true"
+ description="minimum extract length to process"/>
+ <option opt="maxExtractLength" hasArg="true"
+ description="maximum extract length to process"/>
+
</commandline>
@@ -63,6 +68,8 @@
<consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"
+ minExtractLength="-1"
+ maxExtractLength="2000000"
commonTokens="resources/common_tokens"/>
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/test/java/org/apache/tika/MockDBWriter.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/MockDBWriter.java b/tika-eval/src/test/java/org/apache/tika/MockDBWriter.java
index b2edab7..953f4d6 100644
--- a/tika-eval/src/test/java/org/apache/tika/MockDBWriter.java
+++ b/tika-eval/src/test/java/org/apache/tika/MockDBWriter.java
@@ -27,12 +27,15 @@ import org.apache.tika.eval.db.Cols;
import org.apache.tika.eval.db.TableInfo;
import org.apache.tika.eval.io.IDBWriter;
+
public class MockDBWriter implements IDBWriter {
//Map of tableName and tables
//each table consists of a list of rows.
//Each row consists of a map of columns/values
Map<String, List<Map<Cols, String>>> db = new HashMap<String, List<Map<Cols, String>>>();
+ public Map<String, Integer> mimes = new HashMap<>();
+
public MockDBWriter() throws Exception {
}
@@ -53,8 +56,14 @@ public class MockDBWriter implements IDBWriter {
@Override
public int getMimeId(String mimeString) {
- //TODO: fix this
- return 0;
+ Integer val = mimes.get(mimeString);
+ if (val != null) {
+ return val;
+ }
+
+ int id = mimes.size();
+ mimes.put(mimeString, id);
+ return id;
}
public List<Map<Cols, String>> getTable(TableInfo tableInfo) {
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java b/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
index c3f0a7e..8d95622 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
@@ -38,6 +38,7 @@ import java.util.Map;
import org.apache.tika.eval.db.Cols;
import org.apache.tika.eval.db.H2Util;
import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.ExtractReaderException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -116,7 +117,7 @@ public class ProfilerBatchTest {
debugTable(ExtractProfiler.PROFILE_TABLE);
debugTable(ExtractProfiler.CONTENTS_TABLE);
debugTable(ExtractProfiler.EXCEPTION_TABLE);
- debugTable(ExtractProfiler.ERROR_TABLE);
+ debugTable(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
assertEquals(10, fNameList.size());
assertTrue("file1.pdf", fNameList.contains("file1.pdf"));
assertTrue("file2_attachANotB.doc", fNameList.contains("file2_attachANotB.doc"));
@@ -127,7 +128,7 @@ public class ProfilerBatchTest {
@Test
public void testExtractErrors() throws Exception {
- String sql = "select EXTRACT_ERROR_TYPE_ID from errors e" +
+ String sql = "select EXTRACT_EXCEPTION_TYPE_ID from extract_exceptions e" +
" join containers c on c.container_id = e.container_id "+
" where c.file_path='file9_noextract.txt'";
@@ -137,15 +138,15 @@ public class ProfilerBatchTest {
debugTable(ExtractProfiler.PROFILE_TABLE);
debugTable(ExtractProfiler.CONTENTS_TABLE);
debugTable(ExtractProfiler.EXCEPTION_TABLE);
- debugTable(ExtractProfiler.ERROR_TABLE);
+ debugTable(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
- sql = "select EXTRACT_ERROR_TYPE_ID from errors e" +
+ sql = "select EXTRACT_EXCEPTION_TYPE_ID from errors e" +
" join containers c on c.container_id = e.container_id "+
" where c.file_path='file5_emptyA.pdf'";
assertEquals("empty extract: file5_emptyA.pdf", "1",
getSingleResult(sql));
- sql = "select EXTRACT_ERROR_TYPE_ID from errors e" +
+ sql = "select EXTRACT_EXCEPTION_TYPE_ID from errors e" +
" join containers c on c.container_id = e.container_id "+
" where c.file_path='file7_badJson.pdf'";
assertEquals("extract error:file7_badJson.pdf", "2",
@@ -155,18 +156,18 @@ public class ProfilerBatchTest {
@Test
public void testParseErrors() throws Exception {
- debugTable(ExtractProfiler.ERROR_TABLE);
+ debugTable(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
String sql = "select file_path from errors where container_id is null";
assertEquals("file10_permahang.txt",
getSingleResult(sql));
- sql = "select extract_error_type_id from errors where file_path='file11_oom.txt'";
- assertEquals(Integer.toString(AbstractProfiler.
- EXTRACT_ERROR_TYPE.
- ZERO_BYTE_EXTRACT_FILE.ordinal()),
+ sql = "select extract_error_type_id from extract_exceptions " +
+ "where file_path='file11_oom.txt'";
+ assertEquals(Integer.toString(
+ ExtractReaderException.TYPE.ZERO_BYTE_EXTRACT_FILE.ordinal()),
getSingleResult(sql));
- sql = "select parse_error_type_id from errors where file_path='file11_oom.txt'";
+ sql = "select parse_error_type_id from extract_exceptions where file_path='file11_oom.txt'";
assertEquals(Integer.toString(AbstractProfiler.
PARSE_ERROR_TYPE.
OOM.ordinal()),
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java b/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
index 6d4d4ef..e718b33 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
@@ -17,8 +17,8 @@
package org.apache.tika.eval;
import static org.apache.tika.eval.AbstractProfiler.EXCEPTION_TYPE;
-import static org.apache.tika.eval.AbstractProfiler.EXTRACT_ERROR_TYPE;
import static org.apache.tika.eval.AbstractProfiler.getContent;
+import static org.apache.tika.eval.io.ExtractReader.IGNORE_LENGTH;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -36,6 +36,7 @@ import org.apache.tika.TikaTest;
import org.apache.tika.eval.db.Cols;
import org.apache.tika.eval.db.TableInfo;
import org.apache.tika.eval.io.ExtractReader;
+import org.apache.tika.eval.io.ExtractReaderException;
import org.apache.tika.eval.util.LanguageIDWrapper;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.RecursiveParserWrapper;
@@ -57,7 +58,7 @@ public class SimpleComparerTest extends TikaTest {
writer = new MockDBWriter();
comparer = new ExtractComparer(null, null,
Paths.get("extractsA"), Paths.get("extractsB"),
- writer, -1, -1,
+ writer, IGNORE_LENGTH, IGNORE_LENGTH,
ExtractReader.ALTER_METADATA_LIST.AS_IS);
AbstractProfiler.loadCommonTokens(this.getResourceAsFile("/common_tokens").toPath());
LanguageIDWrapper.loadBuiltInModels();
@@ -146,11 +147,11 @@ public class SimpleComparerTest extends TikaTest {
getResourceAsFile("/test-dirs/extractsB/file4_emptyB.pdf.json").toPath()
);
comparer.compareFiles(fpsA, fpsB);
- List<Map<Cols, String>> table = writer.getTable(ExtractComparer.ERROR_TABLE_B);
+ List<Map<Cols, String>> table = writer.getTable(ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
Map<Cols, String> row = table.get(0);
//debugPrintRow(row);
- assertEquals(Integer.toString(EXTRACT_ERROR_TYPE.ZERO_BYTE_EXTRACT_FILE.ordinal()),
- row.get(Cols.EXTRACT_ERROR_TYPE_ID));
+ assertEquals(Integer.toString(ExtractReaderException.TYPE.ZERO_BYTE_EXTRACT_FILE.ordinal()),
+ row.get(Cols.EXTRACT_EXCEPTION_TYPE_ID));
}
@@ -250,8 +251,8 @@ public class SimpleComparerTest extends TikaTest {
comparer.compareFiles(fpsA, fpsB);
for (TableInfo t : new TableInfo[]{
ExtractComparer.COMPARISON_CONTAINERS,
- ExtractComparer.ERROR_TABLE_A,
- ExtractComparer.ERROR_TABLE_B,
+ ExtractComparer.EXTRACT_EXCEPTION_TABLE_A,
+ ExtractComparer.EXTRACT_EXCEPTION_TABLE_B,
ExtractComparer.EXCEPTION_TABLE_A,
ExtractComparer.EXCEPTION_TABLE_B,
ExtractComparer.PROFILES_A,
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java b/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
index 14c0013..a49b24c 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
@@ -45,20 +45,23 @@ public class ExtractReaderTest extends TikaTest {
ExtractReader extractReader = new ExtractReader();
List<Metadata> metadataList = extractReader.loadExtract(testJsonFile,
- ExtractReader.ALTER_METADATA_LIST.AS_IS);
+ ExtractReader.ALTER_METADATA_LIST.AS_IS,
+ ExtractReader.IGNORE_LENGTH, ExtractReader.IGNORE_LENGTH);
assertEquals(2, metadataList.size());
assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
assertEquals(1, metadataList.get(1).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
assertContains("attachment", metadataList.get(1).get(RecursiveParserWrapper.TIKA_CONTENT));
- metadataList = extractReader.loadExtract(testJsonFile, ExtractReader.ALTER_METADATA_LIST.FIRST_ONLY);
+ metadataList = extractReader.loadExtract(testJsonFile, ExtractReader.ALTER_METADATA_LIST.FIRST_ONLY,
+ ExtractReader.IGNORE_LENGTH, ExtractReader.IGNORE_LENGTH);
assertEquals(1, metadataList.size());
assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
assertNotContained("attachment", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
- metadataList = extractReader.loadExtract(testJsonFile, ExtractReader.ALTER_METADATA_LIST.CONCATENATE_CONTENT_INTO_FIRST);
+ metadataList = extractReader.loadExtract(testJsonFile, ExtractReader.ALTER_METADATA_LIST.CONCATENATE_CONTENT_INTO_FIRST,
+ ExtractReader.IGNORE_LENGTH, ExtractReader.IGNORE_LENGTH);
assertEquals(1, metadataList.size());
assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
@@ -69,7 +72,8 @@ public class ExtractReaderTest extends TikaTest {
public void testTextBasic() throws IOException {
ExtractReader extractReader = new ExtractReader();
List<Metadata> metadataList = extractReader.loadExtract(testTxtFile,
- ExtractReader.ALTER_METADATA_LIST.AS_IS);
+ ExtractReader.ALTER_METADATA_LIST.AS_IS,
+ ExtractReader.IGNORE_LENGTH, ExtractReader.IGNORE_LENGTH);
assertEquals(1, metadataList.size());
Metadata m = metadataList.get(0);
assertEquals(1, m.getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml b/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
index 69e4262..aa11d50 100644
--- a/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
+++ b/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
@@ -40,6 +40,10 @@
description="this dir contains the files containing extracted metadata/content" required="false"/>
<option opt="db" hasArg="true"
description="name of db directory or file to which to write results"/>
+ <option opt="minExtractLength" hasArg="true"
+ description="minimum extract length to process"/>
+ <option opt="maxExtractLength" hasArg="true"
+ description="maximum extract length to process"/>
</commandline>
http://git-wip-us.apache.org/repos/asf/tika/blob/a279d039/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml b/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
index 1e37b52..62c7911 100644
--- a/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
+++ b/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
@@ -40,6 +40,10 @@
description="this dir contains the files containing extracted metadata/content" required="false"/>
<option opt="db" hasArg="true"
description="name of db directory or file to which to write results"/>
+ <option opt="minExtractLength" hasArg="true"
+ description="minimum extract length to process"/>
+ <option opt="maxExtractLength" hasArg="true"
+ description="maximum extract length to process"/>
</commandline>