You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/03/03 13:41:52 UTC
[tika] branch master updated: TIKA-2287 -- add general jdbc.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/master by this push:
new 24fec4d TIKA-2287 -- add general jdbc.
24fec4d is described below
commit 24fec4d4537f283ac33fccc1d0b5d7d44f4a78d7
Author: tballison <ta...@mitre.org>
AuthorDate: Fri Mar 3 08:41:43 2017 -0500
TIKA-2287 -- add general jdbc.
---
.../java/org/apache/tika/eval/ExtractComparer.java | 41 ++---
.../java/org/apache/tika/eval/ExtractProfiler.java | 34 ++--
.../java/org/apache/tika/eval/TikaEvalCLI.java | 21 ++-
.../org/apache/tika/eval/XMLErrorLogUpdater.java | 6 +-
.../apache/tika/eval/batch/DBConsumersManager.java | 12 +-
.../tika/eval/batch/EvalConsumerBuilder.java | 32 +++-
.../tika/eval/batch/EvalConsumersBuilder.java | 37 +++--
.../tika/eval/batch/ExtractComparerBuilder.java | 150 ++++++++++++++++++
...merBuilder.java => ExtractProfilerBuilder.java} | 52 +++++--
.../tika/eval/batch/FileComparerBuilder.java | 105 -------------
.../main/java/org/apache/tika/eval/db/H2Util.java | 26 +++-
.../tika/eval/db/{DBUtil.java => JDBCUtil.java} | 171 +++++++++++++++------
.../java/org/apache/tika/eval/db/MimeBuffer.java | 4 +-
.../java/org/apache/tika/eval/db/TableInfo.java | 11 +-
.../java/org/apache/tika/eval/io/DBWriter.java | 6 +-
.../org/apache/tika/eval/io/ExtractReader.java | 31 +++-
.../apache/tika/eval/reports/ResultsReporter.java | 57 ++++---
.../tika/eval/reports/XLSXHREFFormatter.java | 4 +-
tika-eval/src/main/resources/db.properties | 9 ++
.../main/resources/tika-eval-comparison-config.xml | 12 +-
.../main/resources/tika-eval-profiler-config.xml | 12 +-
.../org/apache/tika/eval/ProfilerBatchTest.java | 2 +-
.../org/apache/tika/eval/SimpleComparerTest.java | 5 +-
.../java/org/apache/tika/eval/TikaEvalCLITest.java | 38 ++++-
.../org/apache/tika/eval/io/ExtractReaderTest.java | 17 +-
.../tika/eval/reports/ResultsReporterTest.java | 2 +-
.../single-file-profiler-crawl-extract-config.xml | 2 +-
.../single-file-profiler-crawl-input-config.xml | 2 +-
28 files changed, 593 insertions(+), 308 deletions(-)
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java b/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
index 8e8cb97..a50b710 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
@@ -55,19 +55,14 @@ public class ExtractComparer extends AbstractProfiler {
Option extractsB = new Option("extractsB", true, "directory for extractsB files");
extractsB.setRequired(true);
- Option db = new Option("db", true, "db file to which to write results");
- db.setRequired(true);
-
Option inputDir = new Option("inputDir", true,
"optional: directory of original binary input files if it exists " +
"or can be the same as -extractsA or -extractsB. If not specified, -inputDir=-extractsA");
- inputDir.setRequired(true);
OPTIONS = new Options()
.addOption(extractsA)
.addOption(extractsB)
- .addOption(db)
.addOption(inputDir)
.addOption("bc", "optional: tika-batch config file")
.addOption("numConsumers", true, "optional: number of consumer threads")
@@ -77,7 +72,15 @@ public class ExtractComparer extends AbstractProfiler {
"take just the first/container document ('first_only'), " +
"concatenate all content into the first metadata item ('concatenate_content')"))
.addOption("minExtractLength", true, "minimum extract length to process (in bytes)")
- .addOption("maxExtractLength", true, "maximum extract length to process (in bytes)");
+ .addOption("maxExtractLength", true, "maximum extract length to process (in bytes)")
+ .addOption("db", true, "db file to which to write results")
+ .addOption("jdbc", true, "EXPERT: full jdbc connection string. Must specify this or -db <h2db>")
+ .addOption("jdbcDriver", true, "EXPERT: jdbc driver, or specify via -Djdbc.driver")
+ .addOption("tablePrefixA", true, "EXPERT: optional prefix for table names for A")
+ .addOption("tablePrefixB", true, "EXPERT: optional prefix for table names for B")
+ .addOption("drop", true, "drop tables if they exist")
+ .addOption("maxFilesToAdd", true, "maximum number of files to add to the crawler")
+ ;
}
public static void USAGE() {
@@ -87,7 +90,7 @@ public class ExtractComparer extends AbstractProfiler {
"java -jar tika-eval-x.y.jar Compare -extractsA extractsA -extractsB extractsB -db mydb",
"Tool: Compare",
ExtractComparer.OPTIONS,
- "Note: do not include the .mv.db at the end of the db name.");
+ "Note: for the default h2 db, do not include the .mv.db at the end of the db name.");
}
private final static String FIELD_A = "fa";
@@ -155,24 +158,17 @@ public class ExtractComparer extends AbstractProfiler {
private final Path extractsA;
private final Path extractsB;
- private final long minExtractLength;
- private final long maxExtractLength;
- private final ExtractReader.ALTER_METADATA_LIST alterExtractList;
-
private final TokenContraster tokenContraster = new TokenContraster();
- private final ExtractReader extractReader = new ExtractReader();
+ private final ExtractReader extractReader;
public ExtractComparer(ArrayBlockingQueue<FileResource> queue,
- Path inputDir, Path extractsA, Path extractsB,
- IDBWriter writer, long minJsonLength,
- long maxJsonLength, ExtractReader.ALTER_METADATA_LIST alterExtractList) {
+ Path inputDir, Path extractsA, Path extractsB, ExtractReader extractReader,
+ IDBWriter writer) {
super(queue, writer);
- this.minExtractLength = minJsonLength;
- this.maxExtractLength = maxJsonLength;
this.inputDir = inputDir;
this.extractsA = extractsA;
this.extractsB = extractsB;
- this.alterExtractList = alterExtractList;
+ this.extractReader = extractReader;
}
@Override
@@ -213,8 +209,7 @@ public class ExtractComparer extends AbstractProfiler {
List<Metadata> metadataListA = null;
if (extractExceptionA == null) {
try {
- metadataListA = extractReader.loadExtract(fpsA.getExtractFile(),
- alterExtractList, minExtractLength, maxExtractLength);
+ metadataListA = extractReader.loadExtract(fpsA.getExtractFile());
} catch (ExtractReaderException e) {
extractExceptionA = e.getType();
}
@@ -222,8 +217,7 @@ public class ExtractComparer extends AbstractProfiler {
List<Metadata> metadataListB = null;
try {
- metadataListB = extractReader.loadExtract(fpsB.getExtractFile(),
- alterExtractList, minExtractLength, maxExtractLength);
+ metadataListB = extractReader.loadExtract(fpsB.getExtractFile());
} catch (ExtractReaderException e) {
extractExceptionB = e.getType();
}
@@ -430,9 +424,6 @@ public class ExtractComparer extends AbstractProfiler {
return -1;
}
-
-
-
private void writeContrasts(Map<Cols, String> data, ContrastStatistics contrastStatistics) {
writeContrastString(data, Cols.TOP_10_MORE_IN_A, contrastStatistics.getTopNMoreA());
writeContrastString(data, Cols.TOP_10_MORE_IN_B, contrastStatistics.getTopNMoreB());
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java b/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
index d3f58ef..1f9bfda 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
@@ -42,9 +42,6 @@ public class ExtractProfiler extends AbstractProfiler {
static Options OPTIONS;
static {
- Option db = new Option("db", true, "db file to which to write results");
- db.setRequired(true);
-
//By the time this commandline is parsed, there should be both an extracts and an inputDir
Option extracts = new Option("extracts", true, "directory for extract files");
extracts.setRequired(true);
@@ -52,10 +49,8 @@ public class ExtractProfiler extends AbstractProfiler {
Option inputDir = new Option("inputDir", true,
"optional: directory for original binary input documents."+
" If not specified, -extracts is crawled as is.");
- inputDir.setRequired(true);
OPTIONS = new Options()
- .addOption(db)
.addOption(extracts)
.addOption(inputDir)
.addOption("bc", "optional: tika-batch config file")
@@ -66,7 +61,15 @@ public class ExtractProfiler extends AbstractProfiler {
"take just the first/container document ('first_only'), " +
"concatenate all content into the first metadata item ('concatenate_content')"))
.addOption("minExtractLength", true, "minimum extract length to process (in bytes)")
- .addOption("maxExtractLength", true, "maximum extract length to process (in bytes)");
+ .addOption("maxExtractLength", true, "maximum extract length to process (in bytes)")
+ .addOption("db", true, "db file to which to write results")
+ .addOption("jdbc", true, "EXPERT: full jdbc connection string. Must specify this or -db <h2db>")
+ .addOption("jdbcDriver", true, "EXPERT: jdbc driver, or specify via -Djdbc.driver")
+ .addOption("tablePrefix", true, "EXPERT: optional prefix for table names")
+ .addOption("drop", true, "drop tables if they exist")
+ .addOption("maxFilesToAdd", true, "maximum number of files to add to the crawler")
+
+ ;
}
@@ -77,7 +80,7 @@ public class ExtractProfiler extends AbstractProfiler {
"java -jar tika-eval-x.y.jar Profile -extracts extracts -db mydb [-inputDir input]",
"Tool: Profile",
ExtractProfiler.OPTIONS,
- "Note: for h2 db, do not include the .mv.db at the end of the db name.");
+ "Note: for the default h2 db, do not include the .mv.db at the end of the db name.");
}
private final static String FIELD = "f";
@@ -132,8 +135,8 @@ public class ExtractProfiler extends AbstractProfiler {
new ColInfo(Cols.NUM_UNIQUE_TOKENS, Types.INTEGER),
new ColInfo(Cols.COMMON_TOKENS_LANG, Types.VARCHAR, 12),
new ColInfo(Cols.NUM_COMMON_TOKENS, Types.INTEGER),
- new ColInfo(Cols.TOP_N_TOKENS, Types.VARCHAR, 1024),
new ColInfo(Cols.NUM_ALPHABETIC_TOKENS, Types.INTEGER),
+ new ColInfo(Cols.TOP_N_TOKENS, Types.VARCHAR, 1024),
new ColInfo(Cols.LANG_ID_1, Types.VARCHAR, 12),
new ColInfo(Cols.LANG_ID_PROB_1, Types.FLOAT),
new ColInfo(Cols.LANG_ID_2, Types.VARCHAR, 12),
@@ -147,21 +150,15 @@ public class ExtractProfiler extends AbstractProfiler {
private final Path inputDir;
private final Path extracts;
- private final long minExtractLength;
- private final long maxExtractLength;
- private final ExtractReader.ALTER_METADATA_LIST alterExtractList;
- private final ExtractReader extractReader = new ExtractReader();
+ private final ExtractReader extractReader;
public ExtractProfiler(ArrayBlockingQueue<FileResource> queue,
Path inputDir, Path extracts,
- IDBWriter dbWriter, long minExtractLength,
- long maxExtractLength, ExtractReader.ALTER_METADATA_LIST alterExtractList) {
+ ExtractReader extractReader, IDBWriter dbWriter) {
super(queue, dbWriter);
this.inputDir = inputDir;
this.extracts = extracts;
- this.minExtractLength = minExtractLength;
- this.maxExtractLength = maxExtractLength;
- this.alterExtractList = alterExtractList;
+ this.extractReader = extractReader;
}
@Override
@@ -182,8 +179,7 @@ public class ExtractProfiler extends AbstractProfiler {
List<Metadata> metadataList = null;
try {
- metadataList = extractReader.loadExtract(fps.getExtractFile(),
- alterExtractList, minExtractLength, maxExtractLength);
+ metadataList = extractReader.loadExtract(fps.getExtractFile());
} catch (ExtractReaderException e) {
extractExceptionType = e.getType();
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/TikaEvalCLI.java b/tika-eval/src/main/java/org/apache/tika/eval/TikaEvalCLI.java
index 2fd7f72..fc3f22b 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/TikaEvalCLI.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/TikaEvalCLI.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.ParseException;
import org.apache.tika.batch.fs.FSBatchProcessCLI;
@@ -103,7 +104,8 @@ public class TikaEvalCLI {
i++;
} else if (arg.equals("-alterExtract")) {
if (i+1 >= argList.size()) {
- System.err.println("Must specify directory after -extractsB");
+ System.err.println("Must specify type 'as_is', 'first_only' or " +
+ "'concatenate_content' after -alterExtract");
ExtractComparer.USAGE();
return;
}
@@ -147,7 +149,12 @@ public class TikaEvalCLI {
String[] updatedArgs = argList.toArray(new String[argList.size()]);
DefaultParser defaultCLIParser = new DefaultParser();
try {
- defaultCLIParser.parse(ExtractProfiler.OPTIONS, updatedArgs);
+ CommandLine commandLine = defaultCLIParser.parse(ExtractProfiler.OPTIONS, updatedArgs);
+ if (commandLine.hasOption("db") && commandLine.hasOption("jdbc")) {
+ System.out.println("Please specify either the default -db or the full -jdbc, not both");
+ ExtractProfiler.USAGE();
+ return;
+ }
} catch (ParseException e) {
System.out.println(e.getMessage()+"\n");
ExtractProfiler.USAGE();
@@ -192,7 +199,8 @@ public class TikaEvalCLI {
i++;
} else if (arg.equals("-alterExtract")) {
if (i+1 >= argList.size()) {
- System.err.println("Must specify directory after -extractsB");
+ System.err.println("Must specify type 'as_is', 'first_only' or " +
+ "'concatenate_content' after -alterExtract");
ExtractComparer.USAGE();
return;
}
@@ -232,7 +240,12 @@ public class TikaEvalCLI {
String[] updatedArgs = argList.toArray(new String[argList.size()]);
DefaultParser defaultCLIParser = new DefaultParser();
try {
- defaultCLIParser.parse(ExtractComparer.OPTIONS, updatedArgs);
+ CommandLine commandLine = defaultCLIParser.parse(ExtractComparer.OPTIONS, updatedArgs);
+ if (commandLine.hasOption("db") && commandLine.hasOption("jdbc")) {
+ System.out.println("Please specify either the default -db or the full -jdbc, not both");
+ ExtractComparer.USAGE();
+ return;
+ }
} catch (ParseException e) {
System.out.println(e.getMessage()+"\n");
ExtractComparer.USAGE();
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java b/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java
index 2f28614..42bc4c3 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java
@@ -34,8 +34,8 @@ import java.sql.Statement;
import org.apache.log4j.Level;
import org.apache.tika.eval.db.Cols;
-import org.apache.tika.eval.db.DBUtil;
import org.apache.tika.eval.db.H2Util;
+import org.apache.tika.eval.db.JDBCUtil;
import org.apache.tika.eval.db.TableInfo;
import org.apache.tika.eval.io.XMLLogMsgHandler;
import org.apache.tika.eval.io.XMLLogReader;
@@ -60,8 +60,8 @@ public class XMLErrorLogUpdater {
Path xmlLogFileA = Paths.get(args[0]);
Path xmlLogFileB = Paths.get(args[1]);
Path db = Paths.get(args[2]);
- DBUtil dbUtil = new H2Util(db);
- Connection connection = dbUtil.getConnection(true);
+ JDBCUtil dbUtil = new H2Util(db);
+ Connection connection = dbUtil.getConnection();
writer.update(connection, ExtractComparer.EXTRACT_EXCEPTION_TABLE_A, xmlLogFileA);
writer.update(connection, ExtractComparer.EXTRACT_EXCEPTION_TABLE_B, xmlLogFileB);
connection.commit();
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java
index f353662..2e3dfca 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java
@@ -27,7 +27,7 @@ import org.apache.tika.batch.ConsumersManager;
import org.apache.tika.batch.FileResourceConsumer;
import org.apache.tika.eval.AbstractProfiler;
import org.apache.tika.eval.XMLErrorLogUpdater;
-import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.JDBCUtil;
import org.apache.tika.eval.db.MimeBuffer;
import org.apache.tika.eval.db.TableInfo;
@@ -38,10 +38,10 @@ public class DBConsumersManager extends ConsumersManager {
private final MimeBuffer mimeBuffer;
private final List<LogTablePair> errorLogs = new ArrayList<>();
- public DBConsumersManager(DBUtil dbUtil, MimeBuffer mimeBuffer, List<FileResourceConsumer> consumers)
- throws IOException {
+ public DBConsumersManager(JDBCUtil dbUtil, MimeBuffer mimeBuffer, List<FileResourceConsumer> consumers)
+ throws SQLException {
super(consumers);
- this.conn = dbUtil.getConnection(true);
+ this.conn = dbUtil.getConnection();
this.mimeBuffer = mimeBuffer;
}
@@ -89,10 +89,10 @@ public class DBConsumersManager extends ConsumersManager {
}
}
- public void addErrorLogTablePair(Path log, TableInfo tableName) {
+ public void addErrorLogTablePair(Path log, TableInfo tableInfo) {
LogTablePair p = new LogTablePair();
p.log = log;
- p.tableInfo = tableName;
+ p.tableInfo = tableInfo;
errorLogs.add(p);
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
index d21d1de..c8781e2 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
@@ -22,7 +22,9 @@ import java.sql.Connection;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
@@ -30,7 +32,7 @@ import org.apache.tika.batch.FileResource;
import org.apache.tika.batch.FileResourceConsumer;
import org.apache.tika.eval.AbstractProfiler;
import org.apache.tika.eval.db.Cols;
-import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.JDBCUtil;
import org.apache.tika.eval.db.MimeBuffer;
import org.apache.tika.eval.db.TableInfo;
import org.apache.tika.eval.io.DBWriter;
@@ -42,12 +44,12 @@ public abstract class EvalConsumerBuilder {
private AtomicInteger count = new AtomicInteger(0);
protected ArrayBlockingQueue<FileResource> queue;
Map<String, String> localAttrs;
- DBUtil dbUtil;
+ JDBCUtil dbUtil;
private MimeBuffer mimeBuffer;
AtomicInteger initialized = new AtomicInteger(0);
public void init(ArrayBlockingQueue<FileResource> queue, Map<String, String> localAttrs,
- DBUtil dbUtil, MimeBuffer mimeBuffer) throws IOException, SQLException {
+ JDBCUtil dbUtil, MimeBuffer mimeBuffer) throws IOException, SQLException {
this.queue = queue;
this.localAttrs = localAttrs;
this.dbUtil = dbUtil;
@@ -60,11 +62,29 @@ public abstract class EvalConsumerBuilder {
public abstract FileResourceConsumer build() throws IOException, SQLException;
- protected abstract List<TableInfo> getTableInfo();
+ protected abstract List<TableInfo> getTableInfo(String tableNamePrefixA, String tableNamePrefixB);
protected abstract void addErrorLogTablePairs(DBConsumersManager manager);
public void populateRefTables() throws IOException, SQLException {
+ //test for one ref table. If it exists, don't populate ref tables
+ //TODO: test one at a time
+ boolean tableExists = false;
+ try (Connection connection = dbUtil.getConnection()) {
+ Set<String> tables = dbUtil.getTables(connection);
+ if (tables.contains(
+ AbstractProfiler.REF_PARSE_ERROR_TYPES.getName().toLowerCase(Locale.US)
+ )) {
+ tableExists = true;
+ }
+ } catch (SQLException e) {
+ //swallow
+ }
+
+ if (tableExists) {
+ return;
+ }
+
IDBWriter writer = getDBWriter();
Map<Cols, String> m = new HashMap<>();
for (AbstractProfiler.PARSE_ERROR_TYPE t : AbstractProfiler.PARSE_ERROR_TYPE.values()) {
@@ -92,8 +112,8 @@ public abstract class EvalConsumerBuilder {
}
protected IDBWriter getDBWriter() throws IOException, SQLException {
- Connection conn = dbUtil.getConnection(true);
- return new DBWriter(conn, getTableInfo(), dbUtil, mimeBuffer);
+ Connection conn = dbUtil.getConnection();
+ return new DBWriter(conn, getTableInfo(null, null), dbUtil, mimeBuffer);
}
ExtractReader.ALTER_METADATA_LIST getAlterMetadata(Map<String, String> localAttrs) {
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
index 883f908..d1b6465 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
@@ -33,8 +33,8 @@ import org.apache.tika.batch.builders.AbstractConsumersBuilder;
import org.apache.tika.batch.builders.BatchProcessBuilder;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.eval.AbstractProfiler;
-import org.apache.tika.eval.db.DBUtil;
import org.apache.tika.eval.db.H2Util;
+import org.apache.tika.eval.db.JDBCUtil;
import org.apache.tika.eval.db.MimeBuffer;
import org.apache.tika.eval.util.LanguageIDWrapper;
import org.apache.tika.util.ClassLoaderUtil;
@@ -55,6 +55,9 @@ public class EvalConsumersBuilder extends AbstractConsumersBuilder {
Path db = getPath(localAttrs, "db");
+ String jdbcConnectionString = localAttrs.get("jdbc");
+
+
Path langModelDir = getPath(localAttrs, "langModelDir");
try {
@@ -68,31 +71,43 @@ public class EvalConsumersBuilder extends AbstractConsumersBuilder {
}
Path commonTokens = getPath(localAttrs, "commonTokens");
+ //can be null, in which case will load from memory
try {
AbstractProfiler.loadCommonTokens(commonTokens);
} catch (IOException e) {
throw new RuntimeException(e);
}
- boolean append = PropsUtil.getBoolean(localAttrs.get("dbAppend"), false);
+ boolean forceDrop = PropsUtil.getBoolean(localAttrs.get("drop"), false);
- if (db == null) {
- throw new RuntimeException("Must specify: -db");
+ JDBCUtil jdbcUtil = null;
+ if (db != null) {
+ jdbcUtil = new H2Util(db);
+ } else if (jdbcConnectionString != null) {
+ jdbcUtil = new JDBCUtil(jdbcConnectionString, localAttrs.get("jdbcDriver"));
+ } else {
+ throw new RuntimeException("Must specify: -db or -jdbc");
}
- //parameterize which db util to use
- DBUtil util = new H2Util(db);
EvalConsumerBuilder consumerBuilder = ClassLoaderUtil.buildClass(EvalConsumerBuilder.class,
PropsUtil.getString(localAttrs.get("consumerBuilderClass"), null));
if (consumerBuilder == null) {
throw new RuntimeException("Must specify consumerBuilderClass in config file");
}
+ String tablePrefixA = localAttrs.get("tablePrefix");
+ if (tablePrefixA == null) {
+ tablePrefixA = localAttrs.get("tablePrefixA");
+ }
+ String tablePrefixB = localAttrs.get("tablePrefixB");
+
+ tablePrefixA = (tablePrefixA == null || tablePrefixA.endsWith("_")) ? tablePrefixA : tablePrefixA+"_";
+ tablePrefixB = (tablePrefixB == null || tablePrefixB.endsWith("_")) ? tablePrefixB : tablePrefixB+"_";
MimeBuffer mimeBuffer = null;
try {
- util.createDB(consumerBuilder.getTableInfo(), append);
- mimeBuffer = new MimeBuffer(util.getConnection(true), TikaConfig.getDefaultConfig());
- consumerBuilder.init(queue, localAttrs, util, mimeBuffer);
+ jdbcUtil.createTables(consumerBuilder.getTableInfo(tablePrefixA, tablePrefixB), forceDrop);
+ mimeBuffer = new MimeBuffer(jdbcUtil.getConnection(), TikaConfig.getDefaultConfig());
+ consumerBuilder.init(queue, localAttrs, jdbcUtil, mimeBuffer);
} catch (IOException | SQLException e) {
throw new RuntimeException(e);
}
@@ -107,8 +122,8 @@ public class EvalConsumersBuilder extends AbstractConsumersBuilder {
DBConsumersManager manager;
try {
- manager = new DBConsumersManager(util, mimeBuffer, consumers);
- } catch (IOException e) {
+ manager = new DBConsumersManager(jdbcUtil, mimeBuffer, consumers);
+ } catch (SQLException e) {
throw new RuntimeException(e);
}
consumerBuilder.addErrorLogTablePairs(manager);
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/ExtractComparerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/ExtractComparerBuilder.java
new file mode 100644
index 0000000..09152cd
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/ExtractComparerBuilder.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.batch;
+
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.tika.batch.FileResourceConsumer;
+import org.apache.tika.eval.AbstractProfiler;
+import org.apache.tika.eval.ExtractComparer;
+import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.ExtractReader;
+import org.apache.tika.util.PropsUtil;
+
+public class ExtractComparerBuilder extends EvalConsumerBuilder {
+ private final List<TableInfo> tableInfosA;
+ private final List<TableInfo> tableInfosB;
+ private final List<TableInfo> tableInfosAandB;
+ private final List<TableInfo> refTableInfos;
+
+ public ExtractComparerBuilder() {
+ List<TableInfo> tableInfosA = new ArrayList<>();
+ List<TableInfo> tableInfosB = new ArrayList<>();
+ List<TableInfo> tableInfosAandB = new ArrayList<>();
+ tableInfosA.add(ExtractComparer.PROFILES_A);
+ tableInfosA.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
+ tableInfosA.add(ExtractComparer.EXCEPTION_TABLE_A);
+ tableInfosA.add(ExtractComparer.CONTENTS_TABLE_A);
+ tableInfosA.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
+ tableInfosA.add(ExtractComparer.EMBEDDED_FILE_PATH_TABLE_A);
+
+ tableInfosB.add(ExtractComparer.PROFILES_B);
+ tableInfosB.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
+ tableInfosB.add(ExtractComparer.EXCEPTION_TABLE_B);
+ tableInfosB.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
+ tableInfosB.add(ExtractComparer.CONTENTS_TABLE_B);
+ tableInfosB.add(ExtractComparer.EMBEDDED_FILE_PATH_TABLE_B);
+
+ tableInfosAandB.add(ExtractComparer.COMPARISON_CONTAINERS);
+ tableInfosAandB.add(ExtractComparer.CONTENT_COMPARISONS);
+ tableInfosAandB.add(AbstractProfiler.MIME_TABLE);
+
+ List<TableInfo> refTableInfos = new ArrayList<>();
+ refTableInfos.add(ExtractComparer.REF_PAIR_NAMES);
+ refTableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
+ refTableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
+ refTableInfos.add(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES);
+
+ this.tableInfosA = Collections.unmodifiableList(tableInfosA);
+ this.tableInfosB = Collections.unmodifiableList(tableInfosB);
+ this.tableInfosAandB = Collections.unmodifiableList(tableInfosAandB);
+ this.refTableInfos = Collections.unmodifiableList(refTableInfos);
+ }
+
+ @Override
+ public FileResourceConsumer build() throws IOException, SQLException {
+ Path extractsA = PropsUtil.getPath(localAttrs.get("extractsA"), null);
+ if (extractsA == null) {
+ throw new RuntimeException("Must specify \"extractsA\" -- directory for 'A' extracts");
+ }
+ Path extractsB = PropsUtil.getPath(localAttrs.get("extractsB"), null);
+ if (extractsB == null) {
+ throw new RuntimeException("Must specify \"extractsB\" -- directory for 'B' extracts");
+ }
+
+ Path inputRootDir = PropsUtil.getPath(localAttrs.get("inputDir"), null);
+
+ long minExtractLength = PropsUtil.getLong(localAttrs.get("minExtractLength"), -1L);
+ long maxExtractLength = PropsUtil.getLong(localAttrs.get("maxExtractLength"), -1L);
+
+ ExtractReader.ALTER_METADATA_LIST alterExtractList = getAlterMetadata(localAttrs);
+
+
+ if (inputRootDir == null) {
+ //this is for the sake of the crawler
+ throw new RuntimeException("Must specify an -inputDir");
+ }
+
+ return new ExtractComparer(queue, inputRootDir, extractsA, extractsB,
+ new ExtractReader(alterExtractList, minExtractLength, maxExtractLength),
+ getDBWriter());
+ }
+
+ @Override
+ protected List<TableInfo> getTableInfo(String tableNamePrefixA, String tableNamePrefixB) {
+ List<TableInfo> allTables = new ArrayList<>();
+ if (tableNamePrefixA != null) {
+ for (TableInfo tableInfo : tableInfosA) {
+ tableInfo.setNamePrefix(tableNamePrefixA);
+ }
+ }
+
+ if (tableNamePrefixB != null) {
+ for (TableInfo tableInfo : tableInfosB) {
+ tableInfo.setNamePrefix(tableNamePrefixB);
+ }
+ }
+
+ if (tableNamePrefixA != null || tableNamePrefixB != null) {
+ String aAndB = (tableNamePrefixA == null) ? "" : tableNamePrefixA;
+ aAndB = (tableNamePrefixB == null) ? aAndB : aAndB+tableNamePrefixB;
+ for (TableInfo tableInfo : tableInfosAandB) {
+ tableInfo.setNamePrefix(aAndB);
+ }
+ }
+
+ allTables.addAll(tableInfosA);
+ allTables.addAll(tableInfosB);
+ allTables.addAll(tableInfosAandB);
+ allTables.addAll(refTableInfos);
+ allTables = Collections.unmodifiableList(allTables);
+ return allTables;
+ }
+
+
+ @Override
+ protected void addErrorLogTablePairs(DBConsumersManager manager) {
+ Path errorLogA = PropsUtil.getPath(localAttrs.get("errorLogFileA"), null);
+ if (errorLogA == null) {
+ return;
+ }
+ manager.addErrorLogTablePair(errorLogA, ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
+ Path errorLogB = PropsUtil.getPath(localAttrs.get("errorLogFileB"), null);
+ if (errorLogB == null) {
+ return;
+ }
+ manager.addErrorLogTablePair(errorLogB, ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
+
+ }
+
+}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/ExtractProfilerBuilder.java
similarity index 69%
rename from tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java
rename to tika-eval/src/main/java/org/apache/tika/eval/batch/ExtractProfilerBuilder.java
index c1c25d3..0df225d 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/ExtractProfilerBuilder.java
@@ -21,6 +21,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import org.apache.tika.batch.FileResourceConsumer;
@@ -31,7 +32,26 @@ import org.apache.tika.eval.io.ExtractReader;
import org.apache.tika.util.PropsUtil;
-public class SingleFileConsumerBuilder extends EvalConsumerBuilder {
+public class ExtractProfilerBuilder extends EvalConsumerBuilder {
+ private final List<TableInfo> tableInfos;
+ private final List<TableInfo> refTableInfos;
+ public ExtractProfilerBuilder() {
+ List<TableInfo> tableInfos = new ArrayList();
+ tableInfos.add(AbstractProfiler.MIME_TABLE);
+ tableInfos.add(ExtractProfiler.CONTAINER_TABLE);
+ tableInfos.add(ExtractProfiler.PROFILE_TABLE);
+ tableInfos.add(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
+ tableInfos.add(ExtractProfiler.EXCEPTION_TABLE);
+ tableInfos.add(ExtractProfiler.CONTENTS_TABLE);
+ tableInfos.add(ExtractProfiler.EMBEDDED_FILE_PATH_TABLE);
+ this.tableInfos = Collections.unmodifiableList(tableInfos);
+
+ List<TableInfo> refTableInfos = new ArrayList<>();
+ refTableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
+ refTableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
+ refTableInfos.add(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES);
+ this.refTableInfos = Collections.unmodifiableList(refTableInfos);
+ }
@Override
public FileResourceConsumer build() throws IOException, SQLException {
@@ -60,24 +80,24 @@ public class SingleFileConsumerBuilder extends EvalConsumerBuilder {
if (extracts == null && inputDir != null) {
extracts = inputDir;
}
- return new ExtractProfiler(queue, inputDir, extracts, getDBWriter(),
- minExtractLength, maxExtractLength, alterExtractList);
+ return new ExtractProfiler(queue, inputDir, extracts,
+ new ExtractReader(alterExtractList, minExtractLength, maxExtractLength),
+ getDBWriter());
}
+ //TODO: clean up the interface so we don't need vestigial B
@Override
- protected List<TableInfo> getTableInfo() {
- List<TableInfo> tableInfos = new ArrayList<TableInfo>();
- tableInfos.add(AbstractProfiler.MIME_TABLE);
- tableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
- tableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
- tableInfos.add(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES);
- tableInfos.add(ExtractProfiler.CONTAINER_TABLE);
- tableInfos.add(ExtractProfiler.PROFILE_TABLE);
- tableInfos.add(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
- tableInfos.add(ExtractProfiler.EXCEPTION_TABLE);
- tableInfos.add(ExtractProfiler.CONTENTS_TABLE);
- tableInfos.add(ExtractProfiler.EMBEDDED_FILE_PATH_TABLE);
- return tableInfos;
+ protected List<TableInfo> getTableInfo(String tableNamePrefix, String tableNamePrefixB) {
+ //ignore b
+ if (tableNamePrefix != null && !tableNamePrefix.equals("null")) {
+ for (TableInfo tableInfo : tableInfos) {
+ tableInfo.setNamePrefix(tableNamePrefix);
+ }
+ }
+ List allTables = new ArrayList();
+ allTables.addAll(tableInfos);
+ allTables.addAll(refTableInfos);
+ return Collections.unmodifiableList(allTables);
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java
deleted file mode 100644
index 0b35cb4..0000000
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.eval.batch;
-
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.tika.batch.FileResourceConsumer;
-import org.apache.tika.eval.AbstractProfiler;
-import org.apache.tika.eval.ExtractComparer;
-import org.apache.tika.eval.db.TableInfo;
-import org.apache.tika.eval.io.ExtractReader;
-import org.apache.tika.util.PropsUtil;
-
-public class FileComparerBuilder extends EvalConsumerBuilder {
-
- @Override
- public FileResourceConsumer build() throws IOException, SQLException {
- Path thisRootDir = PropsUtil.getPath(localAttrs.get("extractsA"), null);
- if (thisRootDir == null) {
- throw new RuntimeException("Must specify \"extractsA\" -- directory for 'A' extracts");
- }
- Path thatRootDir = PropsUtil.getPath(localAttrs.get("extractsB"), null);
- if (thatRootDir == null) {
- throw new RuntimeException("Must specify \"extractsB\" -- directory for 'B' extracts");
- }
-
- Path inputRootDir = PropsUtil.getPath(localAttrs.get("inputDir"), null);
-
- long minExtractLength = PropsUtil.getLong(localAttrs.get("minExtractLength"), -1L);
- long maxExtractLength = PropsUtil.getLong(localAttrs.get("maxExtractLength"), -1L);
-
- ExtractReader.ALTER_METADATA_LIST alterExtractList = getAlterMetadata(localAttrs);
-
-
- if (inputRootDir == null) {
- //this is for the sake of the crawler
- throw new RuntimeException("Must specify an -inputDir");
- }
-
- return new ExtractComparer(queue, inputRootDir, thisRootDir, thatRootDir, getDBWriter(),
- minExtractLength, maxExtractLength, alterExtractList);
- }
-
- @Override
- protected List<TableInfo> getTableInfo() {
- List<TableInfo> tableInfos = new ArrayList<>();
- tableInfos.add(ExtractComparer.COMPARISON_CONTAINERS);
- tableInfos.add(ExtractComparer.PROFILES_A);
- tableInfos.add(ExtractComparer.PROFILES_B);
- tableInfos.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
- tableInfos.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
- tableInfos.add(ExtractComparer.EXCEPTION_TABLE_A);
- tableInfos.add(ExtractComparer.EXCEPTION_TABLE_B);
- tableInfos.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
- tableInfos.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
- tableInfos.add(ExtractComparer.CONTENTS_TABLE_A);
- tableInfos.add(ExtractComparer.CONTENTS_TABLE_B);
- tableInfos.add(ExtractComparer.EMBEDDED_FILE_PATH_TABLE_A);
- tableInfos.add(ExtractComparer.EMBEDDED_FILE_PATH_TABLE_B);
-
- tableInfos.add(ExtractComparer.CONTENT_COMPARISONS);
- tableInfos.add(AbstractProfiler.MIME_TABLE);
- tableInfos.add(ExtractComparer.REF_PAIR_NAMES);
- tableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
- tableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
- tableInfos.add(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES);
- return tableInfos;
- }
-
-
- @Override
- protected void addErrorLogTablePairs(DBConsumersManager manager) {
- Path errorLogA = PropsUtil.getPath(localAttrs.get("errorLogFileA"), null);
- if (errorLogA == null) {
- return;
- }
- manager.addErrorLogTablePair(errorLogA, ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
- Path errorLogB = PropsUtil.getPath(localAttrs.get("errorLogFileB"), null);
- if (errorLogB == null) {
- return;
- }
- manager.addErrorLogTablePair(errorLogB, ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
-
- }
-
-}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/H2Util.java b/tika-eval/src/main/java/org/apache/tika/eval/db/H2Util.java
index 0b3ebac..14037be 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/db/H2Util.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/H2Util.java
@@ -18,6 +18,7 @@ package org.apache.tika.eval.db;
import java.nio.file.Path;
import java.sql.Connection;
+import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
@@ -27,12 +28,25 @@ import java.util.Set;
import org.apache.commons.io.FilenameUtils;
-public class H2Util extends DBUtil {
+public class H2Util extends JDBCUtil {
+
+ private final static String DRIVER_CLASS = "org.h2.Driver";
+
+ private final Path db;
public H2Util(Path db) {
- super(db);
+ super(getConnectionString(db, true), DRIVER_CLASS);
+ this.db = db;
}
+ public static boolean databaseExists(Path db) {
+ String connString = getConnectionString(db, false);
+ try (Connection conn = DriverManager.getConnection(connString)) {
+ } catch (SQLException e) {
+ return false;
+ }
+ return true;
+ }
@Override
public String getJDBCDriverClass() {
return "org.h2.Driver";
@@ -48,9 +62,13 @@ public class H2Util extends DBUtil {
}
@Override
- public String getConnectionString(Path db, boolean createIfDoesntExist) {
+ public String getConnectionString() {
+ return getConnectionString(db, true);
+ }
+
+ private static String getConnectionString(Path db, boolean createDBIfItDoesntExist) {
String s = "jdbc:h2:"+ FilenameUtils.separatorsToUnix(db.toAbsolutePath().toString());
- if (! createIfDoesntExist) {
+ if (! createDBIfItDoesntExist) {
s += ";IFEXISTS=TRUE";
}
return s;
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java b/tika-eval/src/main/java/org/apache/tika/eval/db/JDBCUtil.java
similarity index 52%
rename from tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java
rename to tika-eval/src/main/java/org/apache/tika/eval/db/JDBCUtil.java
index 17d898b..0f7f5a5 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/JDBCUtil.java
@@ -19,70 +19,123 @@ package org.apache.tika.eval.db;
import java.io.IOException;
-import java.nio.file.Path;
+import java.io.InputStream;
import java.sql.Connection;
+import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.log4j.Logger;
-import org.apache.tika.io.IOExceptionWithCause;
-public abstract class DBUtil {
+public class JDBCUtil {
- public static Logger logger = Logger.getLogger(DBUtil.class);
- public abstract String getJDBCDriverClass();
- public abstract boolean dropTableIfExists(Connection conn, String tableName) throws SQLException;
- private final Path db;
- public DBUtil(Path db) {
- this.db = db;
+ public static Logger logger = Logger.getLogger(JDBCUtil.class);
+ private final String connectionString;
+ private String driverClass;
+
+ public JDBCUtil(String connectionString, String driverClass) {
+ this.connectionString = connectionString;
+ this.driverClass = driverClass;
+ if (driverClass == null || driverClass.length() == 0) {
+ if (System.getProperty("jdbc.drivers") != null) {
+ //user has specified it on the command line
+ //stop now
+ } else {
+ //try to use the mappings in db.properties to determine the class
+ try (InputStream is = JDBCUtil.class.getResourceAsStream("/db.properties")) {
+ Properties properties = new Properties();
+ properties.load(is);
+ for (String k : properties.stringPropertyNames()) {
+ Matcher m = Pattern.compile("(?i)jdbc:"+k).matcher(connectionString);
+ if (m.find()) {
+ this.driverClass = properties.getProperty(k);
+ }
+ }
+
+ } catch (IOException e) {
+
+ }
+ }
+ }
}
/**
- * This is intended for a file/directory based db.
- * <p>
* Override this any optimizations you want to do on the db
* before writing/reading.
*
* @return
* @throws IOException
*/
- public Connection getConnection(boolean createIfDoesntExist) throws IOException {
- String connectionString = getConnectionString(db, createIfDoesntExist);
+ public Connection getConnection() throws SQLException {
+ String connectionString = getConnectionString();
Connection conn = null;
- try {
+ String jdbcDriver = getJDBCDriverClass();
+ if (jdbcDriver != null) {
try {
Class.forName(getJDBCDriverClass());
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
- conn = DriverManager.getConnection(connectionString);
- conn.setAutoCommit(false);
- } catch (SQLException e) {
- throw new IOExceptionWithCause(e);
}
+ conn = DriverManager.getConnection(connectionString);
+ conn.setAutoCommit(false);
+
return conn;
}
- abstract public String getConnectionString(Path db, boolean createIfDoesntExist);
-
/**
- *
- * @param connection
- * @return a list of uppercased table names
- * @throws SQLException
+ * JDBC driver class. Override as necessary.
+ * @return
*/
- abstract public Set<String> getTables(Connection connection) throws SQLException;
+ public String getJDBCDriverClass() {
+ return driverClass;
+ }
+
+
+ public boolean dropTableIfExists(Connection conn, String tableName) throws SQLException {
+ if (containsTable(tableName)) {
+ try (Statement st = conn.createStatement()) {
+ String sql = "drop table " + tableName;
+ return st.execute(sql);
+ }
+ }
+ return true;
+ }
+
+
+ public String getConnectionString() {
+ return connectionString;
+ }
+
+
+ public Set<String> getTables(Connection connection) throws SQLException {
+ Set<String> tables = new HashSet<>();
+
+ DatabaseMetaData dbMeta = connection.getMetaData();
+
+ try (ResultSet rs = dbMeta.getTables(null, null, "%", null)) {
+ while (rs.next()) {
+ tables.add(rs.getString(3).toLowerCase(Locale.US));
+ }
+ }
+ return tables;
+ }
public static int insert(PreparedStatement insertStatement,
- TableInfo table,
- Map<Cols, String> data) throws SQLException {
+ TableInfo table,
+ Map<Cols, String> data) throws SQLException {
//clear parameters before setting
insertStatement.clearParameters();
@@ -93,21 +146,21 @@ public abstract class DBUtil {
i++;
}
for (Cols c : data.keySet()) {
- if (! table.containsColumn(c)) {
- throw new IllegalArgumentException("Can't add data to "+c +
- " because it doesn't exist in the table: "+table.getName());
+ if (!table.containsColumn(c)) {
+ throw new IllegalArgumentException("Can't add data to " + c +
+ " because it doesn't exist in the table: " + table.getName());
}
}
return insertStatement.executeUpdate();
} catch (SQLException e) {
- logger.warn("couldn't insert data for this row: "+e.getMessage());
+ logger.warn("couldn't insert data for this row: " + e.getMessage());
e.printStackTrace();
return -1;
}
}
public static void updateInsertStatement(int dbColOffset, PreparedStatement st,
- ColInfo colInfo, String value ) throws SQLException {
+ ColInfo colInfo, String value) throws SQLException {
if (value == null) {
st.setNull(dbColOffset, colInfo.getType());
return;
@@ -117,7 +170,7 @@ public abstract class DBUtil {
case Types.VARCHAR:
if (value != null && value.length() > colInfo.getPrecision()) {
value = value.substring(0, colInfo.getPrecision());
- logger.warn("truncated varchar value in " + colInfo.getName() + " : "+value);
+ logger.warn("truncated varchar value in " + colInfo.getName() + " : " + value);
}
st.setString(dbColOffset, value);
break;
@@ -143,38 +196,54 @@ public abstract class DBUtil {
throw new UnsupportedOperationException("Don't yet support type: " + colInfo.getType());
}
} catch (NumberFormatException e) {
- if (! "".equals(value)) {
+ if (!"".equals(value)) {
logger.warn("number format exception: " + colInfo.getName() + " : " + value);
}
st.setNull(dbColOffset, colInfo.getType());
} catch (SQLException e) {
- logger.warn("sqlexception: "+colInfo+ " : " + value);
+ logger.warn("sqlexception: " + colInfo + " : " + value);
st.setNull(dbColOffset, colInfo.getType());
}
}
- public void createDB(List<TableInfo> tableInfos, boolean append) throws SQLException, IOException {
- Connection conn = getConnection(true);
- Set<String> tables = getTables(conn);
+ public void createTables(List<TableInfo> tableInfos, boolean forceDrop) throws SQLException, IOException {
- for (TableInfo tableInfo : tableInfos) {
+ try (Connection conn = getConnection ()) {
+ for (TableInfo tableInfo : tableInfos) {
- if (append && tables.contains(tableInfo.getName().toUpperCase(Locale.ROOT))) {
- continue;
+ if (forceDrop) {
+ dropTableIfExists(conn, tableInfo.getName());
+ }
+ createTable(conn, tableInfo);
}
- if (! append) {
- dropTableIfExists(conn, tableInfo.getName());
+ conn.commit();
+ }
+ }
+
+ public boolean containsTable(String tableName) throws SQLException {
+ try (Connection connection = getConnection()) {
+ Set<String> tables = getTables(connection);
+ if (tables.contains(normalizeTableName(tableName))) {
+ return true;
}
- createTable(conn, tableInfo);
}
+ return false;
+ }
- conn.commit();
- conn.close();
+ /**
+ * Override for custom behavior
+ * @param tableName
+ * @return
+ */
+ String normalizeTableName(String tableName) {
+ tableName = tableName.toLowerCase(Locale.US);
+ return tableName;
}
+ //does not close the connection
private void createTable(Connection conn, TableInfo tableInfo) throws SQLException {
StringBuilder createSql = new StringBuilder();
- createSql.append("CREATE TABLE "+tableInfo.getName());
+ createSql.append("CREATE TABLE " + tableInfo.getName());
createSql.append("(");
int last = 0;
@@ -193,10 +262,10 @@ public abstract class DBUtil {
}
}
createSql.append(")");
- Statement st = conn.createStatement();
- st.execute(createSql.toString());
-
- st.close();
+ try (Statement st = conn.createStatement()) {
+ st.execute(createSql.toString());
+ st.close();
+ }
conn.commit();
}
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java b/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java
index 56f26b7..d557f9f 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java
@@ -39,8 +39,8 @@ public class MimeBuffer extends AbstractDBBuffer {
public MimeBuffer(Connection connection, TikaConfig config) throws SQLException {
st = connection.prepareStatement("insert into " + AbstractProfiler.MIME_TABLE.getName() + "( " +
Cols.MIME_TYPE_ID.name() + ", " +
- Cols.MIME_STRING + ", " +
- Cols.FILE_EXTENSION + ") values (?,?,?);");
+ Cols.MIME_STRING.name() + ", " +
+ Cols.FILE_EXTENSION.name() + ") values (?,?,?)");
this.config = config;
this.connection = connection;
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/TableInfo.java b/tika-eval/src/main/java/org/apache/tika/eval/db/TableInfo.java
index c6e3696..181a028 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/db/TableInfo.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/TableInfo.java
@@ -29,6 +29,8 @@ public class TableInfo {
private final List<ColInfo> colInfos = new ArrayList<>();
private final Set<Cols> colNames = new HashSet<>();
+ private String prefix;
+
public TableInfo(String name, ColInfo... cols) {
Collections.addAll(colInfos, cols);
Collections.unmodifiableList(colInfos);
@@ -50,7 +52,14 @@ public class TableInfo {
}
public String getName() {
- return name;
+ if (prefix == null) {
+ return name;
+ }
+ return prefix+name;
+ }
+
+ public void setNamePrefix(String prefix) {
+ this.prefix = prefix;
}
public List<ColInfo> getColInfos() {
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java b/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java
index 6c4862c..b1f3b36 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java
@@ -29,7 +29,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.log4j.Logger;
import org.apache.tika.eval.db.ColInfo;
import org.apache.tika.eval.db.Cols;
-import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.JDBCUtil;
import org.apache.tika.eval.db.MimeBuffer;
import org.apache.tika.eval.db.TableInfo;
import org.apache.tika.io.IOExceptionWithCause;
@@ -52,14 +52,14 @@ public class DBWriter implements IDBWriter {
private final Long commitEveryX = 1000L;
private final Connection conn;
- private final DBUtil dbUtil;
+ private final JDBCUtil dbUtil;
private final MimeBuffer mimeBuffer;
private final int myId = WRITER_ID.getAndIncrement();
//<tableName, preparedStatement>
private final Map<String, PreparedStatement> inserts = new HashMap<>();
- public DBWriter(Connection connection, List<TableInfo> tableInfos, DBUtil dbUtil, MimeBuffer mimeBuffer)
+ public DBWriter(Connection connection, List<TableInfo> tableInfos, JDBCUtil dbUtil, MimeBuffer mimeBuffer)
throws IOException, SQLException {
this.conn = connection;
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java b/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java
index eebbdbf..fd35363 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java
@@ -55,8 +55,31 @@ public class ExtractReader {
private final static Logger LOGGER = LoggerFactory.getLogger(ExtractReader.class);
TikaConfig tikaConfig = TikaConfig.getDefaultConfig();
- public List<Metadata> loadExtract(Path extractFile, ALTER_METADATA_LIST alterExtractList,
- long minExtractLength, long maxExtractLength) throws ExtractReaderException {
+ private final ALTER_METADATA_LIST alterMetadataList;
+ private final long minExtractLength;
+ private final long maxExtractLength;
+
+ /**
+ * Reads full extract, no modification of metadata list, no min or max extract length checking
+ */
+ public ExtractReader() {
+ this(ALTER_METADATA_LIST.AS_IS, IGNORE_LENGTH, IGNORE_LENGTH);
+ }
+
+ public ExtractReader(ALTER_METADATA_LIST alterMetadataList) {
+ this(alterMetadataList, IGNORE_LENGTH, IGNORE_LENGTH);
+ }
+
+ public ExtractReader(ALTER_METADATA_LIST alterMetadataList, long minExtractLength, long maxExtractLength) {
+ this.alterMetadataList = alterMetadataList;
+ this.minExtractLength = minExtractLength;
+ this.maxExtractLength = maxExtractLength;
+ if (maxExtractLength > IGNORE_LENGTH && minExtractLength >= maxExtractLength) {
+ throw new IllegalArgumentException("minExtractLength("+minExtractLength+
+ ") must be < maxExtractLength("+maxExtractLength+")");
+ }
+ }
+ public List<Metadata> loadExtract(Path extractFile) throws ExtractReaderException {
List<Metadata> metadataList = null;
if (extractFile == null || !Files.isRegularFile(extractFile)) {
@@ -114,11 +137,11 @@ public class ExtractReader {
try {
if (fileSuffixes.txtOrJson.equals("json")) {
metadataList = JsonMetadataList.fromJson(reader);
- if (alterExtractList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) {
+ if (alterMetadataList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) {
while (metadataList.size() > 1) {
metadataList.remove(metadataList.size()-1);
}
- } else if (alterExtractList.equals(ALTER_METADATA_LIST.AS_IS.CONCATENATE_CONTENT_INTO_FIRST) &&
+ } else if (alterMetadataList.equals(ALTER_METADATA_LIST.AS_IS.CONCATENATE_CONTENT_INTO_FIRST) &&
metadataList.size() > 1) {
StringBuilder sb = new StringBuilder();
Metadata containerMetadata = metadataList.get(0);
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/reports/ResultsReporter.java b/tika-eval/src/main/java/org/apache/tika/eval/reports/ResultsReporter.java
index 97b7711..072ba53 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/reports/ResultsReporter.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/reports/ResultsReporter.java
@@ -37,14 +37,13 @@ import java.util.Map;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.poi.common.usermodel.Hyperlink;
import org.apache.tika.eval.ExtractComparer;
import org.apache.tika.eval.ExtractProfiler;
-import org.apache.tika.eval.db.DBUtil;
import org.apache.tika.eval.db.H2Util;
+import org.apache.tika.eval.db.JDBCUtil;
import org.apache.tika.parser.ParseContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,18 +60,15 @@ public class ResultsReporter {
static {
OPTIONS = new Options();
-
- Option db = new Option("db", "database");
- db.setRequired(true);
- db.setArgs(1);
-
- OPTIONS.addOption(db)
- .addOption("rd", "reportsDir", true, "directory for the reports. " +
- "If not specified, will write to 'reports'" +
- "BEWARE: Will overwrite existing reports without warning!"
- )
+ OPTIONS.addOption("rd", "reportsDir", true, "directory for the reports. " +
+ "If not specified, will write to 'reports'" +
+ "BEWARE: Will overwrite existing reports without warning!")
.addOption("rf", "reportsFile", true, "xml specifying sql to call for the reports." +
- "If not specified, will use default reports in resources/tika-eval-*-config.xml");
+ "If not specified, will use default reports in resources/tika-eval-*-config.xml")
+ .addOption("db", true, "default database (in memory H2). Specify a file name for the H2 database.")
+ .addOption("jdbc", true, "EXPERT: full jdbc connection string. Specify this or use -db <h2db_name>")
+ .addOption("jdbcdriver", true, "EXPERT: specify the jdbc driver class if all else fails")
+ .addOption("tablePrefix", true, "EXPERT: if not using the default tables, specify your table name prefix");
}
@@ -111,11 +107,11 @@ public class ResultsReporter {
DocumentBuilder docBuilder = new ParseContext().getDocumentBuilder();
Document doc;
- try(InputStream is = Files.newInputStream(p)) {
+ try (InputStream is = Files.newInputStream(p)) {
doc = docBuilder.parse(is);
}
Node docElement = doc.getDocumentElement();
- assert(docElement.getNodeName().equals("reports"));
+ assert (docElement.getNodeName().equals("reports"));
NodeList children = docElement.getChildNodes();
for (int i = 0; i < children.getLength(); i++) {
Node n = children.item(i);
@@ -158,7 +154,7 @@ public class ResultsReporter {
} else if ("colformats".equals(child.getNodeName())) {
r.cellFormatters = getCellFormatters(child);
} else {
- throw new IllegalArgumentException("Not expecting to see:"+child.getNodeName());
+ throw new IllegalArgumentException("Not expecting to see:" + child.getNodeName());
}
}
return r;
@@ -174,12 +170,12 @@ public class ResultsReporter {
}
NamedNodeMap attrs = child.getAttributes();
String columnName = attrs.getNamedItem("name").getNodeValue();
- assert(!ret.containsKey(columnName));
+ assert (!ret.containsKey(columnName));
String type = attrs.getNamedItem("type").getNodeValue();
if ("numberFormatter".equals(type)) {
String format = attrs.getNamedItem("format").getNodeValue();
XSLXCellFormatter f = new XLSXNumFormatter(format);
- ret.put(columnName,f);
+ ret.put(columnName, f);
} else if ("urlLink".equals(type)) {
String base = "";
Node baseNode = attrs.getNamedItem("base");
@@ -227,10 +223,27 @@ public class ResultsReporter {
USAGE();
return;
}
- Path db = Paths.get(commandLine.getOptionValue("db"));
- DBUtil dbUtil = new H2Util(db);
-
- try (Connection c = dbUtil.getConnection(true)) {
+ JDBCUtil dbUtil = null;
+ if (commandLine.hasOption("db")) {
+ Path db = Paths.get(commandLine.getOptionValue("db"));
+ if (!H2Util.databaseExists(db)) {
+ throw new RuntimeException("I'm sorry, but I couldn't find this h2 database: "
+ + db+ "\nMake sure not to include the .mv.db at the end.");
+ }
+ dbUtil = new H2Util(db);
+ } else if (commandLine.hasOption("jdbc")) {
+ String driverClass = null;
+ if (commandLine.hasOption("jdbcdriver")) {
+ driverClass = commandLine.getOptionValue("jdbcdriver");
+ }
+ dbUtil = new JDBCUtil(commandLine.getOptionValue("jdbc"), driverClass);
+ } else {
+ System.err.println("Must specify either -db for the default in-memory h2 database\n" +
+ "or -jdbc for a full jdbc connection string");
+ USAGE();
+ return;
+ }
+ try (Connection c = dbUtil.getConnection()) {
Path tmpReportsFile = null;
try {
ResultsReporter resultsReporter = null;
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXHREFFormatter.java b/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXHREFFormatter.java
index 52d55e4..f8b9ba2 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXHREFFormatter.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXHREFFormatter.java
@@ -27,9 +27,7 @@ import org.apache.poi.ss.usermodel.Hyperlink;
import org.apache.poi.ss.usermodel.IndexedColors;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
-/**
- * Created by TALLISON on 4/20/2016.
- */
+
public class XLSXHREFFormatter implements XSLXCellFormatter {
//xlsx files can only have this many hyperlinks
//if they have more Excel can't read the file
diff --git a/tika-eval/src/main/resources/db.properties b/tika-eval/src/main/resources/db.properties
new file mode 100644
index 0000000..0f0ad36
--- /dev/null
+++ b/tika-eval/src/main/resources/db.properties
@@ -0,0 +1,9 @@
+postgresql org.postgresql.Driver
+mysql com.mysql.jdbc.Driver
+oracle oracle.jdbc.driver.OracleDriver
+sqlite org.sqlite.JDBC
+odbc sun.jdbc.odbc.JdbcOdbcDriver
+derby:// org.apache.derby.jdbc.ClientDriver
+derby org.apache.derby.jdbc.EmbeddedDriver
+hsqldb org.hsqldb.jdbc.JDBCDriver
+h2 org.h2.Driver
diff --git a/tika-eval/src/main/resources/tika-eval-comparison-config.xml b/tika-eval/src/main/resources/tika-eval-comparison-config.xml
index e0ee34e..298c864 100644
--- a/tika-eval/src/main/resources/tika-eval-comparison-config.xml
+++ b/tika-eval/src/main/resources/tika-eval-comparison-config.xml
@@ -49,6 +49,16 @@
description="minimum extract length to process"/>
<option opt="maxExtractLength" hasArg="true"
description="maximum extract length to process"/>
+ <option opt="jdbc" hasArg="true"
+ description="full jdbc connection string"/>
+ <option opt="jdbcDriver" hasArg="true"
+ description="canonical class name for jdbc driver"/>
+ <option opt="tablePrefixA" hasArg="true"
+ description="EXPERT: prefix for table names for A"/>
+ <option opt="tablePrefixB" hasArg="true"
+ description="EXPERT: prefix for table names for B"/>
+ <option opt="drop" hasArg="false" description="drop tables if they exist"/>
+ <option opt="maxFilesToAdd" hasArg="true" description="maximum number of files to add to the crawler"/>
</commandline>
@@ -67,7 +77,7 @@
/>
<consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
- consumerBuilderClass="org.apache.tika.eval.batch.FileComparerBuilder"
+ consumerBuilderClass="org.apache.tika.eval.batch.ExtractComparerBuilder"
dbAppend="false"
crawlingInputDir="false"
minExtractLength="-1"
diff --git a/tika-eval/src/main/resources/tika-eval-profiler-config.xml b/tika-eval/src/main/resources/tika-eval-profiler-config.xml
index 1425d24..9da2aeb 100644
--- a/tika-eval/src/main/resources/tika-eval-profiler-config.xml
+++ b/tika-eval/src/main/resources/tika-eval-profiler-config.xml
@@ -46,6 +46,16 @@
description="minimum extract length to process"/>
<option opt="maxExtractLength" hasArg="true"
description="maximum extract length to process"/>
+ <option opt="jdbc" hasArg="true"
+ description="full jdbc connection string"/>
+ <option opt="jdbcDriver" hasArg="true"
+ description="canonical class name for jdbc driver"/>
+ <option opt="tablePrefix" hasArg="true"
+ description="EXPERT: prefix for table names"/>
+ <option opt="drop" hasArg="false" description="drop tables if they exist"/>
+ <option opt="maxFilesToAdd" hasArg="true" description="maximum number of files to add to the crawler"/>
+
+
</commandline>
@@ -67,7 +77,7 @@
/>
<consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
- consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"
+ consumerBuilderClass="org.apache.tika.eval.batch.ExtractProfilerBuilder"
minExtractLength="-1"
maxExtractLength="2000000"
commonTokens="resources/common_tokens"/>
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java b/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
index 8d95622..d9ca118 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
@@ -73,7 +73,7 @@ public class ProfilerBatchTest {
System.out.println(streamStrings.getErrString());
System.out.println(streamStrings.getOutString());*/
H2Util dbUtil = new H2Util(db);
- conn = dbUtil.getConnection(true);
+ conn = dbUtil.getConnection();
}
@AfterClass
public static void tearDown() throws IOException {
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java b/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
index e718b33..85e91dd 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
@@ -58,8 +58,9 @@ public class SimpleComparerTest extends TikaTest {
writer = new MockDBWriter();
comparer = new ExtractComparer(null, null,
Paths.get("extractsA"), Paths.get("extractsB"),
- writer, IGNORE_LENGTH, IGNORE_LENGTH,
- ExtractReader.ALTER_METADATA_LIST.AS_IS);
+ new ExtractReader(ExtractReader.ALTER_METADATA_LIST.AS_IS,
+ IGNORE_LENGTH, IGNORE_LENGTH),
+ writer);
AbstractProfiler.loadCommonTokens(this.getResourceAsFile("/common_tokens").toPath());
LanguageIDWrapper.loadBuiltInModels();
}
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java b/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java
index ff0961c..a0c02ff 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java
@@ -27,15 +27,43 @@ public class TikaEvalCLITest {
@Test
@Ignore("TODO: add real tests")
- public void testBasic() throws Exception {
+ public void testBasicCompare() throws Exception {
+ List<String> args = new ArrayList<>();
+ args.add("Compare");
+ args.add("-extractsA");
+ args.add("watson2");
+ args.add("-extractsB");
+ args.add("tika");
+ args.add("-db");
+ args.add("comparedb");
+ args.add("-drop");
+ args.add("-jdbc");
+ args.add("jdbc:postgresql:tika_eval?user=user&password=password");
+ args.add("-maxFilesToAdd");
+ args.add("100");
+ TikaEvalCLI.main(args.toArray(new String[args.size()]));
+ }
+
+ @Ignore("TODO: add real tests")
+ @Test
+ public void testBasicProfile() throws Exception {
List<String> args = new ArrayList<>();
args.add("Profile");
args.add("-extracts");
- args.add("tika");
+ args.add("watson2");
args.add("-db");
- args.add("mydb");
- args.add("-alterExtract");
- args.add("first_only");
+ args.add("testdb");
+ TikaEvalCLI.main(args.toArray(new String[args.size()]));
+ }
+
+ @Test
+ @Ignore("TODO: add real tests")
+ public void testReports() throws Exception {
+ List<String> args = new ArrayList<>();
+ args.add("Report");
+ args.add("-db");
+ args.add("comparedb.mv.db");
+
TikaEvalCLI.main(args.toArray(new String[args.size()]));
}
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java b/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
index a49b24c..f22179a 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
@@ -44,24 +44,23 @@ public class ExtractReaderTest extends TikaTest {
public void testBasic() throws Exception {
ExtractReader extractReader = new ExtractReader();
- List<Metadata> metadataList = extractReader.loadExtract(testJsonFile,
- ExtractReader.ALTER_METADATA_LIST.AS_IS,
- ExtractReader.IGNORE_LENGTH, ExtractReader.IGNORE_LENGTH);
+ List<Metadata> metadataList = extractReader.loadExtract(testJsonFile);
+
assertEquals(2, metadataList.size());
assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
assertEquals(1, metadataList.get(1).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
assertContains("attachment", metadataList.get(1).get(RecursiveParserWrapper.TIKA_CONTENT));
- metadataList = extractReader.loadExtract(testJsonFile, ExtractReader.ALTER_METADATA_LIST.FIRST_ONLY,
- ExtractReader.IGNORE_LENGTH, ExtractReader.IGNORE_LENGTH);
+ extractReader = new ExtractReader(ExtractReader.ALTER_METADATA_LIST.FIRST_ONLY);
+ metadataList = extractReader.loadExtract(testJsonFile);
assertEquals(1, metadataList.size());
assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
assertNotContained("attachment", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
- metadataList = extractReader.loadExtract(testJsonFile, ExtractReader.ALTER_METADATA_LIST.CONCATENATE_CONTENT_INTO_FIRST,
- ExtractReader.IGNORE_LENGTH, ExtractReader.IGNORE_LENGTH);
+ extractReader = new ExtractReader(ExtractReader.ALTER_METADATA_LIST.CONCATENATE_CONTENT_INTO_FIRST);
+ metadataList = extractReader.loadExtract(testJsonFile);
assertEquals(1, metadataList.size());
assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
@@ -71,9 +70,7 @@ public class ExtractReaderTest extends TikaTest {
@Test
public void testTextBasic() throws IOException {
ExtractReader extractReader = new ExtractReader();
- List<Metadata> metadataList = extractReader.loadExtract(testTxtFile,
- ExtractReader.ALTER_METADATA_LIST.AS_IS,
- ExtractReader.IGNORE_LENGTH, ExtractReader.IGNORE_LENGTH);
+ List<Metadata> metadataList = extractReader.loadExtract(testTxtFile);
assertEquals(1, metadataList.size());
Metadata m = metadataList.get(0);
assertEquals(1, m.getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java b/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java
index 5e43303..90d52c0 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java
@@ -39,7 +39,7 @@ public class ResultsReporterTest {
configFile = Paths.get(this.getClass().getResource("/reports.xml").toURI());
tmpDir = Files.createTempDirectory("tika-eval-report-test-");
- connection = new H2Util(tmpDir.resolve(dbName)).getConnection(true);
+ connection = new H2Util(tmpDir.resolve(dbName)).getConnection();
String sql = "CREATE TABLE test_table (ID LONG PRIMARY KEY, STRING VARCHAR(32))";
Statement st = connection.createStatement();
st.execute(sql);
diff --git a/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml b/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
index 8d57ab7..bb83f8b 100644
--- a/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
+++ b/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
@@ -63,7 +63,7 @@
/>
<consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
- consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"
+ consumerBuilderClass="org.apache.tika.eval.batch.ExtractProfilerBuilder"
errorLogFile="src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml"
extracts="src/test/resources/test-dirs/extractsA"
commonTokens="src/test/resources/common_tokens"/>
diff --git a/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml b/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
index 62c7911..22a7dc2 100644
--- a/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
+++ b/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
@@ -63,7 +63,7 @@
/>
<consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
- consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"
+ consumerBuilderClass="org.apache.tika.eval.batch.ExtractProfilerBuilder"
errorLogFile="src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml"
extracts="src/test/resources/test-dirs/extractsA"
inputDir="src/test/resources/test-dirs/raw_input"
--
To stop receiving notification emails like this one, please contact
['"commits@tika.apache.org" <co...@tika.apache.org>'].