You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/03/03 13:41:52 UTC

[tika] branch master updated: TIKA-2287 -- add general jdbc.

This is an automated email from the ASF dual-hosted git repository.

tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git

The following commit(s) were added to refs/heads/master by this push:
       new  24fec4d   TIKA-2287 -- add general jdbc.
24fec4d is described below

commit 24fec4d4537f283ac33fccc1d0b5d7d44f4a78d7
Author: tballison <ta...@mitre.org>
AuthorDate: Fri Mar 3 08:41:43 2017 -0500

    TIKA-2287 -- add general jdbc.
---
 .../java/org/apache/tika/eval/ExtractComparer.java |  41 ++---
 .../java/org/apache/tika/eval/ExtractProfiler.java |  34 ++--
 .../java/org/apache/tika/eval/TikaEvalCLI.java     |  21 ++-
 .../org/apache/tika/eval/XMLErrorLogUpdater.java   |   6 +-
 .../apache/tika/eval/batch/DBConsumersManager.java |  12 +-
 .../tika/eval/batch/EvalConsumerBuilder.java       |  32 +++-
 .../tika/eval/batch/EvalConsumersBuilder.java      |  37 +++--
 .../tika/eval/batch/ExtractComparerBuilder.java    | 150 ++++++++++++++++++
 ...merBuilder.java => ExtractProfilerBuilder.java} |  52 +++++--
 .../tika/eval/batch/FileComparerBuilder.java       | 105 -------------
 .../main/java/org/apache/tika/eval/db/H2Util.java  |  26 +++-
 .../tika/eval/db/{DBUtil.java => JDBCUtil.java}    | 171 +++++++++++++++------
 .../java/org/apache/tika/eval/db/MimeBuffer.java   |   4 +-
 .../java/org/apache/tika/eval/db/TableInfo.java    |  11 +-
 .../java/org/apache/tika/eval/io/DBWriter.java     |   6 +-
 .../org/apache/tika/eval/io/ExtractReader.java     |  31 +++-
 .../apache/tika/eval/reports/ResultsReporter.java  |  57 ++++---
 .../tika/eval/reports/XLSXHREFFormatter.java       |   4 +-
 tika-eval/src/main/resources/db.properties         |   9 ++
 .../main/resources/tika-eval-comparison-config.xml |  12 +-
 .../main/resources/tika-eval-profiler-config.xml   |  12 +-
 .../org/apache/tika/eval/ProfilerBatchTest.java    |   2 +-
 .../org/apache/tika/eval/SimpleComparerTest.java   |   5 +-
 .../java/org/apache/tika/eval/TikaEvalCLITest.java |  38 ++++-
 .../org/apache/tika/eval/io/ExtractReaderTest.java |  17 +-
 .../tika/eval/reports/ResultsReporterTest.java     |   2 +-
 .../single-file-profiler-crawl-extract-config.xml  |   2 +-
 .../single-file-profiler-crawl-input-config.xml    |   2 +-
 28 files changed, 593 insertions(+), 308 deletions(-)

diff --git a/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java b/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
index 8e8cb97..a50b710 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
@@ -55,19 +55,14 @@ public class ExtractComparer extends AbstractProfiler {
         Option extractsB = new Option("extractsB", true, "directory for extractsB files");
         extractsB.setRequired(true);
 
-        Option db = new Option("db", true, "db file to which to write results");
-        db.setRequired(true);
-
         Option inputDir = new Option("inputDir", true,
                 "optional: directory of original binary input files if it exists " +
                         "or can be the same as -extractsA or -extractsB. If not specified, -inputDir=-extractsA");
-        inputDir.setRequired(true);
 
 
         OPTIONS = new Options()
                 .addOption(extractsA)
                 .addOption(extractsB)
-                .addOption(db)
                 .addOption(inputDir)
                 .addOption("bc", "optional: tika-batch config file")
                 .addOption("numConsumers", true, "optional: number of consumer threads")
@@ -77,7 +72,15 @@ public class ExtractComparer extends AbstractProfiler {
                                 "take just the first/container document ('first_only'), " +
                                 "concatenate all content into the first metadata item ('concatenate_content')"))
                 .addOption("minExtractLength", true, "minimum extract length to process (in bytes)")
-                .addOption("maxExtractLength", true, "maximum extract length to process (in bytes)");
+                .addOption("maxExtractLength", true, "maximum extract length to process (in bytes)")
+                .addOption("db", true, "db file to which to write results")
+                .addOption("jdbc", true, "EXPERT: full jdbc connection string. Must specify this or -db <h2db>")
+                .addOption("jdbcDriver", true, "EXPERT: jdbc driver, or specify via -Djdbc.driver")
+                .addOption("tablePrefixA", true, "EXPERT: optional prefix for table names for A")
+                .addOption("tablePrefixB", true, "EXPERT: optional prefix for table names for B")
+                .addOption("drop", true, "drop tables if they exist")
+                .addOption("maxFilesToAdd", true, "maximum number of files to add to the crawler")
+        ;
     }
 
     public static void USAGE() {
@@ -87,7 +90,7 @@ public class ExtractComparer extends AbstractProfiler {
                 "java -jar tika-eval-x.y.jar Compare -extractsA extractsA -extractsB extractsB -db mydb",
                 "Tool: Compare",
                 ExtractComparer.OPTIONS,
-                "Note: do not include the .mv.db at the end of the db name.");
+                "Note: for the default h2 db, do not include the .mv.db at the end of the db name.");
     }
 
     private final static String FIELD_A = "fa";
@@ -155,24 +158,17 @@ public class ExtractComparer extends AbstractProfiler {
     private final Path extractsA;
     private final Path extractsB;
 
-    private final long minExtractLength;
-    private final long maxExtractLength;
-    private final ExtractReader.ALTER_METADATA_LIST alterExtractList;
-
     private final TokenContraster tokenContraster = new TokenContraster();
-    private final ExtractReader extractReader = new ExtractReader();
+    private final ExtractReader extractReader;
 
     public ExtractComparer(ArrayBlockingQueue<FileResource> queue,
-                           Path inputDir, Path extractsA, Path extractsB,
-                           IDBWriter writer, long minJsonLength,
-                           long maxJsonLength, ExtractReader.ALTER_METADATA_LIST alterExtractList) {
+                           Path inputDir, Path extractsA, Path extractsB, ExtractReader extractReader,
+                           IDBWriter writer) {
         super(queue, writer);
-        this.minExtractLength = minJsonLength;
-        this.maxExtractLength = maxJsonLength;
         this.inputDir = inputDir;
         this.extractsA = extractsA;
         this.extractsB = extractsB;
-        this.alterExtractList = alterExtractList;
+        this.extractReader = extractReader;
     }
 
     @Override
@@ -213,8 +209,7 @@ public class ExtractComparer extends AbstractProfiler {
         List<Metadata> metadataListA = null;
         if (extractExceptionA == null) {
             try {
-                metadataListA = extractReader.loadExtract(fpsA.getExtractFile(),
-                        alterExtractList, minExtractLength, maxExtractLength);
+                metadataListA = extractReader.loadExtract(fpsA.getExtractFile());
             } catch (ExtractReaderException e) {
                 extractExceptionA = e.getType();
             }
@@ -222,8 +217,7 @@ public class ExtractComparer extends AbstractProfiler {
 
         List<Metadata> metadataListB = null;
         try {
-            metadataListB = extractReader.loadExtract(fpsB.getExtractFile(),
-                    alterExtractList, minExtractLength, maxExtractLength);
+            metadataListB = extractReader.loadExtract(fpsB.getExtractFile());
         } catch (ExtractReaderException e) {
             extractExceptionB = e.getType();
         }
@@ -430,9 +424,6 @@ public class ExtractComparer extends AbstractProfiler {
         return -1;
     }
 
-
-
-
     private void writeContrasts(Map<Cols, String> data, ContrastStatistics contrastStatistics) {
         writeContrastString(data, Cols.TOP_10_MORE_IN_A, contrastStatistics.getTopNMoreA());
         writeContrastString(data, Cols.TOP_10_MORE_IN_B, contrastStatistics.getTopNMoreB());
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java b/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
index d3f58ef..1f9bfda 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
@@ -42,9 +42,6 @@ public class ExtractProfiler extends AbstractProfiler {
 
     static Options OPTIONS;
     static {
-        Option db = new Option("db", true, "db file to which to write results");
-        db.setRequired(true);
-
         //By the time this commandline is parsed, there should be both an extracts and an inputDir
         Option extracts = new Option("extracts", true, "directory for extract files");
         extracts.setRequired(true);
@@ -52,10 +49,8 @@ public class ExtractProfiler extends AbstractProfiler {
         Option inputDir = new Option("inputDir", true,
                 "optional: directory for original binary input documents."+
         " If not specified, -extracts is crawled as is.");
-        inputDir.setRequired(true);
 
         OPTIONS = new Options()
-                .addOption(db)
                 .addOption(extracts)
                 .addOption(inputDir)
                 .addOption("bc", "optional: tika-batch config file")
@@ -66,7 +61,15 @@ public class ExtractProfiler extends AbstractProfiler {
                                 "take just the first/container document ('first_only'), " +
                                 "concatenate all content into the first metadata item ('concatenate_content')"))
                 .addOption("minExtractLength", true, "minimum extract length to process (in bytes)")
-                .addOption("maxExtractLength", true, "maximum extract length to process (in bytes)");
+                .addOption("maxExtractLength", true, "maximum extract length to process (in bytes)")
+                .addOption("db", true, "db file to which to write results")
+                .addOption("jdbc", true, "EXPERT: full jdbc connection string. Must specify this or -db <h2db>")
+                .addOption("jdbcDriver", true, "EXPERT: jdbc driver, or specify via -Djdbc.driver")
+                .addOption("tablePrefix", true, "EXPERT: optional prefix for table names")
+                .addOption("drop", true, "drop tables if they exist")
+                .addOption("maxFilesToAdd", true, "maximum number of files to add to the crawler")
+
+        ;
 
     }
 
@@ -77,7 +80,7 @@ public class ExtractProfiler extends AbstractProfiler {
                 "java -jar tika-eval-x.y.jar Profile -extracts extracts -db mydb [-inputDir input]",
                 "Tool: Profile",
                 ExtractProfiler.OPTIONS,
-                "Note: for h2 db, do not include the .mv.db at the end of the db name.");
+                "Note: for the default h2 db, do not include the .mv.db at the end of the db name.");
     }
 
     private final static String FIELD = "f";
@@ -132,8 +135,8 @@ public class ExtractProfiler extends AbstractProfiler {
             new ColInfo(Cols.NUM_UNIQUE_TOKENS, Types.INTEGER),
             new ColInfo(Cols.COMMON_TOKENS_LANG, Types.VARCHAR, 12),
             new ColInfo(Cols.NUM_COMMON_TOKENS, Types.INTEGER),
-            new ColInfo(Cols.TOP_N_TOKENS, Types.VARCHAR, 1024),
             new ColInfo(Cols.NUM_ALPHABETIC_TOKENS, Types.INTEGER),
+            new ColInfo(Cols.TOP_N_TOKENS, Types.VARCHAR, 1024),
             new ColInfo(Cols.LANG_ID_1, Types.VARCHAR, 12),
             new ColInfo(Cols.LANG_ID_PROB_1, Types.FLOAT),
             new ColInfo(Cols.LANG_ID_2, Types.VARCHAR, 12),
@@ -147,21 +150,15 @@ public class ExtractProfiler extends AbstractProfiler {
 
     private final Path inputDir;
     private final Path extracts;
-    private final long minExtractLength;
-    private final long maxExtractLength;
-    private final ExtractReader.ALTER_METADATA_LIST alterExtractList;
-    private final ExtractReader extractReader = new ExtractReader();
+    private final ExtractReader extractReader;
 
     public ExtractProfiler(ArrayBlockingQueue<FileResource> queue,
                            Path inputDir, Path extracts,
-                           IDBWriter dbWriter, long minExtractLength,
-                           long maxExtractLength, ExtractReader.ALTER_METADATA_LIST alterExtractList) {
+                           ExtractReader extractReader, IDBWriter dbWriter) {
         super(queue, dbWriter);
         this.inputDir = inputDir;
         this.extracts = extracts;
-        this.minExtractLength = minExtractLength;
-        this.maxExtractLength = maxExtractLength;
-        this.alterExtractList = alterExtractList;
+        this.extractReader = extractReader;
     }
 
     @Override
@@ -182,8 +179,7 @@ public class ExtractProfiler extends AbstractProfiler {
 
         List<Metadata> metadataList = null;
         try {
-            metadataList = extractReader.loadExtract(fps.getExtractFile(),
-                    alterExtractList, minExtractLength, maxExtractLength);
+            metadataList = extractReader.loadExtract(fps.getExtractFile());
         } catch (ExtractReaderException e) {
             extractExceptionType = e.getType();
         }
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/TikaEvalCLI.java b/tika-eval/src/main/java/org/apache/tika/eval/TikaEvalCLI.java
index 2fd7f72..fc3f22b 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/TikaEvalCLI.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/TikaEvalCLI.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.DefaultParser;
 import org.apache.commons.cli.ParseException;
 import org.apache.tika.batch.fs.FSBatchProcessCLI;
@@ -103,7 +104,8 @@ public class TikaEvalCLI {
                 i++;
             } else if (arg.equals("-alterExtract")) {
                 if (i+1 >= argList.size()) {
-                    System.err.println("Must specify directory after -extractsB");
+                    System.err.println("Must specify type 'as_is', 'first_only' or " +
+                            "'concatenate_content' after -alterExtract");
                     ExtractComparer.USAGE();
                     return;
                 }
@@ -147,7 +149,12 @@ public class TikaEvalCLI {
             String[] updatedArgs = argList.toArray(new String[argList.size()]);
             DefaultParser defaultCLIParser = new DefaultParser();
             try {
-                defaultCLIParser.parse(ExtractProfiler.OPTIONS, updatedArgs);
+                CommandLine commandLine = defaultCLIParser.parse(ExtractProfiler.OPTIONS, updatedArgs);
+                if (commandLine.hasOption("db") && commandLine.hasOption("jdbc")) {
+                    System.out.println("Please specify either the default -db or the full -jdbc, not both");
+                    ExtractProfiler.USAGE();
+                    return;
+                }
             } catch (ParseException e) {
                 System.out.println(e.getMessage()+"\n");
                 ExtractProfiler.USAGE();
@@ -192,7 +199,8 @@ public class TikaEvalCLI {
                 i++;
             } else if (arg.equals("-alterExtract")) {
                 if (i+1 >= argList.size()) {
-                    System.err.println("Must specify directory after -extractsB");
+                    System.err.println("Must specify type 'as_is', 'first_only' or " +
+                            "'concatenate_content' after -alterExtract");
                     ExtractComparer.USAGE();
                     return;
                 }
@@ -232,7 +240,12 @@ public class TikaEvalCLI {
             String[] updatedArgs = argList.toArray(new String[argList.size()]);
             DefaultParser defaultCLIParser = new DefaultParser();
             try {
-                defaultCLIParser.parse(ExtractComparer.OPTIONS, updatedArgs);
+                CommandLine commandLine = defaultCLIParser.parse(ExtractComparer.OPTIONS, updatedArgs);
+                if (commandLine.hasOption("db") && commandLine.hasOption("jdbc")) {
+                    System.out.println("Please specify either the default -db or the full -jdbc, not both");
+                    ExtractComparer.USAGE();
+                    return;
+                }
             } catch (ParseException e) {
                 System.out.println(e.getMessage()+"\n");
                 ExtractComparer.USAGE();
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java b/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java
index 2f28614..42bc4c3 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java
@@ -34,8 +34,8 @@ import java.sql.Statement;
 
 import org.apache.log4j.Level;
 import org.apache.tika.eval.db.Cols;
-import org.apache.tika.eval.db.DBUtil;
 import org.apache.tika.eval.db.H2Util;
+import org.apache.tika.eval.db.JDBCUtil;
 import org.apache.tika.eval.db.TableInfo;
 import org.apache.tika.eval.io.XMLLogMsgHandler;
 import org.apache.tika.eval.io.XMLLogReader;
@@ -60,8 +60,8 @@ public class XMLErrorLogUpdater {
         Path xmlLogFileA = Paths.get(args[0]);
         Path xmlLogFileB = Paths.get(args[1]);
         Path db = Paths.get(args[2]);
-        DBUtil dbUtil = new H2Util(db);
-        Connection connection = dbUtil.getConnection(true);
+        JDBCUtil dbUtil = new H2Util(db);
+        Connection connection = dbUtil.getConnection();
         writer.update(connection, ExtractComparer.EXTRACT_EXCEPTION_TABLE_A, xmlLogFileA);
         writer.update(connection, ExtractComparer.EXTRACT_EXCEPTION_TABLE_B, xmlLogFileB);
         connection.commit();
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java
index f353662..2e3dfca 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java
@@ -27,7 +27,7 @@ import org.apache.tika.batch.ConsumersManager;
 import org.apache.tika.batch.FileResourceConsumer;
 import org.apache.tika.eval.AbstractProfiler;
 import org.apache.tika.eval.XMLErrorLogUpdater;
-import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.JDBCUtil;
 import org.apache.tika.eval.db.MimeBuffer;
 import org.apache.tika.eval.db.TableInfo;
 
@@ -38,10 +38,10 @@ public class DBConsumersManager extends ConsumersManager {
     private final MimeBuffer mimeBuffer;
     private final List<LogTablePair> errorLogs = new ArrayList<>();
 
-    public DBConsumersManager(DBUtil dbUtil, MimeBuffer mimeBuffer, List<FileResourceConsumer> consumers)
-            throws IOException {
+    public DBConsumersManager(JDBCUtil dbUtil, MimeBuffer mimeBuffer, List<FileResourceConsumer> consumers)
+            throws SQLException {
         super(consumers);
-        this.conn = dbUtil.getConnection(true);
+        this.conn = dbUtil.getConnection();
         this.mimeBuffer = mimeBuffer;
     }
 
@@ -89,10 +89,10 @@ public class DBConsumersManager extends ConsumersManager {
         }
     }
 
-    public void addErrorLogTablePair(Path log, TableInfo tableName) {
+    public void addErrorLogTablePair(Path log, TableInfo tableInfo) {
         LogTablePair p = new LogTablePair();
         p.log = log;
-        p.tableInfo = tableName;
+        p.tableInfo = tableInfo;
         errorLogs.add(p);
     }
 
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
index d21d1de..c8781e2 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
@@ -22,7 +22,9 @@ import java.sql.Connection;
 import java.sql.SQLException;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -30,7 +32,7 @@ import org.apache.tika.batch.FileResource;
 import org.apache.tika.batch.FileResourceConsumer;
 import org.apache.tika.eval.AbstractProfiler;
 import org.apache.tika.eval.db.Cols;
-import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.JDBCUtil;
 import org.apache.tika.eval.db.MimeBuffer;
 import org.apache.tika.eval.db.TableInfo;
 import org.apache.tika.eval.io.DBWriter;
@@ -42,12 +44,12 @@ public abstract class EvalConsumerBuilder {
     private AtomicInteger count = new AtomicInteger(0);
     protected ArrayBlockingQueue<FileResource> queue;
     Map<String, String> localAttrs;
-    DBUtil dbUtil;
+    JDBCUtil dbUtil;
     private MimeBuffer mimeBuffer;
     AtomicInteger initialized = new AtomicInteger(0);
 
     public void init(ArrayBlockingQueue<FileResource> queue, Map<String, String> localAttrs,
-                     DBUtil dbUtil, MimeBuffer mimeBuffer) throws IOException, SQLException {
+                     JDBCUtil dbUtil, MimeBuffer mimeBuffer) throws IOException, SQLException {
         this.queue = queue;
         this.localAttrs = localAttrs;
         this.dbUtil = dbUtil;
@@ -60,11 +62,29 @@ public abstract class EvalConsumerBuilder {
 
     public abstract FileResourceConsumer build() throws IOException, SQLException;
 
-    protected abstract List<TableInfo> getTableInfo();
+    protected abstract List<TableInfo> getTableInfo(String tableNamePrefixA, String tableNamePrefixB);
 
     protected abstract void addErrorLogTablePairs(DBConsumersManager manager);
 
     public void populateRefTables() throws IOException, SQLException {
+        //test for one ref table.  If it exists, don't populate ref tables
+        //TODO: test one at a time
+        boolean tableExists = false;
+        try (Connection connection = dbUtil.getConnection()) {
+            Set<String> tables = dbUtil.getTables(connection);
+            if (tables.contains(
+                    AbstractProfiler.REF_PARSE_ERROR_TYPES.getName().toLowerCase(Locale.US)
+            )) {
+                tableExists = true;
+            }
+        } catch (SQLException e) {
+            //swallow
+        }
+
+        if (tableExists) {
+            return;
+        }
+
         IDBWriter writer = getDBWriter();
         Map<Cols, String> m = new HashMap<>();
         for (AbstractProfiler.PARSE_ERROR_TYPE t : AbstractProfiler.PARSE_ERROR_TYPE.values()) {
@@ -92,8 +112,8 @@ public abstract class EvalConsumerBuilder {
     }
 
     protected IDBWriter getDBWriter() throws IOException, SQLException {
-        Connection conn = dbUtil.getConnection(true);
-        return new DBWriter(conn, getTableInfo(), dbUtil, mimeBuffer);
+        Connection conn = dbUtil.getConnection();
+        return new DBWriter(conn, getTableInfo(null, null), dbUtil, mimeBuffer);
     }
 
     ExtractReader.ALTER_METADATA_LIST getAlterMetadata(Map<String, String> localAttrs) {
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
index 883f908..d1b6465 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
@@ -33,8 +33,8 @@ import org.apache.tika.batch.builders.AbstractConsumersBuilder;
 import org.apache.tika.batch.builders.BatchProcessBuilder;
 import org.apache.tika.config.TikaConfig;
 import org.apache.tika.eval.AbstractProfiler;
-import org.apache.tika.eval.db.DBUtil;
 import org.apache.tika.eval.db.H2Util;
+import org.apache.tika.eval.db.JDBCUtil;
 import org.apache.tika.eval.db.MimeBuffer;
 import org.apache.tika.eval.util.LanguageIDWrapper;
 import org.apache.tika.util.ClassLoaderUtil;
@@ -55,6 +55,9 @@ public class EvalConsumersBuilder extends AbstractConsumersBuilder {
 
 
         Path db = getPath(localAttrs, "db");
+        String jdbcConnectionString = localAttrs.get("jdbc");
+
+
         Path langModelDir = getPath(localAttrs, "langModelDir");
 
         try {
@@ -68,31 +71,43 @@ public class EvalConsumersBuilder extends AbstractConsumersBuilder {
         }
 
         Path commonTokens = getPath(localAttrs, "commonTokens");
+        //can be null, in which case will load from memory
         try {
             AbstractProfiler.loadCommonTokens(commonTokens);
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
 
-        boolean append = PropsUtil.getBoolean(localAttrs.get("dbAppend"), false);
+        boolean forceDrop = PropsUtil.getBoolean(localAttrs.get("drop"), false);
 
-        if (db == null) {
-            throw new RuntimeException("Must specify: -db");
+        JDBCUtil jdbcUtil = null;
+        if (db != null) {
+            jdbcUtil = new H2Util(db);
+        } else if (jdbcConnectionString != null) {
+            jdbcUtil = new JDBCUtil(jdbcConnectionString, localAttrs.get("jdbcDriver"));
+        } else {
+            throw new RuntimeException("Must specify: -db or -jdbc");
         }
-        //parameterize which db util to use
-        DBUtil util = new H2Util(db);
         EvalConsumerBuilder consumerBuilder = ClassLoaderUtil.buildClass(EvalConsumerBuilder.class,
                 PropsUtil.getString(localAttrs.get("consumerBuilderClass"), null));
         if (consumerBuilder == null) {
             throw new RuntimeException("Must specify consumerBuilderClass in config file");
         }
 
+        String tablePrefixA = localAttrs.get("tablePrefix");
+        if (tablePrefixA == null) {
+            tablePrefixA = localAttrs.get("tablePrefixA");
+        }
+        String tablePrefixB = localAttrs.get("tablePrefixB");
+
+        tablePrefixA = (tablePrefixA == null || tablePrefixA.endsWith("_")) ? tablePrefixA : tablePrefixA+"_";
+        tablePrefixB = (tablePrefixB == null || tablePrefixB.endsWith("_")) ? tablePrefixB : tablePrefixB+"_";
 
         MimeBuffer mimeBuffer = null;
         try {
-            util.createDB(consumerBuilder.getTableInfo(), append);
-            mimeBuffer = new MimeBuffer(util.getConnection(true), TikaConfig.getDefaultConfig());
-            consumerBuilder.init(queue, localAttrs, util, mimeBuffer);
+            jdbcUtil.createTables(consumerBuilder.getTableInfo(tablePrefixA, tablePrefixB), forceDrop);
+            mimeBuffer = new MimeBuffer(jdbcUtil.getConnection(), TikaConfig.getDefaultConfig());
+            consumerBuilder.init(queue, localAttrs, jdbcUtil, mimeBuffer);
         } catch (IOException | SQLException e) {
             throw new RuntimeException(e);
         }
@@ -107,8 +122,8 @@ public class EvalConsumersBuilder extends AbstractConsumersBuilder {
 
         DBConsumersManager manager;
         try {
-            manager = new DBConsumersManager(util, mimeBuffer, consumers);
-        } catch (IOException e) {
+            manager = new DBConsumersManager(jdbcUtil, mimeBuffer, consumers);
+        } catch (SQLException e) {
             throw new RuntimeException(e);
         }
         consumerBuilder.addErrorLogTablePairs(manager);
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/ExtractComparerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/ExtractComparerBuilder.java
new file mode 100644
index 0000000..09152cd
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/ExtractComparerBuilder.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.batch;
+
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.tika.batch.FileResourceConsumer;
+import org.apache.tika.eval.AbstractProfiler;
+import org.apache.tika.eval.ExtractComparer;
+import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.ExtractReader;
+import org.apache.tika.util.PropsUtil;
+
+public class ExtractComparerBuilder extends EvalConsumerBuilder {
+    private final List<TableInfo> tableInfosA;
+    private final List<TableInfo> tableInfosB;
+    private final List<TableInfo> tableInfosAandB;
+    private final List<TableInfo> refTableInfos;
+
+    public ExtractComparerBuilder() {
+        List<TableInfo> tableInfosA = new ArrayList<>();
+        List<TableInfo> tableInfosB = new ArrayList<>();
+        List<TableInfo> tableInfosAandB = new ArrayList<>();
+        tableInfosA.add(ExtractComparer.PROFILES_A);
+        tableInfosA.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
+        tableInfosA.add(ExtractComparer.EXCEPTION_TABLE_A);
+        tableInfosA.add(ExtractComparer.CONTENTS_TABLE_A);
+        tableInfosA.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
+        tableInfosA.add(ExtractComparer.EMBEDDED_FILE_PATH_TABLE_A);
+
+        tableInfosB.add(ExtractComparer.PROFILES_B);
+        tableInfosB.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
+        tableInfosB.add(ExtractComparer.EXCEPTION_TABLE_B);
+        tableInfosB.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
+        tableInfosB.add(ExtractComparer.CONTENTS_TABLE_B);
+        tableInfosB.add(ExtractComparer.EMBEDDED_FILE_PATH_TABLE_B);
+
+        tableInfosAandB.add(ExtractComparer.COMPARISON_CONTAINERS);
+        tableInfosAandB.add(ExtractComparer.CONTENT_COMPARISONS);
+        tableInfosAandB.add(AbstractProfiler.MIME_TABLE);
+
+        List<TableInfo> refTableInfos = new ArrayList<>();
+        refTableInfos.add(ExtractComparer.REF_PAIR_NAMES);
+        refTableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
+        refTableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
+        refTableInfos.add(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES);
+
+        this.tableInfosA = Collections.unmodifiableList(tableInfosA);
+        this.tableInfosB = Collections.unmodifiableList(tableInfosB);
+        this.tableInfosAandB = Collections.unmodifiableList(tableInfosAandB);
+        this.refTableInfos = Collections.unmodifiableList(refTableInfos);
+    }
+
+    @Override
+    public FileResourceConsumer build() throws IOException, SQLException {
+        Path extractsA = PropsUtil.getPath(localAttrs.get("extractsA"), null);
+        if (extractsA == null) {
+            throw new RuntimeException("Must specify \"extractsA\" -- directory for 'A' extracts");
+        }
+        Path extractsB = PropsUtil.getPath(localAttrs.get("extractsB"), null);
+        if (extractsB == null) {
+            throw new RuntimeException("Must specify \"extractsB\" -- directory for 'B' extracts");
+        }
+
+        Path inputRootDir = PropsUtil.getPath(localAttrs.get("inputDir"), null);
+
+        long minExtractLength = PropsUtil.getLong(localAttrs.get("minExtractLength"), -1L);
+        long maxExtractLength = PropsUtil.getLong(localAttrs.get("maxExtractLength"), -1L);
+
+        ExtractReader.ALTER_METADATA_LIST alterExtractList = getAlterMetadata(localAttrs);
+
+
+        if (inputRootDir == null) {
+            //this is for the sake of the crawler
+            throw new RuntimeException("Must specify an -inputDir");
+        }
+
+        return new ExtractComparer(queue, inputRootDir, extractsA, extractsB,
+                new ExtractReader(alterExtractList, minExtractLength, maxExtractLength),
+                getDBWriter());
+    }
+
+    @Override
+    protected List<TableInfo> getTableInfo(String tableNamePrefixA, String tableNamePrefixB) {
+        List<TableInfo> allTables = new ArrayList<>();
+        if (tableNamePrefixA != null) {
+            for (TableInfo tableInfo : tableInfosA) {
+                tableInfo.setNamePrefix(tableNamePrefixA);
+            }
+        }
+
+        if (tableNamePrefixB != null) {
+            for (TableInfo tableInfo : tableInfosB) {
+                tableInfo.setNamePrefix(tableNamePrefixB);
+            }
+        }
+
+        if (tableNamePrefixA != null || tableNamePrefixB != null) {
+            String aAndB = (tableNamePrefixA == null) ? "" : tableNamePrefixA;
+            aAndB = (tableNamePrefixB == null) ? aAndB : aAndB+tableNamePrefixB;
+            for (TableInfo tableInfo : tableInfosAandB) {
+                tableInfo.setNamePrefix(aAndB);
+            }
+        }
+
+        allTables.addAll(tableInfosA);
+        allTables.addAll(tableInfosB);
+        allTables.addAll(tableInfosAandB);
+        allTables.addAll(refTableInfos);
+        allTables = Collections.unmodifiableList(allTables);
+        return allTables;
+    }
+
+
+    @Override
+    protected void addErrorLogTablePairs(DBConsumersManager manager) {
+        Path errorLogA = PropsUtil.getPath(localAttrs.get("errorLogFileA"), null);
+        if (errorLogA == null) {
+            return;
+        }
+        manager.addErrorLogTablePair(errorLogA, ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
+        Path errorLogB = PropsUtil.getPath(localAttrs.get("errorLogFileB"), null);
+        if (errorLogB == null) {
+            return;
+        }
+        manager.addErrorLogTablePair(errorLogB, ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
+
+    }
+
+}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/ExtractProfilerBuilder.java
similarity index 69%
rename from tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java
rename to tika-eval/src/main/java/org/apache/tika/eval/batch/ExtractProfilerBuilder.java
index c1c25d3..0df225d 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/ExtractProfilerBuilder.java
@@ -21,6 +21,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import org.apache.tika.batch.FileResourceConsumer;
@@ -31,7 +32,26 @@ import org.apache.tika.eval.io.ExtractReader;
 import org.apache.tika.util.PropsUtil;
 
 
-public class SingleFileConsumerBuilder extends EvalConsumerBuilder {
+public class ExtractProfilerBuilder extends EvalConsumerBuilder {
+    private final List<TableInfo> tableInfos;
+    private final List<TableInfo> refTableInfos;
+    public ExtractProfilerBuilder() {
+        List<TableInfo> tableInfos = new ArrayList();
+        tableInfos.add(AbstractProfiler.MIME_TABLE);
+        tableInfos.add(ExtractProfiler.CONTAINER_TABLE);
+        tableInfos.add(ExtractProfiler.PROFILE_TABLE);
+        tableInfos.add(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
+        tableInfos.add(ExtractProfiler.EXCEPTION_TABLE);
+        tableInfos.add(ExtractProfiler.CONTENTS_TABLE);
+        tableInfos.add(ExtractProfiler.EMBEDDED_FILE_PATH_TABLE);
+        this.tableInfos = Collections.unmodifiableList(tableInfos);
+
+        List<TableInfo> refTableInfos = new ArrayList<>();
+        refTableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
+        refTableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
+        refTableInfos.add(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES);
+        this.refTableInfos = Collections.unmodifiableList(refTableInfos);
+    }
 
     @Override
     public FileResourceConsumer build() throws IOException, SQLException {
@@ -60,24 +80,24 @@ public class SingleFileConsumerBuilder extends EvalConsumerBuilder {
         if (extracts == null && inputDir != null) {
             extracts = inputDir;
         }
-        return new ExtractProfiler(queue, inputDir, extracts, getDBWriter(),
-                minExtractLength, maxExtractLength, alterExtractList);
+        return new ExtractProfiler(queue, inputDir, extracts,
+                new ExtractReader(alterExtractList, minExtractLength, maxExtractLength),
+                getDBWriter());
     }
 
+    //TODO: clean up the interface so we don't need vestigial B
     @Override
-    protected List<TableInfo> getTableInfo() {
-        List<TableInfo> tableInfos = new ArrayList<TableInfo>();
-        tableInfos.add(AbstractProfiler.MIME_TABLE);
-        tableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
-        tableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
-        tableInfos.add(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES);
-        tableInfos.add(ExtractProfiler.CONTAINER_TABLE);
-        tableInfos.add(ExtractProfiler.PROFILE_TABLE);
-        tableInfos.add(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
-        tableInfos.add(ExtractProfiler.EXCEPTION_TABLE);
-        tableInfos.add(ExtractProfiler.CONTENTS_TABLE);
-        tableInfos.add(ExtractProfiler.EMBEDDED_FILE_PATH_TABLE);
-        return tableInfos;
+    protected List<TableInfo> getTableInfo(String tableNamePrefix, String tableNamePrefixB) {
+        //ignore b
+        if (tableNamePrefix != null && !tableNamePrefix.equals("null")) {
+            for (TableInfo tableInfo : tableInfos) {
+                tableInfo.setNamePrefix(tableNamePrefix);
+            }
+        }
+        List allTables = new ArrayList();
+        allTables.addAll(tableInfos);
+        allTables.addAll(refTableInfos);
+        return Collections.unmodifiableList(allTables);
     }
 
 
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java
deleted file mode 100644
index 0b35cb4..0000000
--- a/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.eval.batch;
-
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.tika.batch.FileResourceConsumer;
-import org.apache.tika.eval.AbstractProfiler;
-import org.apache.tika.eval.ExtractComparer;
-import org.apache.tika.eval.db.TableInfo;
-import org.apache.tika.eval.io.ExtractReader;
-import org.apache.tika.util.PropsUtil;
-
-public class FileComparerBuilder extends EvalConsumerBuilder {
-
-    @Override
-    public FileResourceConsumer build() throws IOException, SQLException {
-        Path thisRootDir = PropsUtil.getPath(localAttrs.get("extractsA"), null);
-        if (thisRootDir == null) {
-            throw new RuntimeException("Must specify \"extractsA\" -- directory for 'A' extracts");
-        }
-        Path thatRootDir = PropsUtil.getPath(localAttrs.get("extractsB"), null);
-        if (thatRootDir == null) {
-            throw new RuntimeException("Must specify \"extractsB\" -- directory for 'B' extracts");
-        }
-
-        Path inputRootDir = PropsUtil.getPath(localAttrs.get("inputDir"), null);
-
-        long minExtractLength = PropsUtil.getLong(localAttrs.get("minExtractLength"), -1L);
-        long maxExtractLength = PropsUtil.getLong(localAttrs.get("maxExtractLength"), -1L);
-
-        ExtractReader.ALTER_METADATA_LIST alterExtractList = getAlterMetadata(localAttrs);
-
-
-        if (inputRootDir == null) {
-            //this is for the sake of the crawler
-            throw new RuntimeException("Must specify an -inputDir");
-        }
-
-        return new ExtractComparer(queue, inputRootDir, thisRootDir, thatRootDir, getDBWriter(),
-                minExtractLength, maxExtractLength, alterExtractList);
-    }
-
-    @Override
-    protected List<TableInfo> getTableInfo() {
-        List<TableInfo> tableInfos = new ArrayList<>();
-        tableInfos.add(ExtractComparer.COMPARISON_CONTAINERS);
-        tableInfos.add(ExtractComparer.PROFILES_A);
-        tableInfos.add(ExtractComparer.PROFILES_B);
-        tableInfos.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
-        tableInfos.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
-        tableInfos.add(ExtractComparer.EXCEPTION_TABLE_A);
-        tableInfos.add(ExtractComparer.EXCEPTION_TABLE_B);
-        tableInfos.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
-        tableInfos.add(ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
-        tableInfos.add(ExtractComparer.CONTENTS_TABLE_A);
-        tableInfos.add(ExtractComparer.CONTENTS_TABLE_B);
-        tableInfos.add(ExtractComparer.EMBEDDED_FILE_PATH_TABLE_A);
-        tableInfos.add(ExtractComparer.EMBEDDED_FILE_PATH_TABLE_B);
-
-        tableInfos.add(ExtractComparer.CONTENT_COMPARISONS);
-        tableInfos.add(AbstractProfiler.MIME_TABLE);
-        tableInfos.add(ExtractComparer.REF_PAIR_NAMES);
-        tableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
-        tableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
-        tableInfos.add(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES);
-        return tableInfos;
-    }
-
-
-    @Override
-    protected void addErrorLogTablePairs(DBConsumersManager manager) {
-        Path errorLogA = PropsUtil.getPath(localAttrs.get("errorLogFileA"), null);
-        if (errorLogA == null) {
-            return;
-        }
-        manager.addErrorLogTablePair(errorLogA, ExtractComparer.EXTRACT_EXCEPTION_TABLE_A);
-        Path errorLogB = PropsUtil.getPath(localAttrs.get("errorLogFileB"), null);
-        if (errorLogB == null) {
-            return;
-        }
-        manager.addErrorLogTablePair(errorLogB, ExtractComparer.EXTRACT_EXCEPTION_TABLE_B);
-
-    }
-
-}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/H2Util.java b/tika-eval/src/main/java/org/apache/tika/eval/db/H2Util.java
index 0b3ebac..14037be 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/db/H2Util.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/H2Util.java
@@ -18,6 +18,7 @@ package org.apache.tika.eval.db;
 
 import java.nio.file.Path;
 import java.sql.Connection;
+import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
@@ -27,12 +28,25 @@ import java.util.Set;
 import org.apache.commons.io.FilenameUtils;
 
 
-public class H2Util extends DBUtil {
+public class H2Util extends JDBCUtil {
+
+    private final static String DRIVER_CLASS = "org.h2.Driver";
+
+    private final Path db;
 
     public H2Util(Path db) {
-        super(db);
+        super(getConnectionString(db, true), DRIVER_CLASS);
+        this.db = db;
     }
 
+    public static boolean databaseExists(Path db) {
+        String connString = getConnectionString(db, false);
+        try (Connection conn = DriverManager.getConnection(connString)) {
+        } catch (SQLException e) {
+            return false;
+        }
+        return true;
+    }
     @Override
     public String getJDBCDriverClass() {
         return "org.h2.Driver";
@@ -48,9 +62,13 @@ public class H2Util extends DBUtil {
     }
 
     @Override
-    public String getConnectionString(Path db, boolean createIfDoesntExist) {
+    public String getConnectionString() {
+        return getConnectionString(db, true);
+    }
+
+    private static String getConnectionString(Path db, boolean createDBIfItDoesntExist) {
         String s = "jdbc:h2:"+ FilenameUtils.separatorsToUnix(db.toAbsolutePath().toString());
-        if (! createIfDoesntExist) {
+        if (! createDBIfItDoesntExist) {
             s += ";IFEXISTS=TRUE";
         }
         return s;
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java b/tika-eval/src/main/java/org/apache/tika/eval/db/JDBCUtil.java
similarity index 52%
rename from tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java
rename to tika-eval/src/main/java/org/apache/tika/eval/db/JDBCUtil.java
index 17d898b..0f7f5a5 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/JDBCUtil.java
@@ -19,70 +19,123 @@ package org.apache.tika.eval.db;
 
 
 import java.io.IOException;
-import java.nio.file.Path;
+import java.io.InputStream;
 import java.sql.Connection;
+import java.sql.DatabaseMetaData;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.sql.Types;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.log4j.Logger;
-import org.apache.tika.io.IOExceptionWithCause;
 
-public abstract class DBUtil {
+public class JDBCUtil {
 
-    public static Logger logger = Logger.getLogger(DBUtil.class);
-    public abstract String getJDBCDriverClass();
-    public abstract boolean dropTableIfExists(Connection conn, String tableName) throws SQLException;
-    private final Path db;
-    public DBUtil(Path db) {
-        this.db = db;
+    public static Logger logger = Logger.getLogger(JDBCUtil.class);
+    private final String connectionString;
+    private String driverClass;
+
+    public JDBCUtil(String connectionString, String driverClass) {
+        this.connectionString = connectionString;
+        this.driverClass = driverClass;
+        if (driverClass == null || driverClass.length() == 0) {
+            if (System.getProperty("jdbc.drivers") != null) {
+                //user has specified it on the command line
+                //stop now
+            } else {
+                //try to use the mappings in db.properties to determine the class
+                try (InputStream is = JDBCUtil.class.getResourceAsStream("/db.properties")) {
+                    Properties properties = new Properties();
+                    properties.load(is);
+                    for (String k : properties.stringPropertyNames()) {
+                        Matcher m = Pattern.compile("(?i)jdbc:"+k).matcher(connectionString);
+                        if (m.find()) {
+                            this.driverClass = properties.getProperty(k);
+                        }
+                    }
+
+                } catch (IOException e) {
+
+                }
+            }
+        }
     }
 
     /**
-     * This is intended for a file/directory based db.
-     * <p>
      * Override this any optimizations you want to do on the db
      * before writing/reading.
      *
      * @return
      * @throws IOException
      */
-    public Connection getConnection(boolean createIfDoesntExist) throws IOException {
-        String connectionString = getConnectionString(db, createIfDoesntExist);
+    public Connection getConnection() throws SQLException {
+        String connectionString = getConnectionString();
         Connection conn = null;
-        try {
+        String jdbcDriver = getJDBCDriverClass();
+        if (jdbcDriver != null) {
             try {
                 Class.forName(getJDBCDriverClass());
             } catch (ClassNotFoundException e) {
                 throw new RuntimeException(e);
             }
-            conn = DriverManager.getConnection(connectionString);
-            conn.setAutoCommit(false);
-        } catch (SQLException e) {
-            throw new IOExceptionWithCause(e);
         }
+        conn = DriverManager.getConnection(connectionString);
+        conn.setAutoCommit(false);
+
         return conn;
     }
 
-    abstract public String getConnectionString(Path db, boolean createIfDoesntExist);
-
     /**
-     *
-     * @param connection
-     * @return a list of uppercased table names
-     * @throws SQLException
+     * JDBC driver class.  Override as necessary.
+     * @return
      */
-    abstract public Set<String> getTables(Connection connection) throws SQLException;
+    public String getJDBCDriverClass() {
+        return driverClass;
+    }
+
+
+    public boolean dropTableIfExists(Connection conn, String tableName) throws SQLException {
+        if (containsTable(tableName)) {
+            try (Statement st = conn.createStatement()) {
+                String sql = "drop table " + tableName;
+                return st.execute(sql);
+            }
+        }
+        return true;
+    }
+
+
+    public String getConnectionString() {
+        return connectionString;
+    }
+
+
+    public Set<String> getTables(Connection connection) throws SQLException {
+        Set<String> tables = new HashSet<>();
+
+        DatabaseMetaData dbMeta = connection.getMetaData();
+
+        try (ResultSet rs = dbMeta.getTables(null, null, "%", null)) {
+            while (rs.next()) {
+                tables.add(rs.getString(3).toLowerCase(Locale.US));
+            }
+        }
+        return tables;
+    }
 
     public static int insert(PreparedStatement insertStatement,
-                              TableInfo table,
-                              Map<Cols, String> data) throws SQLException {
+                             TableInfo table,
+                             Map<Cols, String> data) throws SQLException {
 
         //clear parameters before setting
         insertStatement.clearParameters();
@@ -93,21 +146,21 @@ public abstract class DBUtil {
                 i++;
             }
             for (Cols c : data.keySet()) {
-                if (! table.containsColumn(c)) {
-                    throw new IllegalArgumentException("Can't add data to "+c +
-                    " because it doesn't exist in the table: "+table.getName());
+                if (!table.containsColumn(c)) {
+                    throw new IllegalArgumentException("Can't add data to " + c +
+                            " because it doesn't exist in the table: " + table.getName());
                 }
             }
             return insertStatement.executeUpdate();
         } catch (SQLException e) {
-            logger.warn("couldn't insert data for this row: "+e.getMessage());
+            logger.warn("couldn't insert data for this row: " + e.getMessage());
             e.printStackTrace();
             return -1;
         }
     }
 
     public static void updateInsertStatement(int dbColOffset, PreparedStatement st,
-                                             ColInfo colInfo, String value ) throws SQLException {
+                                             ColInfo colInfo, String value) throws SQLException {
         if (value == null) {
             st.setNull(dbColOffset, colInfo.getType());
             return;
@@ -117,7 +170,7 @@ public abstract class DBUtil {
                 case Types.VARCHAR:
                     if (value != null && value.length() > colInfo.getPrecision()) {
                         value = value.substring(0, colInfo.getPrecision());
-                        logger.warn("truncated varchar value in " + colInfo.getName() + " : "+value);
+                        logger.warn("truncated varchar value in " + colInfo.getName() + " : " + value);
                     }
                     st.setString(dbColOffset, value);
                     break;
@@ -143,38 +196,54 @@ public abstract class DBUtil {
                     throw new UnsupportedOperationException("Don't yet support type: " + colInfo.getType());
             }
         } catch (NumberFormatException e) {
-            if (! "".equals(value)) {
+            if (!"".equals(value)) {
                 logger.warn("number format exception: " + colInfo.getName() + " : " + value);
             }
             st.setNull(dbColOffset, colInfo.getType());
         } catch (SQLException e) {
-            logger.warn("sqlexception: "+colInfo+ " : " + value);
+            logger.warn("sqlexception: " + colInfo + " : " + value);
             st.setNull(dbColOffset, colInfo.getType());
         }
     }
 
-    public void createDB(List<TableInfo> tableInfos, boolean append) throws SQLException, IOException {
-        Connection conn = getConnection(true);
-        Set<String> tables = getTables(conn);
+    public void createTables(List<TableInfo> tableInfos, boolean forceDrop) throws SQLException, IOException {
 
-        for (TableInfo tableInfo : tableInfos) {
+        try (Connection conn = getConnection ()) {
+            for (TableInfo tableInfo : tableInfos) {
 
-            if (append && tables.contains(tableInfo.getName().toUpperCase(Locale.ROOT))) {
-                continue;
+                if (forceDrop) {
+                    dropTableIfExists(conn, tableInfo.getName());
+                }
+                createTable(conn, tableInfo);
             }
-            if (! append) {
-                dropTableIfExists(conn, tableInfo.getName());
+            conn.commit();
+        }
+    }
+
+    public boolean containsTable(String tableName) throws SQLException {
+        try (Connection connection = getConnection()) {
+            Set<String> tables = getTables(connection);
+            if (tables.contains(normalizeTableName(tableName))) {
+                return true;
             }
-            createTable(conn, tableInfo);
         }
+        return false;
+    }
 
-        conn.commit();
-        conn.close();
+    /**
+     * Override for custom behavior
+     * @param tableName
+     * @return
+     */
+    String normalizeTableName(String tableName) {
+        tableName = tableName.toLowerCase(Locale.US);
+        return tableName;
     }
 
+    //does not close the connection
     private void createTable(Connection conn, TableInfo tableInfo) throws SQLException {
         StringBuilder createSql = new StringBuilder();
-        createSql.append("CREATE TABLE "+tableInfo.getName());
+        createSql.append("CREATE TABLE " + tableInfo.getName());
         createSql.append("(");
 
         int last = 0;
@@ -193,10 +262,10 @@ public abstract class DBUtil {
             }
         }
         createSql.append(")");
-        Statement st = conn.createStatement();
-        st.execute(createSql.toString());
-
-        st.close();
+        try (Statement st = conn.createStatement()) {
+            st.execute(createSql.toString());
+            st.close();
+        }
         conn.commit();
     }
 }
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java b/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java
index 56f26b7..d557f9f 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java
@@ -39,8 +39,8 @@ public class MimeBuffer extends AbstractDBBuffer {
     public MimeBuffer(Connection connection, TikaConfig config) throws SQLException {
         st = connection.prepareStatement("insert into " + AbstractProfiler.MIME_TABLE.getName() + "( " +
                 Cols.MIME_TYPE_ID.name() + ", " +
-                Cols.MIME_STRING + ", " +
-                Cols.FILE_EXTENSION + ") values (?,?,?);");
+                Cols.MIME_STRING.name() + ", " +
+                Cols.FILE_EXTENSION.name() + ") values (?,?,?)");
         this.config = config;
         this.connection = connection;
     }
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/TableInfo.java b/tika-eval/src/main/java/org/apache/tika/eval/db/TableInfo.java
index c6e3696..181a028 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/db/TableInfo.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/TableInfo.java
@@ -29,6 +29,8 @@ public class TableInfo {
     private final List<ColInfo> colInfos = new ArrayList<>();
     private final Set<Cols> colNames = new HashSet<>();
 
+    private String prefix;
+
     public TableInfo(String name, ColInfo... cols) {
         Collections.addAll(colInfos, cols);
         Collections.unmodifiableList(colInfos);
@@ -50,7 +52,14 @@ public class TableInfo {
     }
 
     public String getName() {
-        return name;
+        if (prefix == null) {
+            return name;
+        }
+        return prefix+name;
+    }
+
+    public void setNamePrefix(String prefix) {
+        this.prefix = prefix;
     }
 
     public List<ColInfo> getColInfos() {
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java b/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java
index 6c4862c..b1f3b36 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java
@@ -29,7 +29,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import org.apache.log4j.Logger;
 import org.apache.tika.eval.db.ColInfo;
 import org.apache.tika.eval.db.Cols;
-import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.JDBCUtil;
 import org.apache.tika.eval.db.MimeBuffer;
 import org.apache.tika.eval.db.TableInfo;
 import org.apache.tika.io.IOExceptionWithCause;
@@ -52,14 +52,14 @@ public class DBWriter implements IDBWriter {
     private final Long commitEveryX = 1000L;
 
     private final Connection conn;
-    private final DBUtil dbUtil;
+    private final JDBCUtil dbUtil;
     private final MimeBuffer mimeBuffer;
     private final int myId = WRITER_ID.getAndIncrement();
 
     //<tableName, preparedStatement>
     private final Map<String, PreparedStatement> inserts = new HashMap<>();
 
-    public DBWriter(Connection connection, List<TableInfo> tableInfos, DBUtil dbUtil, MimeBuffer mimeBuffer)
+    public DBWriter(Connection connection, List<TableInfo> tableInfos, JDBCUtil dbUtil, MimeBuffer mimeBuffer)
             throws IOException, SQLException {
 
         this.conn = connection;
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java b/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java
index eebbdbf..fd35363 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java
@@ -55,8 +55,31 @@ public class ExtractReader {
     private final static Logger LOGGER = LoggerFactory.getLogger(ExtractReader.class);
     TikaConfig tikaConfig = TikaConfig.getDefaultConfig();
 
-    public List<Metadata> loadExtract(Path extractFile, ALTER_METADATA_LIST alterExtractList,
-                                      long minExtractLength, long maxExtractLength) throws ExtractReaderException {
+    private final ALTER_METADATA_LIST alterMetadataList;
+    private final long minExtractLength;
+    private final long maxExtractLength;
+
+    /**
+     * Reads full extract, no modification of metadata list, no min or max extract length checking
+     */
+    public ExtractReader() {
+        this(ALTER_METADATA_LIST.AS_IS, IGNORE_LENGTH, IGNORE_LENGTH);
+    }
+
+    public ExtractReader(ALTER_METADATA_LIST alterMetadataList) {
+        this(alterMetadataList, IGNORE_LENGTH, IGNORE_LENGTH);
+    }
+
+    public ExtractReader(ALTER_METADATA_LIST alterMetadataList, long minExtractLength, long maxExtractLength) {
+        this.alterMetadataList = alterMetadataList;
+        this.minExtractLength = minExtractLength;
+        this.maxExtractLength = maxExtractLength;
+        if (maxExtractLength > IGNORE_LENGTH && minExtractLength >= maxExtractLength) {
+            throw new IllegalArgumentException("minExtractLength("+minExtractLength+
+                    ") must be < maxExtractLength("+maxExtractLength+")");
+        }
+    }
+    public List<Metadata> loadExtract(Path extractFile) throws ExtractReaderException {
 
         List<Metadata> metadataList = null;
         if (extractFile == null || !Files.isRegularFile(extractFile)) {
@@ -114,11 +137,11 @@ public class ExtractReader {
         try {
             if (fileSuffixes.txtOrJson.equals("json")) {
                 metadataList = JsonMetadataList.fromJson(reader);
-                if (alterExtractList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) {
+                if (alterMetadataList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) {
                     while (metadataList.size() > 1) {
                         metadataList.remove(metadataList.size()-1);
                     }
-                } else if (alterExtractList.equals(ALTER_METADATA_LIST.AS_IS.CONCATENATE_CONTENT_INTO_FIRST) &&
+                } else if (alterMetadataList.equals(ALTER_METADATA_LIST.AS_IS.CONCATENATE_CONTENT_INTO_FIRST) &&
                         metadataList.size() > 1) {
                     StringBuilder sb = new StringBuilder();
                     Metadata containerMetadata = metadataList.get(0);
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/reports/ResultsReporter.java b/tika-eval/src/main/java/org/apache/tika/eval/reports/ResultsReporter.java
index 97b7711..072ba53 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/reports/ResultsReporter.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/reports/ResultsReporter.java
@@ -37,14 +37,13 @@ import java.util.Map;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.DefaultParser;
 import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.poi.common.usermodel.Hyperlink;
 import org.apache.tika.eval.ExtractComparer;
 import org.apache.tika.eval.ExtractProfiler;
-import org.apache.tika.eval.db.DBUtil;
 import org.apache.tika.eval.db.H2Util;
+import org.apache.tika.eval.db.JDBCUtil;
 import org.apache.tika.parser.ParseContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -61,18 +60,15 @@ public class ResultsReporter {
 
     static {
         OPTIONS = new Options();
-
-        Option db = new Option("db", "database");
-        db.setRequired(true);
-        db.setArgs(1);
-
-        OPTIONS.addOption(db)
-                .addOption("rd", "reportsDir", true, "directory for the reports. " +
-                                "If not specified, will write to 'reports'" +
-                        "BEWARE: Will overwrite existing reports without warning!"
-                )
+        OPTIONS.addOption("rd", "reportsDir", true, "directory for the reports. " +
+                "If not specified, will write to 'reports'" +
+                "BEWARE: Will overwrite existing reports without warning!")
                 .addOption("rf", "reportsFile", true, "xml specifying sql to call for the reports." +
-                        "If not specified, will use default reports in resources/tika-eval-*-config.xml");
+                        "If not specified, will use default reports in resources/tika-eval-*-config.xml")
+                .addOption("db", true, "default database (in memory H2). Specify a file name for the H2 database.")
+                .addOption("jdbc", true, "EXPERT: full jdbc connection string. Specify this or use -db <h2db_name>")
+                .addOption("jdbcdriver", true, "EXPERT: specify the jdbc driver class if all else fails")
+                .addOption("tablePrefix", true, "EXPERT: if not using the default tables, specify your table name prefix");
 
     }
 
@@ -111,11 +107,11 @@ public class ResultsReporter {
 
         DocumentBuilder docBuilder = new ParseContext().getDocumentBuilder();
         Document doc;
-        try(InputStream is = Files.newInputStream(p)) {
+        try (InputStream is = Files.newInputStream(p)) {
             doc = docBuilder.parse(is);
         }
         Node docElement = doc.getDocumentElement();
-        assert(docElement.getNodeName().equals("reports"));
+        assert (docElement.getNodeName().equals("reports"));
         NodeList children = docElement.getChildNodes();
         for (int i = 0; i < children.getLength(); i++) {
             Node n = children.item(i);
@@ -158,7 +154,7 @@ public class ResultsReporter {
             } else if ("colformats".equals(child.getNodeName())) {
                 r.cellFormatters = getCellFormatters(child);
             } else {
-                throw new IllegalArgumentException("Not expecting to see:"+child.getNodeName());
+                throw new IllegalArgumentException("Not expecting to see:" + child.getNodeName());
             }
         }
         return r;
@@ -174,12 +170,12 @@ public class ResultsReporter {
             }
             NamedNodeMap attrs = child.getAttributes();
             String columnName = attrs.getNamedItem("name").getNodeValue();
-            assert(!ret.containsKey(columnName));
+            assert (!ret.containsKey(columnName));
             String type = attrs.getNamedItem("type").getNodeValue();
             if ("numberFormatter".equals(type)) {
                 String format = attrs.getNamedItem("format").getNodeValue();
                 XSLXCellFormatter f = new XLSXNumFormatter(format);
-                ret.put(columnName,f);
+                ret.put(columnName, f);
             } else if ("urlLink".equals(type)) {
                 String base = "";
                 Node baseNode = attrs.getNamedItem("base");
@@ -227,10 +223,27 @@ public class ResultsReporter {
             USAGE();
             return;
         }
-        Path db = Paths.get(commandLine.getOptionValue("db"));
-        DBUtil dbUtil = new H2Util(db);
-
-        try (Connection c = dbUtil.getConnection(true)) {
+        JDBCUtil dbUtil = null;
+        if (commandLine.hasOption("db")) {
+            Path db = Paths.get(commandLine.getOptionValue("db"));
+            if (!H2Util.databaseExists(db)) {
+                throw new RuntimeException("I'm sorry, but I couldn't find this h2 database: "
+                        + db+ "\nMake sure not to include the .mv.db at the end.");
+            }
+            dbUtil = new H2Util(db);
+        } else if (commandLine.hasOption("jdbc")) {
+            String driverClass = null;
+            if (commandLine.hasOption("jdbcdriver")) {
+                driverClass = commandLine.getOptionValue("jdbcdriver");
+            }
+            dbUtil = new JDBCUtil(commandLine.getOptionValue("jdbc"), driverClass);
+        } else {
+            System.err.println("Must specify either -db for the default in-memory h2 database\n" +
+                    "or -jdbc for a full jdbc connection string");
+            USAGE();
+            return;
+        }
+        try (Connection c = dbUtil.getConnection()) {
             Path tmpReportsFile = null;
             try {
                 ResultsReporter resultsReporter = null;
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXHREFFormatter.java b/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXHREFFormatter.java
index 52d55e4..f8b9ba2 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXHREFFormatter.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXHREFFormatter.java
@@ -27,9 +27,7 @@ import org.apache.poi.ss.usermodel.Hyperlink;
 import org.apache.poi.ss.usermodel.IndexedColors;
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 
-/**
- * Created by TALLISON on 4/20/2016.
- */
+
 public class XLSXHREFFormatter implements XSLXCellFormatter {
     //xlsx files can only have this many hyperlinks
     //if they have more Excel can't read the file
diff --git a/tika-eval/src/main/resources/db.properties b/tika-eval/src/main/resources/db.properties
new file mode 100644
index 0000000..0f0ad36
--- /dev/null
+++ b/tika-eval/src/main/resources/db.properties
@@ -0,0 +1,9 @@
+postgresql  org.postgresql.Driver
+mysql com.mysql.jdbc.Driver
+oracle oracle.jdbc.driver.OracleDriver
+sqlite org.sqlite.JDBC
+odbc sun.jdbc.odbc.JdbcOdbcDriver
+derby:// org.apache.derby.jdbc.ClientDriver
+derby org.apache.derby.jdbc.EmbeddedDriver
+hsqldb org.hsqldb.jdbc.JDBCDriver
+h2 org.h2.Driver
diff --git a/tika-eval/src/main/resources/tika-eval-comparison-config.xml b/tika-eval/src/main/resources/tika-eval-comparison-config.xml
index e0ee34e..298c864 100644
--- a/tika-eval/src/main/resources/tika-eval-comparison-config.xml
+++ b/tika-eval/src/main/resources/tika-eval-comparison-config.xml
@@ -49,6 +49,16 @@
                 description="minimum extract length to process"/>
         <option opt="maxExtractLength" hasArg="true"
                 description="maximum extract length to process"/>
+        <option opt="jdbc" hasArg="true"
+                description="full jdbc connection string"/>
+        <option opt="jdbcDriver" hasArg="true"
+                description="canonical class name for jdbc driver"/>
+        <option opt="tablePrefixA" hasArg="true"
+                description="EXPERT: prefix for table names for A"/>
+        <option opt="tablePrefixB" hasArg="true"
+                description="EXPERT: prefix for table names for B"/>
+        <option opt="drop" hasArg="false" description="drop tables if they exist"/>
+        <option opt="maxFilesToAdd" hasArg="true" description="maximum number of files to add to the crawler"/>
 
     </commandline>
 
@@ -67,7 +77,7 @@
     />
 
     <consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
-               consumerBuilderClass="org.apache.tika.eval.batch.FileComparerBuilder"
+               consumerBuilderClass="org.apache.tika.eval.batch.ExtractComparerBuilder"
                dbAppend="false"
                crawlingInputDir="false"
                minExtractLength="-1"
diff --git a/tika-eval/src/main/resources/tika-eval-profiler-config.xml b/tika-eval/src/main/resources/tika-eval-profiler-config.xml
index 1425d24..9da2aeb 100644
--- a/tika-eval/src/main/resources/tika-eval-profiler-config.xml
+++ b/tika-eval/src/main/resources/tika-eval-profiler-config.xml
@@ -46,6 +46,16 @@
                 description="minimum extract length to process"/>
         <option opt="maxExtractLength" hasArg="true"
                 description="maximum extract length to process"/>
+        <option opt="jdbc" hasArg="true"
+                description="full jdbc connection string"/>
+        <option opt="jdbcDriver" hasArg="true"
+                description="canonical class name for jdbc driver"/>
+        <option opt="tablePrefix" hasArg="true"
+                description="EXPERT: prefix for table names"/>
+        <option opt="drop" hasArg="false" description="drop tables if they exist"/>
+        <option opt="maxFilesToAdd" hasArg="true" description="maximum number of files to add to the crawler"/>
+
+
 
 
     </commandline>
@@ -67,7 +77,7 @@
     />
 
     <consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
-               consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"
+               consumerBuilderClass="org.apache.tika.eval.batch.ExtractProfilerBuilder"
                minExtractLength="-1"
                maxExtractLength="2000000"
                commonTokens="resources/common_tokens"/>
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java b/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
index 8d95622..d9ca118 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
@@ -73,7 +73,7 @@ public class ProfilerBatchTest {
         System.out.println(streamStrings.getErrString());
         System.out.println(streamStrings.getOutString());*/
         H2Util dbUtil = new H2Util(db);
-        conn = dbUtil.getConnection(true);
+        conn = dbUtil.getConnection();
     }
     @AfterClass
     public static void tearDown() throws IOException {
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java b/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
index e718b33..85e91dd 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
@@ -58,8 +58,9 @@ public class SimpleComparerTest extends TikaTest {
         writer = new MockDBWriter();
         comparer = new ExtractComparer(null, null,
                 Paths.get("extractsA"), Paths.get("extractsB"),
-                writer, IGNORE_LENGTH, IGNORE_LENGTH,
-                ExtractReader.ALTER_METADATA_LIST.AS_IS);
+                new ExtractReader(ExtractReader.ALTER_METADATA_LIST.AS_IS,
+                        IGNORE_LENGTH, IGNORE_LENGTH),
+                writer);
         AbstractProfiler.loadCommonTokens(this.getResourceAsFile("/common_tokens").toPath());
         LanguageIDWrapper.loadBuiltInModels();
     }
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java b/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java
index ff0961c..a0c02ff 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java
@@ -27,15 +27,43 @@ public class TikaEvalCLITest {
 
     @Test
     @Ignore("TODO: add real tests")
-    public void testBasic() throws Exception {
+    public void testBasicCompare() throws Exception {
+        List<String> args = new ArrayList<>();
+        args.add("Compare");
+        args.add("-extractsA");
+        args.add("watson2");
+        args.add("-extractsB");
+        args.add("tika");
+        args.add("-db");
+        args.add("comparedb");
+        args.add("-drop");
+        args.add("-jdbc");
+        args.add("jdbc:postgresql:tika_eval?user=user&password=password");
+        args.add("-maxFilesToAdd");
+        args.add("100");
+        TikaEvalCLI.main(args.toArray(new String[args.size()]));
+    }
+
+    @Ignore("TODO: add real tests")
+    @Test
+    public void testBasicProfile() throws Exception {
         List<String> args = new ArrayList<>();
         args.add("Profile");
         args.add("-extracts");
-        args.add("tika");
+        args.add("watson2");
         args.add("-db");
-        args.add("mydb");
-        args.add("-alterExtract");
-        args.add("first_only");
+        args.add("testdb");
+        TikaEvalCLI.main(args.toArray(new String[args.size()]));
+    }
+
+    @Test
+    @Ignore("TODO: add real tests")
+    public void testReports() throws Exception {
+        List<String> args = new ArrayList<>();
+        args.add("Report");
+        args.add("-db");
+        args.add("comparedb.mv.db");
+
         TikaEvalCLI.main(args.toArray(new String[args.size()]));
     }
 
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java b/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
index a49b24c..f22179a 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
@@ -44,24 +44,23 @@ public class ExtractReaderTest extends TikaTest {
     public void testBasic() throws Exception {
 
         ExtractReader extractReader = new ExtractReader();
-        List<Metadata> metadataList = extractReader.loadExtract(testJsonFile,
-                ExtractReader.ALTER_METADATA_LIST.AS_IS,
-                ExtractReader.IGNORE_LENGTH, ExtractReader.IGNORE_LENGTH);
+        List<Metadata> metadataList = extractReader.loadExtract(testJsonFile);
+
         assertEquals(2, metadataList.size());
         assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
         assertEquals(1, metadataList.get(1).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
         assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
         assertContains("attachment", metadataList.get(1).get(RecursiveParserWrapper.TIKA_CONTENT));
 
-        metadataList = extractReader.loadExtract(testJsonFile, ExtractReader.ALTER_METADATA_LIST.FIRST_ONLY,
-                ExtractReader.IGNORE_LENGTH, ExtractReader.IGNORE_LENGTH);
+        extractReader = new ExtractReader(ExtractReader.ALTER_METADATA_LIST.FIRST_ONLY);
+        metadataList = extractReader.loadExtract(testJsonFile);
         assertEquals(1, metadataList.size());
         assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
         assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
         assertNotContained("attachment", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
 
-        metadataList = extractReader.loadExtract(testJsonFile, ExtractReader.ALTER_METADATA_LIST.CONCATENATE_CONTENT_INTO_FIRST,
-                ExtractReader.IGNORE_LENGTH, ExtractReader.IGNORE_LENGTH);
+        extractReader = new ExtractReader(ExtractReader.ALTER_METADATA_LIST.CONCATENATE_CONTENT_INTO_FIRST);
+        metadataList = extractReader.loadExtract(testJsonFile);
         assertEquals(1, metadataList.size());
         assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
         assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
@@ -71,9 +70,7 @@ public class ExtractReaderTest extends TikaTest {
     @Test
     public void testTextBasic() throws IOException {
         ExtractReader extractReader = new ExtractReader();
-        List<Metadata> metadataList = extractReader.loadExtract(testTxtFile,
-                ExtractReader.ALTER_METADATA_LIST.AS_IS,
-                ExtractReader.IGNORE_LENGTH, ExtractReader.IGNORE_LENGTH);
+        List<Metadata> metadataList = extractReader.loadExtract(testTxtFile);
         assertEquals(1, metadataList.size());
         Metadata m = metadataList.get(0);
         assertEquals(1, m.getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java b/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java
index 5e43303..90d52c0 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java
@@ -39,7 +39,7 @@ public class ResultsReporterTest {
         configFile = Paths.get(this.getClass().getResource("/reports.xml").toURI());
         tmpDir = Files.createTempDirectory("tika-eval-report-test-");
 
-        connection = new H2Util(tmpDir.resolve(dbName)).getConnection(true);
+        connection = new H2Util(tmpDir.resolve(dbName)).getConnection();
         String sql = "CREATE TABLE test_table (ID LONG PRIMARY KEY, STRING VARCHAR(32))";
         Statement st = connection.createStatement();
         st.execute(sql);
diff --git a/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml b/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
index 8d57ab7..bb83f8b 100644
--- a/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
+++ b/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
@@ -63,7 +63,7 @@
     />
 
     <consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
-               consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"
+               consumerBuilderClass="org.apache.tika.eval.batch.ExtractProfilerBuilder"
                errorLogFile="src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml"
                extracts="src/test/resources/test-dirs/extractsA"
                commonTokens="src/test/resources/common_tokens"/>
diff --git a/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml b/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
index 62c7911..22a7dc2 100644
--- a/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
+++ b/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
@@ -63,7 +63,7 @@
     />
 
     <consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
-               consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"
+               consumerBuilderClass="org.apache.tika.eval.batch.ExtractProfilerBuilder"
                errorLogFile="src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml"
                extracts="src/test/resources/test-dirs/extractsA"
                inputDir="src/test/resources/test-dirs/raw_input"

-- 
To stop receiving notification emails like this one, please contact
['"commits@tika.apache.org" <co...@tika.apache.org>'].