You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2025/07/08 19:27:51 UTC

(tika) branch TIKA-4342 created (now 303623e67)

This is an automated email from the ASF dual-hosted git repository.

tallison pushed a change to branch TIKA-4342
in repository https://gitbox.apache.org/repos/asf/tika.git


      at 303623e67 TIKA-4342 -- remove tika-batch from ExtractProfiler

This branch includes the following new commits:

     new 7b94cf0ff TIKA-4342 -- WIP do not merge
     new a5972ca7e WIP -- don't merge
     new 303623e67 TIKA-4342 -- remove tika-batch from ExtractProfiler

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(tika) 03/03: TIKA-4342 -- remove tika-batch from ExtractProfiler

Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tallison pushed a commit to branch TIKA-4342
in repository https://gitbox.apache.org/repos/asf/tika.git

commit 303623e67e451392eab1249d47463f2861507127
Author: tallison <ta...@apache.org>
AuthorDate: Tue Jul 8 15:27:27 2025 -0400

    TIKA-4342 -- remove tika-batch from ExtractProfiler
---
 .../java/org/apache/tika/eval/app/EvalConfig.java  |  89 ++++-
 .../tika/eval/app/ExctractProfileRunner.java       | 204 -----------
 .../apache/tika/eval/app/ExtractProfileRunner.java | 374 +++++++++++++++++++++
 .../org/apache/tika/eval/app/ExtractProfiler.java  |   1 -
 .../org/apache/tika/eval/app/FileProfiler.java     |  19 ++
 .../org/apache/tika/eval/app/ProfilerBase.java     |   3 -
 .../org/apache/tika/eval/app/StatusReporter.java   | 102 ++++++
 .../java/org/apache/tika/eval/app/TikaEvalCLI.java |   4 +-
 .../eval/app/batch/ExtractProfilerBuilder.java     | 120 -------
 .../apache/tika/eval/app/batch/PathResource.java   |  21 +-
 .../org/apache/tika/eval/app/EvalConfigTest.java   |  42 +++
 .../apache/tika/eval/app/ProfilerBatchTest.java    | 117 +++----
 .../org/apache/tika/eval/app/TikaEvalCLITest.java  |  11 +-
 .../resources/eval-configs/eval-config-basic.json  |   3 +
 .../test-dirs/raw_input/file10_permahang.txt       |   0
 .../resources/test-dirs/raw_input/file12_es.txt    |   6 +
 .../test-dirs/raw_input/file13_attachANotB.doc     |  12 +
 .../test-dirs/raw_input/file14_diffAttachOrder     |  21 ++
 .../test/resources/test-dirs/raw_input/file15_tags |  41 +++
 .../resources/test-dirs/raw_input/file16_badTags   |  41 +++
 .../test-dirs/raw_input/file17_tagsOutOfOrder      |  41 +++
 tika-parent/pom.xml                                |   2 +-
 22 files changed, 859 insertions(+), 415 deletions(-)

diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/EvalConfig.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/EvalConfig.java
index 416d7bb6e..5525180ed 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/EvalConfig.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/EvalConfig.java
@@ -1,13 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.tika.eval.app;
 
+import java.nio.file.Path;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 public class EvalConfig {
 
-    long minExtractLength = 0;
-    long maxExtractLength = 10_000_000;
-    String jdbcString = null;
-    int maxFilesToAdd = -1;
-    int maxTokens = 200000;
-    int maxContentLength = 5_000_000;
-    int numThreads = 4;
+    private long minExtractLength = 0;
+    private long maxExtractLength = 2_000_000;
+    private String jdbcString = null;
+    private String jdbcDriverClass = null;
+    private boolean forceDrop = true;
+    private int maxFilesToAdd = -1;
+    private int maxTokens = 200000;
+
+    private int maxContentLength = 5_000_000;
+    private int numWorkers = 4;
+    private Path errorLogFile = null;
+
+
+    public static EvalConfig load(Path path) throws Exception {
+        return new ObjectMapper().readValue(path.toFile(), EvalConfig.class);
+    }
+
+    public long getMinExtractLength() {
+        return minExtractLength;
+    }
+
+    public long getMaxExtractLength() {
+        return maxExtractLength;
+    }
+
+    public String getJdbcString() {
+        return jdbcString;
+    }
+
+    public String getJdbcDriverClass() {
+        return jdbcDriverClass;
+    }
+
+    public boolean isForceDrop() {
+        return forceDrop;
+    }
+
+    public int getMaxFilesToAdd() {
+        return maxFilesToAdd;
+    }
+
+    public int getMaxTokens() {
+        return maxTokens;
+    }
+
+    public int getMaxContentLength() {
+        return maxContentLength;
+    }
+
+    public int getNumWorkers() {
+        return numWorkers;
+    }
+
+    public Path getErrorLogFile() {
+        return errorLogFile;
+    }
 
+    @Override
+    public String toString() {
+        return "EvalConfig{" + "minExtractLength=" + minExtractLength + ", maxExtractLength=" + maxExtractLength + ", jdbcString='" + jdbcString + '\'' + ", jdbcDriverClass='" +
+                jdbcDriverClass + '\'' + ", forceDrop=" + forceDrop + ", maxFilesToAdd=" + maxFilesToAdd + ", maxTokens=" + maxTokens + ", maxContentLength=" + maxContentLength +
+                ", numThreads=" + numWorkers + ", errorLogFile=" + errorLogFile + '}';
+    }
 }
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExctractProfileRunner.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExctractProfileRunner.java
deleted file mode 100644
index 20cb33c1c..000000000
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExctractProfileRunner.java
+++ /dev/null
@@ -1,204 +0,0 @@
-package org.apache.tika.eval.app;
-
-import java.io.IOException;
-import java.nio.file.FileVisitResult;
-import java.nio.file.FileVisitor;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.eval.app.batch.FileResource;
-import org.apache.tika.eval.app.batch.PathResource;
-import org.apache.tika.eval.app.db.JDBCUtil;
-import org.apache.tika.eval.app.db.MimeBuffer;
-import org.apache.tika.eval.app.io.ExtractReader;
-import org.apache.tika.eval.app.io.IDBWriter;
-
-public class ExctractProfileRunner {
-
-    private static final Logger LOG = LoggerFactory.getLogger(ExctractProfileRunner.class);
-    private static final PathResource SEMAPHORE = new PathResource(Paths.get("/"), "STOP");
-
-    static Options OPTIONS;
-
-    static {
-
-        OPTIONS = new Options()
-                .addOption(Option.builder("e").longOpt("extracts").hasArg().desc("required: directory of extracts").build())
-                .addOption(Option.builder("i").longOpt("inputDir").hasArg().desc("optional: directory for original binary input documents."
-                        + " If not specified, -extracts is crawled as is.").build())
-                .addOption(Option.builder("d").longOpt("db").hasArg().desc("optional: db path").build())
-                .addOption(Option.builder("c").longOpt("config").hasArg().desc("tika-eval json config file").build())
-                ;
-    }
-    public static void main(String[] args) throws Exception {
-        DefaultParser defaultCLIParser = new DefaultParser();
-        CommandLine commandLine = defaultCLIParser.parse(OPTIONS, args);
-        EvalConfig evalConfig = commandLine.hasOption('c') ? EvalConfig.load(Paths.get(commandLine.getOptionValue('c'))) : new EvalConfig();
-        Path extractsDir = commandLine.hasOption('e') ? Paths.get(commandLine.getOptionValue('e')) : Paths.get(USAGE_FAIL("Must specify extracts dir: -i"));
-        Path inputDir = commandLine.hasOption('i') ? Paths.get(commandLine.getOptionValue('i')) : extractsDir;
-        String dbPath = commandLine.hasOption('d') ? commandLine.getOptionValue('d') : USAGE_FAIL("Must specify the db name: -d");
-        execute(inputDir, extractsDir, dbPath, evalConfig);
-    }
-
-    private static void execute(Path inputDir, Path extractsDir, String dbPath, EvalConfig evalConfig) {
-
-        ArrayBlockingQueue<FileResource> queue = new ArrayBlockingQueue<>(1000);
-        DirectoryWalker fileWalker = new DirectoryWalker(inputDir, queue);
-        ExecutorService executorService = Executors.newFixedThreadPool(evalConfig.numThreads + 1);
-        ExecutorCompletionService<Integer> executorCompletionService = new ExecutorCompletionService<>(executorService);
-        executorCompletionService.submit(fileWalker);
-        IDBWriter dbWriter = buildDBWriter();
-        for (int i = 0; i < evalConfig.numThreads; i++) {
-            ExtractReader extractReader = new ExtractReader(ExtractReader.ALTER_METADATA_LIST.AS_IS, evalConfig.minExtractLength, evalConfig.maxExtractLength);
-
-            ExtractProfiler extractProfiler = new ExtractProfiler(inputDir, extractsDir, extractReader, dbWriter);
-            executorCompletionService.submit(new ProfileWorker(queue, extractProfiler));
-        }
-
-        int finished = 0;
-        try {
-            while (finished < evalConfig.numThreads + 1) {
-                //blocking
-                Future<Integer> future = executorCompletionService.take();
-                Integer result = future.get();
-                if (result != null) {
-                    finished++;
-                }
-
-            }
-        } catch (InterruptedException e) {
-            LOG.info("interrupted", e);
-        } catch (ExecutionException e) {
-            throw new RuntimeException(e);
-        } finally {
-            executorService.shutdownNow();
-        }
-
-    }
-
-    private static IDBWriter buildDBWriter(String connectionString, String driverClass) {
-        MimeBuffer mimeBuffer = null;
-        JDBCUtil dbUtil = new JDBCUtil(connectionString, driverClass);
-        //Step 1. Used to be update table infos with prefixes
-        updateTableInfosWithPrefixes(localAttrs);
-
-        JDBCUtil.CREATE_TABLE createRegularTable = (forceDrop) ? JDBCUtil.CREATE_TABLE.DROP_IF_EXISTS : JDBCUtil.CREATE_TABLE.THROW_EX_IF_EXISTS;
-
-        JDBCUtil.CREATE_TABLE createRefTable = (forceDrop) ? JDBCUtil.CREATE_TABLE.DROP_IF_EXISTS : JDBCUtil.CREATE_TABLE.SKIP_IF_EXISTS;
-
-        //step 2. create the tables
-        dbUtil.createTables(getNonRefTableInfos(), JDBCUtil.CREATE_TABLE.THROW_EX_IF_EXISTS);
-        dbUtil.createTables(getRefTableInfos(), JDBCUtil.CREATE_TABLE.THROW_EX_IF_EXISTS);
-
-        //step 3. create mime buffer
-        this.mimeBuffer = new MimeBuffer(dbUtil.getConnection(), getMimeTable(), TikaConfig.getDefaultConfig());
-
-        //step 4. populate the reference tables
-        populateRefTables();
-
-        return mimeBuffer;
-
-
-    }
-
-    private static void USAGE() {
-        HelpFormatter helpFormatter = new HelpFormatter();
-        helpFormatter.printHelp(80, "java -jar tika-eval-app-x.y.z.jar FileProfiler -e docs -d mydb [-i inputDir, -c config.json]",
-                "Tool: Profile", OPTIONS, "");
-    }
-
-    private static String USAGE_FAIL(String msg) {
-        USAGE();
-        throw new IllegalArgumentException(msg);
-    }
-
-    private static class ProfileWorker implements Callable<Integer> {
-
-        private final ArrayBlockingQueue<FileResource> queue;
-        private final ExtractProfiler extractProfiler;
-        ProfileWorker(ArrayBlockingQueue<FileResource> queue, ExtractProfiler extractProfiler) {
-            this.queue = queue;
-            this.extractProfiler = extractProfiler;
-        }
-
-        @Override
-        public Integer call() throws Exception {
-            while (true) {
-                FileResource resource = queue.poll(1, TimeUnit.SECONDS);
-                if (resource == null) {
-                    LOG.info("ExtractProfileWorker waiting on queue");
-                    continue;
-                }
-                if (resource == SEMAPHORE) {
-                    LOG.debug("worker hit semaphore and is stopping");
-                    //hangs
-                    queue.put(resource);
-                    return 1;
-                }
-                extractProfiler.processFileResource(resource);
-            }
-        }
-    }
-
-    private static class DirectoryWalker implements Callable<Integer> {
-        private final Path startDir;
-        private final ArrayBlockingQueue<FileResource> queue;
-
-        public DirectoryWalker(Path startDir, ArrayBlockingQueue<FileResource> queue) {
-            this.startDir = startDir;
-            this.queue = queue;
-        }
-
-        @Override
-        public Integer call() throws Exception {
-            Files.walkFileTree(startDir, new FileVisitor<Path>() {
-                @Override
-                public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
-                    return FileVisitResult.CONTINUE;
-                }
-
-                @Override
-                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                    //blocking
-                    try {
-                        queue.put(new PathResource(file, startDir.relativize(file).toString()));
-                    } catch (InterruptedException e) {
-                        return FileVisitResult.TERMINATE;
-                    }
-                    return FileVisitResult.CONTINUE;
-                }
-
-                @Override
-                public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
-                    return FileVisitResult.CONTINUE;
-                }
-
-                @Override
-                public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
-                    return FileVisitResult.CONTINUE;
-                }
-            });
-            return 0;
-        }
-    }
-}
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java
new file mode 100644
index 000000000..cd80a3df3
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.app;
+
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.eval.app.batch.DBConsumersManager;
+import org.apache.tika.eval.app.batch.FileResource;
+import org.apache.tika.eval.app.batch.PathResource;
+import org.apache.tika.eval.app.db.Cols;
+import org.apache.tika.eval.app.db.JDBCUtil;
+import org.apache.tika.eval.app.db.MimeBuffer;
+import org.apache.tika.eval.app.db.TableInfo;
+import org.apache.tika.eval.app.io.DBWriter;
+import org.apache.tika.eval.app.io.ExtractReader;
+import org.apache.tika.eval.app.io.ExtractReaderException;
+import org.apache.tika.eval.app.io.IDBWriter;
+
+public class ExtractProfileRunner {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ExtractProfileRunner.class);
+    private static final PathResource SEMAPHORE = new PathResource(Paths.get("/"), "STOP");
+    private static final int DIR_WALKER_COMPLETED_VALUE = 2;
+    private static final int PROFILE_WORKER_COMPLETED_VALUE = 1;
+
+    static Options OPTIONS;
+
+    static {
+
+        OPTIONS = new Options()
+                .addOption(Option.builder("e").longOpt("extracts").hasArg().desc("required: directory of extracts").build())
+                .addOption(Option.builder("i").longOpt("inputDir").hasArg().desc("optional: directory for original binary input documents."
+                        + " If not specified, -extracts is crawled as is.").build())
+                .addOption(Option.builder("d").longOpt("db").hasArg().desc("optional: db path").build())
+                .addOption(Option.builder("c").longOpt("config").hasArg().desc("tika-eval json config file").build())
+                ;
+    }
+    public static void main(String[] args) throws Exception {
+        DefaultParser defaultCLIParser = new DefaultParser();
+        CommandLine commandLine = defaultCLIParser.parse(OPTIONS, args);
+        EvalConfig evalConfig = commandLine.hasOption('c') ? EvalConfig.load(Paths.get(commandLine.getOptionValue('c'))) : new EvalConfig();
+        Path extractsDir = commandLine.hasOption('e') ? Paths.get(commandLine.getOptionValue('e')) : Paths.get(USAGE_FAIL("Must specify extracts dir: -i"));
+        Path inputDir = commandLine.hasOption('i') ? Paths.get(commandLine.getOptionValue('i')) : extractsDir;
+        String dbPath = commandLine.hasOption('d') ? commandLine.getOptionValue('d') : USAGE_FAIL("Must specify the db name: -d");
+        String jdbcString = getJdbcConnectionString(dbPath);
+        execute(inputDir, extractsDir, jdbcString, evalConfig);
+    }
+
+    private static String getJdbcConnectionString(String dbPath) {
+        if (dbPath.startsWith("jdbc:")) {
+            return dbPath;
+        }
+        //default to h2
+        Path p = Paths.get(dbPath);
+        return "jdbc:h2:file:" + p.toAbsolutePath();
+
+    }
+
+    private static void execute(Path inputDir, Path extractsDir, String dbPath, EvalConfig evalConfig) throws SQLException, IOException {
+
+        //parameterize this? if necesssary
+        try {
+            ProfilerBase.loadCommonTokens(null, null);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+        JDBCUtil jdbcUtil = new JDBCUtil(dbPath, evalConfig.getJdbcDriverClass());
+        ExtractProfilerBuilder builder = new ExtractProfilerBuilder();
+        MimeBuffer mimeBuffer = initTables(jdbcUtil, builder, dbPath, evalConfig);
+        builder.populateRefTables(jdbcUtil, mimeBuffer);
+
+        AtomicInteger enqueued = new AtomicInteger(0);
+        AtomicInteger processed = new AtomicInteger(0);
+        AtomicInteger activeWorkers = new AtomicInteger(evalConfig.getNumWorkers());
+        AtomicBoolean crawlerActive = new AtomicBoolean(true);
+
+
+
+        ArrayBlockingQueue<FileResource> queue = new ArrayBlockingQueue<>(1000);
+        ExecutorService executorService = Executors.newFixedThreadPool(evalConfig.getNumWorkers() + 2);
+        ExecutorCompletionService<Integer> executorCompletionService = new ExecutorCompletionService<>(executorService);
+
+        StatusReporter statusReporter = new StatusReporter(enqueued, processed, activeWorkers, crawlerActive);
+        executorCompletionService.submit(statusReporter);
+
+        DirectoryWalker directoryWalker = new DirectoryWalker(inputDir, queue, enqueued);
+        executorCompletionService.submit(directoryWalker);
+        for (int i = 0; i < evalConfig.getNumWorkers(); i++) {
+            ExtractReader extractReader = new ExtractReader(ExtractReader.ALTER_METADATA_LIST.AS_IS, evalConfig.getMinExtractLength(), evalConfig.getMaxExtractLength());
+            ExtractProfiler extractProfiler = new ExtractProfiler(inputDir, extractsDir, extractReader, builder.getDBWriter(builder.tableInfos, jdbcUtil, mimeBuffer));
+            executorCompletionService.submit(new ProfileWorker(queue, extractProfiler, processed));
+        }
+
+        int finished = 0;
+        try {
+            while (finished < evalConfig.getNumWorkers() + 2) {
+                //blocking
+                Future<Integer> future = executorCompletionService.take();
+                Integer result = future.get();
+                if (result != null) {
+                    //if the dir walker has finished
+                    if (result == DIR_WALKER_COMPLETED_VALUE) {
+                        queue.put(SEMAPHORE);
+                        crawlerActive.set(false);
+                    } else if (result == PROFILE_WORKER_COMPLETED_VALUE) {
+                        activeWorkers.decrementAndGet();
+                    }
+                    finished++;
+                }
+            }
+        } catch (InterruptedException e) {
+            LOG.info("interrupted", e);
+        } catch (ExecutionException e) {
+            throw new RuntimeException(e);
+        } finally {
+            mimeBuffer.close();
+            executorService.shutdownNow();
+        }
+
+    }
+
+    private static MimeBuffer initTables(JDBCUtil jdbcUtil, ExtractProfilerBuilder builder, String connectionString, EvalConfig evalConfig) throws SQLException, IOException {
+
+        //step 1. create the tables
+        jdbcUtil.createTables(builder.getNonRefTableInfos(), JDBCUtil.CREATE_TABLE.THROW_EX_IF_EXISTS);
+        jdbcUtil.createTables(builder.getRefTableInfos(), JDBCUtil.CREATE_TABLE.THROW_EX_IF_EXISTS);
+
+        //step 2. create mime buffer
+        return new MimeBuffer(jdbcUtil.getConnection(), builder.getMimeTable(), TikaConfig.getDefaultConfig());
+    }
+
+    private static void USAGE() {
+        HelpFormatter helpFormatter = new HelpFormatter();
+        helpFormatter.printHelp(80, "java -jar tika-eval-app-x.y.z.jar FileProfiler -e docs -d mydb [-i inputDir, -c config.json]",
+                "Tool: Profile", OPTIONS, "");
+    }
+
+    private static String USAGE_FAIL(String msg) {
+        USAGE();
+        throw new IllegalArgumentException(msg);
+    }
+
+    private static class ProfileWorker implements Callable<Integer> {
+
+        private final ArrayBlockingQueue<FileResource> queue;
+        private final ExtractProfiler extractProfiler;
+        private final AtomicInteger processed;
+
+        ProfileWorker(ArrayBlockingQueue<FileResource> queue, ExtractProfiler extractProfiler, AtomicInteger processed) {
+            this.queue = queue;
+            this.extractProfiler = extractProfiler;
+            this.processed = processed;
+        }
+
+        @Override
+        public Integer call() throws Exception {
+            while (true) {
+                FileResource resource = queue.poll(1, TimeUnit.SECONDS);
+                if (resource == null) {
+                    LOG.info("ExtractProfileWorker waiting on queue");
+                    continue;
+                }
+                if (resource == SEMAPHORE) {
+                    LOG.debug("worker hit semaphore and is stopping");
+                    extractProfiler.closeWriter();
+                    //hangs
+                    queue.put(resource);
+                    return PROFILE_WORKER_COMPLETED_VALUE;
+                }
+                extractProfiler.processFileResource(resource);
+                processed.incrementAndGet();
+            }
+        }
+    }
+
+    private static class DirectoryWalker implements Callable<Integer> {
+        private final Path startDir;
+        private final ArrayBlockingQueue<FileResource> queue;
+        private final AtomicInteger enqueued;
+
+        public DirectoryWalker(Path startDir, ArrayBlockingQueue<FileResource> queue, AtomicInteger enqueued) {
+            this.startDir = startDir;
+            this.queue = queue;
+            this.enqueued = enqueued;
+        }
+
+        @Override
+        public Integer call() throws Exception {
+            Files.walkFileTree(startDir, new FileVisitor<Path>() {
+                @Override
+                public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                    if (Files.isDirectory(file)) {
+                        return FileVisitResult.CONTINUE;
+                    }
+                    try {
+                        //blocking
+                        queue.put(new PathResource(file, startDir.relativize(file).toString()));
+                        enqueued.incrementAndGet();
+                    } catch (InterruptedException e) {
+                        return FileVisitResult.TERMINATE;
+                    }
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+            return DIR_WALKER_COMPLETED_VALUE;
+        }
+    }
+
+    private static class ExtractProfilerBuilder {
+        private final List<TableInfo> tableInfos;
+        private final List<TableInfo> refTableInfos;
+
+        public ExtractProfilerBuilder() {
+            List<TableInfo> tableInfos = new ArrayList();
+            tableInfos.add(AbstractProfiler.MIME_TABLE);
+            tableInfos.add(ExtractProfiler.CONTAINER_TABLE);
+            tableInfos.add(ExtractProfiler.PROFILE_TABLE);
+            tableInfos.add(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
+            tableInfos.add(ExtractProfiler.EXCEPTION_TABLE);
+            tableInfos.add(ExtractProfiler.CONTENTS_TABLE);
+            tableInfos.add(ExtractProfiler.TAGS_TABLE);
+            tableInfos.add(ExtractProfiler.EMBEDDED_FILE_PATH_TABLE);
+            this.tableInfos = Collections.unmodifiableList(tableInfos);
+
+            List<TableInfo> refTableInfos = new ArrayList<>();
+            refTableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
+            refTableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
+            refTableInfos.add(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES);
+            this.refTableInfos = Collections.unmodifiableList(refTableInfos);
+        }
+
+
+        protected List<TableInfo> getRefTableInfos() {
+            return refTableInfos;
+        }
+
+        protected List<TableInfo> getNonRefTableInfos() {
+            return tableInfos;
+        }
+
+        protected TableInfo getMimeTable() {
+            return AbstractProfiler.MIME_TABLE;
+        }
+
+        public void populateRefTables(JDBCUtil dbUtil, MimeBuffer mimeBuffer) throws IOException, SQLException {
+            boolean refTablesPopulated = true;
+            try {
+                Connection connection = dbUtil.getConnection();
+                for (TableInfo tableInfo : getRefTableInfos()) {
+                    int rows = 0;
+                    try (ResultSet rs = connection
+                            .createStatement()
+                            .executeQuery("select * from " + tableInfo.getName())) {
+                        while (rs.next()) {
+                            rows++;
+                        }
+                    }
+                    if (rows == 0) {
+                        refTablesPopulated = false;
+                        break;
+                    }
+
+                }
+            } catch (SQLException e) {
+                //swallow
+            }
+            if (refTablesPopulated) {
+                LOG.info("ref tables are already populated");
+                return;
+            }
+
+            IDBWriter writer = getDBWriter(getRefTableInfos(), dbUtil, mimeBuffer);
+            Map<Cols, String> m = new HashMap<>();
+            for (AbstractProfiler.PARSE_ERROR_TYPE t : AbstractProfiler.PARSE_ERROR_TYPE.values()) {
+                m.clear();
+                m.put(Cols.PARSE_ERROR_ID, Integer.toString(t.ordinal()));
+                m.put(Cols.PARSE_ERROR_DESCRIPTION, t.name());
+                writer.writeRow(AbstractProfiler.REF_PARSE_ERROR_TYPES, m);
+            }
+
+            for (AbstractProfiler.EXCEPTION_TYPE t : AbstractProfiler.EXCEPTION_TYPE.values()) {
+                m.clear();
+                m.put(Cols.PARSE_EXCEPTION_ID, Integer.toString(t.ordinal()));
+                m.put(Cols.PARSE_EXCEPTION_DESCRIPTION, t.name());
+                writer.writeRow(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES, m);
+            }
+
+            for (ExtractReaderException.TYPE t : ExtractReaderException.TYPE.values()) {
+                m.clear();
+                m.put(Cols.EXTRACT_EXCEPTION_ID, Integer.toString(t.ordinal()));
+                m.put(Cols.EXTRACT_EXCEPTION_DESCRIPTION, t.name());
+                writer.writeRow(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES, m);
+            }
+            writer.close();
+        }
+
+        protected IDBWriter getDBWriter(List<TableInfo> tableInfos, JDBCUtil dbUtil, MimeBuffer mimeBuffer) throws IOException, SQLException {
+            Connection conn = dbUtil.getConnection();
+            return new DBWriter(conn, tableInfos, dbUtil, mimeBuffer);
+        }
+
+
+        protected void addErrorLogTablePairs(DBConsumersManager manager, EvalConfig evalConfig) {
+            Path errorLog = evalConfig.getErrorLogFile();
+            if (errorLog == null) {
+                return;
+            }
+            manager.addErrorLogTablePair(errorLog, ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
+        }
+    }
+}
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java
index a2a4986f3..9b0f482f6 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java
@@ -22,7 +22,6 @@ import java.sql.Types;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
 
 import org.apache.commons.cli.Options;
 
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfiler.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfiler.java
index edc431e4a..925452094 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfiler.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfiler.java
@@ -73,7 +73,26 @@ public class FileProfiler extends AbstractProfiler {
     public static TableInfo FILE_MIME_TABLE =
             new TableInfo("file_mimes", new ColInfo(Cols.MIME_ID, Types.INTEGER, "PRIMARY KEY"), new ColInfo(Cols.MIME_STRING, Types.VARCHAR, 256),
                     new ColInfo(Cols.FILE_EXTENSION, Types.VARCHAR, 12));
+    static Options OPTIONS;
 
+    static {
+
+        Option inputDir = new Option("inputDir", true, "optional: directory for original binary input documents." + " If not specified, -extracts is crawled as is.");
+
+        OPTIONS = new Options()
+                .addOption(inputDir)
+                .addOption("bc", "optional: tika-batch config file")
+                .addOption("numConsumers", true, "optional: number of consumer threads")
+                .addOption("db", true, "db file to which to write results")
+                .addOption("jdbc", true, "EXPERT: full jdbc connection string. Must specify this or -db <h2db>")
+                .addOption("jdbcDriver", true, "EXPERT: jdbc driver, or specify via -Djdbc.driver")
+                .addOption("tablePrefix", true, "EXPERT: optional prefix for table names")
+                .addOption("drop", false, "drop tables if they exist")
+                .addOption("maxFilesToAdd", true, "maximum number of files to add to the crawler")
+
+        ;
+
+    }
 
     private final Path inputDir;
 
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java
index f5d5f1a15..19a7d680f 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java
@@ -29,7 +29,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -42,8 +41,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.xml.sax.SAXException;
 
-import org.apache.tika.batch.FileResource;
-import org.apache.tika.batch.FileResourceConsumer;
 import org.apache.tika.batch.fs.FSProperties;
 import org.apache.tika.eval.app.db.ColInfo;
 import org.apache.tika.eval.app.db.Cols;
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java
new file mode 100644
index 000000000..8df9824ed
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.app;
+
+import java.text.NumberFormat;
+import java.util.Locale;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.util.DurationFormatUtils;
+
+public class StatusReporter implements Callable<Integer> {
+
+    public static final int COMPLETED_VAL = 3;
+    private static final Logger LOGGER = LoggerFactory.getLogger(StatusReporter.class);
+    private final AtomicInteger filesQueued;
+    private final AtomicInteger filesProcessed;
+    private final AtomicInteger activeWorkers;
+    private final AtomicBoolean crawlerIsActive;
+    private final long start;
+    private final NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.ROOT);
+
+
+    public StatusReporter(AtomicInteger filesQueued, AtomicInteger filesProcessed, AtomicInteger activeWorkers, AtomicBoolean crawlerIsActive) {
+        this.filesQueued = filesQueued;
+        this.filesProcessed = filesProcessed;
+        this.activeWorkers = activeWorkers;
+        this.crawlerIsActive = crawlerIsActive;
+        this.start = System.currentTimeMillis();
+    }
+
+    @Override
+    public Integer call() throws Exception {
+        while (true) {
+
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException e) {
+                LOGGER.info("Interrupted?");
+                //expected
+                return COMPLETED_VAL;
+            }
+            report();
+            if (activeWorkers.get() == 0) {
+                LOGGER.info("Completed successfully.");
+                return COMPLETED_VAL;
+            }
+        }
+    }
+
+    private void report() {
+        int cnt = filesProcessed.get();
+        long elapsed = System.currentTimeMillis() - start;
+        double elapsedSecs = (double) elapsed / (double) 1000;
+        int avg = (elapsedSecs > 5 || cnt > 100) ? (int) ((double) cnt / elapsedSecs) : -1;
+
+        String elapsedString = DurationFormatUtils.formatMillis(System.currentTimeMillis() - start);
+        String docsPerSec = avg > -1 ? String.format(Locale.ROOT, " (%s docs per sec)", numberFormat.format(avg)) : "";
+        String msg = String.format(Locale.ROOT, "Processed %s documents in %s%s.", numberFormat.format(cnt), elapsedString, docsPerSec);
+        LOGGER.info(msg);
+
+        int stillAlive = activeWorkers.get();
+        if (stillAlive == 1) {
+            msg = "There is one file processor still active.";
+        } else {
+            msg = "There are " + numberFormat.format(stillAlive) + " file processors still active.";
+        }
+        LOGGER.info(msg);
+
+        int queued = filesQueued.get();
+
+        if (queued == 1) {
+            msg = "The crawler has enqueued 1 file.";
+        } else {
+            msg = "The crawler has enqueued " + numberFormat.format(queued) + " files.";
+        }
+        LOGGER.info(msg);
+
+        if (! crawlerIsActive.get()) {
+            msg = "The directory crawler has completed its crawl.\n";
+            LOGGER.info(msg);
+        }
+    }
+}
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/TikaEvalCLI.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/TikaEvalCLI.java
index bae1435e7..91aecd832 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/TikaEvalCLI.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/TikaEvalCLI.java
@@ -117,7 +117,7 @@ public class TikaEvalCLI {
                 CommandLine commandLine = defaultCLIParser.parse(FileProfiler.OPTIONS, updatedArgs);
                 if (commandLine.hasOption("db") && commandLine.hasOption("jdbc")) {
                     System.out.println("Please specify either the default -db or the full -jdbc, not both");
-                    ExtractProfiler.USAGE();
+                    FileProfiler.USAGE();
                     return;
                 }
             } catch (ParseException e) {
@@ -154,7 +154,7 @@ public class TikaEvalCLI {
     }
 
     private void handleProfile(String[] subsetArgs) throws Exception {
-        ExctractProfileRunner.main(subsetArgs);
+        ExtractProfileRunner.main(subsetArgs);
     }
 
     private void handleCompare(String[] subsetArgs) throws Exception {
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/ExtractProfilerBuilder.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/ExtractProfilerBuilder.java
deleted file mode 100644
index c59d53016..000000000
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/ExtractProfilerBuilder.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.eval.app.batch;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.tika.batch.FileResourceConsumer;
-import org.apache.tika.eval.app.AbstractProfiler;
-import org.apache.tika.eval.app.ExtractProfiler;
-import org.apache.tika.eval.app.db.TableInfo;
-import org.apache.tika.util.PropsUtil;
-
-
-public class ExtractProfilerBuilder extends EvalConsumerBuilder {
-
-    public final static String TABLE_PREFIX_KEY = "tablePrefix";
-
-    private final List<TableInfo> tableInfos;
-    private final List<TableInfo> refTableInfos;
-
-    public ExtractProfilerBuilder() {
-        List<TableInfo> tableInfos = new ArrayList();
-        tableInfos.add(AbstractProfiler.MIME_TABLE);
-        tableInfos.add(ExtractProfiler.CONTAINER_TABLE);
-        tableInfos.add(ExtractProfiler.PROFILE_TABLE);
-        tableInfos.add(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
-        tableInfos.add(ExtractProfiler.EXCEPTION_TABLE);
-        tableInfos.add(ExtractProfiler.CONTENTS_TABLE);
-        tableInfos.add(ExtractProfiler.TAGS_TABLE);
-        tableInfos.add(ExtractProfiler.EMBEDDED_FILE_PATH_TABLE);
-        this.tableInfos = Collections.unmodifiableList(tableInfos);
-
-        List<TableInfo> refTableInfos = new ArrayList<>();
-        refTableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
-        refTableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
-        refTableInfos.add(AbstractProfiler.REF_EXTRACT_EXCEPTION_TYPES);
-        this.refTableInfos = Collections.unmodifiableList(refTableInfos);
-    }
-
-    @Override
-    public FileResourceConsumer build() throws IOException, SQLException {
-        Path extracts = PropsUtil.getPath(localAttrs.get("extracts"), null);
-        if (extracts == null) {
-            throw new RuntimeException("Must specify \"extracts\" -- directory to crawl");
-        }
-        if (!Files.isDirectory(extracts)) {
-            throw new RuntimeException("ROOT DIRECTORY DOES NOT EXIST: " + extracts.toAbsolutePath());
-        }
-
-        Path inputDir = PropsUtil.getPath(localAttrs.get("inputDir"), null);
-
-        //we _could_ set this to extracts (if not null)
-        //here, but the Crawler defaults to "input" if nothing is passed
-        //so this won't work
-        if (inputDir == null) {
-            throw new RuntimeException("Must specify -inputDir");
-        }
-        if (extracts == null && inputDir != null) {
-            extracts = inputDir;
-        }
-        return parameterizeProfiler(new ExtractProfiler(queue, inputDir, extracts, buildExtractReader(localAttrs), getDBWriter(tableInfos)));
-    }
-
-
-    @Override
-    protected void updateTableInfosWithPrefixes(Map<String, String> attrs) {
-        String tableNamePrefix = attrs.get(TABLE_PREFIX_KEY);
-        if (tableNamePrefix != null && !tableNamePrefix.equals("null")) {
-            for (TableInfo tableInfo : tableInfos) {
-                tableInfo.setNamePrefix(tableNamePrefix);
-            }
-        }
-    }
-
-
-    @Override
-    protected List<TableInfo> getRefTableInfos() {
-        return refTableInfos;
-    }
-
-    @Override
-    protected List<TableInfo> getNonRefTableInfos() {
-        return tableInfos;
-    }
-
-    @Override
-    protected TableInfo getMimeTable() {
-        return AbstractProfiler.MIME_TABLE;
-    }
-
-    @Override
-    protected void addErrorLogTablePairs(DBConsumersManager manager) {
-        Path errorLog = PropsUtil.getPath(localAttrs.get("errorLogFile"), null);
-        if (errorLog == null) {
-            return;
-        }
-        manager.addErrorLogTablePair(errorLog, ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
-    }
-}
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/PathResource.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/PathResource.java
index 9d64317aa..20f67798a 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/PathResource.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/PathResource.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.tika.eval.app.batch;
 
 
@@ -6,15 +22,18 @@ import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
+import org.apache.tika.batch.fs.FSProperties;
 import org.apache.tika.metadata.Metadata;
 
 public class PathResource implements FileResource {
 
     private final Path path;
     private final String resourceId;
+    private final Metadata metadata = new Metadata();
     public PathResource(Path path, String resourceId) {
         this.path = path;
         this.resourceId = resourceId;
+        metadata.set(FSProperties.FS_REL_PATH, resourceId);
     }
     @Override
     public String getResourceId() {
@@ -23,7 +42,7 @@ public class PathResource implements FileResource {
 
     @Override
     public Metadata getMetadata() {
-        return new Metadata();
+        return metadata;
     }
 
     @Override
diff --git a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/EvalConfigTest.java b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/EvalConfigTest.java
new file mode 100644
index 000000000..395c90fe6
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/EvalConfigTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.app;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.Test;
+
+public class EvalConfigTest {
+
+    @Test
+    public void testBasic() throws Exception {
+        EvalConfig evalConfig = EvalConfig.load(getConfig("eval-config-basic.json"));
+        assertEquals(20000, evalConfig.getMaxExtractLength());
+        assertNull(evalConfig.getErrorLogFile());
+        assertNull(evalConfig.getJdbcString());
+    }
+
+    private Path getConfig(String fileName) throws URISyntaxException {
+        return Paths.get(EvalConfigTest.class.getResource("/eval-configs/" + fileName).toURI());
+    }
+}
diff --git a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ProfilerBatchTest.java b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ProfilerBatchTest.java
index be58e0ed2..3d6e93ad3 100644
--- a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ProfilerBatchTest.java
+++ b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ProfilerBatchTest.java
@@ -21,7 +21,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
-import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -30,79 +29,75 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
-import org.junit.jupiter.api.AfterAll;
+import org.apache.commons.io.FileUtils;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import org.apache.tika.eval.app.db.Cols;
 import org.apache.tika.eval.app.db.H2Util;
 import org.apache.tika.eval.app.db.TableInfo;
-import org.apache.tika.eval.app.io.ExtractReaderException;
 
-@Disabled
 public class ProfilerBatchTest {
 
-    public final static String COMPARER_PROCESS_CLASS = "org.apache.tika.batch.fs.FSBatchProcessCLI";
-    private final static String profileTable = ExtractProfiler.PROFILE_TABLE.getName();
-    private final static String exTable = ExtractProfiler.EXCEPTION_TABLE.getName();
-    private final static String fpCol = Cols.FILE_PATH.name();
-    private static Path dbDir;
-    private static Connection conn;
+    private static Connection CONN;
+    private static Path DB_DIR;
+    private static Path DB;
 
     @BeforeAll
     public static void setUp() throws Exception {
+        DB_DIR = Files.createTempDirectory("profiler-test");
+        Path extractsRoot = Paths.get(ComparerBatchTest.class
+                .getResource("/test-dirs/extractsA")
+                .toURI());
 
         Path inputRoot = Paths.get(ComparerBatchTest.class
-                .getResource("/test-dirs/extractsA")
+                .getResource("/test-dirs/raw_input")
                 .toURI());
-        dbDir = Files.createTempDirectory(inputRoot, "tika-test-db-dir-");
-        Map<String, String> args = new HashMap<>();
-        Path db = dbDir.resolve("profiler_test");
-        args.put("-db", db.toString());
-
-        //for debugging, you can use this to select only one file pair to load
-        //args.put("-includeFilePat", "file8.*");
-
-       /* BatchProcessTestExecutor ex = new BatchProcessTestExecutor(COMPARER_PROCESS_CLASS, args,
-                "/single-file-profiler-crawl-input-config.xml");
-        StreamStrings streamStrings = ex.execute();
-        System.out.println(streamStrings.getErrString());
-        System.out.println(streamStrings.getOutString());*/
-        H2Util dbUtil = new H2Util(db);
-        conn = dbUtil.getConnection();
-    }
 
-    @AfterAll
-    public static void tearDown() throws IOException {
+        DB = DB_DIR.resolve("mydb");
+        String[] args = new String[]{
+            "-i", inputRoot.toAbsolutePath().toString(),
+            "-e", extractsRoot.toAbsolutePath().toString(),
+                "-d", "jdbc:h2:file:" + DB.toAbsolutePath().toString()
+        };
+
+        ExtractProfileRunner.main(args);
+    }
 
+    @AfterEach
+    public void tearDown() throws IOException {
         try {
-            conn.close();
+            CONN.close();
         } catch (SQLException e) {
             throw new RuntimeException(e);
         }
-        //TODO: if/when we turn this back on, use @TempDir instead of this
+        FileUtils.deleteDirectory(DB_DIR.toFile());
 
-        DirectoryStream<Path> dStream = Files.newDirectoryStream(dbDir);
-        for (Path p : dStream) {
-            Files.delete(p);
-        }
-        dStream.close();
-        Files.delete(dbDir);
+    }
+
+    @BeforeEach
+    public void setUpEach() throws SQLException {
+        H2Util dbUtil = new H2Util(DB);
+        CONN = dbUtil.getConnection();
+    }
+
+    @AfterEach
+    public void tearDownEach() throws SQLException {
+        CONN.close();
     }
 
     @Test
     public void testSimpleDBWriteAndRead() throws Exception {
-
         Statement st = null;
         List<String> fNameList = new ArrayList<>();
         try {
             String sql = "select * from " + ExtractProfiler.CONTAINER_TABLE.getName();
-            st = conn.createStatement();
+            st = CONN.createStatement();
             ResultSet rs = st.executeQuery(sql);
             while (rs.next()) {
                 String fileName = rs.getString(Cols.FILE_PATH.name());
@@ -113,17 +108,19 @@ public class ProfilerBatchTest {
                 st.close();
             }
         }
+        /*
         debugTable(ExtractProfiler.CONTAINER_TABLE);
         debugTable(ExtractProfiler.PROFILE_TABLE);
         debugTable(ExtractProfiler.CONTENTS_TABLE);
         debugTable(ExtractProfiler.EXCEPTION_TABLE);
-        debugTable(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
-        assertEquals(10, fNameList.size());
+        debugTable(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);*/
+        assertEquals(17, fNameList.size());
         assertTrue(fNameList.contains("file1.pdf"), "file1.pdf");
         assertTrue(fNameList.contains("file2_attachANotB.doc"), "file2_attachANotB.doc");
         assertTrue(fNameList.contains("file3_attachBNotA.doc"), "file3_attachBNotA.doc");
         assertTrue(fNameList.contains("file4_emptyB.pdf"), "file4_emptyB.pdf");
         assertTrue(fNameList.contains("file7_badJson.pdf"), "file4_emptyB.pdf");
+        assertTrue(fNameList.contains("file9_noextract.txt"), "file9_noextract.txt");
     }
 
     @Test
@@ -131,43 +128,29 @@ public class ProfilerBatchTest {
         String sql =
                 "select EXTRACT_EXCEPTION_ID from extract_exceptions e" + " join containers c on c.container_id = e.container_id " + " where c.file_path='file9_noextract.txt'";
 
-        assertEquals("missing extract: file9_noextract.txt", "0", getSingleResult(sql));
-        debugTable(ExtractProfiler.CONTAINER_TABLE);
+        /*debugTable(ExtractProfiler.CONTAINER_TABLE);
         debugTable(ExtractProfiler.PROFILE_TABLE);
         debugTable(ExtractProfiler.CONTENTS_TABLE);
         debugTable(ExtractProfiler.EXCEPTION_TABLE);
-        debugTable(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
-
-        sql = "select EXTRACT_EXCEPTION_ID from errors e" + " join containers c on c.container_id = e.container_id " + " where c.file_path='file5_emptyA.pdf'";
-        assertEquals("empty extract: file5_emptyA.pdf", "1", getSingleResult(sql));
-
-        sql = "select EXTRACT_EXCEPTION_ID from errors e" + " join containers c on c.container_id = e.container_id " + " where c.file_path='file7_badJson.pdf'";
-        assertEquals("extract error:file7_badJson.pdf", "2", getSingleResult(sql));
-
-    }
-
-    @Test
-    public void testParseErrors() throws Exception {
-        debugTable(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);
-        String sql = "select file_path from errors where container_id is null";
-        assertEquals("file10_permahang.txt", getSingleResult(sql));
-
-        sql = "select extract_error_id from extract_exceptions " + "where file_path='file11_oom.txt'";
-        assertEquals(Integer.toString(ExtractReaderException.TYPE.ZERO_BYTE_EXTRACT_FILE.ordinal()), getSingleResult(sql));
+        debugTable(ExtractProfiler.EXTRACT_EXCEPTION_TABLE);*/
+        assertEquals("0", getSingleResult(sql), "missing extract: file9_noextract.txt");
 
-        sql = "select parse_error_id from extract_exceptions where file_path='file11_oom.txt'";
-        assertEquals(Integer.toString(AbstractProfiler.PARSE_ERROR_TYPE.OOM.ordinal()), getSingleResult(sql));
+        sql = "select EXTRACT_EXCEPTION_ID from extract_exceptions e" + " join containers c on c.container_id = e.container_id " + " where c.file_path='file5_emptyA.pdf'";
+        assertEquals("1", getSingleResult(sql), "empty extract: file5_emptyA.pdf");
 
+        sql = "select EXTRACT_EXCEPTION_ID from extract_exceptions e" + " join containers c on c.container_id = e.container_id " + " where c.file_path='file7_badJson.pdf'";
+        assertEquals("2", getSingleResult(sql), "extract error:file7_badJson.pdf");
     }
 
     @Test
+    @Disabled("create actual unit test")
     public void testParseExceptions() throws Exception {
         debugTable(ExtractProfiler.EXCEPTION_TABLE);
     }
 
     private String getSingleResult(String sql) throws Exception {
         Statement st = null;
-        st = conn.createStatement();
+        st = CONN.createStatement();
         ResultSet rs = st.executeQuery(sql);
         int hits = 0;
         String val = "";
@@ -188,7 +171,7 @@ public class ProfilerBatchTest {
         Statement st = null;
         try {
             String sql = "select * from " + table.getName();
-            st = conn.createStatement();
+            st = CONN.createStatement();
             ResultSet rs = st.executeQuery(sql);
             int colCount = rs
                     .getMetaData()
diff --git a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/TikaEvalCLITest.java b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/TikaEvalCLITest.java
index fff15e3dc..4d7d4bb2b 100644
--- a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/TikaEvalCLITest.java
+++ b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/TikaEvalCLITest.java
@@ -98,20 +98,13 @@ public class TikaEvalCLITest extends TikaTest {
     private static void profile() throws IOException {
         List<String> args = new ArrayList<>();
         args.add("Profile");
-        args.add("-extracts");
+        args.add("-e");
         args.add(ProcessUtils.escapeCommandLine(extractsDir
                 .resolve("extractsA")
                 .toAbsolutePath()
                 .toString()));
-        //add these just to confirm this info doesn't cause problems w cli
-        args.add("-maxTokens");
-        args.add("10000000");
-        args.add("-maxContentLength");
-        args.add("100000000");
-        args.add("-maxContentLengthForLangId");
-        args.add("100000");
 
-        args.add("-db");
+        args.add("-d");
         args.add(ProcessUtils.escapeCommandLine(profileDBDir
                 .toAbsolutePath()
                 .toString() + "/" + dbName));
diff --git a/tika-eval/tika-eval-app/src/test/resources/eval-configs/eval-config-basic.json b/tika-eval/tika-eval-app/src/test/resources/eval-configs/eval-config-basic.json
new file mode 100644
index 000000000..b4af28df3
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/test/resources/eval-configs/eval-config-basic.json
@@ -0,0 +1,3 @@
+{
+  "maxExtractLength" : 20000
+}
\ No newline at end of file
diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file10_permahang.txt b/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file10_permahang.txt
new file mode 100644
index 000000000..e69de29bb
diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file12_es.txt b/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file12_es.txt
new file mode 100644
index 000000000..5ffd824a9
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file12_es.txt
@@ -0,0 +1,6 @@
+[
+  {
+    "Content-Type": "text/plain",
+    "X-TIKA:content": "El zorro marrón rápido saltó sobre el perro. El zorro marrón rápido saltó sobre el perro. El zorro marrón rápido saltó sobre el perro"
+  }
+]
\ No newline at end of file
diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file13_attachANotB.doc b/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file13_attachANotB.doc
new file mode 100644
index 000000000..15bc592a5
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file13_attachANotB.doc
@@ -0,0 +1,12 @@
+[
+  {
+    "Content-Type": "text/plain",
+    "_comment": "simplified",
+    "X-TIKA:content": "调整每一个心脏和每个声音,投标每个护理提取;让大家一起欢乐,赞美老拿骚.调整每一个心脏和每个声音,投标每个护理提取;让大家一起欢乐,赞美老拿骚 狐狸狐狸狐狸 "
+  },
+  {
+    "Content-Type": "text/plain",
+    "X-TIKA:embedded_resource_path": "inner.txt",
+    "X-TIKA:content": "attachment contents"
+  }
+]
\ No newline at end of file
diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file14_diffAttachOrder b/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file14_diffAttachOrder
new file mode 100644
index 000000000..25f0db9a1
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file14_diffAttachOrder
@@ -0,0 +1,21 @@
+[
+  {
+    "Content-Type": "text/plain",
+    "X-TIKA:content": "the quick brown fox fox fox jumped over the lazy lazy dog",
+    "X-TIKA:digest:MD5": "471d98383e9f40444e5ecf821f2c8351"
+  },
+  {
+    "Content-Type": "text/plain",
+    "X-TIKA:embedded_resource_path": "/0",
+    "X-TIKA:content": "a b c d e f g h i j k l m n",
+    "X-TIKA:digest:MD5": "471d98383e9f40444e5ecf821f2c8354",
+    "X-TIKA:embedded_depth": "1"
+  },
+  {
+    "Content-Type": "text/plain",
+    "X-TIKA:embedded_resource_path": "/1",
+    "X-TIKA:content": "o p q r s t u v w x y z",
+    "X-TIKA:digest:MD5": "471d98383e9f40444e5ecf821f2c8353",
+    "X-TIKA:embedded_depth": "1"
+  }
+]
\ No newline at end of file
diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file15_tags b/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file15_tags
new file mode 100644
index 000000000..5af73db80
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file15_tags
@@ -0,0 +1,41 @@
+[
+  {
+    "Content-Length": "34824",
+    "Content-Type": "application/pdf",
+    "Last-Modified": "2007-09-15T09:02:31Z",
+    "X-Parsed-By": [
+      "org.apache.tika.parser.DefaultParser",
+      "org.apache.tika.parser.pdf.PDFParser"
+    ],
+    "X-TIKA:content_handler": "ToXMLContentHandler",
+    "X-TIKA:content": "\u003chtml xmlns\u003d\"http://www.w3.org/1999/xhtml\"\u003e\n\u003chead\u003e\n\u003cmeta name\u003d\"pdf:PDFVersion\" content\u003d\"1.3\" /\u003e\n\u003cmeta name\u003d\"pdf:docinfo:title\" content\u003d\"Apache Tika - Apache Tika\" /\u003e\n\u003cmeta name\u003d\"xmp:CreatorTool\" content\u003d\"Firefox\" /\u003e\n\u003cmeta name\u003d\"access_permission:modify_annotations\" content\u003d\"true\" /\u003e\n\u003cmeta name\u003d\"access_permission:can_print_degra [...]
+    "X-TIKA:parse_time_millis": "500",
+    "access_permission:assemble_document": "true",
+    "access_permission:can_modify": "true",
+    "access_permission:can_print": "true",
+    "access_permission:can_print_degraded": "true",
+    "access_permission:extract_content": "true",
+    "access_permission:extract_for_accessibility": "true",
+    "access_permission:fill_in_form": "true",
+    "access_permission:modify_annotations": "true",
+    "dc:creator": "Bertrand DelacrΘtaz",
+    "dc:format": "application/pdf; version\u003d1.3",
+    "dc:title": "Apache Tika - Apache Tika",
+    "dcterms:created": "2007-09-15T09:02:31Z",
+    "dcterms:modified": "2007-09-15T09:02:31Z",
+    "meta:author": "Bertrand DelacrΘtaz",
+    "meta:creation-date": "2007-09-15T09:02:31Z",
+    "meta:save-date": "2007-09-15T09:02:31Z",
+    "pdf:PDFVersion": "1.3",
+    "pdf:docinfo:created": "2007-09-15T09:02:31Z",
+    "pdf:docinfo:creator": "Bertrand DelacrΘtaz",
+    "pdf:docinfo:creator_tool": "Firefox",
+    "pdf:docinfo:modified": "2007-09-15T09:02:31Z",
+    "pdf:docinfo:producer": "Mac OS X 10.4.10 Quartz PDFContext",
+    "pdf:docinfo:title": "Apache Tika - Apache Tika",
+    "pdf:encrypted": "false",
+    "resourceName": "testPDF.pdf",
+    "xmp:CreatorTool": "Firefox",
+    "xmpTPg:NPages": "1"
+  }
+]
\ No newline at end of file
diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file16_badTags b/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file16_badTags
new file mode 100644
index 000000000..5c6272e43
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file16_badTags
@@ -0,0 +1,41 @@
+[
+  {
+    "Content-Length": "34824",
+    "Content-Type": "application/pdf",
+    "Last-Modified": "2007-09-15T09:02:31Z",
+    "X-Parsed-By": [
+      "org.apache.tika.parser.DefaultParser",
+      "org.apache.tika.parser.pdf.PDFParser"
+    ],
+    "X-TIKA:content_handler": "ToXMLContentHandler",
+    "X-TIKA:content": "\u003chtml xmlns\u003d\"http://www.w3.org/1999/xhtml\"\u003e\n\u003chead\u003e\n\u003cmeta name\u003d\"pdf:PDFVersion\" content\u003d\"1.3\" /\u003e\n\u003cmeta name\u003d\"pdf:docinfo:title\" content\u003d\"Apache Tika - Apache Tika\" meta name\u003d\"xmp:CreatorTool\" content\u003d\"Firefox\" /\u003e\n\u003c\u003c\u003c\u003c\u003cmeta name\u003d\"access_permission:modify_annotations\" content\u003d\"true\" /\u003e\n\u003cmeta name\u003d\"access_permission:can_pr [...]
+    "X-TIKA:parse_time_millis": "500",
+    "access_permission:assemble_document": "true",
+    "access_permission:can_modify": "true",
+    "access_permission:can_print": "true",
+    "access_permission:can_print_degraded": "true",
+    "access_permission:extract_content": "true",
+    "access_permission:extract_for_accessibility": "true",
+    "access_permission:fill_in_form": "true",
+    "access_permission:modify_annotations": "true",
+    "dc:creator": "Bertrand DelacrΘtaz",
+    "dc:format": "application/pdf; version\u003d1.3",
+    "dc:title": "Apache Tika - Apache Tika",
+    "dcterms:created": "2007-09-15T09:02:31Z",
+    "dcterms:modified": "2007-09-15T09:02:31Z",
+    "meta:author": "Bertrand DelacrΘtaz",
+    "meta:creation-date": "2007-09-15T09:02:31Z",
+    "meta:save-date": "2007-09-15T09:02:31Z",
+    "pdf:PDFVersion": "1.3",
+    "pdf:docinfo:created": "2007-09-15T09:02:31Z",
+    "pdf:docinfo:creator": "Bertrand DelacrΘtaz",
+    "pdf:docinfo:creator_tool": "Firefox",
+    "pdf:docinfo:modified": "2007-09-15T09:02:31Z",
+    "pdf:docinfo:producer": "Mac OS X 10.4.10 Quartz PDFContext",
+    "pdf:docinfo:title": "Apache Tika - Apache Tika",
+    "pdf:encrypted": "false",
+    "resourceName": "testPDF.pdf",
+    "xmp:CreatorTool": "Firefox",
+    "xmpTPg:NPages": "1"
+  }
+]
\ No newline at end of file
diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file17_tagsOutOfOrder b/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file17_tagsOutOfOrder
new file mode 100644
index 000000000..97afec8ad
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/raw_input/file17_tagsOutOfOrder
@@ -0,0 +1,41 @@
+[
+  {
+    "Content-Length": "34824",
+    "Content-Type": "application/pdf",
+    "Last-Modified": "2007-09-15T09:02:31Z",
+    "X-Parsed-By": [
+      "org.apache.tika.parser.DefaultParser",
+      "org.apache.tika.parser.pdf.PDFParser"
+    ],
+    "X-TIKA:content_handler": "ToXMLContentHandler",
+    "X-TIKA:content": "\u003chtml xmlns\u003d\"http://www.w3.org/1999/xhtml\"\u003e\n\u003chead\u003e\n\u003cmeta name\u003d\"pdf:PDFVersion\" content\u003d\"1.3\" /\u003e\n\u003cmeta name\u003d\"pdf:docinfo:title\" content\u003d\"Apache Tika - Apache Tika\" /\u003e\n\u003cmeta name\u003d\"xmp:CreatorTool\" content\u003d\"Firefox\" /\u003e\n\u003cmeta name\u003d\"access_permission:modify_annotations\" content\u003d\"true\" /\u003e\n\u003cmeta name\u003d\"access_permission:can_print_degra [...]
+    "X-TIKA:parse_time_millis": "500",
+    "access_permission:assemble_document": "true",
+    "access_permission:can_modify": "true",
+    "access_permission:can_print": "true",
+    "access_permission:can_print_degraded": "true",
+    "access_permission:extract_content": "true",
+    "access_permission:extract_for_accessibility": "true",
+    "access_permission:fill_in_form": "true",
+    "access_permission:modify_annotations": "true",
+    "dc:creator": "Bertrand DelacrΘtaz",
+    "dc:format": "application/pdf; version\u003d1.3",
+    "dc:title": "Apache Tika - Apache Tika",
+    "dcterms:created": "2007-09-15T09:02:31Z",
+    "dcterms:modified": "2007-09-15T09:02:31Z",
+    "meta:author": "Bertrand DelacrΘtaz",
+    "meta:creation-date": "2007-09-15T09:02:31Z",
+    "meta:save-date": "2007-09-15T09:02:31Z",
+    "pdf:PDFVersion": "1.3",
+    "pdf:docinfo:created": "2007-09-15T09:02:31Z",
+    "pdf:docinfo:creator": "Bertrand DelacrΘtaz",
+    "pdf:docinfo:creator_tool": "Firefox",
+    "pdf:docinfo:modified": "2007-09-15T09:02:31Z",
+    "pdf:docinfo:producer": "Mac OS X 10.4.10 Quartz PDFContext",
+    "pdf:docinfo:title": "Apache Tika - Apache Tika",
+    "pdf:encrypted": "false",
+    "resourceName": "testPDF.pdf",
+    "xmp:CreatorTool": "Firefox",
+    "xmpTPg:NPages": "1"
+  }
+]
\ No newline at end of file
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index e0ba1333d..2037f1aba 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -399,7 +399,7 @@
     <jsoup.version>1.21.1</jsoup.version>
     <jsr305.version>3.0.2</jsr305.version>
     <junit4.version>4.13.2</junit4.version>
-    <junit5.version>6.0.0-M1</junit5.version>
+    <junit5.version>5.13.3</junit5.version>
     <juniversalchardet.version>2.5.0</juniversalchardet.version>
     <junrar.version>7.5.5</junrar.version>
     <jwarc.version>0.31.1</jwarc.version>


(tika) 02/03: WIP -- don't merge

Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tallison pushed a commit to branch TIKA-4342
in repository https://gitbox.apache.org/repos/asf/tika.git

commit a5972ca7e53b25b001417e9a13b635dcae7c452b
Author: tallison <ta...@apache.org>
AuthorDate: Wed May 28 16:37:11 2025 -0400

    WIP -- don't merge
---
 .../tika/eval/app/ExctractProfileRunner.java       | 204 ++++++
 .../org/apache/tika/eval/app/ExtractProfiler.java  |  15 +-
 .../apache/tika/eval/app/FileProfileRunner.java    |  54 --
 .../org/apache/tika/eval/app/ProfilerBase.java     | 816 +++++++++++++++++++++
 .../java/org/apache/tika/eval/app/TikaEvalCLI.java |   2 +-
 .../apache/tika/eval/app/batch/FileResource.java   |   2 +-
 .../apache/tika/eval/app/batch/PathResource.java   |  33 +
 7 files changed, 1059 insertions(+), 67 deletions(-)

diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExctractProfileRunner.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExctractProfileRunner.java
new file mode 100644
index 000000000..20cb33c1c
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExctractProfileRunner.java
@@ -0,0 +1,204 @@
+package org.apache.tika.eval.app;
+
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.eval.app.batch.FileResource;
+import org.apache.tika.eval.app.batch.PathResource;
+import org.apache.tika.eval.app.db.JDBCUtil;
+import org.apache.tika.eval.app.db.MimeBuffer;
+import org.apache.tika.eval.app.io.ExtractReader;
+import org.apache.tika.eval.app.io.IDBWriter;
+
+public class ExctractProfileRunner {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ExctractProfileRunner.class);
+    private static final PathResource SEMAPHORE = new PathResource(Paths.get("/"), "STOP");
+
+    static Options OPTIONS;
+
+    static {
+
+        OPTIONS = new Options()
+                .addOption(Option.builder("e").longOpt("extracts").hasArg().desc("required: directory of extracts").build())
+                .addOption(Option.builder("i").longOpt("inputDir").hasArg().desc("optional: directory for original binary input documents."
+                        + " If not specified, -extracts is crawled as is.").build())
+                .addOption(Option.builder("d").longOpt("db").hasArg().desc("optional: db path").build())
+                .addOption(Option.builder("c").longOpt("config").hasArg().desc("tika-eval json config file").build())
+                ;
+    }
+    public static void main(String[] args) throws Exception {
+        DefaultParser defaultCLIParser = new DefaultParser();
+        CommandLine commandLine = defaultCLIParser.parse(OPTIONS, args);
+        EvalConfig evalConfig = commandLine.hasOption('c') ? EvalConfig.load(Paths.get(commandLine.getOptionValue('c'))) : new EvalConfig();
+        Path extractsDir = commandLine.hasOption('e') ? Paths.get(commandLine.getOptionValue('e')) : Paths.get(USAGE_FAIL("Must specify extracts dir: -i"));
+        Path inputDir = commandLine.hasOption('i') ? Paths.get(commandLine.getOptionValue('i')) : extractsDir;
+        String dbPath = commandLine.hasOption('d') ? commandLine.getOptionValue('d') : USAGE_FAIL("Must specify the db name: -d");
+        execute(inputDir, extractsDir, dbPath, evalConfig);
+    }
+
+    private static void execute(Path inputDir, Path extractsDir, String dbPath, EvalConfig evalConfig) {
+
+        ArrayBlockingQueue<FileResource> queue = new ArrayBlockingQueue<>(1000);
+        DirectoryWalker fileWalker = new DirectoryWalker(inputDir, queue);
+        ExecutorService executorService = Executors.newFixedThreadPool(evalConfig.numThreads + 1);
+        ExecutorCompletionService<Integer> executorCompletionService = new ExecutorCompletionService<>(executorService);
+        executorCompletionService.submit(fileWalker);
+        IDBWriter dbWriter = buildDBWriter();
+        for (int i = 0; i < evalConfig.numThreads; i++) {
+            ExtractReader extractReader = new ExtractReader(ExtractReader.ALTER_METADATA_LIST.AS_IS, evalConfig.minExtractLength, evalConfig.maxExtractLength);
+
+            ExtractProfiler extractProfiler = new ExtractProfiler(inputDir, extractsDir, extractReader, dbWriter);
+            executorCompletionService.submit(new ProfileWorker(queue, extractProfiler));
+        }
+
+        int finished = 0;
+        try {
+            while (finished < evalConfig.numThreads + 1) {
+                //blocking
+                Future<Integer> future = executorCompletionService.take();
+                Integer result = future.get();
+                if (result != null) {
+                    finished++;
+                }
+
+            }
+        } catch (InterruptedException e) {
+            LOG.info("interrupted", e);
+        } catch (ExecutionException e) {
+            throw new RuntimeException(e);
+        } finally {
+            executorService.shutdownNow();
+        }
+
+    }
+
+    private static IDBWriter buildDBWriter(String connectionString, String driverClass) {
+        MimeBuffer mimeBuffer = null;
+        JDBCUtil dbUtil = new JDBCUtil(connectionString, driverClass);
+        //Step 1. Used to be update table infos with prefixes
+        updateTableInfosWithPrefixes(localAttrs);
+
+        JDBCUtil.CREATE_TABLE createRegularTable = (forceDrop) ? JDBCUtil.CREATE_TABLE.DROP_IF_EXISTS : JDBCUtil.CREATE_TABLE.THROW_EX_IF_EXISTS;
+
+        JDBCUtil.CREATE_TABLE createRefTable = (forceDrop) ? JDBCUtil.CREATE_TABLE.DROP_IF_EXISTS : JDBCUtil.CREATE_TABLE.SKIP_IF_EXISTS;
+
+        //step 2. create the tables
+        dbUtil.createTables(getNonRefTableInfos(), JDBCUtil.CREATE_TABLE.THROW_EX_IF_EXISTS);
+        dbUtil.createTables(getRefTableInfos(), JDBCUtil.CREATE_TABLE.THROW_EX_IF_EXISTS);
+
+        //step 3. create mime buffer
+        this.mimeBuffer = new MimeBuffer(dbUtil.getConnection(), getMimeTable(), TikaConfig.getDefaultConfig());
+
+        //step 4. populate the reference tables
+        populateRefTables();
+
+        return mimeBuffer;
+
+
+    }
+
+    private static void USAGE() {
+        HelpFormatter helpFormatter = new HelpFormatter();
+        helpFormatter.printHelp(80, "java -jar tika-eval-app-x.y.z.jar FileProfiler -e docs -d mydb [-i inputDir, -c config.json]",
+                "Tool: Profile", OPTIONS, "");
+    }
+
+    private static String USAGE_FAIL(String msg) {
+        USAGE();
+        throw new IllegalArgumentException(msg);
+    }
+
+    private static class ProfileWorker implements Callable<Integer> {
+
+        private final ArrayBlockingQueue<FileResource> queue;
+        private final ExtractProfiler extractProfiler;
+        ProfileWorker(ArrayBlockingQueue<FileResource> queue, ExtractProfiler extractProfiler) {
+            this.queue = queue;
+            this.extractProfiler = extractProfiler;
+        }
+
+        @Override
+        public Integer call() throws Exception {
+            while (true) {
+                FileResource resource = queue.poll(1, TimeUnit.SECONDS);
+                if (resource == null) {
+                    LOG.info("ExtractProfileWorker waiting on queue");
+                    continue;
+                }
+                if (resource == SEMAPHORE) {
+                    LOG.debug("worker hit semaphore and is stopping");
+                    //hangs
+                    queue.put(resource);
+                    return 1;
+                }
+                extractProfiler.processFileResource(resource);
+            }
+        }
+    }
+
+    private static class DirectoryWalker implements Callable<Integer> {
+        private final Path startDir;
+        private final ArrayBlockingQueue<FileResource> queue;
+
+        public DirectoryWalker(Path startDir, ArrayBlockingQueue<FileResource> queue) {
+            this.startDir = startDir;
+            this.queue = queue;
+        }
+
+        @Override
+        public Integer call() throws Exception {
+            Files.walkFileTree(startDir, new FileVisitor<Path>() {
+                @Override
+                public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                    //blocking
+                    try {
+                        queue.put(new PathResource(file, startDir.relativize(file).toString()));
+                    } catch (InterruptedException e) {
+                        return FileVisitResult.TERMINATE;
+                    }
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+            return 0;
+        }
+    }
+}
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java
index 844eac125..a2a4986f3 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java
@@ -16,7 +16,6 @@
  */
 package org.apache.tika.eval.app;
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.sql.Types;
@@ -25,11 +24,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ArrayBlockingQueue;
 
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 
-import org.apache.tika.batch.FileResource;
+import org.apache.tika.eval.app.batch.FileResource;
 import org.apache.tika.eval.app.db.ColInfo;
 import org.apache.tika.eval.app.db.Cols;
 import org.apache.tika.eval.app.db.TableInfo;
@@ -40,7 +37,7 @@ import org.apache.tika.eval.core.util.ContentTags;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaCoreProperties;
 
-public class ExtractProfiler extends AbstractProfiler {
+public class ExtractProfiler extends ProfilerBase {
 
     private final static String FIELD = "f";
     public static TableInfo EXTRACT_EXCEPTION_TABLE =
@@ -81,19 +78,15 @@ public class ExtractProfiler extends AbstractProfiler {
     private final Path extracts;
     private final ExtractReader extractReader;
 
-    public static ExtractProfiler build(ArrayBlockingQueue<FileResource> queue, Path inputDir, Path extracts, ExtractReader extractReader, IDBWriter dbWriter) {
-        return new ExtractProfiler(queue, inputDir, extracts, extractReader, dbWriter);
-    }
 
-    ExtractProfiler(ArrayBlockingQueue<FileResource> queue, Path inputDir, Path extracts, ExtractReader extractReader, IDBWriter dbWriter) {
-        super(queue, dbWriter);
+    ExtractProfiler(Path inputDir, Path extracts, ExtractReader extractReader, IDBWriter dbWriter) {
+        super(dbWriter);
         this.inputDir = inputDir;
         this.extracts = extracts;
         this.extractReader = extractReader;
     }
 
 
-    @Override
     public boolean processFileResource(FileResource fileResource) {
         Metadata metadata = fileResource.getMetadata();
         EvalFilePaths fps = null;
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfileRunner.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfileRunner.java
deleted file mode 100644
index 1e57b35ad..000000000
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfileRunner.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package org.apache.tika.eval.app;
-
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.concurrent.ArrayBlockingQueue;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-
-import org.apache.tika.batch.FileResource;
-
-public class FileProfileRunner {
-    static Options OPTIONS;
-
-    static {
-
-        OPTIONS = new Options()
-                .addOption(Option.builder("e").longOpt("extracts").hasArg().desc("required: directory of extracts").build())
-                .addOption(Option.builder("i").longOpt("inputDir").hasArg().desc("optional: directory for original binary input documents."
-                        + " If not specified, -extracts is crawled as is.").build())
-                .addOption(Option.builder("d").longOpt("db").hasArg().desc("optional: db path").build())
-                .addOption(Option.builder("c").longOpt("config").hasArg().desc("tika-eval json config file").build())
-                ;
-    }
-    public static void main(String[] args) throws Exception {
-        DefaultParser defaultCLIParser = new DefaultParser();
-        CommandLine commandLine = defaultCLIParser.parse(OPTIONS, args);
-        EvalConfig evalConfig = commandLine.hasOption('c') ? EvalConfig.load(Paths.get(commandLine.getOptionValue('c'))) : new EvalConfig();
-        Path extractsDir = commandLine.hasOption('e') ? Paths.get(commandLine.getOptionValue('e')) : Paths.get(USAGE_FAIL("Must specify extracts dir: -i"));
-        Path inputDir = commandLine.hasOption('i') ? Paths.get(commandLine.getOptionValue('i')) : extractsDir;
-        String dbPath = commandLine.hasOption('d') ? commandLine.getOptionValue('d') : USAGE_FAIL("Must specify the db name: -d");
-        execute(inputDir, extractsDir, dbPath, evalConfig);
-    }
-
-    private static void execute(Path inputDir, Path extractsDir, String dbPath, EvalConfig evalConfig) {
-
-        ArrayBlockingQueue<FileResource> queue = new ArrayBlockingQueue<>(1000);
-        FileWalker fileWalker = new FileWalker(queue)
-    }
-
-    private static void USAGE() {
-        HelpFormatter helpFormatter = new HelpFormatter();
-        helpFormatter.printHelp(80, "java -jar tika-eval-app-x.y.z.jar FileProfiler -e docs -d mydb [-i inputDir, -c config.json]",
-                "Tool: Profile", OPTIONS, "");
-    }
-
-    private static String USAGE_FAIL(String msg) {
-        USAGE();
-        throw new IllegalArgumentException(msg);
-    }
-}
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java
new file mode 100644
index 000000000..f5d5f1a15
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java
@@ -0,0 +1,816 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.app;
+
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.SAXException;
+
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.batch.FileResourceConsumer;
+import org.apache.tika.batch.fs.FSProperties;
+import org.apache.tika.eval.app.db.ColInfo;
+import org.apache.tika.eval.app.db.Cols;
+import org.apache.tika.eval.app.db.TableInfo;
+import org.apache.tika.eval.app.io.ExtractReaderException;
+import org.apache.tika.eval.app.io.IDBWriter;
+import org.apache.tika.eval.core.langid.LanguageIDWrapper;
+import org.apache.tika.eval.core.textstats.BasicTokenCountStatsCalculator;
+import org.apache.tika.eval.core.textstats.CommonTokens;
+import org.apache.tika.eval.core.textstats.CompositeTextStatsCalculator;
+import org.apache.tika.eval.core.textstats.ContentLengthCalculator;
+import org.apache.tika.eval.core.textstats.TextStatsCalculator;
+import org.apache.tika.eval.core.textstats.TokenEntropy;
+import org.apache.tika.eval.core.textstats.TokenLengths;
+import org.apache.tika.eval.core.textstats.TopNTokens;
+import org.apache.tika.eval.core.textstats.UnicodeBlockCounter;
+import org.apache.tika.eval.core.tokens.AnalyzerManager;
+import org.apache.tika.eval.core.tokens.CommonTokenCountManager;
+import org.apache.tika.eval.core.tokens.CommonTokenResult;
+import org.apache.tika.eval.core.tokens.TokenCounts;
+import org.apache.tika.eval.core.tokens.TokenIntPair;
+import org.apache.tika.eval.core.util.ContentTagParser;
+import org.apache.tika.eval.core.util.ContentTags;
+import org.apache.tika.eval.core.util.EvalExceptionUtils;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.language.detect.LanguageResult;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.PDF;
+import org.apache.tika.metadata.PagedText;
+import org.apache.tika.metadata.Property;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.sax.ToXMLContentHandler;
+import org.apache.tika.utils.StringUtils;
+
+public abstract class ProfilerBase {
+
+    public static final String TRUE = Boolean.toString(true);
+    public static final String FALSE = Boolean.toString(false);
+    protected static final AtomicInteger ID = new AtomicInteger();
+    static final long NON_EXISTENT_FILE_LENGTH = -1l;
+    final static int FILE_PATH_MAX_LEN = 1024;//max len for varchar for file_path
+    //Container exception key from the 1.x branch
+    private static final Property CONTAINER_EXCEPTION_1X = Property.externalText("X-TIKA" + ":EXCEPTION:runtime");
+    private static final Logger LOG = LoggerFactory.getLogger(ProfilerBase.class);
+    private static final String[] EXTRACT_EXTENSIONS = {".json", ".txt", ""};
+    private static final String[] COMPRESSION_EXTENSIONS = {"", ".bz2", ".gzip", ".zip",};
+    private static final String ZERO = "0";
+    private static final String UNKNOWN_EXTENSION = "unk";
+    //make this configurable
+    private static final String DIGEST_KEY = "X-TIKA:digest:MD5";
+    private static final Map<String, Cols> UC_TAGS_OF_INTEREST = initTags();
+    private final static Pattern ACCESS_PERMISSION_EXCEPTION = Pattern.compile("org\\.apache\\.tika\\.exception\\.AccessPermissionException");
+    private final static Pattern ENCRYPTION_EXCEPTION = Pattern.compile("org\\.apache\\.tika.exception\\.EncryptedDocumentException");
+    public static TableInfo REF_EXTRACT_EXCEPTION_TYPES = new TableInfo("ref_extract_exception_types", new ColInfo(Cols.EXTRACT_EXCEPTION_ID, Types.INTEGER),
+            new ColInfo(Cols.EXTRACT_EXCEPTION_DESCRIPTION, Types.VARCHAR, 128));
+    public static TableInfo REF_PARSE_ERROR_TYPES =
+            new TableInfo("ref_parse_error_types", new ColInfo(Cols.PARSE_ERROR_ID, Types.INTEGER), new ColInfo(Cols.PARSE_ERROR_DESCRIPTION, Types.VARCHAR, 128));
+    public static TableInfo REF_PARSE_EXCEPTION_TYPES =
+            new TableInfo("ref_parse_exception_types", new ColInfo(Cols.PARSE_EXCEPTION_ID, Types.INTEGER), new ColInfo(Cols.PARSE_EXCEPTION_DESCRIPTION, Types.VARCHAR, 128));
+    public static TableInfo MIME_TABLE = new TableInfo("mimes", new ColInfo(Cols.MIME_ID, Types.INTEGER, "PRIMARY KEY"), new ColInfo(Cols.MIME_STRING, Types.VARCHAR, 256),
+            new ColInfo(Cols.FILE_EXTENSION, Types.VARCHAR, 12));
+    private static CommonTokenCountManager COMMON_TOKEN_COUNT_MANAGER;
+    private static Pattern FILE_NAME_CLEANER = Pattern.compile("\\.(json|txt)(\\.(bz2|gz|zip))?$");
+    private static LanguageIDWrapper LANG_ID = new LanguageIDWrapper();
+    protected IDBWriter writer;
+    AnalyzerManager analyzerManager;
+    int maxContentLength = 10000000;
+    int maxContentLengthForLangId = 50000;
+    int maxTokens = 200000;
+
+    CompositeTextStatsCalculator compositeTextStatsCalculator;
+    private String lastExtractExtension = null;
+
+    public ProfilerBase(IDBWriter writer) {
+        this.writer = writer;
+        LanguageIDWrapper.setMaxTextLength(maxContentLengthForLangId);
+        this.compositeTextStatsCalculator = initAnalyzersAndTokenCounter(maxTokens, LANG_ID);
+    }
+
+    private static Map<String, Cols> initTags() {
+        //simplify this mess
+        Map<String, Cols> tmp = new HashMap<>();
+        tmp.put("A", Cols.TAGS_A);
+        tmp.put("B", Cols.TAGS_B);
+        tmp.put("DIV", Cols.TAGS_DIV);
+        tmp.put("I", Cols.TAGS_I);
+        tmp.put("IMG", Cols.TAGS_IMG);
+        tmp.put("LI", Cols.TAGS_LI);
+        tmp.put("OL", Cols.TAGS_OL);
+        tmp.put("P", Cols.TAGS_P);
+        tmp.put("TABLE", Cols.TAGS_TABLE);
+        tmp.put("TD", Cols.TAGS_TD);
+        tmp.put("TITLE", Cols.TAGS_TITLE);
+        tmp.put("TR", Cols.TAGS_TR);
+        tmp.put("U", Cols.TAGS_U);
+        tmp.put("UL", Cols.TAGS_UL);
+        return Collections.unmodifiableMap(tmp);
+    }
+
+    /**
+     * @param p               path to the common_tokens directory.  If this is null, try to load from classPath
+     * @param defaultLangCode this is the language code to use if a common_words list doesn't exist for the
+     *                        detected langauge; can be <code>null</code>
+     * @throws IOException
+     */
+    public static void loadCommonTokens(Path p, String defaultLangCode) throws IOException {
+        COMMON_TOKEN_COUNT_MANAGER = new CommonTokenCountManager(p, defaultLangCode);
+    }
+
+    private static String getFileName(String path) {
+        if (path == null) {
+            return "";
+        }
+        //filenameUtils checks for a null byte in the path.
+        //it will throw an IllegalArgumentException if there is a null byte.
+        //given that we're recording names and not using them on a file path
+        //we should ignore this.
+        try {
+            return FilenameUtils.getName(path);
+        } catch (IllegalArgumentException e) {
+            LOG.warn("{} in {}", e.getMessage(), path);
+        }
+        path = path.replaceAll("\u0000", " ");
+        try {
+            return FilenameUtils.getName(path);
+        } catch (IllegalArgumentException e) {
+            LOG.warn("Again: {} in {}", e.getMessage(), path);
+        }
+        //give up
+        return "";
+    }
+
+    /**
+     * Get the content and record in the data {@link Cols#CONTENT_TRUNCATED_AT_MAX_LEN} whether the string was truncated
+     *
+     * @param contentTags
+     * @param maxLength
+     * @param data
+     * @return
+     */
+    protected static String truncateContent(ContentTags contentTags, int maxLength, Map<Cols, String> data) {
+        data.put(Cols.CONTENT_TRUNCATED_AT_MAX_LEN, "FALSE");
+        if (contentTags == null) {
+            return "";
+        }
+        String c = contentTags.getContent();
+        if (maxLength > -1 && c.length() > maxLength) {
+            c = c.substring(0, maxLength);
+            data.put(Cols.CONTENT_TRUNCATED_AT_MAX_LEN, "TRUE");
+        }
+        return c;
+
+    }
+
+    protected static ContentTags getContent(EvalFilePaths evalFilePaths, Metadata metadata) {
+        if (metadata == null) {
+            return ContentTags.EMPTY_CONTENT_TAGS;
+        }
+        return parseContentAndTags(evalFilePaths, metadata);
+    }
+
+    /**
+     * @param list
+     * @return empty list if input list is empty or null
+     */
+    static List<Integer> countAttachments(List<Metadata> list) {
+        List<Integer> ret = new ArrayList<>();
+        if (list == null || list.size() == 0) {
+            return ret;
+        }
+        //container document attachment count = list.size()-1
+        ret.add(list.size() - 1);
+
+        Map<String, Integer> counts = new HashMap<>();
+        for (int i = 1; i < list.size(); i++) {
+            String path = list
+                    .get(i)
+                    .get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH);
+            if (path == null) {
+                //shouldn't ever happen
+                continue;
+            }
+            String[] parts = path.split("/");
+            StringBuilder parent = new StringBuilder();
+            for (int end = 1; end < parts.length - 1; end++) {
+                parent.setLength(0);
+                join("/", parent, parts, 1, end);
+                String parentPath = parent.toString();
+                Integer count = counts.get(parentPath);
+                if (count == null) {
+                    count = 1;
+                } else {
+                    count++;
+                }
+                counts.put(parentPath, count);
+            }
+        }
+
+        for (int i = 1; i < list.size(); i++) {
+            Integer count = counts.get(list
+                    .get(i)
+                    .get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
+            if (count == null) {
+                count = 0;
+            }
+            ret.add(i, count);
+        }
+        return ret;
+
+
+    }
+
+    private static void join(String delimiter, StringBuilder sb, String[] parts, int start, int end) {
+        for (int i = start; i <= end; i++) {
+            sb.append(delimiter);
+            sb.append(parts[i]);
+        }
+    }
+
+    private static ContentTags parseContentAndTags(EvalFilePaths evalFilePaths, Metadata metadata) {
+        String s = metadata.get(TikaCoreProperties.TIKA_CONTENT);
+        if (s == null || s.isEmpty()) {
+            return ContentTags.EMPTY_CONTENT_TAGS;
+        }
+
+        String handlerClass = metadata.get(TikaCoreProperties.TIKA_CONTENT_HANDLER);
+        if (evalFilePaths
+                .getExtractFile()
+                .getFileName()
+                .toString()
+                .toLowerCase(Locale.ENGLISH)
+                .endsWith(".html")) {
+            try {
+                return ContentTagParser.parseHTML(s, UC_TAGS_OF_INTEREST.keySet());
+            } catch (IOException | SAXException e) {
+                LOG.warn("Problem parsing html in {}; backing off to treat string as text", evalFilePaths
+                        .getExtractFile()
+                        .toAbsolutePath()
+                        .toString(), e);
+
+                return new ContentTags(s, true);
+            }
+        } else if (evalFilePaths
+                .getExtractFile()
+                .getFileName()
+                .toString()
+                .toLowerCase(Locale.ENGLISH)
+                .endsWith(".xhtml") || (handlerClass != null && handlerClass.equals(ToXMLContentHandler.class.getSimpleName()))) {
+            try {
+                return ContentTagParser.parseXML(s, UC_TAGS_OF_INTEREST.keySet());
+            } catch (TikaException | IOException | SAXException e) {
+                LOG.warn("Problem parsing xhtml in {}; backing off to html parser", evalFilePaths
+                        .getExtractFile()
+                        .toAbsolutePath()
+                        .toString(), e);
+                try {
+                    ContentTags contentTags = ContentTagParser.parseHTML(s, UC_TAGS_OF_INTEREST.keySet());
+                    contentTags.setParseException(true);
+                    return contentTags;
+                } catch (IOException | SAXException e2) {
+                    LOG.warn("Problem parsing html in {}; backing off to treat string as text", evalFilePaths
+                            .getExtractFile()
+                            .toAbsolutePath()
+                            .toString(), e2);
+                }
+                return new ContentTags(s, true);
+            }
+        }
+        return new ContentTags(s);
+    }
+
+    private CompositeTextStatsCalculator initAnalyzersAndTokenCounter(int maxTokens, LanguageIDWrapper langIder) {
+        analyzerManager = AnalyzerManager.newInstance(maxTokens);
+        List<TextStatsCalculator> calculators = new ArrayList<>();
+        calculators.add(new CommonTokens(COMMON_TOKEN_COUNT_MANAGER));
+        calculators.add(new TokenEntropy());
+        calculators.add(new TokenLengths());
+        calculators.add(new TopNTokens(10));
+        calculators.add(new BasicTokenCountStatsCalculator());
+        calculators.add(new ContentLengthCalculator());
+        calculators.add(new UnicodeBlockCounter(maxContentLengthForLangId));
+
+        return new CompositeTextStatsCalculator(calculators, analyzerManager.getGeneralAnalyzer(), langIder);
+    }
+
+    /**
+     * Truncate the content string if greater than this length to this length
+     *
+     * @param maxContentLength
+     */
+    public void setMaxContentLength(int maxContentLength) {
+        this.maxContentLength = maxContentLength;
+    }
+
+    /**
+     * Truncate content string if greater than this length to this length for lang id
+     *
+     * @param maxContentLengthForLangId
+     */
+    public void setMaxContentLengthForLangId(int maxContentLengthForLangId) {
+        this.maxContentLengthForLangId = maxContentLengthForLangId;
+        LanguageIDWrapper.setMaxTextLength(maxContentLengthForLangId);
+    }
+
+    /**
+     * Add a LimitTokenCountFilterFactory if &gt; -1
+     *
+     * @param maxTokens
+     */
+    public void setMaxTokens(int maxTokens) {
+        this.maxTokens = maxTokens;
+        initAnalyzersAndTokenCounter(maxTokens, new LanguageIDWrapper());
+    }
+
+    protected void writeExtractException(TableInfo extractExceptionTable, String containerId, String filePath, ExtractReaderException.TYPE type) throws IOException {
+        Map<Cols, String> data = new HashMap<>();
+        data.put(Cols.CONTAINER_ID, containerId);
+        data.put(Cols.FILE_PATH, filePath);
+        data.put(Cols.EXTRACT_EXCEPTION_ID, Integer.toString(type.ordinal()));
+        writer.writeRow(extractExceptionTable, data);
+
+    }
+
+    protected void writeProfileData(EvalFilePaths fps, int i, ContentTags contentTags, Metadata m, String fileId, String containerId, List<Integer> numAttachments,
+                                    TableInfo profileTable) {
+
+        Map<Cols, String> data = new HashMap<>();
+        data.put(Cols.ID, fileId);
+        data.put(Cols.CONTAINER_ID, containerId);
+        data.put(Cols.MD5, m.get(DIGEST_KEY));
+
+        if (i < numAttachments.size()) {
+            data.put(Cols.NUM_ATTACHMENTS, Integer.toString(numAttachments.get(i)));
+        }
+        data.put(Cols.ELAPSED_TIME_MILLIS, getTime(m));
+        data.put(Cols.NUM_METADATA_VALUES, Integer.toString(countMetadataValues(m)));
+
+        Integer nPages = m.getInt(PagedText.N_PAGES);
+        if (nPages != null) {
+            data.put(Cols.NUM_PAGES, Integer.toString(nPages));
+        }
+        Integer nOCRPages = m.getInt(PDF.OCR_PAGE_COUNT);
+        if (nOCRPages != null) {
+            data.put(Cols.NUM_OCR_PAGES, Integer.toString(nOCRPages));
+        }
+
+        //if the outer wrapper document
+        if (i == 0) {
+            data.put(Cols.IS_EMBEDDED, FALSE);
+            data.put(Cols.FILE_NAME, fps
+                    .getRelativeSourceFilePath()
+                    .getFileName()
+                    .toString());
+            data.put(Cols.EMBEDDED_DEPTH, "0");
+        } else {
+            data.put(Cols.IS_EMBEDDED, TRUE);
+            String embeddedFilePath = m.get(TikaCoreProperties.FINAL_EMBEDDED_RESOURCE_PATH);
+            if (!StringUtils.isBlank(embeddedFilePath)) {
+                data.put(Cols.FILE_NAME, getFileName(embeddedFilePath));
+                data.put(Cols.EMBEDDED_FILE_PATH, embeddedFilePath);
+            }
+            if (!StringUtils.isBlank(m.get(TikaCoreProperties.EMBEDDED_DEPTH))) {
+                data.put(Cols.EMBEDDED_DEPTH, m.get(TikaCoreProperties.EMBEDDED_DEPTH));
+            }
+            if (!StringUtils.isBlank(m.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE))) {
+                data.put(Cols.ATTACHMENT_TYPE, m.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
+            }
+        }
+        String ext = FilenameUtils.getExtension(data.get(Cols.FILE_NAME));
+        ext = (ext == null) ? "" : ext.toLowerCase(Locale.US);
+        data.put(Cols.FILE_EXTENSION, ext);
+        long srcFileLen = getSourceFileLength(m);
+        if (srcFileLen > NON_EXISTENT_FILE_LENGTH) {
+            data.put(Cols.LENGTH, Long.toString(srcFileLen));
+        } else {
+            data.put(Cols.LENGTH, "");
+        }
+        int numMetadataValues = countMetadataValues(m);
+        data.put(Cols.NUM_METADATA_VALUES, Integer.toString(numMetadataValues));
+
+        data.put(Cols.ELAPSED_TIME_MILLIS, getTime(m));
+
+        String content = contentTags.getContent();
+        if (content == null || content.isBlank()) {
+            data.put(Cols.HAS_CONTENT, FALSE);
+        } else {
+            data.put(Cols.HAS_CONTENT, TRUE);
+        }
+        getFileTypes(m, data);
+        try {
+            writer.writeRow(profileTable, data);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected void writeExceptionData(String fileId, Metadata m, TableInfo exceptionTable) {
+        Map<Cols, String> data = new HashMap<>();
+        getExceptionStrings(m, data);
+        if (data
+                .keySet()
+                .size() > 0) {
+            try {
+                data.put(Cols.ID, fileId);
+                writer.writeRow(exceptionTable, data);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    protected Map<Class, Object> calcTextStats(ContentTags contentTags) {
+/*        if (contentTags == ContentTags.EMPTY_CONTENT_TAGS) {
+            return Collections.EMPTY_MAP;
+        }*/
+        Map<Cols, String> data = new HashMap<>();
+        String content = truncateContent(contentTags, maxContentLength, data);
+        if (content == null || content.isBlank()) {
+            content = "";
+        }
+        return compositeTextStatsCalculator.calculate(content);
+    }
+
+    /**
+     * Checks to see if metadata is null or content is empty (null or only whitespace).
+     * If any of these, then this does no processing, and the fileId is not
+     * entered into the content table.
+     *
+     * @param fileId
+     * @param textStats
+     * @param contentsTable
+     */
+    protected void writeContentData(String fileId, Map<Class, Object> textStats, TableInfo contentsTable) throws IOException {
+        Map<Cols, String> data = new HashMap<>();
+        data.put(Cols.ID, fileId);
+        if (textStats.containsKey(ContentLengthCalculator.class)) {
+            int length = (int) textStats.get(ContentLengthCalculator.class);
+            if (length == 0) {
+                return;
+            }
+            data.put(Cols.CONTENT_LENGTH, Integer.toString(length));
+        }
+        langid(textStats, data);
+
+        writeTokenCounts(textStats, data);
+        CommonTokenResult commonTokenResult = (CommonTokenResult) textStats.get(CommonTokens.class);
+        if (commonTokenResult != null) {
+            data.put(Cols.COMMON_TOKENS_LANG, commonTokenResult.getLangCode());
+            data.put(Cols.NUM_UNIQUE_COMMON_TOKENS, Integer.toString(commonTokenResult.getUniqueCommonTokens()));
+            data.put(Cols.NUM_COMMON_TOKENS, Integer.toString(commonTokenResult.getCommonTokens()));
+            data.put(Cols.NUM_UNIQUE_ALPHABETIC_TOKENS, Integer.toString(commonTokenResult.getUniqueAlphabeticTokens()));
+            data.put(Cols.NUM_ALPHABETIC_TOKENS, Integer.toString(commonTokenResult.getAlphabeticTokens()));
+            double oov = commonTokenResult.getAlphabeticTokens() > 0 ? commonTokenResult.getOOV() : -1.0;
+            data.put(Cols.OOV, Double.toString(oov));
+        }
+        TokenCounts tokenCounts = (TokenCounts) textStats.get(BasicTokenCountStatsCalculator.class);
+        if (tokenCounts != null) {
+
+            data.put(Cols.NUM_UNIQUE_TOKENS, Integer.toString(tokenCounts.getTotalUniqueTokens()));
+            data.put(Cols.NUM_TOKENS, Integer.toString(tokenCounts.getTotalTokens()));
+        }
+        if (textStats.get(TokenEntropy.class) != null) {
+            data.put(Cols.TOKEN_ENTROPY_RATE, Double.toString((Double) textStats.get(TokenEntropy.class)));
+        }
+
+
+        SummaryStatistics summStats = (SummaryStatistics) textStats.get(TokenLengths.class);
+        if (summStats != null) {
+            data.put(Cols.TOKEN_LENGTH_SUM, Integer.toString((int) summStats.getSum()));
+
+            data.put(Cols.TOKEN_LENGTH_MEAN, Double.toString(summStats.getMean()));
+
+            data.put(Cols.TOKEN_LENGTH_STD_DEV, Double.toString(summStats.getStandardDeviation()));
+        }
+        unicodeBlocks(textStats, data);
+        try {
+            writer.writeRow(contentsTable, data);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    void writeTagData(String fileId, ContentTags contentTags, TableInfo tagsTable) {
+        Map<String, Integer> tags = contentTags.getTags();
+        if (tags.size() == 0 && contentTags.getParseException() == false) {
+            return;
+        }
+        Map<Cols, String> data = new HashMap<>();
+        data.put(Cols.ID, fileId);
+
+        for (Map.Entry<String, Cols> e : UC_TAGS_OF_INTEREST.entrySet()) {
+            Integer count = tags.get(e.getKey());
+            if (count == null) {
+                data.put(e.getValue(), ZERO);
+            } else {
+                data.put(e.getValue(), Integer.toString(count));
+            }
+        }
+
+        if (contentTags.getParseException()) {
+            data.put(Cols.TAGS_PARSE_EXCEPTION, TRUE);
+        } else {
+            data.put(Cols.TAGS_PARSE_EXCEPTION, FALSE);
+        }
+        try {
+            writer.writeRow(tagsTable, data);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    String getTime(Metadata m) {
+        String elapsed = "-1";
+
+        String v = m.get(TikaCoreProperties.PARSE_TIME_MILLIS);
+        if (v != null) {
+            return v;
+        }
+        return elapsed;
+    }
+
+    int countMetadataValues(Metadata m) {
+        if (m == null) {
+            return 0;
+        }
+        int i = 0;
+        for (String n : m.names()) {
+            i += m.getValues(n).length;
+        }
+        return i;
+    }
+
+    void getExceptionStrings(Metadata metadata, Map<Cols, String> data) {
+
+        String fullTrace = metadata.get(TikaCoreProperties.CONTAINER_EXCEPTION);
+        if (fullTrace == null) {
+            fullTrace = metadata.get(CONTAINER_EXCEPTION_1X);
+        }
+
+        if (fullTrace == null) {
+            fullTrace = metadata.get(TikaCoreProperties.EMBEDDED_EXCEPTION);
+        }
+
+        if (fullTrace != null) {
+            //check for "expected" exceptions...exceptions
+            //that can't be fixed.
+            //Do not store trace for "expected" exceptions
+
+            Matcher matcher = ACCESS_PERMISSION_EXCEPTION.matcher(fullTrace);
+            if (matcher.find()) {
+                data.put(Cols.PARSE_EXCEPTION_ID, Integer.toString(EXCEPTION_TYPE.ACCESS_PERMISSION.ordinal()));
+                return;
+            }
+            matcher = ENCRYPTION_EXCEPTION.matcher(fullTrace);
+            if (matcher.find()) {
+                data.put(Cols.PARSE_EXCEPTION_ID, Integer.toString(EXCEPTION_TYPE.ENCRYPTION.ordinal()));
+                return;
+            }
+
+            data.put(Cols.PARSE_EXCEPTION_ID, Integer.toString(EXCEPTION_TYPE.RUNTIME.ordinal()));
+
+            data.put(Cols.ORIG_STACK_TRACE, fullTrace);
+            //TikaExceptions can have object ids, as in the "@2b1ea6ee" in:
+            //org.apache.tika.exception.TikaException: TIKA-198: Illegal
+            //IOException from org.apache.tika.parser.microsoft.OfficeParser@2b1ea6ee
+            //For reporting purposes, let's snip off the object id so that we can more
+            //easily count exceptions.
+            String sortTrace = EvalExceptionUtils.normalize(fullTrace);
+            data.put(Cols.SORT_STACK_TRACE, sortTrace);
+        }
+    }
+
+    void unicodeBlocks(Map<Class, Object> tokenStats, Map<Cols, String> data) {
+
+        Map<String, MutableInt> blocks = (Map<String, MutableInt>) tokenStats.get(UnicodeBlockCounter.class);
+        List<Pair<String, Integer>> pairs = new ArrayList<>();
+        for (Map.Entry<String, MutableInt> e : blocks.entrySet()) {
+            pairs.add(Pair.of(e.getKey(), e
+                    .getValue()
+                    .intValue()));
+        }
+        pairs.sort((o1, o2) -> o2
+                .getValue()
+                .compareTo(o1.getValue()));
+        StringBuilder sb = new StringBuilder();
+
+        for (int i = 0; i < 20 && i < pairs.size(); i++) {
+            if (i > 0) {
+                sb.append(" | ");
+            }
+            sb
+                    .append(pairs
+                            .get(i)
+                            .getKey())
+                    .append(": ")
+                    .append(pairs
+                            .get(i)
+                            .getValue());
+        }
+        data.put(Cols.UNICODE_CHAR_BLOCKS, sb.toString());
+    }
+
+    void langid(Map<Class, Object> stats, Map<Cols, String> data) {
+        List<LanguageResult> probabilities = (List<LanguageResult>) stats.get(LanguageIDWrapper.class);
+
+        if (probabilities.size() > 0) {
+            data.put(Cols.LANG_ID_1, probabilities
+                    .get(0)
+                    .getLanguage());
+            data.put(Cols.LANG_ID_PROB_1, Double.toString(probabilities
+                    .get(0)
+                    .getRawScore()));
+        }
+        if (probabilities.size() > 1) {
+            data.put(Cols.LANG_ID_2, probabilities
+                    .get(1)
+                    .getLanguage());
+            data.put(Cols.LANG_ID_PROB_2, Double.toString(probabilities
+                    .get(1)
+                    .getRawScore()));
+        }
+    }
+
+    void getFileTypes(Metadata metadata, Map<Cols, String> output) {
+        if (metadata == null) {
+            return;
+        }
+        String type = metadata.get(Metadata.CONTENT_TYPE);
+        if (type == null) {
+            return;
+        }
+        int mimeId = writer.getMimeId(type);
+        output.put(Cols.MIME_ID, Integer.toString(mimeId));
+    }
+
+    void writeTokenCounts(Map<Class, Object> textStats, Map<Cols, String> data) {
+        TokenIntPair[] tokenIntPairs = (TokenIntPair[]) textStats.get(TopNTokens.class);
+        int i = 0;
+        StringBuilder sb = new StringBuilder();
+        for (TokenIntPair t : tokenIntPairs) {
+            if (i++ > 0) {
+                sb.append(" | ");
+            }
+            sb
+                    .append(t.getToken())
+                    .append(": ")
+                    .append(t.getValue());
+        }
+
+        data.put(Cols.TOP_N_TOKENS, sb.toString());
+    }
+
+    public void closeWriter() throws IOException {
+        writer.close();
+    }
+
+    /**
+     * @param metadata
+     * @param extracts
+     * @return evalfilepaths for files if crawling an extract directory
+     */
+    protected EvalFilePaths getPathsFromExtractCrawl(Metadata metadata, Path extracts) {
+        String relExtractFilePath = metadata.get(FSProperties.FS_REL_PATH);
+        Matcher m = FILE_NAME_CLEANER.matcher(relExtractFilePath);
+        Path relativeSourceFilePath = Paths.get(m.replaceAll(""));
+        //just try slapping the relextractfilepath on the extractdir
+        Path extractFile = extracts.resolve(relExtractFilePath);
+        if (!Files.isRegularFile(extractFile)) {
+            //if that doesn't work, try to find the right extract file.
+            //This is necessary if crawling extractsA and trying to find a file in
+            //extractsB that is not in the same format: json vs txt or compressed
+            extractFile = findFile(extracts, relativeSourceFilePath);
+        }
+        return new EvalFilePaths(relativeSourceFilePath, extractFile);
+    }
+
+    //call this if the crawler is crawling through the src directory
+    protected EvalFilePaths getPathsFromSrcCrawl(Metadata metadata, Path srcDir, Path extracts) {
+        Path relativeSourceFilePath = Paths.get(metadata.get(FSProperties.FS_REL_PATH));
+        Path extractFile = findFile(extracts, relativeSourceFilePath);
+        Path inputFile = srcDir.resolve(relativeSourceFilePath);
+        long srcLen = -1l;
+        //try to get the length of the source file in case there was an error
+        //in both extracts
+        try {
+            srcLen = Files.size(inputFile);
+        } catch (IOException e) {
+            LOG.warn("Couldn't get length for: {}", inputFile.toAbsolutePath());
+        }
+        return new EvalFilePaths(relativeSourceFilePath, extractFile, srcLen);
+    }
+
+    /**
+     * @param extractRootDir
+     * @param relativeSourceFilePath
+     * @return extractFile or null if couldn't find one.
+     */
+    private Path findFile(Path extractRootDir, Path relativeSourceFilePath) {
+        String relSrcFilePathString = relativeSourceFilePath.toString();
+        if (lastExtractExtension != null) {
+            Path candidate = extractRootDir.resolve(relSrcFilePathString + lastExtractExtension);
+            if (Files.isRegularFile(candidate)) {
+                return candidate;
+            }
+        }
+        for (String ext : EXTRACT_EXTENSIONS) {
+            for (String compress : COMPRESSION_EXTENSIONS) {
+                Path candidate = extractRootDir.resolve(relSrcFilePathString + ext + compress);
+                if (Files.isRegularFile(candidate)) {
+                    lastExtractExtension = ext + compress;
+                    return candidate;
+                }
+            }
+        }
+        return null;
+    }
+
+    protected long getSourceFileLength(EvalFilePaths fps, List<Metadata> metadataList) {
+        if (fps.getSourceFileLength() > NON_EXISTENT_FILE_LENGTH) {
+            return fps.getSourceFileLength();
+        }
+        return getSourceFileLength(metadataList);
+    }
+
+    long getSourceFileLength(List<Metadata> metadataList) {
+        if (metadataList == null || metadataList.size() < 1) {
+            return NON_EXISTENT_FILE_LENGTH;
+        }
+        return getSourceFileLength(metadataList.get(0));
+    }
+
+    long getSourceFileLength(Metadata m) {
+        String lenString = m.get(Metadata.CONTENT_LENGTH);
+        if (lenString == null) {
+            return NON_EXISTENT_FILE_LENGTH;
+        }
+        try {
+            return Long.parseLong(lenString);
+        } catch (NumberFormatException e) {
+            //swallow
+        }
+        return NON_EXISTENT_FILE_LENGTH;
+    }
+
+    protected long getFileLength(Path p) {
+        if (p != null && Files.isRegularFile(p)) {
+            try {
+                return Files.size(p);
+            } catch (IOException e) {
+                //swallow
+            }
+        }
+        return NON_EXISTENT_FILE_LENGTH;
+    }
+
+    public enum EXCEPTION_TYPE {
+        RUNTIME, ENCRYPTION, ACCESS_PERMISSION, UNSUPPORTED_VERSION,
+    }
+
+    /**
+     * If information was gathered from the log file about
+     * a parse error
+     */
+    public enum PARSE_ERROR_TYPE {
+        OOM, TIMEOUT
+    }
+
+
+}
+
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/TikaEvalCLI.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/TikaEvalCLI.java
index 770319708..bae1435e7 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/TikaEvalCLI.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/TikaEvalCLI.java
@@ -154,7 +154,7 @@ public class TikaEvalCLI {
     }
 
     private void handleProfile(String[] subsetArgs) throws Exception {
-        FileProfileRunner.main(subsetArgs);
+        ExctractProfileRunner.main(subsetArgs);
     }
 
     private void handleCompare(String[] subsetArgs) throws Exception {
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/FileResource.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/FileResource.java
index dc0a35d7e..e4702cb8e 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/FileResource.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/FileResource.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.tika.batch;
+package org.apache.tika.eval.app.batch;
 
 
 import java.io.IOException;
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/PathResource.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/PathResource.java
new file mode 100644
index 000000000..9d64317aa
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/PathResource.java
@@ -0,0 +1,33 @@
+package org.apache.tika.eval.app.batch;
+
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.apache.tika.metadata.Metadata;
+
+public class PathResource implements FileResource {
+
+    private final Path path;
+    private final String resourceId;
+    public PathResource(Path path, String resourceId) {
+        this.path = path;
+        this.resourceId = resourceId;
+    }
+    @Override
+    public String getResourceId() {
+        return resourceId;
+    }
+
+    @Override
+    public Metadata getMetadata() {
+        return new Metadata();
+    }
+
+    @Override
+    public InputStream openInputStream() throws IOException {
+        return Files.newInputStream(path);
+    }
+}


(tika) 01/03: TIKA-4342 -- WIP do not merge

Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tallison pushed a commit to branch TIKA-4342
in repository https://gitbox.apache.org/repos/asf/tika.git

commit 7b94cf0ffa6cfbd0ef337946564e516df69c5c85
Author: tallison <ta...@apache.org>
AuthorDate: Thu May 22 09:39:06 2025 -0400

    TIKA-4342 -- WIP do not merge
---
 .../java/org/apache/tika/eval/app/EvalConfig.java  |  13 +++
 .../org/apache/tika/eval/app/ExtractProfiler.java  |  44 ++-------
 .../apache/tika/eval/app/FileProfileRunner.java    |  54 +++++++++++
 .../org/apache/tika/eval/app/FileProfiler.java     |  19 ----
 .../java/org/apache/tika/eval/app/TikaEvalCLI.java | 104 +--------------------
 .../apache/tika/eval/app/batch/FileResource.java   |  66 +++++++++++++
 6 files changed, 140 insertions(+), 160 deletions(-)

diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/EvalConfig.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/EvalConfig.java
new file mode 100644
index 000000000..416d7bb6e
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/EvalConfig.java
@@ -0,0 +1,13 @@
+package org.apache.tika.eval.app;
+
+public class EvalConfig {
+
+    long minExtractLength = 0;
+    long maxExtractLength = 10_000_000;
+    String jdbcString = null;
+    int maxFilesToAdd = -1;
+    int maxTokens = 200000;
+    int maxContentLength = 5_000_000;
+    int numThreads = 4;
+
+}
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java
index 22889d73b..844eac125 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java
@@ -16,6 +16,7 @@
  */
 package org.apache.tika.eval.app;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.sql.Types;
@@ -76,54 +77,21 @@ public class ExtractProfiler extends AbstractProfiler {
                     new ColInfo(Cols.TAGS_PARSE_EXCEPTION, Types.BOOLEAN));
     static Options OPTIONS;
 
-    static {
-        //By the time this commandline is parsed, there should be both an extracts and an inputDir
-        Option extracts = new Option("extracts", true, "directory for extract files");
-        extracts.setRequired(true);
-
-        Option inputDir = new Option("inputDir", true, "optional: directory for original binary input documents." + " If not specified, -extracts is crawled as is.");
-
-        OPTIONS = new Options()
-                .addOption(extracts)
-                .addOption(inputDir)
-                .addOption("bc", "optional: tika-batch config file")
-                .addOption("numConsumers", true, "optional: number of consumer threads")
-                .addOption(new Option("alterExtract", true,
-                        "for json-formatted extract files, " + "process full metadata list ('as_is'=default), " + "take just the first/container document ('first_only'), " +
-                                "concatenate all content into the first metadata item ('concatenate_content')"))
-                .addOption("minExtractLength", true, "minimum extract length to process (in bytes)")
-                .addOption("maxExtractLength", true, "maximum extract length to process (in bytes)")
-                .addOption("db", true, "db file to which to write results")
-                .addOption("jdbc", true, "EXPERT: full jdbc connection string. Must specify this or -db <h2db>")
-                .addOption("jdbcDriver", true, "EXPERT: jdbc driver, or specify via -Djdbc.driver")
-                .addOption("tablePrefix", true, "EXPERT: optional prefix for table names")
-                .addOption("drop", false, "drop tables if they exist")
-                .addOption("maxFilesToAdd", true, "maximum number of files to add to the crawler")
-                .addOption("maxTokens", true, "maximum tokens to process, default=200000")
-                .addOption("maxContentLength", true, "truncate content beyond this length for calculating 'contents' stats, default=1000000")
-                .addOption("maxContentLengthForLangId", true, "truncate content beyond this length for language id, default=50000")
-                .addOption("defaultLangCode", true, "which language to use for common words if no 'common words' file exists for the langid result")
-
-        ;
-
-    }
-
     private final Path inputDir;
     private final Path extracts;
     private final ExtractReader extractReader;
 
-    public ExtractProfiler(ArrayBlockingQueue<FileResource> queue, Path inputDir, Path extracts, ExtractReader extractReader, IDBWriter dbWriter) {
+    public static ExtractProfiler build(ArrayBlockingQueue<FileResource> queue, Path inputDir, Path extracts, ExtractReader extractReader, IDBWriter dbWriter) {
+        return new ExtractProfiler(queue, inputDir, extracts, extractReader, dbWriter);
+    }
+
+    ExtractProfiler(ArrayBlockingQueue<FileResource> queue, Path inputDir, Path extracts, ExtractReader extractReader, IDBWriter dbWriter) {
         super(queue, dbWriter);
         this.inputDir = inputDir;
         this.extracts = extracts;
         this.extractReader = extractReader;
     }
 
-    public static void USAGE() {
-        HelpFormatter helpFormatter = new HelpFormatter();
-        helpFormatter.printHelp(80, "java -jar tika-eval-x.y.jar Profile -extracts extracts -db mydb [-inputDir input]", "Tool: Profile", ExtractProfiler.OPTIONS,
-                "Note: for the default h2 db, do not include the .mv.db at the end of the db name.");
-    }
 
     @Override
     public boolean processFileResource(FileResource fileResource) {
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfileRunner.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfileRunner.java
new file mode 100644
index 000000000..1e57b35ad
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfileRunner.java
@@ -0,0 +1,54 @@
+package org.apache.tika.eval.app;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.ArrayBlockingQueue;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+
+import org.apache.tika.batch.FileResource;
+
+public class FileProfileRunner {
+    static Options OPTIONS;
+
+    static {
+
+        OPTIONS = new Options()
+                .addOption(Option.builder("e").longOpt("extracts").hasArg().desc("required: directory of extracts").build())
+                .addOption(Option.builder("i").longOpt("inputDir").hasArg().desc("optional: directory for original binary input documents."
+                        + " If not specified, -extracts is crawled as is.").build())
+                .addOption(Option.builder("d").longOpt("db").hasArg().desc("optional: db path").build())
+                .addOption(Option.builder("c").longOpt("config").hasArg().desc("tika-eval json config file").build())
+                ;
+    }
+    public static void main(String[] args) throws Exception {
+        DefaultParser defaultCLIParser = new DefaultParser();
+        CommandLine commandLine = defaultCLIParser.parse(OPTIONS, args);
+        EvalConfig evalConfig = commandLine.hasOption('c') ? EvalConfig.load(Paths.get(commandLine.getOptionValue('c'))) : new EvalConfig();
+        Path extractsDir = commandLine.hasOption('e') ? Paths.get(commandLine.getOptionValue('e')) : Paths.get(USAGE_FAIL("Must specify extracts dir: -i"));
+        Path inputDir = commandLine.hasOption('i') ? Paths.get(commandLine.getOptionValue('i')) : extractsDir;
+        String dbPath = commandLine.hasOption('d') ? commandLine.getOptionValue('d') : USAGE_FAIL("Must specify the db name: -d");
+        execute(inputDir, extractsDir, dbPath, evalConfig);
+    }
+
+    private static void execute(Path inputDir, Path extractsDir, String dbPath, EvalConfig evalConfig) {
+
+        ArrayBlockingQueue<FileResource> queue = new ArrayBlockingQueue<>(1000);
+        FileWalker fileWalker = new FileWalker(queue)
+    }
+
+    private static void USAGE() {
+        HelpFormatter helpFormatter = new HelpFormatter();
+        helpFormatter.printHelp(80, "java -jar tika-eval-app-x.y.z.jar FileProfiler -e docs -d mydb [-i inputDir, -c config.json]",
+                "Tool: Profile", OPTIONS, "");
+    }
+
+    private static String USAGE_FAIL(String msg) {
+        USAGE();
+        throw new IllegalArgumentException(msg);
+    }
+}
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfiler.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfiler.java
index 925452094..edc431e4a 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfiler.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/FileProfiler.java
@@ -73,26 +73,7 @@ public class FileProfiler extends AbstractProfiler {
     public static TableInfo FILE_MIME_TABLE =
             new TableInfo("file_mimes", new ColInfo(Cols.MIME_ID, Types.INTEGER, "PRIMARY KEY"), new ColInfo(Cols.MIME_STRING, Types.VARCHAR, 256),
                     new ColInfo(Cols.FILE_EXTENSION, Types.VARCHAR, 12));
-    static Options OPTIONS;
 
-    static {
-
-        Option inputDir = new Option("inputDir", true, "optional: directory for original binary input documents." + " If not specified, -extracts is crawled as is.");
-
-        OPTIONS = new Options()
-                .addOption(inputDir)
-                .addOption("bc", "optional: tika-batch config file")
-                .addOption("numConsumers", true, "optional: number of consumer threads")
-                .addOption("db", true, "db file to which to write results")
-                .addOption("jdbc", true, "EXPERT: full jdbc connection string. Must specify this or -db <h2db>")
-                .addOption("jdbcDriver", true, "EXPERT: jdbc driver, or specify via -Djdbc.driver")
-                .addOption("tablePrefix", true, "EXPERT: optional prefix for table names")
-                .addOption("drop", false, "drop tables if they exist")
-                .addOption("maxFilesToAdd", true, "maximum number of files to add to the crawler")
-
-        ;
-
-    }
 
     private final Path inputDir;
 
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/TikaEvalCLI.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/TikaEvalCLI.java
index a897461ee..770319708 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/TikaEvalCLI.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/TikaEvalCLI.java
@@ -154,109 +154,7 @@ public class TikaEvalCLI {
     }
 
     private void handleProfile(String[] subsetArgs) throws Exception {
-        List<String> argList = new ArrayList(Arrays.asList(subsetArgs));
-
-        boolean containsBC = false;
-        String inputDir = null;
-        String extracts = null;
-        String alterExtract = null;
-        //confirm there's a batch-config file
-        for (int i = 0; i < argList.size(); i++) {
-            String arg = argList.get(i);
-            switch (arg) {
-                case "-bc":
-                    containsBC = true;
-                    break;
-                case "-inputDir":
-                    if (i + 1 >= argList.size()) {
-                        System.err.println("Must specify directory after -inputDir");
-                        ExtractProfiler.USAGE();
-                        return;
-                    }
-                    inputDir = argList.get(i + 1);
-                    i++;
-                    break;
-                case "-extracts":
-                    if (i + 1 >= argList.size()) {
-                        System.err.println("Must specify directory after -extracts");
-                        ExtractProfiler.USAGE();
-                        return;
-                    }
-                    extracts = argList.get(i + 1);
-                    i++;
-                    break;
-                case "-alterExtract":
-                    if (i + 1 >= argList.size()) {
-                        System.err.println("Must specify type 'as_is', 'first_only' or " + "'concatenate_content' after -alterExtract");
-                        ExtractComparer.USAGE();
-                        return;
-                    }
-                    alterExtract = argList.get(i + 1);
-                    i++;
-                    break;
-            }
-        }
-
-        if (alterExtract != null && !alterExtract.equals("as_is") && !alterExtract.equals("concatenate_content") && !alterExtract.equals("first_only")) {
-            System.out.println("Sorry, I don't understand:" + alterExtract + ". The values must be one of: as_is, first_only, concatenate_content");
-            ExtractProfiler.USAGE();
-            return;
-        }
-
-        //need to specify each in this commandline
-        //if only extracts is passed to tika-batch,
-        //the crawler will see no inputDir and start crawling "input".
-        //this allows the user to specify either extracts or inputDir
-        if (extracts == null && inputDir != null) {
-            argList.add("-extracts");
-            argList.add(inputDir);
-        } else if (inputDir == null && extracts != null) {
-            argList.add("-inputDir");
-            argList.add(extracts);
-        }
-
-        Path tmpBCConfig = null;
-        try {
-            tmpBCConfig = Files.createTempFile("tika-eval-profiler", ".xml");
-            if (!containsBC) {
-                try (InputStream is = this
-                        .getClass()
-                        .getResourceAsStream("/tika-eval-profiler-config.xml")) {
-                    Files.copy(is, tmpBCConfig, StandardCopyOption.REPLACE_EXISTING);
-                }
-                argList.add("-bc");
-                argList.add(tmpBCConfig
-                        .toAbsolutePath()
-                        .toString());
-            }
-
-            String[] updatedArgs = argList.toArray(new String[0]);
-            DefaultParser defaultCLIParser = new DefaultParser();
-            try {
-                CommandLine commandLine = defaultCLIParser.parse(ExtractProfiler.OPTIONS, updatedArgs);
-                if (commandLine.hasOption("db") && commandLine.hasOption("jdbc")) {
-                    System.out.println("Please specify either the default -db or the full -jdbc, not both");
-                    ExtractProfiler.USAGE();
-                    return;
-                }
-            } catch (ParseException e) {
-                System.out.println(e.getMessage() + "\n");
-                ExtractProfiler.USAGE();
-                return;
-            }
-
-            // lazy delete because main() calls System.exit()
-            if (tmpBCConfig != null && Files.isRegularFile(tmpBCConfig)) {
-                tmpBCConfig
-                        .toFile()
-                        .deleteOnExit();
-            }
-            FSBatchProcessCLI.main(updatedArgs);
-        } finally {
-            if (tmpBCConfig != null && Files.isRegularFile(tmpBCConfig)) {
-                Files.delete(tmpBCConfig);
-            }
-        }
+        FileProfileRunner.main(subsetArgs);
     }
 
     private void handleCompare(String[] subsetArgs) throws Exception {
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/FileResource.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/FileResource.java
new file mode 100644
index 000000000..dc0a35d7e
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/batch/FileResource.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.batch;
+
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
+
+
+/**
+ * This is a basic interface to handle a logical "file".
+ * This should enable code-agnostic handling of files from different
+ * sources: file system, database, etc.
+ */
+public interface FileResource {
+
+    //The literal lowercased extension of a file.  This may or may not
+    //have any relationship to the actual type of the file.
+    public static final Property FILE_EXTENSION = Property.internalText("tika:file_ext");
+
+    /**
+     * This is only used in logging to identify which file
+     * may have caused problems.  While it is probably best
+     * to use unique ids for the sake of debugging, it is not
+     * necessary that the ids be unique.  This id
+     * is never used as a hashkey by the batch processors, for example.
+     *
+     * @return an id for a FileResource
+     */
+    public String getResourceId();
+
+    /**
+     * This gets the metadata available before the parsing of the file.
+     * This will typically be "external" metadata: file name,
+     * file size, file location, data stream, etc.  That is, things
+     * that are known about the file from outside information, not
+     * file-internal metadata.
+     *
+     * @return Metadata
+     */
+    public Metadata getMetadata();
+
+    /**
+     * @return an InputStream for the FileResource
+     * @throws java.io.IOException
+     */
+    public InputStream openInputStream() throws IOException;
+
+}