You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/02/16 17:18:42 UTC
[2/6] tika git commit: TIKA-1332 -- initial commit for tika-eval
module. More work remains.
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/java/org/apache/tika/eval/ComparerBatchTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/ComparerBatchTest.java b/tika-eval/src/test/java/org/apache/tika/eval/ComparerBatchTest.java
new file mode 100644
index 0000000..0d925cf
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/ComparerBatchTest.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.tika.batch.fs.FSBatchTestBase;
+import org.apache.tika.eval.db.Cols;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore("need to fix tika-batch tests to make this work")
+public class ComparerBatchTest extends FSBatchTestBase {
+
+ public final static String COMPARER_PROCESS_CLASS = "org.apache.tika.batch.fs.FSBatchProcessCLI";
+
+ private static Path dbDir;
+ private static Connection conn;
+
+ private final static String compJoinCont = "";
+ /*ExtractComparer.COMPARISONS_TABLE+" cmp " +
+ "join "+ExtractComparer.CONTAINERS_TABLE + " cnt "+
+ "on cmp."+AbstractProfiler.CONTAINER_HEADERS.CONTAINER_ID+
+ " = cnt."+AbstractProfiler.CONTAINER_HEADERS.CONTAINER_ID;*/
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+
+ File inputRoot = new File(ComparerBatchTest.class.getResource("/test-dirs").toURI());
+ dbDir = Files.createTempDirectory(inputRoot.toPath(), "tika-test-db-dir-");
+ Map<String, String> args = new HashMap<>();
+ Path db = FileSystems.getDefault().getPath(dbDir.toString(), "comparisons_test");
+ args.put("-db", db.toString());
+
+ //for debugging, you can use this to select only one file pair to load
+ //args.put("-includeFilePat", "file8.*");
+/*
+ BatchProcessTestExecutor ex = new BatchProcessTestExecutor(COMPARER_PROCESS_CLASS, args,
+ "/tika-batch-comparison-eval-config.xml");
+ StreamStrings streamStrings = ex.execute();
+ System.out.println(streamStrings.getErrString());
+ System.out.println(streamStrings.getOutString());
+ H2Util dbUtil = new H2Util(db);
+ conn = dbUtil.getConnection();*/
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+
+ conn.close();
+
+ FileUtils.deleteDirectory(dbDir.toFile());
+ }
+
+
+ @Test
+ public void testSimpleDBWriteAndRead() throws Exception {
+ Set<String> set = new HashSet<>();
+ //filenames
+ List<String> list = getColStrings(Cols.FILE_NAME.name(),
+ ExtractComparer.PROFILES_A.getName(), "");
+ assertEquals(7, list.size());
+ assertTrue(list.contains("file1.pdf"));
+
+ //container ids in comparisons table
+ list = getColStrings(Cols.CONTAINER_ID.name(),
+ ExtractComparer.COMPARISON_CONTAINERS.getName(),"");
+ assertEquals(10, list.size());
+ set.clear(); set.addAll(list);
+ assertEquals(10, set.size());
+/*
+ //ids in comparisons table
+ list = getColStrings(AbstractProfiler.HEADERS.ID.name(),
+ compTable,"");
+ assertEquals(9, list.size());
+ set.clear(); set.addAll(list);
+ assertEquals(9, set.size());*/
+ }
+
+
+
+ /*
+ @Test
+ public void testFile1PDFRow() throws Exception {
+ String where = fp+"='file1.pdf'";
+ Map<String, String> data = getRow(compJoinCont, where);
+ String result = data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_UNIQUE_TOKEN_DIFFS + "_A");
+ assertTrue(result.startsWith("over: 1"));
+
+ result = data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_UNIQUE_TOKEN_DIFFS + "_B");
+ assertTrue(result.startsWith("aardvark: 3 | bear: 2"));
+
+
+ assertEquals("aardvark: 3 | bear: 2",
+ data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_MORE_IN_B.toString()));
+ assertEquals("fox: 2 | lazy: 1 | over: 1",
+ data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_MORE_IN_A.toString()));
+ assertEquals("12", data.get(ExtractComparer.HEADERS.NUM_TOKENS+"_A"));
+ assertEquals("13", data.get(ExtractComparer.HEADERS.NUM_TOKENS+"_B"));
+ assertEquals("8", data.get(ExtractComparer.HEADERS.NUM_UNIQUE_TOKENS+"_A"));
+ assertEquals("9", data.get(ExtractComparer.HEADERS.NUM_UNIQUE_TOKENS+"_B"));
+
+ assertEquals(ExtractComparer.COMPARISON_HEADERS.OVERLAP.name(),
+ 0.64f, Float.parseFloat(data.get("OVERLAP")), 0.0001f);
+
+ assertEquals(ExtractComparer.COMPARISON_HEADERS.DICE_COEFFICIENT.name(),
+ 0.8235294f, Float.parseFloat(data.get("DICE_COEFFICIENT")), 0.0001f);
+
+ assertEquals(ExtractComparer.HEADERS.TOKEN_LENGTH_MEAN+"_A", 3.83333d,
+ Double.parseDouble(
+ data.get(ExtractComparer.HEADERS.TOKEN_LENGTH_MEAN+"_A")), 0.0001d);
+
+ assertEquals(ExtractComparer.HEADERS.TOKEN_LENGTH_MEAN+"_B", 4.923d,
+ Double.parseDouble(
+ data.get(ExtractComparer.HEADERS.TOKEN_LENGTH_MEAN+"_B")), 0.0001d);
+
+ assertEquals(ExtractComparer.HEADERS.TOKEN_LENGTH_STD_DEV+"_A", 1.0298d,
+ Double.parseDouble(
+ data.get(ExtractComparer.HEADERS.TOKEN_LENGTH_STD_DEV+"_A")), 0.0001d);
+
+ assertEquals(ExtractComparer.HEADERS.TOKEN_LENGTH_STD_DEV+"_B", 1.9774d,
+ Double.parseDouble(data.get(ExtractComparer.HEADERS.TOKEN_LENGTH_STD_DEV+"_B")), 0.0001d);
+
+ assertEquals(ExtractComparer.HEADERS.TOKEN_LENGTH_SUM+"_A", 46,
+ Integer.parseInt(
+ data.get(ExtractComparer.HEADERS.TOKEN_LENGTH_SUM+"_A")));
+
+ assertEquals(ExtractComparer.HEADERS.TOKEN_LENGTH_SUM+"_B", 64,
+ Integer.parseInt(data.get(ExtractComparer.HEADERS.TOKEN_LENGTH_SUM+"_B")));
+
+ assertEquals("TOKEN_ENTROPY_RATE_A", 0.237949,
+ Double.parseDouble(data.get("TOKEN_ENTROPY_RATE_A")), 0.0001d);
+
+ assertEquals("TOKEN_ENTROPY_RATE_B", 0.232845,
+ Double.parseDouble(data.get("TOKEN_ENTROPY_RATE_B")), 0.0001d);
+
+ }
+
+
+ @Test
+ public void testEmpty() throws Exception {
+ String where = fp+"='file4_emptyB.pdf'";
+ Map<String, String> data = getRow(contTable, where);
+ assertNull(data.get(AbstractProfiler.CONTAINER_HEADERS.JSON_EX +
+ ExtractComparer.aExtension));
+ assertTrue(data.get(AbstractProfiler.CONTAINER_HEADERS.JSON_EX +
+ ExtractComparer.bExtension).equals(AbstractProfiler.JSON_PARSE_EXCEPTION));
+
+ where = fp+"='file5_emptyA.pdf'";
+ data = getRow(contTable, where);
+ assertNull(data.get(AbstractProfiler.CONTAINER_HEADERS.JSON_EX +
+ ExtractComparer.bExtension));
+ assertTrue(data.get(AbstractProfiler.CONTAINER_HEADERS.JSON_EX+
+ ExtractComparer.aExtension).equals(AbstractProfiler.JSON_PARSE_EXCEPTION));
+ }
+
+ @Test
+ public void testMissingAttachment() throws Exception {
+ String where = fp+"='file2_attachANotB.doc' and "+AbstractProfiler.HEADERS.EMBEDDED_FILE_PATH+
+ "='inner.txt'";
+ Map<String, String> data = getRow(compJoinCont, where);
+ assertContains("attachment: 1", data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_MORE_IN_A.name()));
+ assertNotContained("fox", data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_MORE_IN_B.name()));
+ assertNull(data.get(ExtractComparer.HEADERS.TOP_N_TOKENS +
+ ExtractComparer.bExtension));
+ assertNotContained("fox", data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_UNIQUE_TOKEN_DIFFS +
+ ExtractComparer.bExtension));
+
+ assertEquals("3", data.get("NUM_METADATA_VALUES_A"));
+ assertNull(data.get("DIFF_NUM_ATTACHMENTS"));
+ assertNull(data.get("NUM_METADATA_VALUES_B"));
+ assertEquals("0", data.get("NUM_UNIQUE_TOKENS_B"));
+ assertNull(data.get("TOKEN_ENTROPY_RATE_B"));
+ assertNull(data.get("NUM_EN_STOPS_TOP_N_B"));
+
+ where = fp+"='file3_attachBNotA.doc' and "+AbstractProfiler.HEADERS.EMBEDDED_FILE_PATH+
+ "='inner.txt'";
+ data = getRow(compJoinCont, where);
+ assertContains("attachment: 1", data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_MORE_IN_B.name()));
+ assertNotContained("fox", data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_MORE_IN_A.name()));
+ assertNull(data.get(ExtractComparer.HEADERS.TOP_N_TOKENS +
+ ExtractComparer.aExtension));
+ assertNotContained("fox", data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_UNIQUE_TOKEN_DIFFS +
+ ExtractComparer.aExtension));
+
+ assertEquals("3", data.get("NUM_METADATA_VALUES_B"));
+ assertNull(data.get("DIFF_NUM_ATTACHMENTS"));
+ assertNull(data.get("NUM_METADATA_VALUES_A"));
+ assertEquals("0", data.get("NUM_UNIQUE_TOKENS_A"));
+ assertNull(data.get("TOKEN_ENTROPY_RATE_A"));
+ assertNull(data.get("NUM_EN_STOPS_TOP_N_A"));
+
+ }
+
+ @Test
+ public void testBothBadJson() throws Exception {
+ debugDumpAll(contTable);
+ String where = fp+"='file7_badJson.pdf'";
+ Map<String, String> data = getRow(contTable, where);
+ assertEquals(AbstractProfiler.JSON_PARSE_EXCEPTION,
+ data.get(AbstractProfiler.CONTAINER_HEADERS.JSON_EX+ ExtractComparer.aExtension));
+ assertEquals(AbstractProfiler.JSON_PARSE_EXCEPTION,
+ data.get(AbstractProfiler.CONTAINER_HEADERS.JSON_EX+ ExtractComparer.bExtension));
+ assertEquals("file7_badJson.pdf",
+ data.get(AbstractProfiler.CONTAINER_HEADERS.FILE_PATH.name()));
+ assertEquals("61", data.get("JSON_FILE_LENGTH_A"));
+ assertEquals("0", data.get("JSON_FILE_LENGTH_B"));
+ assertEquals("pdf", data.get(AbstractProfiler.CONTAINER_HEADERS.FILE_EXTENSION.name()));
+
+ }
+
+ @Test
+ public void testAccessPermissionException() throws Exception {
+ String sql = "select "+
+ AbstractProfiler.EXCEPTION_HEADERS.ACCESS_PERMISSION_EXCEPTION.name() +
+ " from " + AbstractProfiler.EXCEPTIONS_TABLE+"_A exA "+
+ " join " + ExtractComparer.COMPARISONS_TABLE + " cmp on cmp.ID=exA.ID "+
+ " join " + ExtractComparer.CONTAINERS_TABLE + " cont on cmp.CONTAINER_ID=cont.CONTAINER_ID "+
+ " where "+fp+"='file6_accessEx.pdf'";
+ Statement st = conn.createStatement();
+ ResultSet rs = st.executeQuery(sql);
+ List<String> results = new ArrayList<String>();
+ while (rs.next()) {
+ results.add(rs.getString(1));
+ }
+ assertEquals(1, results.size());
+ assertEquals("TRUE", results.get(0));
+
+ sql = "select "+
+ AbstractProfiler.EXCEPTION_HEADERS.ACCESS_PERMISSION_EXCEPTION.name() +
+ " from " + AbstractProfiler.EXCEPTIONS_TABLE+"_B exB "+
+ " join " + ExtractComparer.COMPARISONS_TABLE + " cmp on cmp.ID=exB.ID "+
+ " join " + ExtractComparer.CONTAINERS_TABLE + " cont on cmp.CONTAINER_ID=cont.CONTAINER_ID "+
+ " where "+fp+"='file6_accessEx.pdf'";
+ st = conn.createStatement();
+ rs = st.executeQuery(sql);
+ results = new ArrayList<String>();
+ while (rs.next()) {
+ results.add(rs.getString(1));
+ }
+ assertEquals(1, results.size());
+ assertEquals("TRUE", results.get(0));
+
+ }
+
+ @Test
+ public void testContainerException() throws Exception {
+ String sql = "select * "+
+ " from " + AbstractProfiler.EXCEPTIONS_TABLE+"_A exA "+
+ " join " + ExtractComparer.COMPARISONS_TABLE + " cmp on cmp.ID=exA.ID "+
+ " join " + ExtractComparer.CONTAINERS_TABLE + " cont on cmp.CONTAINER_ID=cont.CONTAINER_ID "+
+ "where "+fp+"='file8_IOEx.pdf'";
+ Statement st = conn.createStatement();
+ ResultSet rs = st.executeQuery(sql);
+
+ Map<String, String> data = new HashMap<String,String>();
+ ResultSetMetaData rsM = rs.getMetaData();
+ while (rs.next()) {
+ for (int i = 1; i <= rsM.getColumnCount(); i++)
+ data.put(rsM.getColumnName(i), rs.getString(i));
+ }
+
+ String sortStack = data.get(AbstractProfiler.EXCEPTION_HEADERS.SORT_STACK_TRACE.name());
+ sortStack = sortStack.replaceAll("[\r\n]", "<N>");
+ assertTrue(sortStack.startsWith("java.lang.RuntimeException<N>"));
+
+ String fullStack = data.get(AbstractProfiler.EXCEPTION_HEADERS.ORIG_STACK_TRACE.name());
+ assertTrue(
+ fullStack.startsWith("java.lang.RuntimeException: java.io.IOException: Value is not an integer"));
+ }
+
+ private void debugDumpAll(String table) throws Exception {
+ Statement st = conn.createStatement();
+ String sql = "select * from "+table;
+ ResultSet rs = st.executeQuery(sql);
+ ResultSetMetaData m = rs.getMetaData();
+ for (int i = 1; i <= m.getColumnCount(); i++) {
+ System.out.print(m.getColumnName(i) + ", ");
+ }
+ System.out.println("\n");
+ while (rs.next()) {
+ for (int i = 1; i <= m.getColumnCount(); i++) {
+ System.out.print(rs.getString(i)+", ");
+ }
+ System.out.println("\n");
+ }
+ st.close();
+
+ }
+ */
+ private void debugShowColumns(String table) throws Exception {
+ Statement st = conn.createStatement();
+ String sql = "select * from "+table;
+ ResultSet rs = st.executeQuery(sql);
+ ResultSetMetaData m = rs.getMetaData();
+ for (int i = 1; i <= m.getColumnCount(); i++) {
+ System.out.println(i+" : "+m.getColumnName(i));
+ }
+ st.close();
+ }
+
+ //return the string value for one cell
+ private String getString(String colName, String table, String where) throws Exception {
+ List<String> results = getColStrings(colName, table, where);
+ if (results.size() > 1) {
+ throw new RuntimeException("more than one result");
+ } else if (results.size() == 0) {
+ throw new RuntimeException("no results");
+ }
+
+ return results.get(0);
+ }
+
+
+ private Map<String, String> getRow(String table, String where) throws Exception {
+ String sql = getSql("*", table, where);
+ Map<String, String> results = new HashMap<String, String>();
+ Statement st = null;
+
+ try {
+ st = conn.createStatement();
+ ResultSet rs = st.executeQuery(sql);
+ ResultSetMetaData m = rs.getMetaData();
+ int rows = 0;
+ while (rs.next()) {
+ if (rows > 0) {
+ throw new RuntimeException("returned more than one row!");
+ }
+ for (int i = 1; i <= m.getColumnCount(); i++) {
+ results.put(m.getColumnName(i), rs.getString(i));
+ }
+ rows++;
+ }
+ } finally {
+ if (st != null) {
+ st.close();
+ }
+ }
+ return results;
+
+ }
+
+ //return the string representations of the column values for one column
+ //as a list of strings
+ private List<String> getColStrings(String colName) throws Exception {
+ return getColStrings(colName, ExtractComparer.CONTENT_COMPARISONS.getName(), null);
+ }
+
+ private List<String> getColStrings(String colName, String table, String where) throws Exception {
+ String sql = getSql(colName, table, where);
+ List<String> results = new ArrayList<>();
+ Statement st = null;
+ try {
+ st = conn.createStatement();
+ System.out.println("SQL: "+sql);
+ ResultSet rs = st.executeQuery(sql);
+ while (rs.next()) {
+ results.add(rs.getString(1));
+ }
+ } finally {
+ if (st != null) {
+ st.close();
+ }
+ }
+ return results;
+ }
+
+ private String getSql(String colName, String table, String where) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("select ").append(colName).append(" from ").append(table);
+ if (where != null && ! where.equals("")) {
+ sb.append(" where ").append(where);
+ }
+ return sb.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java b/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
new file mode 100644
index 0000000..c3f0a7e
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.tika.eval.db.Cols;
+import org.apache.tika.eval.db.H2Util;
+import org.apache.tika.eval.db.TableInfo;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+@Ignore
+public class ProfilerBatchTest {
+
+ public final static String COMPARER_PROCESS_CLASS = "org.apache.tika.batch.fs.FSBatchProcessCLI";
+
+ private static Path dbDir;
+ private static Connection conn;
+
+ private final static String profileTable = ExtractProfiler.PROFILE_TABLE.getName();
+ private final static String exTable = ExtractProfiler.EXCEPTION_TABLE.getName();
+ private final static String fpCol = Cols.FILE_PATH.name();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+
+ Path inputRoot = Paths.get(new ComparerBatchTest().getClass().getResource("/test-dirs/extractsA").toURI());
+ dbDir = Files.createTempDirectory(inputRoot, "tika-test-db-dir-");
+ Map<String, String> args = new HashMap<>();
+ Path db = dbDir.resolve("profiler_test");
+ args.put("-db", db.toString());
+
+ //for debugging, you can use this to select only one file pair to load
+ //args.put("-includeFilePat", "file8.*");
+
+ /* BatchProcessTestExecutor ex = new BatchProcessTestExecutor(COMPARER_PROCESS_CLASS, args,
+ "/single-file-profiler-crawl-input-config.xml");
+ StreamStrings streamStrings = ex.execute();
+ System.out.println(streamStrings.getErrString());
+ System.out.println(streamStrings.getOutString());*/
+ H2Util dbUtil = new H2Util(db);
+ conn = dbUtil.getConnection(true);
+ }
+ @AfterClass
+ public static void tearDown() throws IOException {
+
+ try{
+ conn.close();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+
+
+ DirectoryStream<Path> dStream = Files.newDirectoryStream(dbDir);
+ Iterator<Path> it = dStream.iterator();
+ while (it.hasNext()) {
+ Path p = it.next();
+ Files.delete(p);
+ }
+ dStream.close();
+ Files.delete(dbDir);
+ }
+
+ @Test
+ public void testSimpleDBWriteAndRead() throws Exception {
+
+ Statement st = null;
+ List<String> fNameList = new ArrayList<>();
+ try {
+ String sql = "select * from "+ ExtractProfiler.CONTAINER_TABLE.getName();
+ st = conn.createStatement();
+ ResultSet rs = st.executeQuery(sql);
+ while (rs.next()) {
+ String fileName = rs.getString(Cols.FILE_PATH.name());
+ fNameList.add(fileName);
+ }
+ } finally {
+ if (st != null) {
+ st.close();
+ }
+ }
+ debugTable(ExtractProfiler.CONTAINER_TABLE);
+ debugTable(ExtractProfiler.PROFILE_TABLE);
+ debugTable(ExtractProfiler.CONTENTS_TABLE);
+ debugTable(ExtractProfiler.EXCEPTION_TABLE);
+ debugTable(ExtractProfiler.ERROR_TABLE);
+ assertEquals(10, fNameList.size());
+ assertTrue("file1.pdf", fNameList.contains("file1.pdf"));
+ assertTrue("file2_attachANotB.doc", fNameList.contains("file2_attachANotB.doc"));
+ assertTrue("file3_attachBNotA.doc", fNameList.contains("file3_attachBNotA.doc"));
+ assertTrue("file4_emptyB.pdf", fNameList.contains("file4_emptyB.pdf"));
+ assertTrue("file7_badJson.pdf", fNameList.contains("file7_badJson.pdf"));
+ }
+
+ @Test
+ public void testExtractErrors() throws Exception {
+ String sql = "select EXTRACT_ERROR_TYPE_ID from errors e" +
+ " join containers c on c.container_id = e.container_id "+
+ " where c.file_path='file9_noextract.txt'";
+
+ assertEquals("missing extract: file9_noextract.txt", "0",
+ getSingleResult(sql));
+ debugTable(ExtractProfiler.CONTAINER_TABLE);
+ debugTable(ExtractProfiler.PROFILE_TABLE);
+ debugTable(ExtractProfiler.CONTENTS_TABLE);
+ debugTable(ExtractProfiler.EXCEPTION_TABLE);
+ debugTable(ExtractProfiler.ERROR_TABLE);
+
+ sql = "select EXTRACT_ERROR_TYPE_ID from errors e" +
+ " join containers c on c.container_id = e.container_id "+
+ " where c.file_path='file5_emptyA.pdf'";
+ assertEquals("empty extract: file5_emptyA.pdf", "1",
+ getSingleResult(sql));
+
+ sql = "select EXTRACT_ERROR_TYPE_ID from errors e" +
+ " join containers c on c.container_id = e.container_id "+
+ " where c.file_path='file7_badJson.pdf'";
+ assertEquals("extract error:file7_badJson.pdf", "2",
+ getSingleResult(sql));
+
+ }
+
+ @Test
+ public void testParseErrors() throws Exception {
+ debugTable(ExtractProfiler.ERROR_TABLE);
+ String sql = "select file_path from errors where container_id is null";
+ assertEquals("file10_permahang.txt",
+ getSingleResult(sql));
+
+ sql = "select extract_error_type_id from errors where file_path='file11_oom.txt'";
+ assertEquals(Integer.toString(AbstractProfiler.
+ EXTRACT_ERROR_TYPE.
+ ZERO_BYTE_EXTRACT_FILE.ordinal()),
+ getSingleResult(sql));
+
+ sql = "select parse_error_type_id from errors where file_path='file11_oom.txt'";
+ assertEquals(Integer.toString(AbstractProfiler.
+ PARSE_ERROR_TYPE.
+ OOM.ordinal()),
+ getSingleResult(sql));
+
+ }
+
+ @Test
+ public void testParseExceptions() throws Exception {
+ debugTable(ExtractProfiler.EXCEPTION_TABLE);
+ }
+
+ private String getSingleResult(String sql) throws Exception {
+ Statement st = null;
+ st = conn.createStatement();
+ ResultSet rs = st.executeQuery(sql);
+ int hits = 0;
+ String val = "";
+ while (rs.next()) {
+ assertEquals("must have only one column in result",
+ 1, rs.getMetaData().getColumnCount());
+ val = rs.getString(1);
+ hits++;
+ }
+ assertEquals("must have only one hit", 1, hits);
+ return val;
+ }
+
+ //TODO: lots more testing!
+
+ public void debugTable(TableInfo table) throws Exception {
+ Statement st = null;
+ try {
+ String sql = "select * from "+table.getName();
+ st = conn.createStatement();
+ ResultSet rs = st.executeQuery(sql);
+ int colCount = rs.getMetaData().getColumnCount();
+ System.out.println("TABLE: "+table.getName());
+ for (int i = 1; i <= colCount; i++) {
+ if (i > 1) {
+ System.out.print(" | ");
+ }
+ System.out.print(rs.getMetaData().getColumnName(i));
+ }
+ System.out.println("");
+ int rowCount = 0;
+ while (rs.next()) {
+ for (int i = 1; i <= colCount; i++) {
+ if (i > 1) {
+ System.out.print(" | ");
+ }
+ System.out.print(rs.getString(i));
+ rowCount++;
+ }
+ System.out.println("");
+ }
+ if (rowCount == 0) {
+ System.out.println(table.getName() + " was empty");
+ }
+ } finally {
+ if (st != null) {
+ st.close();
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java b/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
new file mode 100644
index 0000000..72e8008
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval;
+
+import static org.apache.tika.eval.AbstractProfiler.EXCEPTION_TYPE;
+import static org.apache.tika.eval.AbstractProfiler.EXTRACT_ERROR_TYPE;
+import static org.apache.tika.eval.AbstractProfiler.getContent;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.tika.MockDBWriter;
+import org.apache.tika.TikaTest;
+import org.apache.tika.eval.db.Cols;
+import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.ExtractReader;
+import org.apache.tika.eval.util.LanguageIDWrapper;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.RecursiveParserWrapper;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+//These tests ensure that the comparer is extracting the right information
+//into a Map<String,String>. A full integration test
+//should also ensure that the elements are properly being written to the db
+
+public class SimpleComparerTest extends TikaTest {
+
+ private ExtractComparer comparer = null;
+ private MockDBWriter writer = null;
+
+ @Before
+ public void setUp() throws Exception {
+ writer = new MockDBWriter();
+ comparer = new ExtractComparer(null, null,
+ Paths.get("extractsA"), Paths.get("extractsB"),
+ writer, -1, -1,
+ ExtractReader.ALTER_METADATA_LIST.AS_IS);
+ AbstractProfiler.loadCommonTokens(this.getResourceAsFile("/commontokens").toPath());
+ LanguageIDWrapper.loadBuiltInModels();
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+ EvalFilePaths fpsA = new EvalFilePaths(
+ Paths.get("file1.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsA/file1.pdf.json").toPath()
+ );
+ EvalFilePaths fpsB = new EvalFilePaths(
+ Paths.get("file1.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsB/file1.pdf.json").toPath());
+
+ comparer.compareFiles(fpsA, fpsB);
+
+ List<Map<Cols, String>> tableInfos = writer.getTable(ExtractComparer.CONTENT_COMPARISONS);
+ Map<Cols, String> row = tableInfos.get(0);
+ assertEquals("0", row.get(Cols.ID));
+ assertTrue(
+ row.get(Cols.TOP_10_UNIQUE_TOKEN_DIFFS_A)
+ .startsWith("1,200: 1 | 120000: 1 | over: 1"));
+
+ tableInfos = writer.getTable(ExtractComparer.CONTENTS_TABLE_A);
+ row = tableInfos.get(0);
+ assertEquals("0", row.get(Cols.ID));
+ assertEquals("70", row.get(Cols.CONTENT_LENGTH));
+ assertEquals("10", row.get(Cols.NUM_UNIQUE_TOKENS));
+ assertEquals("14", row.get(Cols.NUM_TOKENS));
+ assertEquals("12", row.get(Cols.NUM_ALPHABETIC_TOKENS));
+ assertEquals("6", row.get(Cols.NUM_COMMON_TOKENS));
+ assertEquals("57", row.get(Cols.TOKEN_LENGTH_SUM));
+ assertEquals("en", row.get(Cols.COMMON_TOKENS_LANG));
+
+ tableInfos = writer.getTable(ExtractComparer.CONTENTS_TABLE_B);
+ row = tableInfos.get(0);
+ assertEquals("0", row.get(Cols.ID));
+ assertEquals("76", row.get(Cols.CONTENT_LENGTH));
+ assertEquals("9", row.get(Cols.NUM_UNIQUE_TOKENS));
+ assertEquals("13", row.get(Cols.NUM_TOKENS));
+ assertEquals("4", row.get(Cols.NUM_COMMON_TOKENS));
+ assertEquals("64", row.get(Cols.TOKEN_LENGTH_SUM));
+ assertEquals("en", row.get(Cols.COMMON_TOKENS_LANG));
+
+ tableInfos = writer.getTable(ExtractComparer.PROFILES_A);
+ row = tableInfos.get(0);
+ assertEquals("2", row.get(Cols.NUM_PAGES));
+
+ }
+
+ @Test
+ public void testBasicSpanish() throws Exception {
+ EvalFilePaths fpsA = new EvalFilePaths(
+ Paths.get("file1.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsA/file12_es.txt.json").toPath()
+ );
+ EvalFilePaths fpsB = new EvalFilePaths(
+ Paths.get("file1.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsB/file12_es.txt.json").toPath());
+
+ comparer.compareFiles(fpsA, fpsB);
+
+ List<Map<Cols, String>> tableInfos = writer.getTable(ExtractComparer.CONTENTS_TABLE_A);
+
+ Map<Cols, String> row = tableInfos.get(0);
+ assertEquals("133", row.get(Cols.CONTENT_LENGTH));
+ assertEquals("7", row.get(Cols.NUM_UNIQUE_TOKENS));
+ assertEquals("24", row.get(Cols.NUM_TOKENS));
+ assertEquals("3", row.get(Cols.NUM_COMMON_TOKENS));
+ assertEquals("108", row.get(Cols.TOKEN_LENGTH_SUM));
+ assertEquals("es", row.get(Cols.COMMON_TOKENS_LANG));
+ assertEquals("24", row.get(Cols.NUM_ALPHABETIC_TOKENS));
+
+ }
+
+
+ @Test
+ public void testEmpty() throws Exception {
+ EvalFilePaths fpsA = new EvalFilePaths(
+ Paths.get("file1.pdf"),
+ getResourceAsFile("/test-dirs/extractsA/file1.pdf.json").toPath()
+ );
+ EvalFilePaths fpsB = new EvalFilePaths(
+ Paths.get("file1.pdf"),
+ getResourceAsFile("/test-dirs/extractsB/file4_emptyB.pdf.json").toPath()
+ );
+ comparer.compareFiles(fpsA, fpsB);
+ List<Map<Cols, String>> table = writer.getTable(ExtractComparer.ERROR_TABLE_B);
+ Map<Cols, String> row = table.get(0);
+ //debugPrintRow(row);
+ assertEquals(Integer.toString(EXTRACT_ERROR_TYPE.ZERO_BYTE_EXTRACT_FILE.ordinal()),
+ row.get(Cols.EXTRACT_ERROR_TYPE_ID));
+ }
+
+
+ @Test
+ public void testGetContent() throws Exception {
+ Metadata m = new Metadata();
+ m.add(RecursiveParserWrapper.TIKA_CONTENT, "0123456789");
+
+ String content = getContent(m, 10);
+ assertEquals(10, content.length());
+
+ content = getContent(m, 4);
+ assertEquals(4, content.length());
+
+ //test Metadata with no content
+ content = getContent(new Metadata(), 10);
+ assertEquals(0, content.length());
+
+ //test null Metadata
+ content = getContent(null, 10);
+ assertEquals(0, content.length());
+ }
+
+ @Test
+ public void testAccessException() throws Exception {
+ EvalFilePaths fpsA = new EvalFilePaths(
+ Paths.get("file6_accessEx.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsA/file6_accessEx.pdf.json").toPath()
+ );
+ EvalFilePaths fpsB = new EvalFilePaths(
+ Paths.get("file6_accessEx.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsB/file6_accessEx.pdf.json").toPath()
+ );
+ comparer.compareFiles(fpsA, fpsB);
+ for (TableInfo t : new TableInfo[]{ExtractComparer.EXCEPTION_TABLE_A, ExtractComparer.EXCEPTION_TABLE_B}) {
+ List<Map<Cols, String>> table = writer.getTable(t);
+
+ Map<Cols, String> rowA = table.get(0);
+ //debugPrintRow(rowA);
+ assertEquals(Integer.toString(EXCEPTION_TYPE.ACCESS_PERMISSION.ordinal()),
+ rowA.get(Cols.PARSE_EXCEPTION_TYPE_ID));
+ assertNull(rowA.get(Cols.ORIG_STACK_TRACE));
+ assertNull(rowA.get(Cols.SORT_STACK_TRACE));
+ }
+ }
+
+
+ @Test
+ public void testAttachmentCounts() {
+ List<Metadata> list = new ArrayList<>();
+ Metadata m0 = new Metadata();
+ m0.set(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH, "dir1/dir2/file.zip");//bad data should be ignored
+ //in the first metadata object
+ list.add(m0);
+ Metadata m1 = new Metadata();
+ m1.set(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH, "/f1.docx/f2.zip/text1.txt");
+ list.add(m1);
+ Metadata m2 = new Metadata();
+ m2.set(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH, "/f1.docx/f2.zip/text2.txt");
+ list.add(m2);
+ Metadata m3 = new Metadata();
+ m3.set(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH, "/f1.docx/f2.zip");
+ list.add(m3);
+ Metadata m4 = new Metadata();
+ m4.set(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH, "/f1.docx");
+ list.add(m4);
+ Metadata m5 = new Metadata();
+ m5.set(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH, "/f1.docx/text3.txt");
+ list.add(m5);
+
+ List<Integer> counts = AbstractProfiler.countAttachments(list);
+
+ List<Integer> expected = new ArrayList<>();
+ expected.add(5);
+ expected.add(0);
+ expected.add(0);
+ expected.add(2);
+ expected.add(4);
+ expected.add(0);
+ assertEquals(expected, counts);
+ }
+
+
+ @Test
+ @Ignore
+ public void testDebug() throws Exception {
+ Path commonTokens = Paths.get(getResourceAsFile("/common_tokens_short.txt").toURI());
+ AbstractProfiler.loadCommonTokens(commonTokens);
+ EvalFilePaths fpsA = new EvalFilePaths(
+ Paths.get("file1.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsA/file1.pdf.json").toPath()
+ );
+ EvalFilePaths fpsB = new EvalFilePaths(
+ Paths.get("file1.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsB/file1.pdf.json").toPath()
+ );
+ comparer.compareFiles(fpsA, fpsB);
+ for (TableInfo t : new TableInfo[]{
+ ExtractComparer.COMPARISON_CONTAINERS,
+ ExtractComparer.ERROR_TABLE_A,
+ ExtractComparer.ERROR_TABLE_B,
+ ExtractComparer.EXCEPTION_TABLE_A,
+ ExtractComparer.EXCEPTION_TABLE_B,
+ ExtractComparer.PROFILES_A,
+ ExtractComparer.PROFILES_B,
+ ExtractComparer.CONTENTS_TABLE_A,
+ ExtractComparer.CONTENTS_TABLE_B,
+ ExtractComparer.CONTENT_COMPARISONS}) {
+ //debugPrintTable(t);
+ }
+ }
+
+ private void debugPrintTable(TableInfo tableInfo) {
+ List<Map<Cols, String>> table = writer.getTable(tableInfo);
+ if (table == null) {
+ return;
+ }
+ int i = 0;
+ System.out.println("TABLE: "+tableInfo.getName());
+ for (Map<Cols, String> row : table) {
+ SortedSet<Cols> keys = new TreeSet<Cols>(row.keySet());
+ for (Cols key : keys) {
+ System.out.println( i + " :: " + key + " : " + row.get(key));
+ }
+ i++;
+ }
+ System.out.println("");
+ }
+
+ private void debugPrintRow(Map<Cols, String> row) {
+ SortedSet<Cols> keys = new TreeSet<Cols>(row.keySet());
+ for (Cols key : keys) {
+ System.out.println(key + " : " + row.get(key));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java b/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java
new file mode 100644
index 0000000..c358149
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TikaEvalCLITest {
+
+ @Test
+ @Ignore("TODO: add real tests")
+ public void testBasic() throws Exception {
+ List<String> args = new ArrayList<>();
+ args.add("Profile");
+ args.add("-extractDir");
+ args.add("tika");
+ args.add("-db");
+ args.add("mydb");
+ args.add("-alterExtract");
+ args.add("first_only");
+ TikaEvalCLI.main(args.toArray(new String[args.size()]));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/java/org/apache/tika/eval/db/AbstractBufferTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/db/AbstractBufferTest.java b/tika-eval/src/test/java/org/apache/tika/eval/db/AbstractBufferTest.java
new file mode 100644
index 0000000..810425b
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/db/AbstractBufferTest.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.db;
+
+
+import static org.junit.Assert.assertEquals;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+
+public class AbstractBufferTest {
+
+
+ @Test(timeout = 30000)
+ public void runTest() throws InterruptedException, ExecutionException {
+ List<String> keys = new ArrayList<>();
+ Collections.addAll(keys, new String[]{
+ "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"});
+
+ int numGets = 100;
+ int numTesters = 20;
+ AbstractDBBuffer b = new TestBuffer();
+
+
+ ExecutorService ex = Executors.newFixedThreadPool(numTesters);
+ CompletionService<MyTestResult> completionService =
+ new ExecutorCompletionService<>(
+ ex);
+ for (int i = 0; i < numTesters; i++) {
+ completionService.submit(new Tester(keys, b, numGets));
+ }
+
+ int results = 0;
+ Map<String, Integer> combined = new HashMap<>();
+ while (results < numTesters) {
+ Future<MyTestResult> futureResult =
+ completionService.poll(1, TimeUnit.SECONDS);
+ if (futureResult != null) {
+ results++;
+ assertEquals(keys.size(), futureResult.get().getMap().keySet().size());
+ for (Map.Entry<String, Integer> e : futureResult.get().getMap().entrySet()) {
+ if (!combined.containsKey(e.getKey())) {
+ combined.put(e.getKey(), e.getValue());
+ } else {
+ assertEquals(combined.get(e.getKey()), e.getValue());
+ }
+ }
+ }
+ }
+ assertEquals(keys.size(), b.getNumWrites());
+ }
+
+ private class Tester implements Callable<MyTestResult> {
+
+ private Random r = new Random();
+ private Map<String, Integer> m = new HashMap<>();
+ List<String> keys = new ArrayList<>();
+ private final AbstractDBBuffer dbBuffer;
+ private final int numGets;
+
+ private Tester(List<String> inputKeys, AbstractDBBuffer buffer, int numGets) {
+ keys.addAll(inputKeys);
+ dbBuffer = buffer;
+ this.numGets = numGets;
+ }
+
+ @Override
+ public MyTestResult call() throws Exception {
+
+
+ for (int i = 0; i < numGets; i++) {
+ int index = r.nextInt(keys.size());
+ String k = keys.get(index);
+ if (k == null) {
+ throw new RuntimeException("keys can't be null");
+ }
+ Integer expected = m.get(k);
+ Integer val = dbBuffer.getId(k);
+ if (val == null) {
+ throw new RuntimeException("Val can't be null!");
+ }
+ if (expected != null) {
+ assertEquals(expected, val);
+ }
+ m.put(k, val);
+ }
+
+ //now add the val for every key
+ //just in case the rand() process didn't hit
+ //all indices
+ for (String k : keys) {
+ Integer val = dbBuffer.getId(k);
+ m.put(k, val);
+ }
+ MyTestResult r = new MyTestResult(m);
+ return r;
+ }
+ }
+
+ private class MyTestResult {
+ Map<String, Integer> m;
+ private MyTestResult(Map<String, Integer> m) {
+ this.m = m;
+ }
+ private Map<String, Integer> getMap() {
+ return m;
+ }
+
+ @Override
+ public String toString() {
+ return "MyTester: "+m.size();
+ }
+ }
+
+ private class TestBuffer extends AbstractDBBuffer {
+ @Override
+ public void write(int id, String value) throws RuntimeException {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void close() throws SQLException {
+ //no-op
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java b/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
new file mode 100644
index 0000000..14c0013
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.io;
+
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.RecursiveParserWrapper;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ExtractReaderTest extends TikaTest {
+
+ private Path testJsonFile;
+ private Path testTxtFile;
+
+ @Before
+ public void setUp() throws Exception {
+ testJsonFile = getResourceAsFile("/test-dirs/extractsA/file2_attachANotB.doc.json").toPath();
+ testTxtFile = getResourceAsFile("/test-dirs/extractsB/file13_attachANotB.doc.txt").toPath();
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+
+ ExtractReader extractReader = new ExtractReader();
+ List<Metadata> metadataList = extractReader.loadExtract(testJsonFile,
+ ExtractReader.ALTER_METADATA_LIST.AS_IS);
+ assertEquals(2, metadataList.size());
+ assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
+ assertEquals(1, metadataList.get(1).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
+ assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
+ assertContains("attachment", metadataList.get(1).get(RecursiveParserWrapper.TIKA_CONTENT));
+
+ metadataList = extractReader.loadExtract(testJsonFile, ExtractReader.ALTER_METADATA_LIST.FIRST_ONLY);
+ assertEquals(1, metadataList.size());
+ assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
+ assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
+ assertNotContained("attachment", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
+
+ metadataList = extractReader.loadExtract(testJsonFile, ExtractReader.ALTER_METADATA_LIST.CONCATENATE_CONTENT_INTO_FIRST);
+ assertEquals(1, metadataList.size());
+ assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
+ assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
+ assertContains("attachment", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
+ }
+
+ @Test
+ public void testTextBasic() throws IOException {
+ ExtractReader extractReader = new ExtractReader();
+ List<Metadata> metadataList = extractReader.loadExtract(testTxtFile,
+ ExtractReader.ALTER_METADATA_LIST.AS_IS);
+ assertEquals(1, metadataList.size());
+ Metadata m = metadataList.get(0);
+ assertEquals(1, m.getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
+ assertEquals("the quick brown fox fox fox jumped over the lazy lazy dog\n",
+ m.get(RecursiveParserWrapper.TIKA_CONTENT));
+
+ //test that the mime is inferred from the file extension
+ assertEquals("application/msword", m.get(Metadata.CONTENT_TYPE));
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/java/org/apache/tika/eval/io/FatalExceptionReaderTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/io/FatalExceptionReaderTest.java b/tika-eval/src/test/java/org/apache/tika/eval/io/FatalExceptionReaderTest.java
new file mode 100644
index 0000000..3b99a76
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/io/FatalExceptionReaderTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.io;
+
+
+import java.io.InputStream;
+
+import org.junit.Test;
+
+public class FatalExceptionReaderTest {
+ @Test
+ public void testSimpleRead() throws Exception {
+ InputStream is = this.getClass().getResourceAsStream("/test-dirs/batch-logs/batch-process-fatal.xml");
+ XMLLogReader reader = new XMLLogReader();
+ //reader.read(is);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java b/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java
new file mode 100644
index 0000000..5e43303
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.reports;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.Statement;
+
+import org.apache.tika.eval.db.H2Util;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class ResultsReporterTest {
+ private Path configFile;
+ private Path tmpDir;
+ private String dbName = "test-db";
+ private Connection connection;
+
+ @Before
+ public void setUp() throws Exception {
+ configFile = Paths.get(this.getClass().getResource("/reports.xml").toURI());
+ tmpDir = Files.createTempDirectory("tika-eval-report-test-");
+
+ connection = new H2Util(tmpDir.resolve(dbName)).getConnection(true);
+ String sql = "CREATE TABLE test_table (ID LONG PRIMARY KEY, STRING VARCHAR(32))";
+ Statement st = connection.createStatement();
+ st.execute(sql);
+ sql = "INSERT into test_table values ( 100000, 'the quick brown')";
+ st.execute(sql);
+ sql = "INSERT into test_table values (123456789, 'fox jumped over')";
+ st.execute(sql);
+ connection.commit();
+ }
+
+ @Test
+ @Ignore("add a real test here")
+ public void testBuilder() throws Exception {
+ ResultsReporter r = ResultsReporter.build(configFile);
+ r.execute(connection, Paths.get("reports"));
+ System.out.println("finished: "+ tmpDir.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/java/org/apache/tika/eval/tokens/LuceneTokenCounter.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/tokens/LuceneTokenCounter.java b/tika-eval/src/test/java/org/apache/tika/eval/tokens/LuceneTokenCounter.java
new file mode 100644
index 0000000..486cac7
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/tokens/LuceneTokenCounter.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.tokens;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.commons.math3.util.FastMath;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Experimental class uses Lucene's MemoryIndex to effectively build the
+ * token info.
+ */
+public class LuceneTokenCounter {
+ private static final String ALPHA_IDEOGRAPH_SUFFIX = "_a";
+
+ private final LeafReader leafReader;
+ private final MemoryIndex memoryIndex;
+ private final Analyzer generalAnalyzer;
+ private final Analyzer alphaIdeographAnalyzer;
+ private int topN = 10;
+
+ Map<String, TokenStatistics> fieldStats = new HashMap<>();
+
+ public LuceneTokenCounter(Analyzer generalAnalyzer, Analyzer alphaIdeographAnalyzer) throws IOException {
+ memoryIndex = new MemoryIndex();
+ IndexSearcher searcher = memoryIndex.createSearcher();
+ leafReader = (LeafReader)searcher.getIndexReader();
+ this.generalAnalyzer = generalAnalyzer;
+ this.alphaIdeographAnalyzer = alphaIdeographAnalyzer;
+ }
+
+ public void add(String field, String content) throws IOException {
+ memoryIndex.addField(field, content, generalAnalyzer);
+ //memoryIndex.addField(field+ALPHA_IDEOGRAPH_SUFFIX,
+ // content, alphaIdeographAnalyzer);
+ count(field);
+ //count(field+ALPHA_IDEOGRAPH_SUFFIX);
+
+ }
+
+
+ void count(String field) throws IOException {
+ long tokenCount = leafReader.getSumTotalTermFreq(field);
+ if (tokenCount > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("can't handle longs");
+ }
+ int tokenCountInt = (int)tokenCount;
+ int uniqueTokenCount = 0;
+ SummaryStatistics summStats = new SummaryStatistics();
+ double ent = 0.0d;
+ double p = 0.0d;
+ double base = 2.0;
+
+ Terms terms = leafReader.terms(field);
+ if (terms == null) {
+ //if there were no terms
+ fieldStats.put(field, new TokenStatistics(uniqueTokenCount, tokenCountInt,
+ new TokenIntPair[0], ent, summStats));
+ return;
+
+ }
+ TermsEnum termsEnum = terms.iterator();
+ BytesRef bytesRef = termsEnum.next();
+ TokenCountPriorityQueue queue= new TokenCountPriorityQueue(topN);
+
+ while (bytesRef != null) {
+
+ long termFreq = termsEnum.totalTermFreq();
+ if (termFreq > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("Sorry can't handle longs yet");
+ }
+ int tf = (int)termFreq;
+ //TODO: figure out how to avoid Stringifying this
+ //to get codepoint count
+ String t = bytesRef.utf8ToString();
+ int len = t.codePointCount(0, t.length());
+ for (int i = 0; i < tf; i++) {
+ summStats.addValue(len);
+ }
+ p = (double) tf / (double) tokenCount;
+ ent += p * FastMath.log(base, p);
+
+ if (queue.top() == null || queue.size() < topN ||
+ tf >= queue.top().getValue()) {
+ queue.insertWithOverflow(new TokenIntPair(t, tf));
+ }
+
+ uniqueTokenCount++;
+ bytesRef = termsEnum.next();
+ }
+ if (tokenCountInt > 0) {
+ ent = (-1.0d / (double)tokenCountInt) * ent;
+ }
+
+ fieldStats.put(field, new TokenStatistics(uniqueTokenCount, tokenCountInt,
+ queue.getArray(), ent, summStats));
+ }
+
+ public void setTopN(int topN) {
+ this.topN = topN;
+ }
+
+ public TokenStatistics getTokenStatistics(String field) {
+ return fieldStats.get(field);
+ }
+ public Terms getAlphaTerms(String field) throws IOException {
+ return leafReader.terms(field+ALPHA_IDEOGRAPH_SUFFIX);
+ }
+ public Terms getTerms(String field) throws IOException {
+ return leafReader.terms(field);
+ }
+
+
+ public void clear() {
+ memoryIndex.reset();
+ fieldStats.clear();
+ }
+/*
+ public ContrastStatistics contrast(String fieldA, String fieldB) throws IOException {
+ long diceDenom = getUniqueTokenCount(fieldA) +
+ getUniqueTokenCount(fieldB);
+
+ long diceNum = 0;
+ long overlapNum = 0;
+
+ Terms termsA = getTerms(fieldA);
+ Terms termsB = getTerms(fieldB);
+
+ TermsEnum termsEnumA = termsA.iterator();
+ TermsEnum termsEnumB = termsB.iterator();
+
+ BytesRef bytesRefA = termsEnumA.next();
+ BytesRef bytesRefB = termsEnumB.next();
+
+ while (bytesRefA != null) {
+ int compare = bytesRefA.compareTo(bytesRefB);
+ while (compare > 0) {
+ if (bytesRefB == null) {
+ break;
+ }
+ //handle term in B, but not A
+
+ compare = bytesRefA.compareTo(bytesRefB);
+ bytesRefB = termsEnumB.next();
+ }
+ if (compare == 0) {
+ diceNum += 2;
+ overlapNum += 2 * Math.min(termsEnumA.totalTermFreq(), termsEnumB.totalTermFreq());
+ }
+
+ bytesRefA = termsEnumA.next();
+ }
+
+
+ for (PairCount p : tokens.values()) {
+ if (p.a > 0 && p.b > 0) {
+ diceNum += 2;
+ overlapNum += 2 * Math.min(p.a, p.b);
+ }
+ }
+
+ float dice = (float) diceNum / (float) diceDenom;
+ float overlap = (float) overlapNum / (float) (theseTokens.getTokenCount() + thoseTokens.getTokenCount());
+ }
+*/
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/java/org/apache/tika/eval/tokens/TokenCounterTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/tokens/TokenCounterTest.java b/tika-eval/src/test/java/org/apache/tika/eval/tokens/TokenCounterTest.java
new file mode 100644
index 0000000..719b56c
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/tokens/TokenCounterTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.tokens;
+
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TokenCounterTest {
+ private final static String FIELD = "f";
+ private static AnalyzerManager analyzerManager;
+
+ private final int topN = 10;
+
+ @BeforeClass
+ public static void setUp() throws IOException {
+ analyzerManager = AnalyzerManager.newInstance();
+
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+ String s = " bde cde def abc efg f f f f ghijklmnop a a a a a a a a a a a a a a a a a b b b b b b b b b b b b b";
+ TokenCounter counter = new TokenCounter(analyzerManager.getGeneralAnalyzer(),
+ analyzerManager.getAlphaIdeoAnalyzer());
+ counter.add(FIELD, s);
+ TokenStatistics simpleTokenStatistics = counter.getTokenStatistics(FIELD);
+ LuceneTokenCounter tokenCounter = new LuceneTokenCounter(analyzerManager.getGeneralAnalyzer(),
+ analyzerManager.getAlphaIdeoAnalyzer());
+ tokenCounter.add(FIELD, s);
+ assertEquals(simpleTokenStatistics, tokenCounter.getTokenStatistics(FIELD));
+ }
+
+ @Test
+ public void testRandom() throws Exception {
+
+ long simple = 0;
+ long lucene = 0;
+ int numberOfTests = 100;
+ for (int i = 0; i < numberOfTests; i++) {
+ String s = generateString();
+ long start = new Date().getTime();
+ TokenCounter counter = new TokenCounter(analyzerManager.getGeneralAnalyzer(),
+ analyzerManager.getAlphaIdeoAnalyzer());
+ counter.add(FIELD, s);
+ simple += new Date().getTime()-start;
+ TokenStatistics simpleTokenStatistics = counter.getTokenStatistics(FIELD);
+
+ start = new Date().getTime();
+ LuceneTokenCounter tokenCounter = new LuceneTokenCounter(analyzerManager.getGeneralAnalyzer(),
+ analyzerManager.getAlphaIdeoAnalyzer());
+ tokenCounter.add(FIELD, s);
+ lucene += new Date().getTime()-start;
+ assertEquals(s, simpleTokenStatistics, tokenCounter.getTokenStatistics(FIELD));
+ }
+
+ //System.out.println("SIMPLE: " + simple + " lucene: "+lucene);
+ }
+
+ @Test
+ public void testCommonTokens() throws Exception {
+ TokenCounter tokenCounter = new TokenCounter(analyzerManager.getGeneralAnalyzer(),
+ analyzerManager.getAlphaIdeoAnalyzer());
+ String s = "the http://www.cnn.com and blahdeblah@apache.org are in valuable www.sites.org \u666e\u6797\u65af\u987f\u5927\u5b66";
+ tokenCounter.add(FIELD, s);
+ Map<String, MutableInt> tokens = tokenCounter.getAlphaTokens(FIELD);
+ assertEquals(new MutableInt(2), tokens.get("___url___"));
+ assertEquals(new MutableInt(1), tokens.get("___email___"));
+ }
+
+ @Test
+ public void testCJKFilter() throws Exception {
+ String s = "then quickbrownfoxjumpedoverthelazy dogss dog \u666e\u6797\u65af\u987f\u5927\u5b66";
+ Analyzer analyzer = analyzerManager.getCommonTokensAnalyzer();
+ TokenStream ts = analyzer.tokenStream(FIELD, s);
+ CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
+ ts.reset();
+ Map<String, Integer> tokens = new HashMap<>();
+ while (ts.incrementToken()) {
+ String t = termAtt.toString();
+ Integer count = tokens.get(t);
+ count = (count == null) ? count = 0 : count;
+ count++;
+ tokens.put(t, count);
+ }
+ ts.end();
+ ts.close();
+ assertEquals(7, tokens.size());
+ assertEquals(new Integer(1), tokens.get("\u6797\u65af"));
+ }
+
+ private String generateString() {
+
+ Random r = new Random();
+ int len = r.nextInt(1000);
+ int uniqueVocabTerms = 10000;
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < len; i++) {
+ sb.append(Integer.toString(r.nextInt(uniqueVocabTerms)+100000));
+ sb.append(" ");
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/java/org/apache/tika/eval/util/MimeUtilTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/util/MimeUtilTest.java b/tika-eval/src/test/java/org/apache/tika/eval/util/MimeUtilTest.java
new file mode 100644
index 0000000..40b7484
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/util/MimeUtilTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.util;
+
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.mime.MimeType;
+import org.apache.tika.mime.MimeTypeException;
+import org.apache.tika.mime.MimeTypes;
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore("Fix mimetype.getExtension to work with these and then we can get rid of MimeUtil")
+public class MimeUtilTest {
+
+ private final TikaConfig config = TikaConfig.getDefaultConfig();
+
+ @Test
+ public void testBasic() throws Exception {
+ assertResult("application/pdf", ".pdf");
+ assertResult("APPLICATION/PDF", ".pdf");
+ assertResult("text/plain; charset=ISO-8859-1", ".txt");
+ assertResult("application/xhtml+xml; charset=UTF-8\n", ".html");
+ assertResult("application/xml; charset=UTF-8\n", ".xml");
+
+ assertException("bogosity", "xml");
+ }
+
+ private void assertException(String contentType, String expected) {
+ boolean ex = false;
+ try {
+ assertResult(contentType, expected);
+ } catch (MimeTypeException e) {
+ ex = true;
+ }
+ assertTrue("Should have had exception for: " + contentType, ex);
+ }
+
+ private void assertResult(String contentType, String expected) throws MimeTypeException {
+ TikaConfig tikaConfig = TikaConfig.getDefaultConfig();
+ MimeTypes r = tikaConfig.getMimeRepository();
+ MimeType mt = r.forName(contentType);
+
+// String ext = MimeUtil.getExtension(contentType, config);
+ assertEquals(expected, mt.getExtension());
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/resources/commontokens/en
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/commontokens/en b/tika-eval/src/test/resources/commontokens/en
new file mode 100644
index 0000000..8d442fe
--- /dev/null
+++ b/tika-eval/src/test/resources/commontokens/en
@@ -0,0 +1,8 @@
+the
+of
+and
+a
+or
+#quick
+brown
+fox
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/resources/commontokens/es
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/commontokens/es b/tika-eval/src/test/resources/commontokens/es
new file mode 100644
index 0000000..b9bfd03
--- /dev/null
+++ b/tika-eval/src/test/resources/commontokens/es
@@ -0,0 +1,10 @@
+la
+de
+y
+una
+
+
+o
+r�pido
+marr�n
+zorro
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/resources/commontokens/zh-cn
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/commontokens/zh-cn b/tika-eval/src/test/resources/commontokens/zh-cn
new file mode 100644
index 0000000..bec617d
--- /dev/null
+++ b/tika-eval/src/test/resources/commontokens/zh-cn
@@ -0,0 +1,8 @@
+\u7684
+\u7684
+\u548c
+\u4e00\u500b
+\u8981\u4e48
+\u5feb
+\u68d5\u8272
+\u72d0\u72f8
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/resources/commontokens/zh-tw
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/commontokens/zh-tw b/tika-eval/src/test/resources/commontokens/zh-tw
new file mode 100644
index 0000000..bc91291
--- /dev/null
+++ b/tika-eval/src/test/resources/commontokens/zh-tw
@@ -0,0 +1,8 @@
+\u7684
+\u7684
+\u548c
+\u4e00\u4e2a
+\u8981\u4e48
+\u5feb
+\u68d5\u8272
+\u72d0\u72f8
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/log4j.properties b/tika-eval/src/test/resources/log4j.properties
new file mode 100644
index 0000000..925f9f2
--- /dev/null
+++ b/tika-eval/src/test/resources/log4j.properties
@@ -0,0 +1,11 @@
+
+log4j.rootLogger=WARN,A1
+
+#for debugging
+#log4j.rootLogger=TRACE,A1
+
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/resources/log4j_process.properties
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/log4j_process.properties b/tika-eval/src/test/resources/log4j_process.properties
new file mode 100644
index 0000000..cca8871
--- /dev/null
+++ b/tika-eval/src/test/resources/log4j_process.properties
@@ -0,0 +1,11 @@
+
+log4j.rootLogger=TRACE,A1
+
+#for debugging
+#log4j.rootLogger=TRACE,A1
+
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml b/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
new file mode 100644
index 0000000..e8b9d6c
--- /dev/null
+++ b/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<tika-batch-config
+ maxAliveTimeSeconds="-1"
+ pauseOnEarlyTerminationMillis="500"
+ timeoutCheckPulseMillis="1000"
+ maxQueueSize="10000"
+ numConsumers="5"
+ timeoutThresholdMillis="300000">
+
+ <commandline>
+ <option opt="c" longOpt="tika-config" hasArg="true"
+ description="TikaConfig file"/>
+
+ <option opt="bc" longOpt="batch-config" hasArg="true"
+ description="xml batch config file" required="true"/>
+ <option opt="inputDir" hasArg="true"
+ description="dir to start crawling"/>
+ <option opt="numConsumers" hasArg="true"
+ description="number of fileConsumers threads"/>
+ <option opt="extractDir" hasArg="true"
+ description="this dir contains the files containing extracted metadata/content" required="false"/>
+ <option opt="db" hasArg="true"
+ description="name of db directory or file to which to write results"/>
+ </commandline>
+
+
+ <!--
+ Can also add startDir: this tells the crawler to start indexing a
+ child directory of the inputDir directory.
+ -->
+ <crawler builderClass="org.apache.tika.batch.fs.builders.FSCrawlerBuilder"
+ inputDir="src/test/resources/test-dirs/extractsA"
+ crawlOrder="sorted"
+ maxConsecWaitMillis="5000"
+ maxFilesToAdd="-1"
+ maxFilesToConsider="-1"
+ includeFilePat=""
+ excludeFilePat=""
+ maxFileSizeBytes="-1"
+ />
+
+ <consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
+ consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"
+ errorLogFile="src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml"
+ extractDir="src/test/resources/test-dirs/extractsA"
+ commonTokens="src/test/resources/common_tokens"/>
+
+
+ <!-- reporter and interrupter are optional -->
+ <reporter builderClass="org.apache.tika.batch.builders.SimpleLogReporterBuilder" sleepMillis="1000"
+ staleThresholdMillis="500000"/>
+ <interrupter builderClass="org.apache.tika.batch.builders.InterrupterBuilder"/>
+</tika-batch-config>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml b/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
new file mode 100644
index 0000000..da59d03
--- /dev/null
+++ b/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<tika-batch-config
+ maxAliveTimeSeconds="-1"
+ pauseOnEarlyTerminationMillis="500"
+ timeoutCheckPulseMillis="1000"
+ maxQueueSize="10000"
+ numConsumers="5"
+ timeoutThresholdMillis="300000">
+
+ <commandline>
+ <option opt="c" longOpt="tika-config" hasArg="true"
+ description="TikaConfig file"/>
+
+ <option opt="bc" longOpt="batch-config" hasArg="true"
+ description="xml batch config file" required="true"/>
+ <option opt="inputDir" hasArg="true"
+ description="dir to start crawling"/>
+ <option opt="numConsumers" hasArg="true"
+ description="number of fileConsumers threads"/>
+ <option opt="extractDir" hasArg="true"
+ description="this dir contains the files containing extracted metadata/content" required="false"/>
+ <option opt="db" hasArg="true"
+ description="name of db directory or file to which to write results"/>
+ </commandline>
+
+
+ <!--
+ Can also add startDir: this tells the crawler to start indexing a
+ child directory of the inputDir directory.
+ -->
+ <crawler builderClass="org.apache.tika.batch.fs.builders.FSCrawlerBuilder"
+ inputDir="src/test/resources/test-dirs/raw_input"
+ crawlOrder="sorted"
+ maxConsecWaitMillis="5000"
+ maxFilesToAdd="-1"
+ maxFilesToConsider="-1"
+ includeFilePat=""
+ excludeFilePat=""
+ maxFileSizeBytes="-1"
+ />
+
+ <consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
+ consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"
+ errorLogFile="src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml"
+ extractDir="src/test/resources/test-dirs/extractsA"
+ inputDir="src/test/resources/test-dirs/raw_input"
+ commonTokens="src/test/resources/common_tokens"/>
+
+
+ <!-- reporter and interrupter are optional -->
+ <reporter builderClass="org.apache.tika.batch.builders.SimpleLogReporterBuilder" sleepMillis="1000"
+ staleThresholdMillis="500000"/>
+ <interrupter builderClass="org.apache.tika.batch.builders.InterrupterBuilder"/>
+</tika-batch-config>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/aa7a0c35/tika-eval/src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml b/tika-eval/src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml
new file mode 100644
index 0000000..520306b
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml
@@ -0,0 +1,59 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<log4j:event logger="org.apache.tika.batch.FileResourceConsumer" timestamp="1436376775762" level="ERROR" thread="pool-2-thread-11">
+<log4j:message><![CDATA[<?xml version="1.0" ?><timed_out resourceId="file10_permahang.txt" elapsedMS="340302"></timed_out>]]></log4j:message>
+</log4j:event>
+
+<log4j:event logger="org.apache.tika.batch.FileResourceConsumer" timestamp="1436376775758" level="ERROR" thread="pool-2-thread-10">
+<log4j:message><![CDATA[<?xml version="1.0" ?><oom resourceId="file11_oom.txt">java.lang.OutOfMemoryError: Java heap space
+ at java.io.ByteArrayOutputStream.<init>(ByteArrayOutputStream.java:77)
+ at org.apache.fontbox.ttf.MemoryTTFDataStream.<init>(MemoryTTFDataStream.java:45)
+ at org.apache.fontbox.ttf.TTFParser.parse(TTFParser.java:96)
+ at org.apache.pdfbox.pdmodel.font.PDTrueTypeFont.<init>(PDTrueTypeFont.java:135)
+ at org.apache.pdfbox.pdmodel.font.PDFontFactory.createFont(PDFontFactory.java:75)
+ at org.apache.pdfbox.pdmodel.PDResources.getFont(PDResources.java:96)
+ at org.apache.pdfbox.contentstream.operator.text.SetFontAndSize.process(SetFontAndSize.java:50)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processOperator(PDFStreamEngine.java:795)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processStreamOperators(PDFStreamEngine.java:462)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processStream(PDFStreamEngine.java:438)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.showForm(PDFStreamEngine.java:178)
+ at org.apache.pdfbox.contentstream.operator.DrawObject.process(DrawObject.java:49)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processOperator(PDFStreamEngine.java:795)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processStreamOperators(PDFStreamEngine.java:462)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processStream(PDFStreamEngine.java:438)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processPage(PDFStreamEngine.java:149)
+ at org.apache.pdfbox.text.PDFTextStreamEngine.processPage(PDFTextStreamEngine.java:117)
+ at org.apache.pdfbox.text.PDFTextStripper.processPage(PDFTextStripper.java:369)
+ at org.apache.pdfbox.text.PDFTextStripper.processPages(PDFTextStripper.java:305)
+ at org.apache.pdfbox.text.PDFTextStripper.writeText(PDFTextStripper.java:249)
+ at org.apache.tika.parser.pdf.PDF2XHTML.process(PDF2XHTML.java:137)
+ at org.apache.tika.parser.pdf.PDFParser.parse(PDFParser.java:132)
+ at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:281)
+ at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:281)
+ at org.apache.tika.parser.AutoDetectParser.parse(AutoDetectParser.java:120)
+ at org.apache.tika.parser.ParserDecorator.parse(ParserDecorator.java:177)
+ at org.apache.tika.parser.DigestingParser.parse(DigestingParser.java:74)
+ at org.apache.tika.parser.RecursiveParserWrapper.parse(RecursiveParserWrapper.java:158)
+ at org.apache.tika.batch.FileResourceConsumer.parse(FileResourceConsumer.java:410)
+ at org.apache.tika.batch.fs.RecursiveParserWrapperFSConsumer.processFileResource(RecursiveParserWrapperFSConsumer.java:104)
+ at org.apache.tika.batch.FileResourceConsumer._processFileResource(FileResourceConsumer.java:182)
+ at org.apache.tika.batch.FileResourceConsumer.call(FileResourceConsumer.java:115)
+</oom>]]></log4j:message>
+</log4j:event>
\ No newline at end of file