You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/02/16 17:20:02 UTC
[1/6] tika git commit: TIKA-1332 initial commit of tika-eval. More
work remains.
Repository: tika
Updated Branches:
refs/heads/2.x 6bfe5d565 -> 5e49c3308
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsA/file1.pdf.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsA/file1.pdf.json b/tika-eval/src/test/resources/test-dirs/extractsA/file1.pdf.json
new file mode 100644
index 0000000..6ef09de
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsA/file1.pdf.json
@@ -0,0 +1,5 @@
+[{
+ "Content-Type":"text/plain",
+ "X-TIKA:content":"the quick brown fox fox fox jumped over the lazy lazy dog 1,200 120000",
+ "xmpTPg:NPages":2
+}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsA/file10_permahang.txt.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsA/file10_permahang.txt.json b/tika-eval/src/test/resources/test-dirs/extractsA/file10_permahang.txt.json
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsA/file11_oom.txt.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsA/file11_oom.txt.json b/tika-eval/src/test/resources/test-dirs/extractsA/file11_oom.txt.json
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsA/file12_es.txt.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsA/file12_es.txt.json b/tika-eval/src/test/resources/test-dirs/extractsA/file12_es.txt.json
new file mode 100644
index 0000000..0e2558b
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsA/file12_es.txt.json
@@ -0,0 +1,4 @@
+[{
+ "Content-Type":"text/plain",
+ "X-TIKA:content":"El zorro marr�n r�pido salt� sobre el perro. El zorro marr�n r�pido salt� sobre el perro. El zorro marr�n r�pido salt� sobre el perro"
+}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsA/file13_attachANotB.doc.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsA/file13_attachANotB.doc.json b/tika-eval/src/test/resources/test-dirs/extractsA/file13_attachANotB.doc.json
new file mode 100644
index 0000000..5371c87
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsA/file13_attachANotB.doc.json
@@ -0,0 +1,10 @@
+[{
+ "Content-Type":"text/plain",
+ "X-TIKA:content":"the quick brown fox fox fox jumped over the lazy lazy dog"
+ },
+ {
+ "Content-Type":"text/plain",
+ "X-TIKA:embedded_resource_path":"inner.txt",
+ "X-TIKA:content":"attachment contents"
+ }
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsA/file2_attachANotB.doc.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsA/file2_attachANotB.doc.json b/tika-eval/src/test/resources/test-dirs/extractsA/file2_attachANotB.doc.json
new file mode 100644
index 0000000..5371c87
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsA/file2_attachANotB.doc.json
@@ -0,0 +1,10 @@
+[{
+ "Content-Type":"text/plain",
+ "X-TIKA:content":"the quick brown fox fox fox jumped over the lazy lazy dog"
+ },
+ {
+ "Content-Type":"text/plain",
+ "X-TIKA:embedded_resource_path":"inner.txt",
+ "X-TIKA:content":"attachment contents"
+ }
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsA/file3_attachBNotA.doc.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsA/file3_attachBNotA.doc.json b/tika-eval/src/test/resources/test-dirs/extractsA/file3_attachBNotA.doc.json
new file mode 100644
index 0000000..18763d1
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsA/file3_attachBNotA.doc.json
@@ -0,0 +1,4 @@
+[{
+ "Content-Type":"text/plain",
+ "X-TIKA:content":"the quick brown fox fox fox jumped over the lazy lazy dog"
+}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsA/file4_emptyB.pdf.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsA/file4_emptyB.pdf.json b/tika-eval/src/test/resources/test-dirs/extractsA/file4_emptyB.pdf.json
new file mode 100644
index 0000000..18763d1
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsA/file4_emptyB.pdf.json
@@ -0,0 +1,4 @@
+[{
+ "Content-Type":"text/plain",
+ "X-TIKA:content":"the quick brown fox fox fox jumped over the lazy lazy dog"
+}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsA/file5_emptyA.pdf.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsA/file5_emptyA.pdf.json b/tika-eval/src/test/resources/test-dirs/extractsA/file5_emptyA.pdf.json
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsA/file6_accessEx.pdf.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsA/file6_accessEx.pdf.json b/tika-eval/src/test/resources/test-dirs/extractsA/file6_accessEx.pdf.json
new file mode 100644
index 0000000..ded29af
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsA/file6_accessEx.pdf.json
@@ -0,0 +1 @@
+[{"Content-Type":"application/pdf","X-Parsed-By":["org.apache.tika.parser.DefaultParser","org.apache.tika.parser.pdf.PDFParser"],"X-TIKA:EXCEPTION:runtime":"org.apache.tika.exception.AccessPermissionException: Content extraction is not allowed.\n\tat org.apache.tika.parser.pdf.AccessChecker.check(AccessChecker.java:77)\n\tat org.apache.tika.parser.pdf.PDFParser.parse(PDFParser.java:147)\n\tat org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:270)\n\tat org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:270)\n\tat org.apache.tika.parser.AutoDetectParser.parse(AutoDetectParser.java:120)\n\tat org.apache.tika.parser.RecursiveParserWrapper.parse(RecursiveParserWrapper.java:130)\n\tat org.apache.tika.batch.fs.RecursiveParserWrapperFSConsumer.processFileResource(RecursiveParserWrapperFSConsumer.java:123)\n\tat org.apache.tika.batch.FileResourceConsumer._processFileResource(FileResourceConsumer.java:171)\n\tat org.apache.tika.batch.FileResourceConsumer.call(F
ileResourceConsumer.java:104)\n\tat org.apache.tika.batch.FileResourceConsumer.call(FileResourceConsumer.java:44)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:262)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:262)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n","access_permission:assemble_document":"false","access_permission:can_modify":"false","access_permission:can_print":"true","access_permission:can_print_degraded":"true","access_permission:extract_content":"false","access_permission:extract_for_accessibility":"true","access_permission:fill_in_form":"false","access_permission:modify_annotations":"false","pdf:encrypted":"true","resourceName":"file3_accessEx","tika:file_ext":"pdf","tika_batch_fs:relative_path":
"file3_accessEx","xmpTPg:NPages":"4"}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsA/file7_badJson.pdf.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsA/file7_badJson.pdf.json b/tika-eval/src/test/resources/test-dirs/extractsA/file7_badJson.pdf.json
new file mode 100644
index 0000000..8cf61da
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsA/file7_badJson.pdf.json
@@ -0,0 +1,4 @@
+[{
+ "Content-Type":"text/plain",
+ "X-TIKA:content":2,100
+}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsA/file8_IOEx.pdf.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsA/file8_IOEx.pdf.json b/tika-eval/src/test/resources/test-dirs/extractsA/file8_IOEx.pdf.json
new file mode 100644
index 0000000..4ecf0e8
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsA/file8_IOEx.pdf.json
@@ -0,0 +1 @@
+[{"Content-Length":"479562","Content-Type":"application/pdf","X-Parsed-By":["org.apache.tika.parser.DefaultParser","org.apache.tika.parser.pdf.PDFParser"],"X-TIKA:EXCEPTION:runtime":"java.lang.RuntimeException: java.io.IOException: Value is not an integer: 8546736428538085463808\n\tat org.apache.pdfbox.pdfparser.PDFStreamParser$1.tryNext(PDFStreamParser.java:186)\n\tat org.apache.pdfbox.pdfparser.PDFStreamParser$1.hasNext(PDFStreamParser.java:193)\n\tat org.apache.pdfbox.util.PDFStreamEngine.processSubStream(PDFStreamEngine.java:255)\n\tat org.apache.pdfbox.util.PDFStreamEngine.processSubStream(PDFStreamEngine.java:235)\n\tat org.apache.pdfbox.util.PDFStreamEngine.processStream(PDFStreamEngine.java:215)\n\tat org.apache.pdfbox.util.PDFTextStripper.processPage(PDFTextStripper.java:456)\n\tat org.apache.pdfbox.util.PDFTextStripper.processPages(PDFTextStripper.java:381)\n\tat org.apache.pdfbox.util.PDFTextStripper.writeText(PDFTextStripper.java:340)\n\tat org.apache.tika.parser.pdf.PDF
2XHTML.process(PDF2XHTML.java:106)\n\tat org.apache.tika.parser.pdf.PDFParser.parse(PDFParser.java:148)\n\tat org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:247)\n\tat org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:247)\n\tat org.apache.tika.parser.AutoDetectParser.parse(AutoDetectParser.java:120)\n\tat org.apache.tika.parser.RecursiveParserWrapper.parse(RecursiveParserWrapper.java:130)\n\tat org.apache.tika.batch.FileResourceConsumer.parse(FileResourceConsumer.java:410)\n\tat org.apache.tika.batch.fs.RecursiveParserWrapperFSConsumer.processFileResource(RecursiveParserWrapperFSConsumer.java:106)\n\tat org.apache.tika.batch.FileResourceConsumer._processFileResource(FileResourceConsumer.java:182)\n\tat org.apache.tika.batch.FileResourceConsumer.call(FileResourceConsumer.java:115)\n\tat org.apache.tika.batch.FileResourceConsumer.call(FileResourceConsumer.java:49)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:262)\n\tat java.util.concurr
ent.Executors$RunnableAdapter.call(Executors.java:471)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:262)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\nCaused by: java.io.IOException: Value is not an integer: 8546736428538085463808\n\tat org.apache.pdfbox.cos.COSNumber.get(COSNumber.java:104)\n\tat org.apache.pdfbox.pdfparser.PDFStreamParser.parseNextToken(PDFStreamParser.java:350)\n\tat org.apache.pdfbox.pdfparser.PDFStreamParser.access$000(PDFStreamParser.java:46)\n\tat org.apache.pdfbox.pdfparser.PDFStreamParser$1.tryNext(PDFStreamParser.java:181)\n\t... 24 more\n","access_permission:assemble_document":"true","access_permission:can_modify":"true","access_permission:can_print":"true","access_permission:can_print_degraded":"true","access_permission:extract_content":"true","access_permission:extract_for
_accessibility":"true","access_permission:fill_in_form":"true","access_permission:modify_annotations":"true", "resourceName":"file8_IOEx.pdf","tika:file_ext":"pdf","tika_batch_fs:relative_path":"file8_IOEx.pdf"}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsB/file1.pdf.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsB/file1.pdf.json b/tika-eval/src/test/resources/test-dirs/extractsB/file1.pdf.json
new file mode 100644
index 0000000..cbb51cf
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsB/file1.pdf.json
@@ -0,0 +1,2 @@
+[{ "Content-Type":"text/plain",
+ "X-TIKA:content":"the quick brown fox jumped the lazy dog aardvark aardvark aardvark bear bear"}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsB/file11_oom.txt.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsB/file11_oom.txt.json b/tika-eval/src/test/resources/test-dirs/extractsB/file11_oom.txt.json
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsB/file12_es.txt.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsB/file12_es.txt.json b/tika-eval/src/test/resources/test-dirs/extractsB/file12_es.txt.json
new file mode 100644
index 0000000..0e2558b
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsB/file12_es.txt.json
@@ -0,0 +1,4 @@
+[{
+ "Content-Type":"text/plain",
+ "X-TIKA:content":"El zorro marr�n r�pido salt� sobre el perro. El zorro marr�n r�pido salt� sobre el perro. El zorro marr�n r�pido salt� sobre el perro"
+}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsB/file13_attachANotB.doc.txt
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsB/file13_attachANotB.doc.txt b/tika-eval/src/test/resources/test-dirs/extractsB/file13_attachANotB.doc.txt
new file mode 100644
index 0000000..240a94e
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsB/file13_attachANotB.doc.txt
@@ -0,0 +1 @@
+the quick brown fox fox fox jumped over the lazy lazy dog
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsB/file2_attachANotB.doc.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsB/file2_attachANotB.doc.json b/tika-eval/src/test/resources/test-dirs/extractsB/file2_attachANotB.doc.json
new file mode 100644
index 0000000..18763d1
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsB/file2_attachANotB.doc.json
@@ -0,0 +1,4 @@
+[{
+ "Content-Type":"text/plain",
+ "X-TIKA:content":"the quick brown fox fox fox jumped over the lazy lazy dog"
+}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsB/file3_attachBNotA.doc.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsB/file3_attachBNotA.doc.json b/tika-eval/src/test/resources/test-dirs/extractsB/file3_attachBNotA.doc.json
new file mode 100644
index 0000000..5371c87
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsB/file3_attachBNotA.doc.json
@@ -0,0 +1,10 @@
+[{
+ "Content-Type":"text/plain",
+ "X-TIKA:content":"the quick brown fox fox fox jumped over the lazy lazy dog"
+ },
+ {
+ "Content-Type":"text/plain",
+ "X-TIKA:embedded_resource_path":"inner.txt",
+ "X-TIKA:content":"attachment contents"
+ }
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsB/file4_emptyB.pdf.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsB/file4_emptyB.pdf.json b/tika-eval/src/test/resources/test-dirs/extractsB/file4_emptyB.pdf.json
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsB/file5_emptyA.pdf.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsB/file5_emptyA.pdf.json b/tika-eval/src/test/resources/test-dirs/extractsB/file5_emptyA.pdf.json
new file mode 100644
index 0000000..18763d1
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsB/file5_emptyA.pdf.json
@@ -0,0 +1,4 @@
+[{
+ "Content-Type":"text/plain",
+ "X-TIKA:content":"the quick brown fox fox fox jumped over the lazy lazy dog"
+}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsB/file6_accessEx.pdf.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsB/file6_accessEx.pdf.json b/tika-eval/src/test/resources/test-dirs/extractsB/file6_accessEx.pdf.json
new file mode 100644
index 0000000..ded29af
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsB/file6_accessEx.pdf.json
@@ -0,0 +1 @@
+[{"Content-Type":"application/pdf","X-Parsed-By":["org.apache.tika.parser.DefaultParser","org.apache.tika.parser.pdf.PDFParser"],"X-TIKA:EXCEPTION:runtime":"org.apache.tika.exception.AccessPermissionException: Content extraction is not allowed.\n\tat org.apache.tika.parser.pdf.AccessChecker.check(AccessChecker.java:77)\n\tat org.apache.tika.parser.pdf.PDFParser.parse(PDFParser.java:147)\n\tat org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:270)\n\tat org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:270)\n\tat org.apache.tika.parser.AutoDetectParser.parse(AutoDetectParser.java:120)\n\tat org.apache.tika.parser.RecursiveParserWrapper.parse(RecursiveParserWrapper.java:130)\n\tat org.apache.tika.batch.fs.RecursiveParserWrapperFSConsumer.processFileResource(RecursiveParserWrapperFSConsumer.java:123)\n\tat org.apache.tika.batch.FileResourceConsumer._processFileResource(FileResourceConsumer.java:171)\n\tat org.apache.tika.batch.FileResourceConsumer.call(F
ileResourceConsumer.java:104)\n\tat org.apache.tika.batch.FileResourceConsumer.call(FileResourceConsumer.java:44)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:262)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:262)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n","access_permission:assemble_document":"false","access_permission:can_modify":"false","access_permission:can_print":"true","access_permission:can_print_degraded":"true","access_permission:extract_content":"false","access_permission:extract_for_accessibility":"true","access_permission:fill_in_form":"false","access_permission:modify_annotations":"false","pdf:encrypted":"true","resourceName":"file3_accessEx","tika:file_ext":"pdf","tika_batch_fs:relative_path":
"file3_accessEx","xmpTPg:NPages":"4"}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsB/file7_badJson.pdf.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsB/file7_badJson.pdf.json b/tika-eval/src/test/resources/test-dirs/extractsB/file7_badJson.pdf.json
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/extractsB/file8_IOEx.pdf.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/extractsB/file8_IOEx.pdf.json b/tika-eval/src/test/resources/test-dirs/extractsB/file8_IOEx.pdf.json
new file mode 100644
index 0000000..4ecf0e8
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/extractsB/file8_IOEx.pdf.json
@@ -0,0 +1 @@
+[{"Content-Length":"479562","Content-Type":"application/pdf","X-Parsed-By":["org.apache.tika.parser.DefaultParser","org.apache.tika.parser.pdf.PDFParser"],"X-TIKA:EXCEPTION:runtime":"java.lang.RuntimeException: java.io.IOException: Value is not an integer: 8546736428538085463808\n\tat org.apache.pdfbox.pdfparser.PDFStreamParser$1.tryNext(PDFStreamParser.java:186)\n\tat org.apache.pdfbox.pdfparser.PDFStreamParser$1.hasNext(PDFStreamParser.java:193)\n\tat org.apache.pdfbox.util.PDFStreamEngine.processSubStream(PDFStreamEngine.java:255)\n\tat org.apache.pdfbox.util.PDFStreamEngine.processSubStream(PDFStreamEngine.java:235)\n\tat org.apache.pdfbox.util.PDFStreamEngine.processStream(PDFStreamEngine.java:215)\n\tat org.apache.pdfbox.util.PDFTextStripper.processPage(PDFTextStripper.java:456)\n\tat org.apache.pdfbox.util.PDFTextStripper.processPages(PDFTextStripper.java:381)\n\tat org.apache.pdfbox.util.PDFTextStripper.writeText(PDFTextStripper.java:340)\n\tat org.apache.tika.parser.pdf.PDF
2XHTML.process(PDF2XHTML.java:106)\n\tat org.apache.tika.parser.pdf.PDFParser.parse(PDFParser.java:148)\n\tat org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:247)\n\tat org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:247)\n\tat org.apache.tika.parser.AutoDetectParser.parse(AutoDetectParser.java:120)\n\tat org.apache.tika.parser.RecursiveParserWrapper.parse(RecursiveParserWrapper.java:130)\n\tat org.apache.tika.batch.FileResourceConsumer.parse(FileResourceConsumer.java:410)\n\tat org.apache.tika.batch.fs.RecursiveParserWrapperFSConsumer.processFileResource(RecursiveParserWrapperFSConsumer.java:106)\n\tat org.apache.tika.batch.FileResourceConsumer._processFileResource(FileResourceConsumer.java:182)\n\tat org.apache.tika.batch.FileResourceConsumer.call(FileResourceConsumer.java:115)\n\tat org.apache.tika.batch.FileResourceConsumer.call(FileResourceConsumer.java:49)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:262)\n\tat java.util.concurr
ent.Executors$RunnableAdapter.call(Executors.java:471)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:262)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\nCaused by: java.io.IOException: Value is not an integer: 8546736428538085463808\n\tat org.apache.pdfbox.cos.COSNumber.get(COSNumber.java:104)\n\tat org.apache.pdfbox.pdfparser.PDFStreamParser.parseNextToken(PDFStreamParser.java:350)\n\tat org.apache.pdfbox.pdfparser.PDFStreamParser.access$000(PDFStreamParser.java:46)\n\tat org.apache.pdfbox.pdfparser.PDFStreamParser$1.tryNext(PDFStreamParser.java:181)\n\t... 24 more\n","access_permission:assemble_document":"true","access_permission:can_modify":"true","access_permission:can_print":"true","access_permission:can_print_degraded":"true","access_permission:extract_content":"true","access_permission:extract_for
_accessibility":"true","access_permission:fill_in_form":"true","access_permission:modify_annotations":"true", "resourceName":"file8_IOEx.pdf","tika:file_ext":"pdf","tika_batch_fs:relative_path":"file8_IOEx.pdf"}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/raw_input/file1.pdf
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/raw_input/file1.pdf b/tika-eval/src/test/resources/test-dirs/raw_input/file1.pdf
new file mode 100644
index 0000000..ef9ddba
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/raw_input/file1.pdf
@@ -0,0 +1,13 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+dummy source file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/raw_input/file11_oom.txt
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/raw_input/file11_oom.txt b/tika-eval/src/test/resources/test-dirs/raw_input/file11_oom.txt
new file mode 100644
index 0000000..d3cf2f9
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/raw_input/file11_oom.txt
@@ -0,0 +1,2 @@
+dummy
+dummy
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/raw_input/file2_attachANotB.doc
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/raw_input/file2_attachANotB.doc b/tika-eval/src/test/resources/test-dirs/raw_input/file2_attachANotB.doc
new file mode 100644
index 0000000..ef9ddba
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/raw_input/file2_attachANotB.doc
@@ -0,0 +1,13 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+dummy source file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/raw_input/file3_attachBNotA.doc
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/raw_input/file3_attachBNotA.doc b/tika-eval/src/test/resources/test-dirs/raw_input/file3_attachBNotA.doc
new file mode 100644
index 0000000..ef9ddba
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/raw_input/file3_attachBNotA.doc
@@ -0,0 +1,13 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+dummy source file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/raw_input/file4_emptyB.pdf
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/raw_input/file4_emptyB.pdf b/tika-eval/src/test/resources/test-dirs/raw_input/file4_emptyB.pdf
new file mode 100644
index 0000000..ef9ddba
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/raw_input/file4_emptyB.pdf
@@ -0,0 +1,13 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+dummy source file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/raw_input/file5_emptyA.pdf
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/raw_input/file5_emptyA.pdf b/tika-eval/src/test/resources/test-dirs/raw_input/file5_emptyA.pdf
new file mode 100644
index 0000000..ef9ddba
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/raw_input/file5_emptyA.pdf
@@ -0,0 +1,13 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+dummy source file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/raw_input/file6_accessEx.pdf
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/raw_input/file6_accessEx.pdf b/tika-eval/src/test/resources/test-dirs/raw_input/file6_accessEx.pdf
new file mode 100644
index 0000000..ef9ddba
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/raw_input/file6_accessEx.pdf
@@ -0,0 +1,13 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+dummy source file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/raw_input/file7_badJson.pdf
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/raw_input/file7_badJson.pdf b/tika-eval/src/test/resources/test-dirs/raw_input/file7_badJson.pdf
new file mode 100644
index 0000000..ef9ddba
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/raw_input/file7_badJson.pdf
@@ -0,0 +1,13 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+dummy source file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/raw_input/file8_IOEx.pdf
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/raw_input/file8_IOEx.pdf b/tika-eval/src/test/resources/test-dirs/raw_input/file8_IOEx.pdf
new file mode 100644
index 0000000..ef9ddba
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/raw_input/file8_IOEx.pdf
@@ -0,0 +1,13 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+dummy source file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/raw_input/file9_noextract.txt
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/raw_input/file9_noextract.txt b/tika-eval/src/test/resources/test-dirs/raw_input/file9_noextract.txt
new file mode 100644
index 0000000..5c3118d
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/raw_input/file9_noextract.txt
@@ -0,0 +1 @@
+dummy file
[2/6] tika git commit: TIKA-1332 initial commit of tika-eval. More
work remains.
Posted by ta...@apache.org.
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/java/org/apache/tika/eval/ComparerBatchTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/ComparerBatchTest.java b/tika-eval/src/test/java/org/apache/tika/eval/ComparerBatchTest.java
new file mode 100644
index 0000000..0d925cf
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/ComparerBatchTest.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.tika.batch.fs.FSBatchTestBase;
+import org.apache.tika.eval.db.Cols;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore("need to fix tika-batch tests to make this work")
+public class ComparerBatchTest extends FSBatchTestBase {
+
+ public final static String COMPARER_PROCESS_CLASS = "org.apache.tika.batch.fs.FSBatchProcessCLI";
+
+ private static Path dbDir;
+ private static Connection conn;
+
+ private final static String compJoinCont = "";
+ /*ExtractComparer.COMPARISONS_TABLE+" cmp " +
+ "join "+ExtractComparer.CONTAINERS_TABLE + " cnt "+
+ "on cmp."+AbstractProfiler.CONTAINER_HEADERS.CONTAINER_ID+
+ " = cnt."+AbstractProfiler.CONTAINER_HEADERS.CONTAINER_ID;*/
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+
+ File inputRoot = new File(ComparerBatchTest.class.getResource("/test-dirs").toURI());
+ dbDir = Files.createTempDirectory(inputRoot.toPath(), "tika-test-db-dir-");
+ Map<String, String> args = new HashMap<>();
+ Path db = FileSystems.getDefault().getPath(dbDir.toString(), "comparisons_test");
+ args.put("-db", db.toString());
+
+ //for debugging, you can use this to select only one file pair to load
+ //args.put("-includeFilePat", "file8.*");
+/*
+ BatchProcessTestExecutor ex = new BatchProcessTestExecutor(COMPARER_PROCESS_CLASS, args,
+ "/tika-batch-comparison-eval-config.xml");
+ StreamStrings streamStrings = ex.execute();
+ System.out.println(streamStrings.getErrString());
+ System.out.println(streamStrings.getOutString());
+ H2Util dbUtil = new H2Util(db);
+ conn = dbUtil.getConnection();*/
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+
+ conn.close();
+
+ FileUtils.deleteDirectory(dbDir.toFile());
+ }
+
+
+ @Test
+ public void testSimpleDBWriteAndRead() throws Exception {
+ Set<String> set = new HashSet<>();
+ //filenames
+ List<String> list = getColStrings(Cols.FILE_NAME.name(),
+ ExtractComparer.PROFILES_A.getName(), "");
+ assertEquals(7, list.size());
+ assertTrue(list.contains("file1.pdf"));
+
+ //container ids in comparisons table
+ list = getColStrings(Cols.CONTAINER_ID.name(),
+ ExtractComparer.COMPARISON_CONTAINERS.getName(),"");
+ assertEquals(10, list.size());
+ set.clear(); set.addAll(list);
+ assertEquals(10, set.size());
+/*
+ //ids in comparisons table
+ list = getColStrings(AbstractProfiler.HEADERS.ID.name(),
+ compTable,"");
+ assertEquals(9, list.size());
+ set.clear(); set.addAll(list);
+ assertEquals(9, set.size());*/
+ }
+
+
+
+ /*
+ @Test
+ public void testFile1PDFRow() throws Exception {
+ String where = fp+"='file1.pdf'";
+ Map<String, String> data = getRow(compJoinCont, where);
+ String result = data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_UNIQUE_TOKEN_DIFFS + "_A");
+ assertTrue(result.startsWith("over: 1"));
+
+ result = data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_UNIQUE_TOKEN_DIFFS + "_B");
+ assertTrue(result.startsWith("aardvark: 3 | bear: 2"));
+
+
+ assertEquals("aardvark: 3 | bear: 2",
+ data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_MORE_IN_B.toString()));
+ assertEquals("fox: 2 | lazy: 1 | over: 1",
+ data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_MORE_IN_A.toString()));
+ assertEquals("12", data.get(ExtractComparer.HEADERS.NUM_TOKENS+"_A"));
+ assertEquals("13", data.get(ExtractComparer.HEADERS.NUM_TOKENS+"_B"));
+ assertEquals("8", data.get(ExtractComparer.HEADERS.NUM_UNIQUE_TOKENS+"_A"));
+ assertEquals("9", data.get(ExtractComparer.HEADERS.NUM_UNIQUE_TOKENS+"_B"));
+
+ assertEquals(ExtractComparer.COMPARISON_HEADERS.OVERLAP.name(),
+ 0.64f, Float.parseFloat(data.get("OVERLAP")), 0.0001f);
+
+ assertEquals(ExtractComparer.COMPARISON_HEADERS.DICE_COEFFICIENT.name(),
+ 0.8235294f, Float.parseFloat(data.get("DICE_COEFFICIENT")), 0.0001f);
+
+ assertEquals(ExtractComparer.HEADERS.TOKEN_LENGTH_MEAN+"_A", 3.83333d,
+ Double.parseDouble(
+ data.get(ExtractComparer.HEADERS.TOKEN_LENGTH_MEAN+"_A")), 0.0001d);
+
+ assertEquals(ExtractComparer.HEADERS.TOKEN_LENGTH_MEAN+"_B", 4.923d,
+ Double.parseDouble(
+ data.get(ExtractComparer.HEADERS.TOKEN_LENGTH_MEAN+"_B")), 0.0001d);
+
+ assertEquals(ExtractComparer.HEADERS.TOKEN_LENGTH_STD_DEV+"_A", 1.0298d,
+ Double.parseDouble(
+ data.get(ExtractComparer.HEADERS.TOKEN_LENGTH_STD_DEV+"_A")), 0.0001d);
+
+ assertEquals(ExtractComparer.HEADERS.TOKEN_LENGTH_STD_DEV+"_B", 1.9774d,
+ Double.parseDouble(data.get(ExtractComparer.HEADERS.TOKEN_LENGTH_STD_DEV+"_B")), 0.0001d);
+
+ assertEquals(ExtractComparer.HEADERS.TOKEN_LENGTH_SUM+"_A", 46,
+ Integer.parseInt(
+ data.get(ExtractComparer.HEADERS.TOKEN_LENGTH_SUM+"_A")));
+
+ assertEquals(ExtractComparer.HEADERS.TOKEN_LENGTH_SUM+"_B", 64,
+ Integer.parseInt(data.get(ExtractComparer.HEADERS.TOKEN_LENGTH_SUM+"_B")));
+
+ assertEquals("TOKEN_ENTROPY_RATE_A", 0.237949,
+ Double.parseDouble(data.get("TOKEN_ENTROPY_RATE_A")), 0.0001d);
+
+ assertEquals("TOKEN_ENTROPY_RATE_B", 0.232845,
+ Double.parseDouble(data.get("TOKEN_ENTROPY_RATE_B")), 0.0001d);
+
+ }
+
+
+ @Test
+ public void testEmpty() throws Exception {
+ String where = fp+"='file4_emptyB.pdf'";
+ Map<String, String> data = getRow(contTable, where);
+ assertNull(data.get(AbstractProfiler.CONTAINER_HEADERS.JSON_EX +
+ ExtractComparer.aExtension));
+ assertTrue(data.get(AbstractProfiler.CONTAINER_HEADERS.JSON_EX +
+ ExtractComparer.bExtension).equals(AbstractProfiler.JSON_PARSE_EXCEPTION));
+
+ where = fp+"='file5_emptyA.pdf'";
+ data = getRow(contTable, where);
+ assertNull(data.get(AbstractProfiler.CONTAINER_HEADERS.JSON_EX +
+ ExtractComparer.bExtension));
+ assertTrue(data.get(AbstractProfiler.CONTAINER_HEADERS.JSON_EX+
+ ExtractComparer.aExtension).equals(AbstractProfiler.JSON_PARSE_EXCEPTION));
+ }
+
+ @Test
+ public void testMissingAttachment() throws Exception {
+ String where = fp+"='file2_attachANotB.doc' and "+AbstractProfiler.HEADERS.EMBEDDED_FILE_PATH+
+ "='inner.txt'";
+ Map<String, String> data = getRow(compJoinCont, where);
+ assertContains("attachment: 1", data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_MORE_IN_A.name()));
+ assertNotContained("fox", data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_MORE_IN_B.name()));
+ assertNull(data.get(ExtractComparer.HEADERS.TOP_N_TOKENS +
+ ExtractComparer.bExtension));
+ assertNotContained("fox", data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_UNIQUE_TOKEN_DIFFS +
+ ExtractComparer.bExtension));
+
+ assertEquals("3", data.get("NUM_METADATA_VALUES_A"));
+ assertNull(data.get("DIFF_NUM_ATTACHMENTS"));
+ assertNull(data.get("NUM_METADATA_VALUES_B"));
+ assertEquals("0", data.get("NUM_UNIQUE_TOKENS_B"));
+ assertNull(data.get("TOKEN_ENTROPY_RATE_B"));
+ assertNull(data.get("NUM_EN_STOPS_TOP_N_B"));
+
+ where = fp+"='file3_attachBNotA.doc' and "+AbstractProfiler.HEADERS.EMBEDDED_FILE_PATH+
+ "='inner.txt'";
+ data = getRow(compJoinCont, where);
+ assertContains("attachment: 1", data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_MORE_IN_B.name()));
+ assertNotContained("fox", data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_MORE_IN_A.name()));
+ assertNull(data.get(ExtractComparer.HEADERS.TOP_N_TOKENS +
+ ExtractComparer.aExtension));
+ assertNotContained("fox", data.get(ExtractComparer.COMPARISON_HEADERS.TOP_10_UNIQUE_TOKEN_DIFFS +
+ ExtractComparer.aExtension));
+
+ assertEquals("3", data.get("NUM_METADATA_VALUES_B"));
+ assertNull(data.get("DIFF_NUM_ATTACHMENTS"));
+ assertNull(data.get("NUM_METADATA_VALUES_A"));
+ assertEquals("0", data.get("NUM_UNIQUE_TOKENS_A"));
+ assertNull(data.get("TOKEN_ENTROPY_RATE_A"));
+ assertNull(data.get("NUM_EN_STOPS_TOP_N_A"));
+
+ }
+
+ @Test
+ public void testBothBadJson() throws Exception {
+ debugDumpAll(contTable);
+ String where = fp+"='file7_badJson.pdf'";
+ Map<String, String> data = getRow(contTable, where);
+ assertEquals(AbstractProfiler.JSON_PARSE_EXCEPTION,
+ data.get(AbstractProfiler.CONTAINER_HEADERS.JSON_EX+ ExtractComparer.aExtension));
+ assertEquals(AbstractProfiler.JSON_PARSE_EXCEPTION,
+ data.get(AbstractProfiler.CONTAINER_HEADERS.JSON_EX+ ExtractComparer.bExtension));
+ assertEquals("file7_badJson.pdf",
+ data.get(AbstractProfiler.CONTAINER_HEADERS.FILE_PATH.name()));
+ assertEquals("61", data.get("JSON_FILE_LENGTH_A"));
+ assertEquals("0", data.get("JSON_FILE_LENGTH_B"));
+ assertEquals("pdf", data.get(AbstractProfiler.CONTAINER_HEADERS.FILE_EXTENSION.name()));
+
+ }
+
+ @Test
+ public void testAccessPermissionException() throws Exception {
+ String sql = "select "+
+ AbstractProfiler.EXCEPTION_HEADERS.ACCESS_PERMISSION_EXCEPTION.name() +
+ " from " + AbstractProfiler.EXCEPTIONS_TABLE+"_A exA "+
+ " join " + ExtractComparer.COMPARISONS_TABLE + " cmp on cmp.ID=exA.ID "+
+ " join " + ExtractComparer.CONTAINERS_TABLE + " cont on cmp.CONTAINER_ID=cont.CONTAINER_ID "+
+ " where "+fp+"='file6_accessEx.pdf'";
+ Statement st = conn.createStatement();
+ ResultSet rs = st.executeQuery(sql);
+ List<String> results = new ArrayList<String>();
+ while (rs.next()) {
+ results.add(rs.getString(1));
+ }
+ assertEquals(1, results.size());
+ assertEquals("TRUE", results.get(0));
+
+ sql = "select "+
+ AbstractProfiler.EXCEPTION_HEADERS.ACCESS_PERMISSION_EXCEPTION.name() +
+ " from " + AbstractProfiler.EXCEPTIONS_TABLE+"_B exB "+
+ " join " + ExtractComparer.COMPARISONS_TABLE + " cmp on cmp.ID=exB.ID "+
+ " join " + ExtractComparer.CONTAINERS_TABLE + " cont on cmp.CONTAINER_ID=cont.CONTAINER_ID "+
+ " where "+fp+"='file6_accessEx.pdf'";
+ st = conn.createStatement();
+ rs = st.executeQuery(sql);
+ results = new ArrayList<String>();
+ while (rs.next()) {
+ results.add(rs.getString(1));
+ }
+ assertEquals(1, results.size());
+ assertEquals("TRUE", results.get(0));
+
+ }
+
+ @Test
+ public void testContainerException() throws Exception {
+ String sql = "select * "+
+ " from " + AbstractProfiler.EXCEPTIONS_TABLE+"_A exA "+
+ " join " + ExtractComparer.COMPARISONS_TABLE + " cmp on cmp.ID=exA.ID "+
+ " join " + ExtractComparer.CONTAINERS_TABLE + " cont on cmp.CONTAINER_ID=cont.CONTAINER_ID "+
+ "where "+fp+"='file8_IOEx.pdf'";
+ Statement st = conn.createStatement();
+ ResultSet rs = st.executeQuery(sql);
+
+ Map<String, String> data = new HashMap<String,String>();
+ ResultSetMetaData rsM = rs.getMetaData();
+ while (rs.next()) {
+ for (int i = 1; i <= rsM.getColumnCount(); i++)
+ data.put(rsM.getColumnName(i), rs.getString(i));
+ }
+
+ String sortStack = data.get(AbstractProfiler.EXCEPTION_HEADERS.SORT_STACK_TRACE.name());
+ sortStack = sortStack.replaceAll("[\r\n]", "<N>");
+ assertTrue(sortStack.startsWith("java.lang.RuntimeException<N>"));
+
+ String fullStack = data.get(AbstractProfiler.EXCEPTION_HEADERS.ORIG_STACK_TRACE.name());
+ assertTrue(
+ fullStack.startsWith("java.lang.RuntimeException: java.io.IOException: Value is not an integer"));
+ }
+
+ private void debugDumpAll(String table) throws Exception {
+ Statement st = conn.createStatement();
+ String sql = "select * from "+table;
+ ResultSet rs = st.executeQuery(sql);
+ ResultSetMetaData m = rs.getMetaData();
+ for (int i = 1; i <= m.getColumnCount(); i++) {
+ System.out.print(m.getColumnName(i) + ", ");
+ }
+ System.out.println("\n");
+ while (rs.next()) {
+ for (int i = 1; i <= m.getColumnCount(); i++) {
+ System.out.print(rs.getString(i)+", ");
+ }
+ System.out.println("\n");
+ }
+ st.close();
+
+ }
+ */
+ private void debugShowColumns(String table) throws Exception {
+ Statement st = conn.createStatement();
+ String sql = "select * from "+table;
+ ResultSet rs = st.executeQuery(sql);
+ ResultSetMetaData m = rs.getMetaData();
+ for (int i = 1; i <= m.getColumnCount(); i++) {
+ System.out.println(i+" : "+m.getColumnName(i));
+ }
+ st.close();
+ }
+
+ //return the string value for one cell
+ private String getString(String colName, String table, String where) throws Exception {
+ List<String> results = getColStrings(colName, table, where);
+ if (results.size() > 1) {
+ throw new RuntimeException("more than one result");
+ } else if (results.size() == 0) {
+ throw new RuntimeException("no results");
+ }
+
+ return results.get(0);
+ }
+
+
+ private Map<String, String> getRow(String table, String where) throws Exception {
+ String sql = getSql("*", table, where);
+ Map<String, String> results = new HashMap<String, String>();
+ Statement st = null;
+
+ try {
+ st = conn.createStatement();
+ ResultSet rs = st.executeQuery(sql);
+ ResultSetMetaData m = rs.getMetaData();
+ int rows = 0;
+ while (rs.next()) {
+ if (rows > 0) {
+ throw new RuntimeException("returned more than one row!");
+ }
+ for (int i = 1; i <= m.getColumnCount(); i++) {
+ results.put(m.getColumnName(i), rs.getString(i));
+ }
+ rows++;
+ }
+ } finally {
+ if (st != null) {
+ st.close();
+ }
+ }
+ return results;
+
+ }
+
+ //return the string representations of the column values for one column
+ //as a list of strings
+ private List<String> getColStrings(String colName) throws Exception {
+ return getColStrings(colName, ExtractComparer.CONTENT_COMPARISONS.getName(), null);
+ }
+
+ private List<String> getColStrings(String colName, String table, String where) throws Exception {
+ String sql = getSql(colName, table, where);
+ List<String> results = new ArrayList<>();
+ Statement st = null;
+ try {
+ st = conn.createStatement();
+ System.out.println("SQL: "+sql);
+ ResultSet rs = st.executeQuery(sql);
+ while (rs.next()) {
+ results.add(rs.getString(1));
+ }
+ } finally {
+ if (st != null) {
+ st.close();
+ }
+ }
+ return results;
+ }
+
+ private String getSql(String colName, String table, String where) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("select ").append(colName).append(" from ").append(table);
+ if (where != null && ! where.equals("")) {
+ sb.append(" where ").append(where);
+ }
+ return sb.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java b/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
new file mode 100644
index 0000000..c3f0a7e
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/ProfilerBatchTest.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.tika.eval.db.Cols;
+import org.apache.tika.eval.db.H2Util;
+import org.apache.tika.eval.db.TableInfo;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+@Ignore
+public class ProfilerBatchTest {
+
+ public final static String COMPARER_PROCESS_CLASS = "org.apache.tika.batch.fs.FSBatchProcessCLI";
+
+ private static Path dbDir;
+ private static Connection conn;
+
+ private final static String profileTable = ExtractProfiler.PROFILE_TABLE.getName();
+ private final static String exTable = ExtractProfiler.EXCEPTION_TABLE.getName();
+ private final static String fpCol = Cols.FILE_PATH.name();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+
+ Path inputRoot = Paths.get(new ComparerBatchTest().getClass().getResource("/test-dirs/extractsA").toURI());
+ dbDir = Files.createTempDirectory(inputRoot, "tika-test-db-dir-");
+ Map<String, String> args = new HashMap<>();
+ Path db = dbDir.resolve("profiler_test");
+ args.put("-db", db.toString());
+
+ //for debugging, you can use this to select only one file pair to load
+ //args.put("-includeFilePat", "file8.*");
+
+ /* BatchProcessTestExecutor ex = new BatchProcessTestExecutor(COMPARER_PROCESS_CLASS, args,
+ "/single-file-profiler-crawl-input-config.xml");
+ StreamStrings streamStrings = ex.execute();
+ System.out.println(streamStrings.getErrString());
+ System.out.println(streamStrings.getOutString());*/
+ H2Util dbUtil = new H2Util(db);
+ conn = dbUtil.getConnection(true);
+ }
+ @AfterClass
+ public static void tearDown() throws IOException {
+
+ try{
+ conn.close();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+
+
+ DirectoryStream<Path> dStream = Files.newDirectoryStream(dbDir);
+ Iterator<Path> it = dStream.iterator();
+ while (it.hasNext()) {
+ Path p = it.next();
+ Files.delete(p);
+ }
+ dStream.close();
+ Files.delete(dbDir);
+ }
+
+ @Test
+ public void testSimpleDBWriteAndRead() throws Exception {
+
+ Statement st = null;
+ List<String> fNameList = new ArrayList<>();
+ try {
+ String sql = "select * from "+ ExtractProfiler.CONTAINER_TABLE.getName();
+ st = conn.createStatement();
+ ResultSet rs = st.executeQuery(sql);
+ while (rs.next()) {
+ String fileName = rs.getString(Cols.FILE_PATH.name());
+ fNameList.add(fileName);
+ }
+ } finally {
+ if (st != null) {
+ st.close();
+ }
+ }
+ debugTable(ExtractProfiler.CONTAINER_TABLE);
+ debugTable(ExtractProfiler.PROFILE_TABLE);
+ debugTable(ExtractProfiler.CONTENTS_TABLE);
+ debugTable(ExtractProfiler.EXCEPTION_TABLE);
+ debugTable(ExtractProfiler.ERROR_TABLE);
+ assertEquals(10, fNameList.size());
+ assertTrue("file1.pdf", fNameList.contains("file1.pdf"));
+ assertTrue("file2_attachANotB.doc", fNameList.contains("file2_attachANotB.doc"));
+ assertTrue("file3_attachBNotA.doc", fNameList.contains("file3_attachBNotA.doc"));
+ assertTrue("file4_emptyB.pdf", fNameList.contains("file4_emptyB.pdf"));
+ assertTrue("file7_badJson.pdf", fNameList.contains("file7_badJson.pdf"));
+ }
+
+ @Test
+ public void testExtractErrors() throws Exception {
+ String sql = "select EXTRACT_ERROR_TYPE_ID from errors e" +
+ " join containers c on c.container_id = e.container_id "+
+ " where c.file_path='file9_noextract.txt'";
+
+ assertEquals("missing extract: file9_noextract.txt", "0",
+ getSingleResult(sql));
+ debugTable(ExtractProfiler.CONTAINER_TABLE);
+ debugTable(ExtractProfiler.PROFILE_TABLE);
+ debugTable(ExtractProfiler.CONTENTS_TABLE);
+ debugTable(ExtractProfiler.EXCEPTION_TABLE);
+ debugTable(ExtractProfiler.ERROR_TABLE);
+
+ sql = "select EXTRACT_ERROR_TYPE_ID from errors e" +
+ " join containers c on c.container_id = e.container_id "+
+ " where c.file_path='file5_emptyA.pdf'";
+ assertEquals("empty extract: file5_emptyA.pdf", "1",
+ getSingleResult(sql));
+
+ sql = "select EXTRACT_ERROR_TYPE_ID from errors e" +
+ " join containers c on c.container_id = e.container_id "+
+ " where c.file_path='file7_badJson.pdf'";
+ assertEquals("extract error:file7_badJson.pdf", "2",
+ getSingleResult(sql));
+
+ }
+
+ @Test
+ public void testParseErrors() throws Exception {
+ debugTable(ExtractProfiler.ERROR_TABLE);
+ String sql = "select file_path from errors where container_id is null";
+ assertEquals("file10_permahang.txt",
+ getSingleResult(sql));
+
+ sql = "select extract_error_type_id from errors where file_path='file11_oom.txt'";
+ assertEquals(Integer.toString(AbstractProfiler.
+ EXTRACT_ERROR_TYPE.
+ ZERO_BYTE_EXTRACT_FILE.ordinal()),
+ getSingleResult(sql));
+
+ sql = "select parse_error_type_id from errors where file_path='file11_oom.txt'";
+ assertEquals(Integer.toString(AbstractProfiler.
+ PARSE_ERROR_TYPE.
+ OOM.ordinal()),
+ getSingleResult(sql));
+
+ }
+
+ @Test
+ public void testParseExceptions() throws Exception {
+ debugTable(ExtractProfiler.EXCEPTION_TABLE);
+ }
+
+ private String getSingleResult(String sql) throws Exception {
+ Statement st = null;
+ st = conn.createStatement();
+ ResultSet rs = st.executeQuery(sql);
+ int hits = 0;
+ String val = "";
+ while (rs.next()) {
+ assertEquals("must have only one column in result",
+ 1, rs.getMetaData().getColumnCount());
+ val = rs.getString(1);
+ hits++;
+ }
+ assertEquals("must have only one hit", 1, hits);
+ return val;
+ }
+
+ //TODO: lots more testing!
+
+ public void debugTable(TableInfo table) throws Exception {
+ Statement st = null;
+ try {
+ String sql = "select * from "+table.getName();
+ st = conn.createStatement();
+ ResultSet rs = st.executeQuery(sql);
+ int colCount = rs.getMetaData().getColumnCount();
+ System.out.println("TABLE: "+table.getName());
+ for (int i = 1; i <= colCount; i++) {
+ if (i > 1) {
+ System.out.print(" | ");
+ }
+ System.out.print(rs.getMetaData().getColumnName(i));
+ }
+ System.out.println("");
+ int rowCount = 0;
+ while (rs.next()) {
+ for (int i = 1; i <= colCount; i++) {
+ if (i > 1) {
+ System.out.print(" | ");
+ }
+ System.out.print(rs.getString(i));
+ rowCount++;
+ }
+ System.out.println("");
+ }
+ if (rowCount == 0) {
+ System.out.println(table.getName() + " was empty");
+ }
+ } finally {
+ if (st != null) {
+ st.close();
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java b/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
new file mode 100644
index 0000000..72e8008
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval;
+
+import static org.apache.tika.eval.AbstractProfiler.EXCEPTION_TYPE;
+import static org.apache.tika.eval.AbstractProfiler.EXTRACT_ERROR_TYPE;
+import static org.apache.tika.eval.AbstractProfiler.getContent;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.tika.MockDBWriter;
+import org.apache.tika.TikaTest;
+import org.apache.tika.eval.db.Cols;
+import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.ExtractReader;
+import org.apache.tika.eval.util.LanguageIDWrapper;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.RecursiveParserWrapper;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+//These tests ensure that the comparer is extracting the right information
+//into a Map<String,String>. A full integration test
+//should also ensure that the elements are properly being written to the db
+
+public class SimpleComparerTest extends TikaTest {
+
+ private ExtractComparer comparer = null;
+ private MockDBWriter writer = null;
+
+ @Before
+ public void setUp() throws Exception {
+ writer = new MockDBWriter();
+ comparer = new ExtractComparer(null, null,
+ Paths.get("extractsA"), Paths.get("extractsB"),
+ writer, -1, -1,
+ ExtractReader.ALTER_METADATA_LIST.AS_IS);
+ AbstractProfiler.loadCommonTokens(this.getResourceAsFile("/commontokens").toPath());
+ LanguageIDWrapper.loadBuiltInModels();
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+ EvalFilePaths fpsA = new EvalFilePaths(
+ Paths.get("file1.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsA/file1.pdf.json").toPath()
+ );
+ EvalFilePaths fpsB = new EvalFilePaths(
+ Paths.get("file1.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsB/file1.pdf.json").toPath());
+
+ comparer.compareFiles(fpsA, fpsB);
+
+ List<Map<Cols, String>> tableInfos = writer.getTable(ExtractComparer.CONTENT_COMPARISONS);
+ Map<Cols, String> row = tableInfos.get(0);
+ assertEquals("0", row.get(Cols.ID));
+ assertTrue(
+ row.get(Cols.TOP_10_UNIQUE_TOKEN_DIFFS_A)
+ .startsWith("1,200: 1 | 120000: 1 | over: 1"));
+
+ tableInfos = writer.getTable(ExtractComparer.CONTENTS_TABLE_A);
+ row = tableInfos.get(0);
+ assertEquals("0", row.get(Cols.ID));
+ assertEquals("70", row.get(Cols.CONTENT_LENGTH));
+ assertEquals("10", row.get(Cols.NUM_UNIQUE_TOKENS));
+ assertEquals("14", row.get(Cols.NUM_TOKENS));
+ assertEquals("12", row.get(Cols.NUM_ALPHABETIC_TOKENS));
+ assertEquals("6", row.get(Cols.NUM_COMMON_TOKENS));
+ assertEquals("57", row.get(Cols.TOKEN_LENGTH_SUM));
+ assertEquals("en", row.get(Cols.COMMON_TOKENS_LANG));
+
+ tableInfos = writer.getTable(ExtractComparer.CONTENTS_TABLE_B);
+ row = tableInfos.get(0);
+ assertEquals("0", row.get(Cols.ID));
+ assertEquals("76", row.get(Cols.CONTENT_LENGTH));
+ assertEquals("9", row.get(Cols.NUM_UNIQUE_TOKENS));
+ assertEquals("13", row.get(Cols.NUM_TOKENS));
+ assertEquals("4", row.get(Cols.NUM_COMMON_TOKENS));
+ assertEquals("64", row.get(Cols.TOKEN_LENGTH_SUM));
+ assertEquals("en", row.get(Cols.COMMON_TOKENS_LANG));
+
+ tableInfos = writer.getTable(ExtractComparer.PROFILES_A);
+ row = tableInfos.get(0);
+ assertEquals("2", row.get(Cols.NUM_PAGES));
+
+ }
+
+ @Test
+ public void testBasicSpanish() throws Exception {
+ EvalFilePaths fpsA = new EvalFilePaths(
+ Paths.get("file1.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsA/file12_es.txt.json").toPath()
+ );
+ EvalFilePaths fpsB = new EvalFilePaths(
+ Paths.get("file1.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsB/file12_es.txt.json").toPath());
+
+ comparer.compareFiles(fpsA, fpsB);
+
+ List<Map<Cols, String>> tableInfos = writer.getTable(ExtractComparer.CONTENTS_TABLE_A);
+
+ Map<Cols, String> row = tableInfos.get(0);
+ assertEquals("133", row.get(Cols.CONTENT_LENGTH));
+ assertEquals("7", row.get(Cols.NUM_UNIQUE_TOKENS));
+ assertEquals("24", row.get(Cols.NUM_TOKENS));
+ assertEquals("3", row.get(Cols.NUM_COMMON_TOKENS));
+ assertEquals("108", row.get(Cols.TOKEN_LENGTH_SUM));
+ assertEquals("es", row.get(Cols.COMMON_TOKENS_LANG));
+ assertEquals("24", row.get(Cols.NUM_ALPHABETIC_TOKENS));
+
+ }
+
+
+ @Test
+ public void testEmpty() throws Exception {
+ EvalFilePaths fpsA = new EvalFilePaths(
+ Paths.get("file1.pdf"),
+ getResourceAsFile("/test-dirs/extractsA/file1.pdf.json").toPath()
+ );
+ EvalFilePaths fpsB = new EvalFilePaths(
+ Paths.get("file1.pdf"),
+ getResourceAsFile("/test-dirs/extractsB/file4_emptyB.pdf.json").toPath()
+ );
+ comparer.compareFiles(fpsA, fpsB);
+ List<Map<Cols, String>> table = writer.getTable(ExtractComparer.ERROR_TABLE_B);
+ Map<Cols, String> row = table.get(0);
+ //debugPrintRow(row);
+ assertEquals(Integer.toString(EXTRACT_ERROR_TYPE.ZERO_BYTE_EXTRACT_FILE.ordinal()),
+ row.get(Cols.EXTRACT_ERROR_TYPE_ID));
+ }
+
+
+ @Test
+ public void testGetContent() throws Exception {
+ Metadata m = new Metadata();
+ m.add(RecursiveParserWrapper.TIKA_CONTENT, "0123456789");
+
+ String content = getContent(m, 10);
+ assertEquals(10, content.length());
+
+ content = getContent(m, 4);
+ assertEquals(4, content.length());
+
+ //test Metadata with no content
+ content = getContent(new Metadata(), 10);
+ assertEquals(0, content.length());
+
+ //test null Metadata
+ content = getContent(null, 10);
+ assertEquals(0, content.length());
+ }
+
+ @Test
+ public void testAccessException() throws Exception {
+ EvalFilePaths fpsA = new EvalFilePaths(
+ Paths.get("file6_accessEx.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsA/file6_accessEx.pdf.json").toPath()
+ );
+ EvalFilePaths fpsB = new EvalFilePaths(
+ Paths.get("file6_accessEx.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsB/file6_accessEx.pdf.json").toPath()
+ );
+ comparer.compareFiles(fpsA, fpsB);
+ for (TableInfo t : new TableInfo[]{ExtractComparer.EXCEPTION_TABLE_A, ExtractComparer.EXCEPTION_TABLE_B}) {
+ List<Map<Cols, String>> table = writer.getTable(t);
+
+ Map<Cols, String> rowA = table.get(0);
+ //debugPrintRow(rowA);
+ assertEquals(Integer.toString(EXCEPTION_TYPE.ACCESS_PERMISSION.ordinal()),
+ rowA.get(Cols.PARSE_EXCEPTION_TYPE_ID));
+ assertNull(rowA.get(Cols.ORIG_STACK_TRACE));
+ assertNull(rowA.get(Cols.SORT_STACK_TRACE));
+ }
+ }
+
+
+ @Test
+ public void testAttachmentCounts() {
+ List<Metadata> list = new ArrayList<>();
+ Metadata m0 = new Metadata();
+ m0.set(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH, "dir1/dir2/file.zip");//bad data should be ignored
+ //in the first metadata object
+ list.add(m0);
+ Metadata m1 = new Metadata();
+ m1.set(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH, "/f1.docx/f2.zip/text1.txt");
+ list.add(m1);
+ Metadata m2 = new Metadata();
+ m2.set(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH, "/f1.docx/f2.zip/text2.txt");
+ list.add(m2);
+ Metadata m3 = new Metadata();
+ m3.set(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH, "/f1.docx/f2.zip");
+ list.add(m3);
+ Metadata m4 = new Metadata();
+ m4.set(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH, "/f1.docx");
+ list.add(m4);
+ Metadata m5 = new Metadata();
+ m5.set(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH, "/f1.docx/text3.txt");
+ list.add(m5);
+
+ List<Integer> counts = AbstractProfiler.countAttachments(list);
+
+ List<Integer> expected = new ArrayList<>();
+ expected.add(5);
+ expected.add(0);
+ expected.add(0);
+ expected.add(2);
+ expected.add(4);
+ expected.add(0);
+ assertEquals(expected, counts);
+ }
+
+
+ @Test
+ @Ignore
+ public void testDebug() throws Exception {
+ Path commonTokens = Paths.get(getResourceAsFile("/common_tokens_short.txt").toURI());
+ AbstractProfiler.loadCommonTokens(commonTokens);
+ EvalFilePaths fpsA = new EvalFilePaths(
+ Paths.get("file1.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsA/file1.pdf.json").toPath()
+ );
+ EvalFilePaths fpsB = new EvalFilePaths(
+ Paths.get("file1.pdf.json"),
+ getResourceAsFile("/test-dirs/extractsB/file1.pdf.json").toPath()
+ );
+ comparer.compareFiles(fpsA, fpsB);
+ for (TableInfo t : new TableInfo[]{
+ ExtractComparer.COMPARISON_CONTAINERS,
+ ExtractComparer.ERROR_TABLE_A,
+ ExtractComparer.ERROR_TABLE_B,
+ ExtractComparer.EXCEPTION_TABLE_A,
+ ExtractComparer.EXCEPTION_TABLE_B,
+ ExtractComparer.PROFILES_A,
+ ExtractComparer.PROFILES_B,
+ ExtractComparer.CONTENTS_TABLE_A,
+ ExtractComparer.CONTENTS_TABLE_B,
+ ExtractComparer.CONTENT_COMPARISONS}) {
+ //debugPrintTable(t);
+ }
+ }
+
+ private void debugPrintTable(TableInfo tableInfo) {
+ List<Map<Cols, String>> table = writer.getTable(tableInfo);
+ if (table == null) {
+ return;
+ }
+ int i = 0;
+ System.out.println("TABLE: "+tableInfo.getName());
+ for (Map<Cols, String> row : table) {
+ SortedSet<Cols> keys = new TreeSet<Cols>(row.keySet());
+ for (Cols key : keys) {
+ System.out.println( i + " :: " + key + " : " + row.get(key));
+ }
+ i++;
+ }
+ System.out.println("");
+ }
+
+ private void debugPrintRow(Map<Cols, String> row) {
+ SortedSet<Cols> keys = new TreeSet<Cols>(row.keySet());
+ for (Cols key : keys) {
+ System.out.println(key + " : " + row.get(key));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java b/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java
new file mode 100644
index 0000000..c358149
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/TikaEvalCLITest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TikaEvalCLITest {
+
+ @Test
+ @Ignore("TODO: add real tests")
+ public void testBasic() throws Exception {
+ List<String> args = new ArrayList<>();
+ args.add("Profile");
+ args.add("-extractDir");
+ args.add("tika");
+ args.add("-db");
+ args.add("mydb");
+ args.add("-alterExtract");
+ args.add("first_only");
+ TikaEvalCLI.main(args.toArray(new String[args.size()]));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/java/org/apache/tika/eval/db/AbstractBufferTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/db/AbstractBufferTest.java b/tika-eval/src/test/java/org/apache/tika/eval/db/AbstractBufferTest.java
new file mode 100644
index 0000000..7b5c3cb
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/db/AbstractBufferTest.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.db;
+
+
+import static org.junit.Assert.assertEquals;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+
+public class AbstractBufferTest {
+
+
+ @Test(timeout = 30000)
+ public void runTest() throws InterruptedException, ExecutionException {
+ List<String> keys = new ArrayList<>();
+ Collections.addAll(keys, new String[]{
+ "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"});
+
+ int numGets = 1000;
+ int numTesters = 20;
+ AbstractDBBuffer b = new TestBuffer();
+
+
+ ExecutorService ex = Executors.newFixedThreadPool(numTesters);
+ CompletionService<MyTestResult> completionService =
+ new ExecutorCompletionService<>(
+ ex);
+ for (int i = 0; i < numTesters; i++) {
+ completionService.submit(new Tester(keys, b, numGets));
+ }
+
+ int results = 0;
+ Map<String, Integer> combined = new HashMap<>();
+ while (results < numTesters) {
+ Future<MyTestResult> futureResult =
+ completionService.poll(1, TimeUnit.SECONDS);
+ if (futureResult != null) {
+ results++;
+ assertEquals(keys.size(), futureResult.get().getMap().keySet().size());
+ for (Map.Entry<String, Integer> e : futureResult.get().getMap().entrySet()) {
+ if (!combined.containsKey(e.getKey())) {
+ combined.put(e.getKey(), e.getValue());
+ } else {
+ assertEquals(combined.get(e.getKey()), e.getValue());
+ }
+ }
+ }
+ }
+ assertEquals(keys.size(), b.getNumWrites());
+ }
+
+ private class Tester implements Callable<MyTestResult> {
+
+ private Random r = new Random();
+ private Map<String, Integer> m = new HashMap<>();
+ List<String> keys = new ArrayList<>();
+ private final AbstractDBBuffer dbBuffer;
+ private final int numGets;
+
+ private Tester(List<String> inputKeys, AbstractDBBuffer buffer, int numGets) {
+ keys.addAll(inputKeys);
+ dbBuffer = buffer;
+ this.numGets = numGets;
+ }
+
+ @Override
+ public MyTestResult call() throws Exception {
+
+
+ for (int i = 0; i < numGets; i++) {
+ int index = r.nextInt(keys.size());
+ String k = keys.get(index);
+ if (k == null) {
+ throw new RuntimeException("keys can't be null");
+ }
+ Integer expected = m.get(k);
+ Integer val = dbBuffer.getId(k);
+ if (val == null) {
+ throw new RuntimeException("Val can't be null!");
+ }
+ if (expected != null) {
+ assertEquals(expected, val);
+ }
+ m.put(k, val);
+ }
+
+ //now add the val for every key
+ //just in case the rand() process didn't hit
+ //all indices
+ for (String k : keys) {
+ Integer val = dbBuffer.getId(k);
+ m.put(k, val);
+ }
+ MyTestResult r = new MyTestResult(m);
+ return r;
+ }
+ }
+
+ private class MyTestResult {
+ Map<String, Integer> m;
+ private MyTestResult(Map<String, Integer> m) {
+ this.m = m;
+ }
+ private Map<String, Integer> getMap() {
+ return m;
+ }
+
+ @Override
+ public String toString() {
+ return "MyTester: "+m.size();
+ }
+ }
+
+ private class TestBuffer extends AbstractDBBuffer {
+ @Override
+ public void write(int id, String value) throws RuntimeException {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void close() throws SQLException {
+ //no-op
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java b/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
new file mode 100644
index 0000000..14c0013
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.io;
+
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.RecursiveParserWrapper;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ExtractReaderTest extends TikaTest {
+
+ private Path testJsonFile;
+ private Path testTxtFile;
+
+ @Before
+ public void setUp() throws Exception {
+ testJsonFile = getResourceAsFile("/test-dirs/extractsA/file2_attachANotB.doc.json").toPath();
+ testTxtFile = getResourceAsFile("/test-dirs/extractsB/file13_attachANotB.doc.txt").toPath();
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+
+ ExtractReader extractReader = new ExtractReader();
+ List<Metadata> metadataList = extractReader.loadExtract(testJsonFile,
+ ExtractReader.ALTER_METADATA_LIST.AS_IS);
+ assertEquals(2, metadataList.size());
+ assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
+ assertEquals(1, metadataList.get(1).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
+ assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
+ assertContains("attachment", metadataList.get(1).get(RecursiveParserWrapper.TIKA_CONTENT));
+
+ metadataList = extractReader.loadExtract(testJsonFile, ExtractReader.ALTER_METADATA_LIST.FIRST_ONLY);
+ assertEquals(1, metadataList.size());
+ assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
+ assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
+ assertNotContained("attachment", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
+
+ metadataList = extractReader.loadExtract(testJsonFile, ExtractReader.ALTER_METADATA_LIST.CONCATENATE_CONTENT_INTO_FIRST);
+ assertEquals(1, metadataList.size());
+ assertEquals(1, metadataList.get(0).getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
+ assertContains("fox", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
+ assertContains("attachment", metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT));
+ }
+
+ @Test
+ public void testTextBasic() throws IOException {
+ ExtractReader extractReader = new ExtractReader();
+ List<Metadata> metadataList = extractReader.loadExtract(testTxtFile,
+ ExtractReader.ALTER_METADATA_LIST.AS_IS);
+ assertEquals(1, metadataList.size());
+ Metadata m = metadataList.get(0);
+ assertEquals(1, m.getValues(RecursiveParserWrapper.TIKA_CONTENT).length);
+ assertEquals("the quick brown fox fox fox jumped over the lazy lazy dog\n",
+ m.get(RecursiveParserWrapper.TIKA_CONTENT));
+
+ //test that the mime is inferred from the file extension
+ assertEquals("application/msword", m.get(Metadata.CONTENT_TYPE));
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/java/org/apache/tika/eval/io/FatalExceptionReaderTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/io/FatalExceptionReaderTest.java b/tika-eval/src/test/java/org/apache/tika/eval/io/FatalExceptionReaderTest.java
new file mode 100644
index 0000000..3b99a76
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/io/FatalExceptionReaderTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.io;
+
+
+import java.io.InputStream;
+
+import org.junit.Test;
+
+public class FatalExceptionReaderTest {
+ @Test
+ public void testSimpleRead() throws Exception {
+ InputStream is = this.getClass().getResourceAsStream("/test-dirs/batch-logs/batch-process-fatal.xml");
+ XMLLogReader reader = new XMLLogReader();
+ //reader.read(is);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java b/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java
new file mode 100644
index 0000000..5e43303
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/reports/ResultsReporterTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.reports;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.Statement;
+
+import org.apache.tika.eval.db.H2Util;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class ResultsReporterTest {
+ private Path configFile;
+ private Path tmpDir;
+ private String dbName = "test-db";
+ private Connection connection;
+
+ @Before
+ public void setUp() throws Exception {
+ configFile = Paths.get(this.getClass().getResource("/reports.xml").toURI());
+ tmpDir = Files.createTempDirectory("tika-eval-report-test-");
+
+ connection = new H2Util(tmpDir.resolve(dbName)).getConnection(true);
+ String sql = "CREATE TABLE test_table (ID LONG PRIMARY KEY, STRING VARCHAR(32))";
+ Statement st = connection.createStatement();
+ st.execute(sql);
+ sql = "INSERT into test_table values ( 100000, 'the quick brown')";
+ st.execute(sql);
+ sql = "INSERT into test_table values (123456789, 'fox jumped over')";
+ st.execute(sql);
+ connection.commit();
+ }
+
+ @Test
+ @Ignore("add a real test here")
+ public void testBuilder() throws Exception {
+ ResultsReporter r = ResultsReporter.build(configFile);
+ r.execute(connection, Paths.get("reports"));
+ System.out.println("finished: "+ tmpDir.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/java/org/apache/tika/eval/tokens/LuceneTokenCounter.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/tokens/LuceneTokenCounter.java b/tika-eval/src/test/java/org/apache/tika/eval/tokens/LuceneTokenCounter.java
new file mode 100644
index 0000000..486cac7
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/tokens/LuceneTokenCounter.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.tokens;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.commons.math3.util.FastMath;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Experimental class uses Lucene's MemoryIndex to effectively build the
+ * token info.
+ */
+public class LuceneTokenCounter {
+ private static final String ALPHA_IDEOGRAPH_SUFFIX = "_a";
+
+ private final LeafReader leafReader;
+ private final MemoryIndex memoryIndex;
+ private final Analyzer generalAnalyzer;
+ private final Analyzer alphaIdeographAnalyzer;
+ private int topN = 10;
+
+ Map<String, TokenStatistics> fieldStats = new HashMap<>();
+
+ public LuceneTokenCounter(Analyzer generalAnalyzer, Analyzer alphaIdeographAnalyzer) throws IOException {
+ memoryIndex = new MemoryIndex();
+ IndexSearcher searcher = memoryIndex.createSearcher();
+ leafReader = (LeafReader)searcher.getIndexReader();
+ this.generalAnalyzer = generalAnalyzer;
+ this.alphaIdeographAnalyzer = alphaIdeographAnalyzer;
+ }
+
+ public void add(String field, String content) throws IOException {
+ memoryIndex.addField(field, content, generalAnalyzer);
+ //memoryIndex.addField(field+ALPHA_IDEOGRAPH_SUFFIX,
+ // content, alphaIdeographAnalyzer);
+ count(field);
+ //count(field+ALPHA_IDEOGRAPH_SUFFIX);
+
+ }
+
+
+ void count(String field) throws IOException {
+ long tokenCount = leafReader.getSumTotalTermFreq(field);
+ if (tokenCount > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("can't handle longs");
+ }
+ int tokenCountInt = (int)tokenCount;
+ int uniqueTokenCount = 0;
+ SummaryStatistics summStats = new SummaryStatistics();
+ double ent = 0.0d;
+ double p = 0.0d;
+ double base = 2.0;
+
+ Terms terms = leafReader.terms(field);
+ if (terms == null) {
+ //if there were no terms
+ fieldStats.put(field, new TokenStatistics(uniqueTokenCount, tokenCountInt,
+ new TokenIntPair[0], ent, summStats));
+ return;
+
+ }
+ TermsEnum termsEnum = terms.iterator();
+ BytesRef bytesRef = termsEnum.next();
+ TokenCountPriorityQueue queue= new TokenCountPriorityQueue(topN);
+
+ while (bytesRef != null) {
+
+ long termFreq = termsEnum.totalTermFreq();
+ if (termFreq > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("Sorry can't handle longs yet");
+ }
+ int tf = (int)termFreq;
+ //TODO: figure out how to avoid Stringifying this
+ //to get codepoint count
+ String t = bytesRef.utf8ToString();
+ int len = t.codePointCount(0, t.length());
+ for (int i = 0; i < tf; i++) {
+ summStats.addValue(len);
+ }
+ p = (double) tf / (double) tokenCount;
+ ent += p * FastMath.log(base, p);
+
+ if (queue.top() == null || queue.size() < topN ||
+ tf >= queue.top().getValue()) {
+ queue.insertWithOverflow(new TokenIntPair(t, tf));
+ }
+
+ uniqueTokenCount++;
+ bytesRef = termsEnum.next();
+ }
+ if (tokenCountInt > 0) {
+ ent = (-1.0d / (double)tokenCountInt) * ent;
+ }
+
+ fieldStats.put(field, new TokenStatistics(uniqueTokenCount, tokenCountInt,
+ queue.getArray(), ent, summStats));
+ }
+
+ public void setTopN(int topN) {
+ this.topN = topN;
+ }
+
+ public TokenStatistics getTokenStatistics(String field) {
+ return fieldStats.get(field);
+ }
+ public Terms getAlphaTerms(String field) throws IOException {
+ return leafReader.terms(field+ALPHA_IDEOGRAPH_SUFFIX);
+ }
+ public Terms getTerms(String field) throws IOException {
+ return leafReader.terms(field);
+ }
+
+
+ public void clear() {
+ memoryIndex.reset();
+ fieldStats.clear();
+ }
+/*
+ public ContrastStatistics contrast(String fieldA, String fieldB) throws IOException {
+ long diceDenom = getUniqueTokenCount(fieldA) +
+ getUniqueTokenCount(fieldB);
+
+ long diceNum = 0;
+ long overlapNum = 0;
+
+ Terms termsA = getTerms(fieldA);
+ Terms termsB = getTerms(fieldB);
+
+ TermsEnum termsEnumA = termsA.iterator();
+ TermsEnum termsEnumB = termsB.iterator();
+
+ BytesRef bytesRefA = termsEnumA.next();
+ BytesRef bytesRefB = termsEnumB.next();
+
+ while (bytesRefA != null) {
+ int compare = bytesRefA.compareTo(bytesRefB);
+ while (compare > 0) {
+ if (bytesRefB == null) {
+ break;
+ }
+ //handle term in B, but not A
+
+ compare = bytesRefA.compareTo(bytesRefB);
+ bytesRefB = termsEnumB.next();
+ }
+ if (compare == 0) {
+ diceNum += 2;
+ overlapNum += 2 * Math.min(termsEnumA.totalTermFreq(), termsEnumB.totalTermFreq());
+ }
+
+ bytesRefA = termsEnumA.next();
+ }
+
+
+ for (PairCount p : tokens.values()) {
+ if (p.a > 0 && p.b > 0) {
+ diceNum += 2;
+ overlapNum += 2 * Math.min(p.a, p.b);
+ }
+ }
+
+ float dice = (float) diceNum / (float) diceDenom;
+ float overlap = (float) overlapNum / (float) (theseTokens.getTokenCount() + thoseTokens.getTokenCount());
+ }
+*/
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/java/org/apache/tika/eval/tokens/TokenCounterTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/tokens/TokenCounterTest.java b/tika-eval/src/test/java/org/apache/tika/eval/tokens/TokenCounterTest.java
new file mode 100644
index 0000000..719b56c
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/tokens/TokenCounterTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.tokens;
+
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TokenCounterTest {
+ private final static String FIELD = "f";
+ private static AnalyzerManager analyzerManager;
+
+ private final int topN = 10;
+
+ @BeforeClass
+ public static void setUp() throws IOException {
+ analyzerManager = AnalyzerManager.newInstance();
+
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+ String s = " bde cde def abc efg f f f f ghijklmnop a a a a a a a a a a a a a a a a a b b b b b b b b b b b b b";
+ TokenCounter counter = new TokenCounter(analyzerManager.getGeneralAnalyzer(),
+ analyzerManager.getAlphaIdeoAnalyzer());
+ counter.add(FIELD, s);
+ TokenStatistics simpleTokenStatistics = counter.getTokenStatistics(FIELD);
+ LuceneTokenCounter tokenCounter = new LuceneTokenCounter(analyzerManager.getGeneralAnalyzer(),
+ analyzerManager.getAlphaIdeoAnalyzer());
+ tokenCounter.add(FIELD, s);
+ assertEquals(simpleTokenStatistics, tokenCounter.getTokenStatistics(FIELD));
+ }
+
+ @Test
+ public void testRandom() throws Exception {
+
+ long simple = 0;
+ long lucene = 0;
+ int numberOfTests = 100;
+ for (int i = 0; i < numberOfTests; i++) {
+ String s = generateString();
+ long start = new Date().getTime();
+ TokenCounter counter = new TokenCounter(analyzerManager.getGeneralAnalyzer(),
+ analyzerManager.getAlphaIdeoAnalyzer());
+ counter.add(FIELD, s);
+ simple += new Date().getTime()-start;
+ TokenStatistics simpleTokenStatistics = counter.getTokenStatistics(FIELD);
+
+ start = new Date().getTime();
+ LuceneTokenCounter tokenCounter = new LuceneTokenCounter(analyzerManager.getGeneralAnalyzer(),
+ analyzerManager.getAlphaIdeoAnalyzer());
+ tokenCounter.add(FIELD, s);
+ lucene += new Date().getTime()-start;
+ assertEquals(s, simpleTokenStatistics, tokenCounter.getTokenStatistics(FIELD));
+ }
+
+ //System.out.println("SIMPLE: " + simple + " lucene: "+lucene);
+ }
+
+ @Test
+ public void testCommonTokens() throws Exception {
+ TokenCounter tokenCounter = new TokenCounter(analyzerManager.getGeneralAnalyzer(),
+ analyzerManager.getAlphaIdeoAnalyzer());
+ String s = "the http://www.cnn.com and blahdeblah@apache.org are in valuable www.sites.org \u666e\u6797\u65af\u987f\u5927\u5b66";
+ tokenCounter.add(FIELD, s);
+ Map<String, MutableInt> tokens = tokenCounter.getAlphaTokens(FIELD);
+ assertEquals(new MutableInt(2), tokens.get("___url___"));
+ assertEquals(new MutableInt(1), tokens.get("___email___"));
+ }
+
+ @Test
+ public void testCJKFilter() throws Exception {
+ String s = "then quickbrownfoxjumpedoverthelazy dogss dog \u666e\u6797\u65af\u987f\u5927\u5b66";
+ Analyzer analyzer = analyzerManager.getCommonTokensAnalyzer();
+ TokenStream ts = analyzer.tokenStream(FIELD, s);
+ CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
+ ts.reset();
+ Map<String, Integer> tokens = new HashMap<>();
+ while (ts.incrementToken()) {
+ String t = termAtt.toString();
+ Integer count = tokens.get(t);
+ count = (count == null) ? count = 0 : count;
+ count++;
+ tokens.put(t, count);
+ }
+ ts.end();
+ ts.close();
+ assertEquals(7, tokens.size());
+ assertEquals(new Integer(1), tokens.get("\u6797\u65af"));
+ }
+
+ private String generateString() {
+
+ Random r = new Random();
+ int len = r.nextInt(1000);
+ int uniqueVocabTerms = 10000;
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < len; i++) {
+ sb.append(Integer.toString(r.nextInt(uniqueVocabTerms)+100000));
+ sb.append(" ");
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/java/org/apache/tika/eval/util/MimeUtilTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/util/MimeUtilTest.java b/tika-eval/src/test/java/org/apache/tika/eval/util/MimeUtilTest.java
new file mode 100644
index 0000000..40b7484
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/util/MimeUtilTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.util;
+
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.mime.MimeType;
+import org.apache.tika.mime.MimeTypeException;
+import org.apache.tika.mime.MimeTypes;
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore("Fix mimetype.getExtension to work with these and then we can get rid of MimeUtil")
+public class MimeUtilTest {
+
+ private final TikaConfig config = TikaConfig.getDefaultConfig();
+
+ @Test
+ public void testBasic() throws Exception {
+ assertResult("application/pdf", ".pdf");
+ assertResult("APPLICATION/PDF", ".pdf");
+ assertResult("text/plain; charset=ISO-8859-1", ".txt");
+ assertResult("application/xhtml+xml; charset=UTF-8\n", ".html");
+ assertResult("application/xml; charset=UTF-8\n", ".xml");
+
+ assertException("bogosity", "xml");
+ }
+
+ private void assertException(String contentType, String expected) {
+ boolean ex = false;
+ try {
+ assertResult(contentType, expected);
+ } catch (MimeTypeException e) {
+ ex = true;
+ }
+ assertTrue("Should have had exception for: " + contentType, ex);
+ }
+
+ private void assertResult(String contentType, String expected) throws MimeTypeException {
+ TikaConfig tikaConfig = TikaConfig.getDefaultConfig();
+ MimeTypes r = tikaConfig.getMimeRepository();
+ MimeType mt = r.forName(contentType);
+
+// String ext = MimeUtil.getExtension(contentType, config);
+ assertEquals(expected, mt.getExtension());
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/commontokens/en
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/commontokens/en b/tika-eval/src/test/resources/commontokens/en
new file mode 100644
index 0000000..8d442fe
--- /dev/null
+++ b/tika-eval/src/test/resources/commontokens/en
@@ -0,0 +1,8 @@
+the
+of
+and
+a
+or
+#quick
+brown
+fox
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/commontokens/es
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/commontokens/es b/tika-eval/src/test/resources/commontokens/es
new file mode 100644
index 0000000..b9bfd03
--- /dev/null
+++ b/tika-eval/src/test/resources/commontokens/es
@@ -0,0 +1,10 @@
+la
+de
+y
+una
+
+
+o
+r�pido
+marr�n
+zorro
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/commontokens/zh-cn
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/commontokens/zh-cn b/tika-eval/src/test/resources/commontokens/zh-cn
new file mode 100644
index 0000000..bec617d
--- /dev/null
+++ b/tika-eval/src/test/resources/commontokens/zh-cn
@@ -0,0 +1,8 @@
+\u7684
+\u7684
+\u548c
+\u4e00\u500b
+\u8981\u4e48
+\u5feb
+\u68d5\u8272
+\u72d0\u72f8
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/commontokens/zh-tw
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/commontokens/zh-tw b/tika-eval/src/test/resources/commontokens/zh-tw
new file mode 100644
index 0000000..bc91291
--- /dev/null
+++ b/tika-eval/src/test/resources/commontokens/zh-tw
@@ -0,0 +1,8 @@
+\u7684
+\u7684
+\u548c
+\u4e00\u4e2a
+\u8981\u4e48
+\u5feb
+\u68d5\u8272
+\u72d0\u72f8
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/log4j.properties b/tika-eval/src/test/resources/log4j.properties
new file mode 100644
index 0000000..925f9f2
--- /dev/null
+++ b/tika-eval/src/test/resources/log4j.properties
@@ -0,0 +1,11 @@
+
+log4j.rootLogger=WARN,A1
+
+#for debugging
+#log4j.rootLogger=TRACE,A1
+
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/log4j_process.properties
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/log4j_process.properties b/tika-eval/src/test/resources/log4j_process.properties
new file mode 100644
index 0000000..cca8871
--- /dev/null
+++ b/tika-eval/src/test/resources/log4j_process.properties
@@ -0,0 +1,11 @@
+
+log4j.rootLogger=TRACE,A1
+
+#for debugging
+#log4j.rootLogger=TRACE,A1
+
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml b/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
new file mode 100644
index 0000000..9993ff1
--- /dev/null
+++ b/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<tika-batch-config
+ maxAliveTimeSeconds="-1"
+ pauseOnEarlyTerminationMillis="500"
+ timeoutCheckPulseMillis="1000"
+ maxQueueSize="10000"
+ numConsumers="5"
+ timeoutThresholdMillis="300000">
+
+ <commandline>
+ <option opt="c" longOpt="tika-config" hasArg="true"
+ description="TikaConfig file"/>
+
+ <option opt="bc" longOpt="batch-config" hasArg="true"
+ description="xml batch config file" required="true"/>
+ <option opt="inputDir" hasArg="true"
+ description="dir to start crawling"/>
+ <option opt="numConsumers" hasArg="true"
+ description="number of fileConsumers threads"/>
+ <option opt="extractDir" hasArg="true"
+ description="this dir contains the files containing extracted metadata/content" required="false"/>
+ <option opt="db" hasArg="true"
+ description="name of db directory or file to which to write results"/>
+ </commandline>
+
+
+ <!--
+ Can also add startDir: this tells the crawler to start indexing a
+ child directory of the inputDir directory.
+ -->
+ <crawler builderClass="org.apache.tika.batch.fs.builders.FSCrawlerBuilder"
+ inputDir="src/test/resources/test-dirs/extractsA"
+ crawlOrder="sorted"
+ maxConsecWaitMillis="5000"
+ maxFilesToAdd="-1"
+ maxFilesToConsider="-1"
+ includeFilePat=""
+ excludeFilePat=""
+ maxFileSizeBytes="-1"
+ />
+
+ <consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
+ consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"
+ errorLogFile="src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml"
+ extractDir="src/test/resources/test-dirs/extractsA"
+ commonTokens="src/test/resources/common_tokens_short.txt"/>
+
+
+ <!-- reporter and interrupter are optional -->
+ <reporter builderClass="org.apache.tika.batch.builders.SimpleLogReporterBuilder" sleepMillis="1000"
+ staleThresholdMillis="500000"/>
+ <interrupter builderClass="org.apache.tika.batch.builders.InterrupterBuilder"/>
+</tika-batch-config>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml b/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
new file mode 100644
index 0000000..da59d03
--- /dev/null
+++ b/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<tika-batch-config
+ maxAliveTimeSeconds="-1"
+ pauseOnEarlyTerminationMillis="500"
+ timeoutCheckPulseMillis="1000"
+ maxQueueSize="10000"
+ numConsumers="5"
+ timeoutThresholdMillis="300000">
+
+ <commandline>
+ <option opt="c" longOpt="tika-config" hasArg="true"
+ description="TikaConfig file"/>
+
+ <option opt="bc" longOpt="batch-config" hasArg="true"
+ description="xml batch config file" required="true"/>
+ <option opt="inputDir" hasArg="true"
+ description="dir to start crawling"/>
+ <option opt="numConsumers" hasArg="true"
+ description="number of fileConsumers threads"/>
+ <option opt="extractDir" hasArg="true"
+ description="this dir contains the files containing extracted metadata/content" required="false"/>
+ <option opt="db" hasArg="true"
+ description="name of db directory or file to which to write results"/>
+ </commandline>
+
+
+ <!--
+ Can also add startDir: this tells the crawler to start indexing a
+ child directory of the inputDir directory.
+ -->
+ <crawler builderClass="org.apache.tika.batch.fs.builders.FSCrawlerBuilder"
+ inputDir="src/test/resources/test-dirs/raw_input"
+ crawlOrder="sorted"
+ maxConsecWaitMillis="5000"
+ maxFilesToAdd="-1"
+ maxFilesToConsider="-1"
+ includeFilePat=""
+ excludeFilePat=""
+ maxFileSizeBytes="-1"
+ />
+
+ <consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
+ consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"
+ errorLogFile="src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml"
+ extractDir="src/test/resources/test-dirs/extractsA"
+ inputDir="src/test/resources/test-dirs/raw_input"
+ commonTokens="src/test/resources/common_tokens"/>
+
+
+ <!-- reporter and interrupter are optional -->
+ <reporter builderClass="org.apache.tika.batch.builders.SimpleLogReporterBuilder" sleepMillis="1000"
+ staleThresholdMillis="500000"/>
+ <interrupter builderClass="org.apache.tika.batch.builders.InterrupterBuilder"/>
+</tika-batch-config>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml b/tika-eval/src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml
new file mode 100644
index 0000000..520306b
--- /dev/null
+++ b/tika-eval/src/test/resources/test-dirs/batch-logs/batch-process-fatal.xml
@@ -0,0 +1,59 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<log4j:event logger="org.apache.tika.batch.FileResourceConsumer" timestamp="1436376775762" level="ERROR" thread="pool-2-thread-11">
+<log4j:message><![CDATA[<?xml version="1.0" ?><timed_out resourceId="file10_permahang.txt" elapsedMS="340302"></timed_out>]]></log4j:message>
+</log4j:event>
+
+<log4j:event logger="org.apache.tika.batch.FileResourceConsumer" timestamp="1436376775758" level="ERROR" thread="pool-2-thread-10">
+<log4j:message><![CDATA[<?xml version="1.0" ?><oom resourceId="file11_oom.txt">java.lang.OutOfMemoryError: Java heap space
+ at java.io.ByteArrayOutputStream.<init>(ByteArrayOutputStream.java:77)
+ at org.apache.fontbox.ttf.MemoryTTFDataStream.<init>(MemoryTTFDataStream.java:45)
+ at org.apache.fontbox.ttf.TTFParser.parse(TTFParser.java:96)
+ at org.apache.pdfbox.pdmodel.font.PDTrueTypeFont.<init>(PDTrueTypeFont.java:135)
+ at org.apache.pdfbox.pdmodel.font.PDFontFactory.createFont(PDFontFactory.java:75)
+ at org.apache.pdfbox.pdmodel.PDResources.getFont(PDResources.java:96)
+ at org.apache.pdfbox.contentstream.operator.text.SetFontAndSize.process(SetFontAndSize.java:50)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processOperator(PDFStreamEngine.java:795)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processStreamOperators(PDFStreamEngine.java:462)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processStream(PDFStreamEngine.java:438)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.showForm(PDFStreamEngine.java:178)
+ at org.apache.pdfbox.contentstream.operator.DrawObject.process(DrawObject.java:49)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processOperator(PDFStreamEngine.java:795)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processStreamOperators(PDFStreamEngine.java:462)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processStream(PDFStreamEngine.java:438)
+ at org.apache.pdfbox.contentstream.PDFStreamEngine.processPage(PDFStreamEngine.java:149)
+ at org.apache.pdfbox.text.PDFTextStreamEngine.processPage(PDFTextStreamEngine.java:117)
+ at org.apache.pdfbox.text.PDFTextStripper.processPage(PDFTextStripper.java:369)
+ at org.apache.pdfbox.text.PDFTextStripper.processPages(PDFTextStripper.java:305)
+ at org.apache.pdfbox.text.PDFTextStripper.writeText(PDFTextStripper.java:249)
+ at org.apache.tika.parser.pdf.PDF2XHTML.process(PDF2XHTML.java:137)
+ at org.apache.tika.parser.pdf.PDFParser.parse(PDFParser.java:132)
+ at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:281)
+ at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:281)
+ at org.apache.tika.parser.AutoDetectParser.parse(AutoDetectParser.java:120)
+ at org.apache.tika.parser.ParserDecorator.parse(ParserDecorator.java:177)
+ at org.apache.tika.parser.DigestingParser.parse(DigestingParser.java:74)
+ at org.apache.tika.parser.RecursiveParserWrapper.parse(RecursiveParserWrapper.java:158)
+ at org.apache.tika.batch.FileResourceConsumer.parse(FileResourceConsumer.java:410)
+ at org.apache.tika.batch.fs.RecursiveParserWrapperFSConsumer.processFileResource(RecursiveParserWrapperFSConsumer.java:104)
+ at org.apache.tika.batch.FileResourceConsumer._processFileResource(FileResourceConsumer.java:182)
+ at org.apache.tika.batch.FileResourceConsumer.call(FileResourceConsumer.java:115)
+</oom>]]></log4j:message>
+</log4j:event>
\ No newline at end of file
[3/6] tika git commit: TIKA-1332 initial commit of tika-eval. More
work remains.
Posted by ta...@apache.org.
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCounter.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCounter.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCounter.java
new file mode 100644
index 0000000..28e1c78
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCounter.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.tokens;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.commons.math3.util.FastMath;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+public class TokenCounter {
+
+ private static final String ALPHA_IDEOGRAPH_SUFFIX = "_a";
+
+
+ Map<String, Map<String, MutableInt>> map = new HashMap<>(); //Map<field, Map<token, count>>
+ Map<String, TokenStatistics> tokenStatistics = new HashMap<>();
+
+ private final TokenStatistics NULL_TOKEN_STAT = new TokenStatistics(
+ 0, 0, new TokenIntPair[0], 0.0d, new SummaryStatistics());
+
+ private final Analyzer generalAnalyzer;
+ private final Analyzer alphaIdeoAnalyzer;
+
+ private int topN = 10;
+
+ public TokenCounter(Analyzer generalAnalyzer, Analyzer alphaIdeoAnalyzer) throws IOException {
+ this.generalAnalyzer = generalAnalyzer;
+ this.alphaIdeoAnalyzer = alphaIdeoAnalyzer;
+ }
+
+ public void add(String field, String content) throws IOException {
+ _add(field, generalAnalyzer, content);
+ _add(field+ALPHA_IDEOGRAPH_SUFFIX, alphaIdeoAnalyzer, content);
+ }
+
+ private void _add(String field, Analyzer analyzer, String content) throws IOException {
+ int totalTokens = 0;
+
+ TokenStream ts = analyzer.tokenStream(field, content);
+ CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
+ ts.reset();
+ Map<String, MutableInt> tokenMap = map.get(field);
+ if (tokenMap == null) {
+ tokenMap = new HashMap<>();
+ map.put(field, tokenMap);
+ }
+ while (ts.incrementToken()) {
+ String token = termAtt.toString();
+ MutableInt cnt = tokenMap.get(token);
+ if (cnt == null) {
+ cnt = new MutableInt(1);
+ tokenMap.put(token, cnt);
+ } else {
+ cnt.increment();
+ }
+ totalTokens++;
+ }
+ ts.close();
+ ts.end();
+
+ int totalUniqueTokens = tokenMap.size();
+
+ double ent = 0.0d;
+ double p = 0.0d;
+ double base = 2.0;
+
+ TokenCountPriorityQueue queue = new TokenCountPriorityQueue(topN);
+
+ SummaryStatistics summaryStatistics = new SummaryStatistics();
+ for (Map.Entry<String, MutableInt> e : tokenMap.entrySet()) {
+ String token = e.getKey();
+ int termFreq = e.getValue().intValue();
+
+ p = (double) termFreq / (double) totalTokens;
+ ent += p * FastMath.log(base, p);
+ int len = token.codePointCount(0, token.length());
+ for (int i = 0; i < e.getValue().intValue(); i++) {
+ summaryStatistics.addValue(len);
+ }
+ if (queue.top() == null || queue.size() < topN ||
+ termFreq >= queue.top().getValue()) {
+ queue.insertWithOverflow(new TokenIntPair(token, termFreq));
+ }
+
+ }
+ if (totalTokens > 0) {
+ ent = (-1.0d / (double)totalTokens) * ent;
+ }
+
+/* Collections.sort(allTokens);
+ List<TokenIntPair> topNList = new ArrayList<>(topN);
+ for (int i = 0; i < topN && i < allTokens.size(); i++) {
+ topNList.add(allTokens.get(i));
+ }*/
+
+ tokenStatistics.put(field, new TokenStatistics(totalUniqueTokens, totalTokens,
+ queue.getArray(), ent, summaryStatistics));
+
+ }
+
+ public TokenStatistics getTokenStatistics(String field) {
+ TokenStatistics tokenStat = tokenStatistics.get(field);
+ if (tokenStat == null) {
+ return NULL_TOKEN_STAT;
+ }
+ return tokenStat;
+ }
+
+ public void setTopN(int topN) {
+ this.topN = topN;
+ }
+
+ public void clear(String field) {
+ Map<String, MutableInt> tokenMap = map.get(field);
+ if (tokenMap != null) {
+ tokenMap.clear();
+ }
+ Map<String, MutableInt> alphaMap = map.get(field+ALPHA_IDEOGRAPH_SUFFIX);
+ if (alphaMap != null) {
+ alphaMap.clear();
+ }
+
+ tokenStatistics.put(field+ALPHA_IDEOGRAPH_SUFFIX, NULL_TOKEN_STAT);
+ tokenStatistics.put(field, NULL_TOKEN_STAT);
+ }
+
+ public Map<String, MutableInt> getAlphaTokens(String field) {
+ Map<String, MutableInt> ret = map.get(field+ALPHA_IDEOGRAPH_SUFFIX);
+ if (ret == null) {
+ return Collections.emptyMap();
+ }
+ return ret;
+ }
+
+ public Map<String, MutableInt> getTokens(String field) {
+ Map<String, MutableInt> ret = map.get(field);
+ if (ret == null) {
+ return Collections.emptyMap();
+ }
+ return ret;
+ }
+
+ public TokenStatistics getAlphaTokenStatistics(String fieldName) {
+ return getTokenStatistics(fieldName+ALPHA_IDEOGRAPH_SUFFIX);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenIntPair.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenIntPair.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenIntPair.java
new file mode 100644
index 0000000..4b57d25
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenIntPair.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.tokens;
+
+
+import org.jetbrains.annotations.NotNull;
+
+public class TokenIntPair implements Comparable<TokenIntPair> {
+
+ final String token;
+ final int value;
+
+ public TokenIntPair(String token, int value) {
+ this.token = token;
+ this.value = value;
+ }
+
+ public long getValue() {
+ return value;
+ }
+
+ public String getToken() {
+ return token;
+ }
+
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ TokenIntPair that = (TokenIntPair) o;
+
+ if (value != that.value) return false;
+ return token.equals(that.token);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = token.hashCode();
+ result = 31 * result + value;
+ return result;
+ }
+
+ /**
+ * Descending by value, ascending by token
+ *
+ * @param o other tokenlong pair
+ * @return comparison
+ */
+ @Override
+ public int compareTo(@NotNull TokenIntPair o) {
+ if (this.value > o.value) {
+ return -1;
+ } else if (this.value < o.value) {
+ return 1;
+ }
+ return this.token.compareTo(o.token);
+ }
+
+ @Override
+ public String toString() {
+ return "TokenIntPair{" +
+ "token='" + token + '\'' +
+ ", value=" + value +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenStatistics.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenStatistics.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenStatistics.java
new file mode 100644
index 0000000..62d3d20
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenStatistics.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.tokens;
+
+import java.util.Arrays;
+
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+
+
+public class TokenStatistics {
+
+ private final int totalTokens;
+ private final int totalUniqueTokens;
+ private final TokenIntPair[] topN;
+ private final double entropy;
+ private final SummaryStatistics summaryStatistics;
+
+ public TokenStatistics(int totalUniqueTokens, int totalTokens,
+ TokenIntPair[] topN,
+ double entropy, SummaryStatistics summaryStatistics) {
+ this.totalUniqueTokens = totalUniqueTokens;
+ this.totalTokens = totalTokens;
+ this.topN = topN;
+ this.entropy = entropy;
+ this.summaryStatistics = summaryStatistics;
+ }
+
+
+ public int getTotalTokens() {
+
+ return totalTokens;
+ }
+
+ public int getTotalUniqueTokens() {
+ return totalUniqueTokens;
+ }
+
+ public TokenIntPair[] getTopN() {
+ return topN;
+ }
+
+ public double getEntropy() {
+ return entropy;
+ }
+
+ public SummaryStatistics getSummaryStatistics() {
+ return summaryStatistics;
+ }
+
+
+ @Override
+ public String toString() {
+ return "TokenStatistics{" +
+ "totalTokens=" + totalTokens +
+ ", totalUniqueTokens=" + totalUniqueTokens +
+ ", topN=" + Arrays.toString(topN) +
+ ", entropy=" + entropy +
+ ", summaryStatistics=" + summaryStatistics +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ TokenStatistics that = (TokenStatistics) o;
+
+ if (totalTokens != that.totalTokens) return false;
+ if (totalUniqueTokens != that.totalUniqueTokens) return false;
+ if (!doubleEquals(that.entropy, entropy)) return false;
+ // Probably incorrect - comparing Object[] arrays with Arrays.equals
+ if (!Arrays.equals(topN, that.topN)) return false;
+
+ SummaryStatistics thatS = ((TokenStatistics) o).summaryStatistics;
+ if (summaryStatistics.getN() != thatS.getN()) return false;
+
+ //if both have n==0, don't bother with the stats
+ if (summaryStatistics.getN() ==0L) return true;
+ //TODO: consider adding others...
+ if (!doubleEquals(summaryStatistics.getGeometricMean(), thatS.getGeometricMean())) return false;
+ if (!doubleEquals(summaryStatistics.getMax(), thatS.getMax())) return false;
+ if (!doubleEquals(summaryStatistics.getMean(), thatS.getMean())) return false;
+ if (!doubleEquals(summaryStatistics.getMin(), thatS.getMin())) return false;
+ if (!doubleEquals(summaryStatistics.getSum(), thatS.getSum())) return false;
+ if (!doubleEquals(summaryStatistics.getStandardDeviation(), thatS.getStandardDeviation())) return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result;
+ long temp;
+ result = (int) (totalTokens ^ (totalTokens >>> 32));
+ result = 31 * result + (int) (totalUniqueTokens ^ (totalUniqueTokens >>> 32));
+ result = 31 * result + Arrays.hashCode(topN);
+ temp = Double.doubleToLongBits(entropy);
+ result = 31 * result + (int) (temp ^ (temp >>> 32));
+ result = 31 * result + summaryStatistics.hashCode();
+ return result;
+ }
+
+ private static boolean doubleEquals(double a, double b) {
+ return doubleEquals(a, b, 0.000000000001d);
+ }
+
+ private static boolean doubleEquals(double a, double b, double epsilon) {
+ return a == b ? true : Math.abs(a - b) < epsilon;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/util/LanguageIDWrapper.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/util/LanguageIDWrapper.java b/tika-eval/src/main/java/org/apache/tika/eval/util/LanguageIDWrapper.java
new file mode 100644
index 0000000..59d032a
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/util/LanguageIDWrapper.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.util;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+
+import com.google.common.base.Optional;
+import com.optimaize.langdetect.DetectedLanguage;
+import com.optimaize.langdetect.LanguageDetector;
+import com.optimaize.langdetect.LanguageDetectorBuilder;
+import com.optimaize.langdetect.i18n.LdLocale;
+import com.optimaize.langdetect.ngram.NgramExtractors;
+import com.optimaize.langdetect.profiles.LanguageProfile;
+import com.optimaize.langdetect.profiles.LanguageProfileReader;
+import com.optimaize.langdetect.text.CommonTextObjectFactories;
+import com.optimaize.langdetect.text.TextObjectFactory;
+
+
+public class LanguageIDWrapper {
+ static List<LanguageProfile> languageProfiles;
+ static LanguageDetector detector;
+ static TextObjectFactory textObjectFactory;
+
+ public static void loadBuiltInModels() throws IOException {
+
+ languageProfiles = new LanguageProfileReader().readAllBuiltIn();
+ detector = LanguageDetectorBuilder.create(NgramExtractors.standard())
+ .withProfiles(languageProfiles)
+ .build();
+ textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
+ }
+
+ public static void loadModels(Path path) throws IOException {
+
+ languageProfiles = new LanguageProfileReader().readAll(path.toFile());
+ detector = LanguageDetectorBuilder.create(NgramExtractors.standard())
+ .withProfiles(languageProfiles)
+ .build();
+ textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
+ }
+
+
+
+ public static Optional<LdLocale> detect(String s) {
+ return detector.detect(textObjectFactory.forText(s));
+ }
+
+ public static List<DetectedLanguage> getProbabilities(String s) {
+
+ return detector.getProbabilities(textObjectFactory.forText(s));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory b/tika-eval/src/main/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
new file mode 100644
index 0000000..1d21002
--- /dev/null
+++ b/tika-eval/src/main/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.tika.eval.tokens.AlphaIdeographFilterFactory
+org.apache.tika.eval.tokens.CJKBigramAwareLengthFilterFactory
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/resources/comparison-reports.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/comparison-reports.xml b/tika-eval/src/main/resources/comparison-reports.xml
new file mode 100644
index 0000000..cb7befd
--- /dev/null
+++ b/tika-eval/src/main/resources/comparison-reports.xml
@@ -0,0 +1,791 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<reports>
+
+
+ <before>
+
+ <sql>drop table if exists md5_multiples_tmp_a</sql>
+ <sql>create table md5_multiples_tmp_a (MD5 char(32), cnt int)
+ as
+ select md5, count(1) cnt
+ from profiles_a
+ where md5 is not null
+ group by md5
+ having cnt > 1
+ order by cnt desc
+ </sql>
+
+ <sql>drop table if exists md5_multiples_tmp_b</sql>
+ <sql>create table md5_multiples_tmp_b (MD5 char(32), cnt int)
+ as
+ select md5, count(1) cnt
+ from profiles_b
+ where md5 is not null
+ group by md5
+ having cnt > 1
+ order by cnt desc
+ </sql>
+ <sql>create index if not exists pa_m_idx
+ on profiles_a (mime_type_id);
+ </sql>
+ <sql>
+ create index if not exists pb_m_idx
+ on profiles_b (mime_type_id);
+ </sql>
+
+ <sql>drop table if exists exceptions_compared</sql>
+ <sql>
+ create table exceptions_compared
+ (mime_type_id integer primary key,
+ exceptions_a integer,
+ total_a integer,
+ percent_exceptions_a double,
+ exceptions_b integer,
+ total_b integer,
+ percent_exceptions_b double)
+ </sql>
+ <sql>
+ insert into exceptions_compared (mime_type_id)
+ select mime_type_id from mimes;
+ </sql>
+
+ <sql>
+ update exceptions_compared ec set total_a=(
+ select count(1) as cnt from profiles_a
+ where profiles_a.mime_type_id= ec.mime_type_id
+ group by mime_type_id
+ )
+ </sql>
+ <sql>
+ update exceptions_compared ec set total_b=(
+ select count(1) as cnt from profiles_b
+ where profiles_b.mime_type_id= ec.mime_type_id
+ group by mime_type_id
+ )
+ </sql>
+ <sql>
+ update exceptions_compared ec set exceptions_a=( select count(1) as
+ cnt from exceptions_a ea
+ join profiles_a pa on ea.id=pa.id
+ where pa.mime_type_id= ec.mime_type_id
+ and parse_exception_type_id=0
+ group by mime_type_id )
+ </sql>
+ <sql>
+ update exceptions_compared ec set exceptions_b=(
+ select count(1) as cnt from exceptions_b eb
+ join profiles_b pb on eb.id=pb.id
+ where pb.mime_type_id= ec.mime_type_id
+ and parse_exception_type_id=0
+ group by mime_type_id )
+ </sql>
+
+ </before>
+
+ <!-- MIMES -->
+ <report reportName="All Mimes In A"
+ reportFilename="mimes/all_mimes_A.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string, count(1) cnt from
+ profiles_a p
+ join mimes m on m.mime_type_id = p.mime_type_id
+ group by mime_string
+ order by cnt desc
+ </sql>
+ </report>
+
+ <report reportName="All Mimes In B"
+ reportFilename="mimes/all_mimes_B.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string, count(1) cnt from
+ profiles_b p
+ join mimes m on m.mime_type_id = p.mime_type_id
+ group by mime_string
+ order by cnt desc
+ </sql>
+ </report>
+ <report reportName="Container Mimes In A"
+ reportFilename="mimes/container_mimes_A.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string, count(1) cnt from
+ profiles_a p
+ join mimes m on m.mime_type_id = p.mime_type_id
+ where is_embedded=false
+ group by mime_string
+ order by cnt desc
+ </sql>
+ </report>
+
+ <report reportName="Container Mimes In B"
+ reportFilename="mimes/container_mimes_B.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string, count(1) cnt from
+ profiles_b p
+ join mimes m on m.mime_type_id = p.mime_type_id
+ where is_embedded=false
+ group by mime_string
+ order by cnt desc
+ </sql>
+ </report>
+ <report reportName="Embedded Mimes In A"
+ reportFilename="mimes/embedded_mimes_A.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string, count(1) cnt from
+ profiles_a p
+ join mimes m on m.mime_type_id = p.mime_type_id
+ where is_embedded=true
+ group by mime_string
+ order by cnt desc
+ </sql>
+ </report>
+
+ <report reportName="Embedded Mimes In B"
+ reportFilename="mimes/embedded_mimes_B.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string, count(1) cnt from
+ profiles_b p
+ join mimes m on m.mime_type_id = p.mime_type_id
+ where is_embedded=true
+ group by mime_string
+ order by cnt desc
+ </sql>
+ </report>
+ <report reportName="Mime Differences A -> B"
+ reportFilename="mimes/mime_diffs_A_to_B.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select concat(ma.mime_string, ' -> ', mb.mime_string) as
+ MIME_A_TO_MIME_B, count(1) as COUNT
+ from profiles_a a
+ join profiles_b b on a.id=b.id
+ join mimes ma on ma.mime_type_id=a.mime_type_id
+ join mimes mb on mb.mime_type_id=b.mime_type_id
+ where a.mime_type_id <> b.mime_type_id
+ group by MIME_A_TO_MIME_B
+ order by COUNT DESC
+ </sql>
+ </report>
+
+ <report reportName="Mime Differences A -> B Details"
+ reportFilename="mimes/mime_diffs_A_to_B_details.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select concat(ma.mime_string, ' -> ', mb.mime_string) as
+ MIME_A_TO_MIME_B, file_path, a.file_name
+ from profiles_a a
+ join profiles_b b on a.id=b.id
+ join mimes ma on ma.mime_type_id=a.mime_type_id
+ join mimes mb on mb.mime_type_id=b.mime_type_id
+ join containers c on a.container_id=c.container_id
+ where a.mime_type_id <> b.mime_type_id
+ order by MIME_A_TO_MIME_B
+ </sql>
+ </report>
+
+ <report reportName="AllExceptionsByMimeA"
+ reportFilename="exceptions/exceptions_by_mime_A.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string, count(1) cnt from
+ exceptions_a e
+ join profiles_a p on p.id=e.id
+ join mimes m on m.mime_type_id = p.mime_type_id
+ group by mime_string
+ order by cnt desc
+ </sql>
+ </report>
+ <report reportName="AllExceptionsByMimeB"
+ reportFilename="exceptions/exceptions_by_mime_B.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string, count(1) cnt from
+ exceptions_b e
+ join profiles_b p on p.id=e.id
+ join mimes m on m.mime_type_id = p.mime_type_id
+ group by mime_string
+ order by cnt desc
+ </sql>
+ </report>
+
+ <report reportName="ContainerExceptionsByMimeA"
+ reportFilename="exceptions/container_exceptions_by_mime_A.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string, count(1) cnt from
+ exceptions_a e
+ join profiles_a p on p.id=e.id
+ join mimes m on m.mime_type_id = p.mime_type_id
+ where is_embedded=false
+ and parse_exception_type_id=0
+ group by mime_string
+ order by cnt desc
+ </sql>
+ </report>
+
+ <report reportName="ContainerExceptionsByMimeB"
+ reportFilename="exceptions/container_exceptions_by_mime_B.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string, count(1) cnt from
+ exceptions_b e
+ join profiles_b p on p.id=e.id
+ join mimes m on m.mime_type_id = p.mime_type_id
+ where is_embedded=false
+ and parse_exception_type_id=0
+ group by mime_string
+ order by cnt desc
+ </sql>
+ </report>
+ <report reportName="AllExceptionsByMimeByTypeA"
+ reportFilename="exceptions/exceptions_by_mime_by_typeA.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string as MIME_TYPE,
+ parse_exception_description as EXCEPTION_TYPE, count(1) as COUNT
+ from exceptions_a e
+ join profiles_a p on p.id=e.id
+ join containers c on p.container_id=c.container_id
+ join mimes m on m.mime_type_id=p.mime_type_id
+ join ref_parse_exception_types r on
+ r.parse_exception_type_id=e.parse_exception_type_id
+ group by p.mime_type_id, parse_exception_description
+ order by MIME_TYPE, EXCEPTION_TYPE
+ </sql>
+ </report>
+
+ <report reportName="AllExceptionsByMimeByTypeB"
+ reportFilename="exceptions/exceptions_by_mime_by_typeB.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string as MIME_TYPE,
+ parse_exception_description as EXCEPTION_TYPE, count(1) as COUNT
+ from exceptions_b e
+ join profiles_b p on p.id=e.id
+ join containers c on p.container_id=c.container_id
+ join mimes m on m.mime_type_id=p.mime_type_id
+ join ref_parse_exception_types r on
+ r.parse_exception_type_id=e.parse_exception_type_id
+ group by p.mime_type_id, parse_exception_description
+ order by MIME_TYPE, EXCEPTION_TYPE
+ </sql>
+ </report>
+
+ <report reportName="TextLostFromACausedByNewExceptionsInB"
+ reportFilename="exceptions/textLostFromACausedByNewExceptionsInB.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select file_path as FILE_PATH, ca.NUM_TOKENS as NUM_TOKENS_A,
+ cb.NUM_TOKENS as NUM_TOKENS_B,
+ ca.NUM_UNIQUE_TOKENS as NUM_UNIQUE_TOKENS_A, cb.NUM_UNIQUE_TOKENS
+ as NUM_UNIQUE_TOKENS_B,
+ ca.common_tokens_lang as COMMON_TOKENS_LANG_A,
+ ca.num_common_tokens as NUM_COMMON_TOKENS_A,
+ cb.common_tokens_lang as COMMON_TOKENS_LANG_B,
+ cb.num_common_tokens as NUM_COMMON_TOKENS_B,
+ ca.top_n_tokens as TOP_N_TOKENS_A, cb.top_n_tokens as TOP_N_TOKENS_B,
+ eb.ORIG_STACK_TRACE as ORIG_STACK_TRACE_B
+ from contents_a ca
+ join profiles_a pa on ca.id = pa.id
+ join containers c on pa.container_id=c.container_id
+ left join contents_b cb on ca.id=cb.id
+ left join exceptions_b eb on ca.id = eb.id
+ left join exceptions_a ea on ca.id = ea.id
+ where eb.orig_stack_trace is not null
+ and ea.orig_stack_trace is null
+ order by ca.num_common_tokens - ifnull(cb.num_common_tokens,0) desc
+ </sql>
+ </report>
+
+ <report reportName="FixedExceptionsInBByMimeType"
+ reportFilename="exceptions/fixedExceptionsInBByMimeType.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string as MIME_TYPE, count(1) as COUNT
+ from exceptions_a ea
+ left join exceptions_b eb on ea.id = eb.id
+ join profiles_a pa on pa.id=ea.id
+ join profiles_b pb on pa.id=pb.id
+ join containers c on pa.container_id=c.container_id
+ join mimes m on m.mime_type_id=pa.mime_type_id
+ where eb.id is null
+ and ea.parse_exception_type_id=0
+ group by mime_string
+ </sql>
+ </report>
+
+ <report reportName="FixedExceptionsInByDetails"
+ reportFilename="exceptions/fixedExceptionsInBDetails.xlsx"
+ format="xlsx"
+ includeSql="true">
+ <sql>
+ select mime_string as MIME_TYPE,
+ file_path, pa.file_name, pa.is_embedded
+ from exceptions_a ea
+ left join exceptions_b eb on ea.id = eb.id
+ join profiles_a pa on pa.id=ea.id
+ join profiles_b pb on pb.id=pa.id //this ensures that files were actually processed in both runs
+ join containers c on pa.container_id=c.container_id
+ join mimes m on m.mime_type_id=pa.mime_type_id
+ where eb.id is null
+ and ea.parse_exception_type_id=0
+ order by mime_string
+ </sql>
+ </report>
+ <report reportName="ContentsOfFixedExceptionsInB"
+ reportFilename="exceptions/contentsOfFixedExceptionsInB.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select file_path, mime_string as MIME_TYPE,
+ CONTENT_LENGTH,
+ NUM_TOKENS, NUM_UNIQUE_TOKENS,
+ TOP_N_TOKENS, LANG_ID_1,TOKEN_LENGTH_MEAN, TOKEN_LENGTH_STD_DEV
+ from exceptions_a ea
+ left join exceptions_b eb on ea.id = eb.id
+ join profiles_a p on p.id=ea.id
+ join contents_b cb on cb.id=ea.id
+ join containers c on p.container_id=c.container_id
+ join mimes m on m.mime_type_id=p.mime_type_id
+ where eb.id is null
+ and ea.parse_exception_type_id=0
+ </sql>
+ </report>
+
+ <report reportName="NewExceptionsByMimeType"
+ reportFilename="exceptions/newExceptionsInBByMimeType.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string as MIME_TYPE_A, count(1) as COUNT
+ from exceptions_b eb
+ left join exceptions_a ea on ea.id = eb.id
+ join profiles_a pa on pa.id=eb.id
+ join profiles_b pb on pb.id=pa.id
+ join containers c on pa.container_id=c.container_id
+ join mimes m on m.mime_type_id=pa.mime_type_id
+ where ea.id is null
+ and eb.parse_exception_type_id=0
+ group by mime_string
+ order by COUNT desc
+ </sql>
+ </report>
+
+ <report reportName="NewExceptionsInBByMimeTypeByStackTrace"
+ reportFilename="exceptions/newExceptionsInBByMimeTypeByStackTrace.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select MIME_STRING as MIME_TYPE, eb.sort_stack_trace, count(1) as
+ COUNT
+ from exceptions_b eb
+ left join exceptions_a ea on ea.id = eb.id
+ join profiles_a p on p.id=eb.id
+ join mimes m on m.mime_type_id=p.mime_type_id
+ where ea.id is null
+ and eb.parse_exception_type_id=0
+ group by MIME_TYPE, eb.sort_stack_trace
+ order by MIME_TYPE asc, COUNT desc
+ </sql>
+ </report>
+
+ <report reportName="NewExceptionsInBDetails"
+ reportFilename="exceptions/newExceptionsInBDetails.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select file_path, MIME_STRING as MIME_TYPE, p.length,
+ eb.orig_stack_trace, eb.sort_stack_trace
+ from exceptions_b eb
+ left join exceptions_a ea on ea.id = eb.id
+ join profiles_a p on p.id=eb.id
+ join containers c on p.container_id=c.container_id
+ join mimes m on m.mime_type_id=p.mime_type_id
+ where ea.id is null
+ and eb.parse_exception_type_id=0
+ order by MIME_TYPE asc, eb.ORIG_STACK_TRACE
+ </sql>
+ </report>
+
+ <report reportName="StackTracesByMimeInA"
+ reportFilename="exceptions/stackTracesByMimeInA.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select MIME_STRING as MIME_TYPE, e.sort_stack_trace, count(1) as
+ COUNT
+ from exceptions_a e
+ join profiles_a p on p.id=e.id
+ join mimes m on m.mime_type_id=p.mime_type_id
+ and e.parse_exception_type_id=0
+ group by MIME_TYPE, e.sort_stack_trace
+ order by MIME_TYPE asc, COUNT desc
+ </sql>
+ </report>
+
+ <report reportName="AllStackTracesInA"
+ reportFilename="exceptions/stackTracesInA.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select file_path, c.length as FILE_LENGTH, MIME_STRING as MIME_TYPE,
+ orig_stack_trace, sort_stack_trace
+ from exceptions_a e
+ join profiles_a p on p.id=e.id
+ join containers c on p.container_id=c.container_id
+ join mimes m on m.mime_type_id=p.mime_type_id
+ and e.parse_exception_type_id=0
+ order by MIME_TYPE asc, sort_stack_trace, orig_stack_trace,
+ FILE_LENGTH asc
+ </sql>
+ </report>
+ <report reportName="AllStackTracesInB"
+ reportFilename="exceptions/stackTracesInB.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select file_path, c.length as FILE_LENGTH, MIME_STRING as MIME_TYPE,
+ orig_stack_trace, sort_stack_trace
+ from exceptions_b e
+ join profiles_b p on p.id=e.id
+ join containers c on p.container_id=c.container_id
+ join mimes m on m.mime_type_id=p.mime_type_id
+ and e.parse_exception_type_id=0
+ order by MIME_TYPE asc, sort_stack_trace, orig_stack_trace,
+ FILE_LENGTH asc
+ </sql>
+ </report>
+
+ <report reportName="StackTracesByMimeInB"
+ reportFilename="exceptions/stackTracesByMimeInB.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select MIME_STRING as MIME_TYPE, e.sort_stack_trace, count(1) as
+ COUNT
+ from exceptions_b e
+ join profiles_b p on p.id=e.id
+ join mimes m on m.mime_type_id=p.mime_type_id
+ and e.parse_exception_type_id=0
+ group by MIME_TYPE, e.sort_stack_trace
+ order by MIME_TYPE asc, COUNT desc
+ </sql>
+ </report>
+ <report reportName="extractErrorsA"
+ reportFilename="exceptions/extract_errors_a.xlsx"
+ format="xlsx"
+ includeSql="true">
+ <sql>
+ select file_path, extract_error_description
+ from extract_errors_a e
+ join ref_extract_error_types t
+ on e.extract_error_type_id=t.extract_error_type_id
+ </sql>
+ </report>
+ <report reportName="extractErrorsB"
+ reportFilename="exceptions/extract_errors_b.xlsx"
+ format="xlsx"
+ includeSql="true">
+ <sql>
+ select file_path, extract_error_description
+ from extract_errors_b e
+ join ref_extract_error_types t
+ on e.extract_error_type_id=t.extract_error_type_id
+ </sql>
+ </report>
+ <report reportName="parseExceptionTypesA"
+ reportFilename="exceptions/overall_exception_types_a.xlsx"
+ format="xlsx"
+ includeSql="true">
+ <sql>
+ select parse_exception_description, count(1)
+ from exceptions_a e
+ join ref_parse_exception_types t on
+ t.parse_exception_type_id=e.parse_exception_type_id
+ group by e.parse_exception_type_id
+ </sql>
+ </report>
+ <report reportName="parseExceptionTypesB"
+ reportFilename="exceptions/overall_exception_types_b.xlsx"
+ format="xlsx"
+ includeSql="true">
+ <sql>
+ select parse_exception_description, count(1)
+ from exceptions_b e
+ join ref_parse_exception_types t on
+ t.parse_exception_type_id=e.parse_exception_type_id
+ group by e.parse_exception_type_id
+ </sql>
+ </report>
+
+ <report reportName="contentDiffsWExceptions"
+ reportFilename="content/content_diffs_with_exceptions.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select file_path,
+ ma.mime_string as mime_string_a,
+ mb.mime_string as mime_string_b,
+ ca.num_unique_tokens as NUM_UNIQUE_TOKENS_A,
+ cb.num_unique_tokens as NUM_UNIQUE_TOKENS_B,
+ ca.num_tokens as NUM_TOKENS_A,
+ cb.num_tokens as NUM_TOKENS_B,
+ ca.common_tokens_lang as COMMON_TOKENS_LANG_A,
+ ca.num_common_tokens as NUM_COMMON_TOKENS_A,
+ cb.common_tokens_lang as COMMON_TOKENS_LANG_B,
+ cb.num_common_tokens as NUM_COMMON_TOKENS_B,
+ ca.top_n_tokens as TOP_N_TOKENS_A,
+ cb.top_n_tokens as TOP_N_TOKENS_B,
+ ca.unicode_char_blocks as UNICODE_CHAR_BLOCKS_A,
+ cb.unicode_char_blocks as UNICODE_CHAR_BLOCKS_B,
+ top_10_unique_token_diffs_a,
+ top_10_unique_token_diffs_b,
+ top_10_more_in_a, top_10_more_in_b, dice_coefficient, overlap
+ from content_comparisons cc
+ join contents_a ca on ca.id=cc.id
+ left join contents_b cb on cb.id=cc.id
+ join profiles_a pa on pa.id = cc.id
+ join profiles_b pb on pb.id=cc.id
+ join containers c on c.container_id=pa.container_id
+ join mimes ma on ma.mime_type_id=pa.mime_type_id
+ join mimes mb on mb.mime_type_id=pb.mime_type_id
+ left join exceptions_a ea on ea.id=cc.id
+ left join exceptions_b eb on eb.id=cc.id
+ where (overlap < 0.95 or abs(ca.NUM_TOKENS-cb.NUM_TOKENS) >30)
+ and (ea.parse_exception_type_id is null or
+ ea.parse_exception_type_id <> 2)
+ and (eb.parse_exception_type_id is null or
+ eb.parse_exception_type_id <> 2)
+ order by ma.mime_string, overlap asc
+ limit 100000
+ </sql>
+ </report>
+ <report reportName="contentDiffsIgnoreExceptions"
+ reportFilename="content/content_diffs_ignore_exceptions.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select file_path,
+ ma.mime_string as mime_string_a,
+ mb.mime_string as mime_string_b,
+ ca.NUM_UNIQUE_TOKENS as NUM_UNIQUE_TOKENS_A,
+ cb.NUM_UNIQUE_TOKENS as NUM_UNIQUE_TOKENS_B,
+ ca.NUM_TOKENS as NUM_TOKENS_A,
+ cb.NUM_TOKENS as NUM_TOKENS_B,
+ ca.common_tokens_lang as COMMON_TOKENS_LANG_A,
+ ca.num_common_tokens as NUM_COMMON_TOKENS_A,
+ cb.common_tokens_lang as COMMON_TOKENS_LANG_B,
+ cb.num_common_tokens as NUM_COMMON_TOKENS_B,
+ ca.top_n_tokens as TOP_N_TOKENS_A,
+ cb.top_n_tokens as TOP_N_TOKENS_B,
+ ca.unicode_char_blocks as UNICODE_CHAR_BLOCKS_A,
+ cb.unicode_char_blocks as UNICODE_CHAR_BLOCKS_B,
+ top_10_unique_token_diffs_a,
+ top_10_unique_token_diffs_b,
+ top_10_more_in_a, top_10_more_in_b, dice_coefficient, overlap
+ from content_comparisons cc
+ join contents_a ca on ca.id=cc.id
+ join contents_b cb on cb.id=cc.id
+ join profiles_a pa on pa.id = cc.id
+ join profiles_b pb on pb.id=cc.id
+ join containers c on c.container_id=pa.container_id
+ join mimes ma on ma.mime_type_id=pa.mime_type_id
+ join mimes mb on mb.mime_type_id=pb.mime_type_id
+ left join exceptions_a ea on ea.id=cc.id
+ left join exceptions_b eb on eb.id=cc.id
+ where (overlap < 0.95 or abs(ca.NUM_TOKENS-cb.NUM_TOKENS) >30)
+ and (ea.parse_exception_type_id is null)
+ and (eb.parse_exception_type_id is null)
+ order by ma.mime_string, overlap asc
+ limit 100000
+ </sql>
+ </report>
+ <!-- <report reportName="MD5 Duplicate Counts A"
+ reportFilename="md5/md5_duplicate_counts_A.xlsx"
+ format="xlsx"
+ includeSql="true">
+ <sql>
+ select md5, count(1) cnt
+ from profiles_a
+ group by md5
+ having cnt > 2
+ order by cnt desc
+ </sql>
+ </report>
+
+ <report reportName="MD5 Duplicate Counts B"
+ reportFilename="md5/md5_duplicate_counts_B.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select md5, count(1) cnt
+ from profiles_b
+ group by md5
+ having cnt > 2
+ order by cnt desc
+ </sql>
+ </report>
+
+ <report reportName="MD5 Duplicates A"
+ reportFilename="md5/md5_duplicates_A.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select file_path, file_name, is_embedded, content_length, NUM_TOKENS, p.md5
+ from md5_multiples_tmp_a t
+ join profiles_a p on p.md5 = t.md5
+ join containers c on p.container_id = c.container_id
+ join contents_a cb on p.id=cb.id
+ order by t.cnt desc
+ </sql>
+ </report>
+
+ <report reportName="MD5 Duplicates B"
+ reportFilename="md5/md5_duplicates_B.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select file_path, file_name, is_embedded, content_length, NUM_TOKENS, p.md5
+ from md5_multiples_tmp_b t
+ join profiles_b p on p.md5 = t.md5
+ join containers c on p.container_id = c.container_id
+ join contents_b cb on p.id=cb.id
+ order by t.cnt desc
+ </sql>
+ </report>
+ -->
+
+ <report reportName="Attachment Diffs"
+ reportFilename="attachments/attachment_diffs.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select file_path,
+ ma.mime_string as mime_string_a,
+ mb.mime_string as mime_string_b,
+ pa.num_attachments as num_attachments_a,
+ pb.num_attachments as num_attachments_b,
+ ea.parse_exception_type_id as exception_type_id_a,
+ eb.parse_exception_type_id as exception_type_id_b
+ from profiles_a pa
+ join profiles_b pb on pa.id= pb.id
+ join containers c on pa.container_id=c.container_id
+ join mimes ma on pa.mime_type_id=ma.mime_type_id
+ join mimes mb on pb.mime_type_id=mb.mime_type_id
+ left join exceptions_a ea on ea.id=pa.id
+ left join exceptions_b eb on eb.id=pb.id
+ where pa.is_embedded=false and
+ ea.parse_exception_type_id is null and
+ eb.parse_exception_type_id is null
+ and pa.num_attachments <> pb.num_attachments
+ order by ma.mime_string, pb.num_attachments-pa.num_attachments
+ limit 1000;
+ </sql>
+ </report>
+
+ <!-- metadata values -->
+ <report reportName="Metadata Value Diffs"
+ reportFilename="metadata/metadata_value_count_diffs.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select file_path,
+ ma.mime_string as mime_string_a,
+ mb.mime_string as mime_string_b,
+ pa.num_metadata_values as num_metadata_values_a,
+ pb.num_metadata_values as num_metadata_values_b,
+ ea.parse_exception_type_id as parse_ex_type_id_a,
+ eb.parse_exception_type_id as parse_ex_type_id_b
+ from profiles_a pa
+ join profiles_b pb on pa.id= pb.id
+ join containers c on pa.container_id=c.container_id
+ join mimes ma on pa.mime_type_id=ma.mime_type_id
+ join mimes mb on pb.mime_type_id=mb.mime_type_id
+ left join exceptions_a ea on ea.id=pa.id
+ left join exceptions_b eb on eb.id=pb.id
+ where
+ ea.parse_exception_type_id is null and
+ eb.parse_exception_type_id is null
+ and pa.num_metadata_values <> pb.num_metadata_values
+ order by ma.mime_string,
+ pb.num_metadata_values-pa.num_metadata_values
+ </sql>
+ </report>
+
+ <after>
+ <sql>drop table if exists md5_multiples_tmp_a</sql>
+ <sql>drop table if exists md5_multiples_tmp_b</sql>
+ </after>
+</reports>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/resources/lucene-analyzers.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/lucene-analyzers.json b/tika-eval/src/main/resources/lucene-analyzers.json
new file mode 100644
index 0000000..268494f
--- /dev/null
+++ b/tika-eval/src/main/resources/lucene-analyzers.json
@@ -0,0 +1,107 @@
+{
+ "analyzers": {
+ "general" :
+ {
+ "charfilters": [
+ {
+ "factory": "oala.charfilter.MappingCharFilterFactory",
+ "params": {
+ "mapping" : "/lucene-char-mapping.txt"
+ }
+ }
+ ],
+ "tokenizer": {
+ "factory": "oala.standard.UAX29URLEmailTokenizerFactory",
+ "params": {}
+ },
+ "tokenfilters": [
+ {
+ "factory": "oala.icu.ICUFoldingFilterFactory",
+ "params": {}
+ },
+ {
+ "factory": "oala.cjk.CJKBigramFilterFactory",
+ "params": {
+ "outputUnigrams" : "false"
+ }
+ }
+ ]
+
+ },
+
+ "alpha" :
+ {
+ "charfilters": [
+ {
+ "factory": "oala.charfilter.MappingCharFilterFactory",
+ "params": {
+ "mapping" : "/lucene-char-mapping.txt"
+ }
+ }
+ ],
+ "tokenizer": {
+ "factory": "oala.standard.UAX29URLEmailTokenizerFactory",
+ "params": {}
+ },
+ "tokenfilters": [
+ {
+ "factory": "oala.icu.ICUFoldingFilterFactory",
+ "params": {}
+ },
+ {
+ "factory": "oala.pattern.PatternReplaceFilterFactory",
+ "params": {
+ "pattern": "^[\\w+\\.]{1,30}@(?:\\w+\\.){1,10}\\w+$",
+ "replacement": "___email___",
+ "replace": "all"
+ }
+ },
+ {
+ "factory": "oala.pattern.PatternReplaceFilterFactory",
+ "params": {
+ "pattern": "^(?:(?:ftp|https?):\\/\\/)?(?:\\w+\\.){1,10}\\w+$",
+ "replacement": "___url___",
+ "replace": "all"
+ }
+ },
+ {
+ "factory": "oala.cjk.CJKBigramFilterFactory",
+ "params": {
+ "outputUnigrams" : "false"
+ }
+ },
+ {
+ "factory": "org.apache.tika.eval.tokens.AlphaIdeographFilterFactory",
+ "params": {}
+ }
+ ]
+
+ },
+ "common_tokens" :
+ {
+ "tokenizer": {
+ "factory": "oala.standard.UAX29URLEmailTokenizerFactory",
+ "params": {}
+ },
+
+ "tokenfilters": [
+ {
+ "factory": "oala.cjk.CJKBigramFilterFactory",
+ "params": {
+ "outputUnigrams" : "false"
+ }
+ },
+ {
+ "factory": "org.apache.tika.eval.tokens.CJKBigramAwareLengthFilterFactory",
+ "params": {
+ "min" : 4,
+ "max" : 20
+ }
+ }
+
+ ]
+
+ }
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/resources/lucene-char-mapping.txt
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/lucene-char-mapping.txt b/tika-eval/src/main/resources/lucene-char-mapping.txt
new file mode 100644
index 0000000..06db6b9
--- /dev/null
+++ b/tika-eval/src/main/resources/lucene-char-mapping.txt
@@ -0,0 +1,2 @@
+"\u2018" => "'"
+"\u2019" => "'"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/resources/profile-reports.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/profile-reports.xml b/tika-eval/src/main/resources/profile-reports.xml
new file mode 100644
index 0000000..2a94a97
--- /dev/null
+++ b/tika-eval/src/main/resources/profile-reports.xml
@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<reports>
+
+
+ <before>
+ <!-- <sql>create index on x</sql>-->
+ </before>
+
+
+ <!-- MIMES -->
+ <report reportName="All Mimes"
+ reportFilename="mimes/all_mimes.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string, count(1) cnt from
+ profiles p
+ join mimes m on m.mime_type_id = p.mime_type_id
+ group by mime_string
+ order by cnt desc
+ </sql>
+ </report>
+ <report reportName="Container Mimes"
+ reportFilename="mimes/container_mimes.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string, count(1) cnt from
+ profiles p
+ join mimes m on m.mime_type_id = p.mime_type_id
+ where is_embedded=false
+ group by mime_string
+ order by cnt desc
+ </sql>
+ </report>
+
+ <report reportName="Embedded Mimes"
+ reportFilename="mimes/embedded_mimes.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select mime_string, count(1) cnt from
+ profiles p
+ join mimes m on m.mime_type_id = p.mime_type_id
+ where is_embedded=true
+ group by mime_string
+ order by cnt desc
+ </sql>
+ </report>
+
+ <!-- content -->
+ <report reportName="Common Tokens by Lang"
+ reportFilename="content/common_tokens_by_lang.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select common_tokens_lang, sum(num_common_tokens) as cnt
+ from contents
+ group by common_tokens_lang
+ order by cnt desc;
+ </sql>
+ </report>
+
+ <report reportName="Detected Languages"
+ reportFilename="content/detected_langs.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select LANG_ID_1 as DetectedLang, count(1) as cnt
+ from contents
+ group by LANG_ID_1
+ order by cnt desc
+ </sql>
+ </report>
+
+
+
+ <report reportName="Token Count by Detected Language"
+ reportFilename="content/num_tokens_by_detected_langs.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select LANG_ID_1 as DetectedLang, sum(num_tokens) as cnt
+ from contents
+ group by LANG_ID_1
+ order by cnt desc;
+ </sql>
+ </report>
+ <report reportName="Exceptions by Type"
+ reportFilename="exceptions/exceptions_by_type.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select LANG_ID_1 as DetectedLang, count(1) as cnt
+ from contents
+ group by LANG_ID_1
+ order by cnt desc
+ </sql>
+ </report>
+
+
+ <report reportName="Embedded Exceptions by Type"
+ reportFilename="exceptions/exceptions_by_type_embedded.xlsx"
+ format="xlsx"
+ includeSql="true">
+
+ <sql>
+ select parse_exception_description, count(1) cnt
+ from parse_exceptions e
+ join profiles p on p.id = e.id
+ join ref_parse_exception_types et on et.parse_exception_type_id=e.parse_exception_type_id
+ where is_embedded=true
+ group by parse_exception_description
+ order by cnt desc;
+ </sql>
+ </report>
+ <after>
+
+ <!--<sql>drop index on x</sql>
+ -->
+ </after>
+</reports>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/resources/tika-eval-comparison-config.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/tika-eval-comparison-config.xml b/tika-eval/src/main/resources/tika-eval-comparison-config.xml
new file mode 100644
index 0000000..85cde71
--- /dev/null
+++ b/tika-eval/src/main/resources/tika-eval-comparison-config.xml
@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<tika-batch-config
+ maxAliveTimeSeconds="-1"
+ pauseOnEarlyTerminationMillis = "500"
+ timeoutCheckPulseMillis="1000"
+ maxQueueSize="10000"
+ numConsumers="5"
+ timeoutThresholdMillis="300000"
+ >
+
+ <commandline>
+ <option opt="c" longOpt="tika-config" hasArg="true"
+ description="TikaConfig file"/>
+ <option opt="bc" longOpt="batch-config" hasArg="true"
+ description="xml batch config file" required="true"/>
+ <option opt="inputDir" hasArg="true"
+ description="dir to start crawling"/>
+ <option opt="numConsumers" hasArg="true"
+ description="number of fileConsumers threads"/>
+ <option opt="extractsA" hasArg="true"
+ description="this dir for analysis" required="false"/>
+ <option opt="extractsB" hasArg="true"
+ description="thatDir for analysis"/>
+ <option opt="db" hasArg="true"
+ description="name of db directory or file to which to write results"/>
+ <option opt="alterExtract" hasArg="true"
+ description="for json-formatted extract files
+ process full metadata list ('as_is'=default),
+ take just the first/container document ('first_only'),
+ concatenate all content into the first metadata item ('concatenate_content')"/>
+ <option opt="includeFilePat" hasArg="true"
+ description="regex for files to include"/>
+ </commandline>
+
+
+ <!--
+ Can also add startDir: this tells the crawler to start indexing a
+ child directory of the srcDir directory.
+ -->
+ <crawler builderClass="org.apache.tika.batch.fs.builders.FSCrawlerBuilder"
+ crawlOrder="sorted"
+ maxConsecWaitMillis="30000"
+ maxFilesToAdd="-1"
+ maxFilesToConsider="-1"
+ includeFilePat=""
+ excludeFilePat=""
+ maxFileSizeBytes="10000000"
+ />
+
+ <consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
+ consumerBuilderClass="org.apache.tika.eval.batch.FileComparerBuilder"
+ dbAppend="false"
+ crawlingInputDir="false"
+ minJsonFileSizeBytes="-1"
+ maxJsonFileSizeBytes="2000000"
+ commonTokens="resources/commontokens"
+ />
+
+ <!-- reporter and interrupter are optional -->
+ <reporter builderClass="org.apache.tika.batch.builders.SimpleLogReporterBuilder" sleepMillis="1000" staleThresholdMillis="500000"/>
+ <interrupter builderClass="org.apache.tika.batch.builders.InterrupterBuilder"/>
+</tika-batch-config>
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/resources/tika-eval-profiler-config.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/tika-eval-profiler-config.xml b/tika-eval/src/main/resources/tika-eval-profiler-config.xml
new file mode 100644
index 0000000..bd94b25
--- /dev/null
+++ b/tika-eval/src/main/resources/tika-eval-profiler-config.xml
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<tika-batch-config
+ maxAliveTimeSeconds="-1"
+ pauseOnEarlyTerminationMillis="500"
+ timeoutCheckPulseMillis="1000"
+ maxQueueSize="10000"
+ numConsumers="5"
+ timeoutThresholdMillis="300000">
+
+ <commandline>
+ <option opt="c" longOpt="tika-config" hasArg="true"
+ description="TikaConfig file"/>
+
+ <option opt="bc" longOpt="batch-config" hasArg="true"
+ description="xml batch config file" required="true"/>
+ <option opt="inputDir" hasArg="true"
+ description="dir to start crawling"/>
+ <option opt="numConsumers" hasArg="true"
+ description="number of fileConsumers threads"/>
+ <option opt="extractDir" hasArg="true"
+ description="this dir for analysis" required="false"/>
+ <option opt="db" hasArg="true"
+ description="name of db directory or file to which to write results"/>
+ <option opt="alterExtract" hasArg="true"
+ description="for json-formatted extract files
+ process full metadata list ('as_is'=default),
+ take just the first/container document ('first_only'),
+ concatenate all content into the first metadata item ('concatenate_content')"/>
+
+ </commandline>
+
+
+ <!--
+ Can also add startDir: this tells the crawler to start indexing a
+ child directory of the inputDir directory.
+ -->
+ <crawler builderClass="org.apache.tika.batch.fs.builders.FSCrawlerBuilder"
+
+ crawlOrder="sorted"
+ maxConsecWaitMillis="5000"
+ maxFilesToAdd="-1"
+ maxFilesToConsider="-1"
+ includeFilePat=""
+ excludeFilePat=""
+ maxFileSizeBytes="-1"
+ />
+
+ <consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
+ consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"
+ commonTokens="resources/commontokens"/>
+
+
+ <!-- reporter and interrupter are optional -->
+ <reporter builderClass="org.apache.tika.batch.builders.SimpleLogReporterBuilder" sleepMillis="1000"
+ staleThresholdMillis="500000"/>
+ <interrupter builderClass="org.apache.tika.batch.builders.InterrupterBuilder"/>
+</tika-batch-config>
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/java/org/apache/tika/MockDBWriter.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/MockDBWriter.java b/tika-eval/src/test/java/org/apache/tika/MockDBWriter.java
new file mode 100644
index 0000000..b2edab7
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/MockDBWriter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.tika.eval.db.Cols;
+import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.IDBWriter;
+
+public class MockDBWriter implements IDBWriter {
+ //Map of tableName and tables
+ //each table consists of a list of rows.
+ //Each row consists of a map of columns/values
+ Map<String, List<Map<Cols, String>>> db = new HashMap<String, List<Map<Cols, String>>>();
+
+ public MockDBWriter() throws Exception {
+ }
+
+ @Override
+ public void writeRow(TableInfo tableInfo, Map<Cols, String> row) throws IOException {
+ List<Map<Cols, String>> table = db.get(tableInfo.getName());
+ if (table == null) {
+ table = new ArrayList<Map<Cols, String>>();
+ }
+ table.add(row);
+ db.put(tableInfo.getName(), table);
+ }
+
+ @Override
+ public void close() throws IOException {
+ //no-op
+ }
+
+ @Override
+ public int getMimeId(String mimeString) {
+ //TODO: fix this
+ return 0;
+ }
+
+ public List<Map<Cols, String>> getTable(TableInfo tableInfo) {
+ if (db.get(tableInfo.getName()) == null) {
+ System.err.println("I can't seem to find: "+ tableInfo.getName() + ", but I do see:");
+ for (String table : db.keySet()) {
+ System.err.println(table);
+ }
+ }
+ return db.get(tableInfo.getName());
+ }
+
+ public void clear() {
+ db.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/test/java/org/apache/tika/eval/AnalyzerManagerTest.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/AnalyzerManagerTest.java b/tika-eval/src/test/java/org/apache/tika/eval/AnalyzerManagerTest.java
new file mode 100644
index 0000000..f1b9163
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/AnalyzerManagerTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.tika.eval.tokens.AnalyzerManager;
+import org.junit.Test;
+
+public class AnalyzerManagerTest {
+
+ @Test
+ public void testGeneral() throws Exception {
+ AnalyzerManager analyzerManager = AnalyzerManager.newInstance();
+ Analyzer general = analyzerManager.getGeneralAnalyzer();
+ TokenStream ts = general.tokenStream("f", "tHe quick aaaa aaa anD dirty dog");
+ ts.reset();
+
+ CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
+ Set<String> seen = new HashSet<>();
+ while (ts.incrementToken()) {
+ seen.add(termAtt.toString());
+ }
+ ts.end();
+ ts.close();
+
+ assertTrue(seen.contains("the"));
+ assertTrue(seen.contains("and"));
+ assertTrue(seen.contains("dog"));
+
+ }
+
+ @Test
+ public void testCommon() throws Exception {
+ AnalyzerManager analyzerManager = AnalyzerManager.newInstance();
+ Analyzer common = analyzerManager.getAlphaIdeoAnalyzer();
+ TokenStream ts = common.tokenStream("f", "the 5,000.12 and dirty dog");
+ ts.reset();
+ CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
+ Set<String> seen = new HashSet<>();
+ while (ts.incrementToken()) {
+ if (termAtt.toString().contains("5")) {
+ fail("Shouldn't have found a numeric");
+ }
+ seen.add(termAtt.toString());
+ }
+ ts.end();
+ ts.close();
+
+ assertTrue(seen.contains("the"));
+ assertTrue(seen.contains("and"));
+ assertTrue(seen.contains("dog"));
+
+
+ }
+
+}
[6/6] tika git commit: TIKA-1332 initial commit of tika-eval. More
work remains.
Posted by ta...@apache.org.
TIKA-1332 initial commit of tika-eval. More work remains.
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/5e49c330
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/5e49c330
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/5e49c330
Branch: refs/heads/2.x
Commit: 5e49c33087bbf03763b05efda3bbb96d8cc20ea4
Parents: 6bfe5d5
Author: tballison <ta...@mitre.org>
Authored: Thu Feb 16 12:19:54 2017 -0500
Committer: tballison <ta...@mitre.org>
Committed: Thu Feb 16 12:19:54 2017 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 +
LICENSE.txt | 8 +
pom.xml | 1 +
tika-eval/pom.xml | 281 +++++++
.../org/apache/tika/eval/AbstractProfiler.java | 693 ++++++++++++++++
.../org/apache/tika/eval/EvalFilePaths.java | 108 +++
.../org/apache/tika/eval/ExtractComparer.java | 455 +++++++++++
.../org/apache/tika/eval/ExtractProfiler.java | 238 ++++++
.../java/org/apache/tika/eval/TikaEvalCLI.java | 262 ++++++
.../apache/tika/eval/XMLErrorLogUpdater.java | 226 ++++++
.../tika/eval/batch/DBConsumersManager.java | 92 +++
.../tika/eval/batch/EvalConsumerBuilder.java | 134 ++++
.../tika/eval/batch/EvalConsumersBuilder.java | 133 ++++
.../tika/eval/batch/FileComparerBuilder.java | 122 +++
.../eval/batch/SingleFileConsumerBuilder.java | 108 +++
.../apache/tika/eval/db/AbstractDBBuffer.java | 77 ++
.../java/org/apache/tika/eval/db/ColInfo.java | 116 +++
.../main/java/org/apache/tika/eval/db/Cols.java | 90 +++
.../java/org/apache/tika/eval/db/DBBuffer.java | 54 ++
.../java/org/apache/tika/eval/db/DBUtil.java | 201 +++++
.../java/org/apache/tika/eval/db/H2Util.java | 71 ++
.../org/apache/tika/eval/db/MimeBuffer.java | 144 ++++
.../java/org/apache/tika/eval/db/TableInfo.java | 64 ++
.../java/org/apache/tika/eval/io/DBWriter.java | 141 ++++
.../org/apache/tika/eval/io/ExtractReader.java | 161 ++++
.../java/org/apache/tika/eval/io/IDBWriter.java | 31 +
.../apache/tika/eval/io/XMLLogMsgHandler.java | 26 +
.../org/apache/tika/eval/io/XMLLogReader.java | 120 +++
.../org/apache/tika/eval/reports/Report.java | 197 +++++
.../tika/eval/reports/ResultsReporter.java | 295 +++++++
.../tika/eval/reports/XLSXHREFFormatter.java | 79 ++
.../tika/eval/reports/XLSXNumFormatter.java | 54 ++
.../tika/eval/reports/XSLXCellFormatter.java | 30 +
.../tokens/AlphaIdeographFilterFactory.java | 74 ++
.../tika/eval/tokens/AnalyzerDeserializer.java | 345 ++++++++
.../tika/eval/tokens/AnalyzerManager.java | 95 +++
.../CJKBigramAwareLengthFilterFactory.java | 74 ++
.../eval/tokens/CommonTokenCountManager.java | 141 ++++
.../tika/eval/tokens/CommonTokenResult.java | 37 +
.../tika/eval/tokens/ContrastStatistics.java | 78 ++
.../tika/eval/tokens/TokenContraster.java | 183 +++++
.../eval/tokens/TokenCountPriorityQueue.java | 49 ++
.../apache/tika/eval/tokens/TokenCounter.java | 167 ++++
.../apache/tika/eval/tokens/TokenIntPair.java | 82 ++
.../tika/eval/tokens/TokenStatistics.java | 127 +++
.../tika/eval/util/LanguageIDWrapper.java | 69 ++
...ache.lucene.analysis.util.TokenFilterFactory | 17 +
.../src/main/resources/comparison-reports.xml | 791 +++++++++++++++++++
.../src/main/resources/lucene-analyzers.json | 107 +++
.../src/main/resources/lucene-char-mapping.txt | 2 +
.../src/main/resources/profile-reports.xml | 148 ++++
.../resources/tika-eval-comparison-config.xml | 81 ++
.../resources/tika-eval-profiler-config.xml | 76 ++
.../test/java/org/apache/tika/MockDBWriter.java | 73 ++
.../apache/tika/eval/AnalyzerManagerTest.java | 79 ++
.../org/apache/tika/eval/ComparerBatchTest.java | 411 ++++++++++
.../org/apache/tika/eval/ProfilerBatchTest.java | 236 ++++++
.../apache/tika/eval/SimpleComparerTest.java | 289 +++++++
.../org/apache/tika/eval/TikaEvalCLITest.java | 42 +
.../apache/tika/eval/db/AbstractBufferTest.java | 160 ++++
.../apache/tika/eval/io/ExtractReaderTest.java | 85 ++
.../tika/eval/io/FatalExceptionReaderTest.java | 32 +
.../tika/eval/reports/ResultsReporterTest.java | 60 ++
.../tika/eval/tokens/LuceneTokenCounter.java | 191 +++++
.../tika/eval/tokens/TokenCounterTest.java | 131 +++
.../org/apache/tika/eval/util/MimeUtilTest.java | 65 ++
tika-eval/src/test/resources/commontokens/en | 8 +
tika-eval/src/test/resources/commontokens/es | 10 +
tika-eval/src/test/resources/commontokens/zh-cn | 8 +
tika-eval/src/test/resources/commontokens/zh-tw | 8 +
tika-eval/src/test/resources/log4j.properties | 11 +
.../src/test/resources/log4j_process.properties | 11 +
...ingle-file-profiler-crawl-extract-config.xml | 72 ++
.../single-file-profiler-crawl-input-config.xml | 73 ++
.../batch-logs/batch-process-fatal.xml | 59 ++
.../test-dirs/extractsA/file1.pdf.json | 5 +
.../extractsA/file10_permahang.txt.json | 0
.../test-dirs/extractsA/file11_oom.txt.json | 0
.../test-dirs/extractsA/file12_es.txt.json | 4 +
.../extractsA/file13_attachANotB.doc.json | 10 +
.../extractsA/file2_attachANotB.doc.json | 10 +
.../extractsA/file3_attachBNotA.doc.json | 4 +
.../test-dirs/extractsA/file4_emptyB.pdf.json | 4 +
.../test-dirs/extractsA/file5_emptyA.pdf.json | 0
.../test-dirs/extractsA/file6_accessEx.pdf.json | 1 +
.../test-dirs/extractsA/file7_badJson.pdf.json | 4 +
.../test-dirs/extractsA/file8_IOEx.pdf.json | 1 +
.../test-dirs/extractsB/file1.pdf.json | 2 +
.../test-dirs/extractsB/file11_oom.txt.json | 0
.../test-dirs/extractsB/file12_es.txt.json | 4 +
.../extractsB/file13_attachANotB.doc.txt | 1 +
.../extractsB/file2_attachANotB.doc.json | 4 +
.../extractsB/file3_attachBNotA.doc.json | 10 +
.../test-dirs/extractsB/file4_emptyB.pdf.json | 0
.../test-dirs/extractsB/file5_emptyA.pdf.json | 4 +
.../test-dirs/extractsB/file6_accessEx.pdf.json | 1 +
.../test-dirs/extractsB/file7_badJson.pdf.json | 0
.../test-dirs/extractsB/file8_IOEx.pdf.json | 1 +
.../resources/test-dirs/raw_input/file1.pdf | 13 +
.../test-dirs/raw_input/file11_oom.txt | 2 +
.../test-dirs/raw_input/file2_attachANotB.doc | 13 +
.../test-dirs/raw_input/file3_attachBNotA.doc | 13 +
.../test-dirs/raw_input/file4_emptyB.pdf | 13 +
.../test-dirs/raw_input/file5_emptyA.pdf | 13 +
.../test-dirs/raw_input/file6_accessEx.pdf | 13 +
.../test-dirs/raw_input/file7_badJson.pdf | 13 +
.../test-dirs/raw_input/file8_IOEx.pdf | 13 +
.../test-dirs/raw_input/file9_noextract.txt | 1 +
108 files changed, 9848 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 72fc96c..785edfd 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -17,6 +17,8 @@ Release 2.0 - ???
Release 1.15 -???
+ * Added tika-eval module (TIKA-1332).
+
* Add parsers for EMF/WMF files (TIKA-2246/TIKA-2247).
* Official mime types for BMP, EMF and WMF have been registered with
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
index 9576237..0673358 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -370,3 +370,11 @@ JUnRAR (https://github.com/edmund-wagner/junrar/)
Sqlite (bundled in org.xerial's sqlite-jdbc)
This product bundles Sqlite, which is in the Public Domain. For details
see: https://www.sqlite.org/copyright.html
+
+H2 Database in tika-eval
+ This software contains unmodified binary redistributions for
+ H2 database engine (http://www.h2database.com/),
+ which is dual licensed and available under the MPL 2.0
+ (Mozilla Public License) or under the EPL 1.0 (Eclipse Public License).
+ An original copy of the license agreement can be found at:
+ http://www.h2database.com/html/license.html
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7df81eb..8e7ff7b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -61,6 +61,7 @@
<module>tika-langdetect</module>
<module>tika-example</module>
<module>tika-java7</module>
+ <module>tika-eval</module>
</modules>
<profiles>
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/pom.xml
----------------------------------------------------------------------
diff --git a/tika-eval/pom.xml b/tika-eval/pom.xml
new file mode 100644
index 0000000..ee0940c
--- /dev/null
+++ b/tika-eval/pom.xml
@@ -0,0 +1,281 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <properties>
+ <cli.version>1.3.1</cli.version> <!--sync version with tika-server or move to parent? -->
+ <lucene.version>6.2.1</lucene.version>
+ <poi.version>3.16-beta2</poi.version>
+
+ </properties>
+
+ <parent>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-parent</artifactId>
+ <version>1.15-SNAPSHOT</version>
+ <relativePath>../tika-parent/pom.xml</relativePath>
+ </parent>
+
+ <artifactId>tika-eval</artifactId>
+ <name>Apache Tika eval</name>
+ <url>http://tika.apache.org/</url>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-batch</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-serialization</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.4.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-csv</artifactId>
+ <version>1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <version>1.4.193</version>
+ </dependency>
+ <dependency>
+ <groupId>com.optimaize.languagedetector</groupId>
+ <artifactId>language-detector</artifactId>
+ <version>0.5</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>${cli.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-icu</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-memory</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.poi</groupId>
+ <artifactId>poi</artifactId>
+ <version>${poi.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.poi</groupId>
+ <artifactId>poi-ooxml</artifactId>
+ <version>${poi.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.poi</groupId>
+ <artifactId>poi-ooxml-schemas</artifactId>
+ <version>${poi.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.poi</groupId>
+ <artifactId>poi-scratchpad</artifactId>
+ <version>${poi.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-batch</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>
+ false
+ </createDependencyReducedPom>
+ <!-- <filters> -->
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.apache.tika.eval.TikaEvalCLI</mainClass>
+ </transformer>
+
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-remote-resources-plugin</artifactId>
+ <version>1.5</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>bundle</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <includes>
+ <include>**/*.xml</include>
+ </includes>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <Bundle-DocURL>${project.url}</Bundle-DocURL>
+ <Bundle-Activator>
+ org.apache.tika.config.TikaActivator
+ </Bundle-Activator>
+ <Bundle-ActivationPolicy>lazy</Bundle-ActivationPolicy>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/test/resources/org/apache/tika/**</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>2.10</version>
+ <configuration>
+ <additionalClasspathElements>
+ <additionalClasspathElement>
+ ${project.build.directory}/${project.build.finalName}.jar
+ </additionalClasspathElement>
+ </additionalClasspathElements>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+
+ </build>
+
+
+
+ <organization>
+ <name>The Apache Software Foundation</name>
+ <url>http://www.apache.org</url>
+ </organization>
+ <scm>
+ <url>http://svn.apache.org/viewvc/tika/trunk/tika-batch</url>
+ <connection>scm:svn:http://svn.apache.org/repos/asf/tika/trunk/tika-batch</connection>
+ <developerConnection>scm:svn:https://svn.apache.org/repos/asf/tika/trunk/tika-batch</developerConnection>
+ </scm>
+ <issueManagement>
+ <system>JIRA</system>
+ <url>https://issues.apache.org/jira/browse/TIKA</url>
+ </issueManagement>
+ <ciManagement>
+ <system>Jenkins</system>
+ <url>https://builds.apache.org/job/Tika-trunk/</url>
+ </ciManagement>
+
+
+</project>
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java b/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
new file mode 100644
index 0000000..24f7358
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
@@ -0,0 +1,693 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval;
+
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.optimaize.langdetect.DetectedLanguage;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.batch.FileResourceConsumer;
+import org.apache.tika.batch.fs.FSProperties;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.eval.db.ColInfo;
+import org.apache.tika.eval.db.Cols;
+import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.IDBWriter;
+import org.apache.tika.eval.tokens.AnalyzerManager;
+import org.apache.tika.eval.tokens.CommonTokenCountManager;
+import org.apache.tika.eval.tokens.CommonTokenResult;
+import org.apache.tika.eval.tokens.TokenCounter;
+import org.apache.tika.eval.tokens.TokenIntPair;
+import org.apache.tika.eval.tokens.TokenStatistics;
+import org.apache.tika.eval.util.LanguageIDWrapper;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.PagedText;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.RecursiveParserWrapper;
+import org.apache.tika.utils.ExceptionUtils;
+
+public abstract class AbstractProfiler extends FileResourceConsumer {
+
+ private static final String[] EXTRACT_EXTENSIONS = {
+ ".json",
+ ".txt",
+ ""
+ };
+
+ private static final String[] COMPRESSION_EXTENSIONS = {
+ "",
+ ".bz2",
+ ".gzip",
+ ".zip",
+ };
+ static final long NON_EXISTENT_FILE_LENGTH = -1l;
+
+ public static TableInfo REF_EXTRACT_ERROR_TYPES = new TableInfo("ref_extract_error_types",
+ new ColInfo(Cols.EXTRACT_ERROR_TYPE_ID, Types.INTEGER),
+ new ColInfo(Cols.EXTRACT_ERROR_DESCRIPTION, Types.VARCHAR, 128)
+ );
+
+
+ public static TableInfo REF_PARSE_ERROR_TYPES = new TableInfo("ref_parse_error_types",
+ new ColInfo(Cols.PARSE_ERROR_TYPE_ID, Types.INTEGER),
+ new ColInfo(Cols.PARSE_ERROR_DESCRIPTION, Types.VARCHAR, 128)
+ );
+
+ public static TableInfo REF_PARSE_EXCEPTION_TYPES = new TableInfo("ref_parse_exception_types",
+ new ColInfo(Cols.PARSE_EXCEPTION_TYPE_ID, Types.INTEGER),
+ new ColInfo(Cols.PARSE_EXCEPTION_DESCRIPTION, Types.VARCHAR, 128)
+ );
+
+ public static final String TRUE = Boolean.toString(true);
+ public static final String FALSE = Boolean.toString(false);
+
+
+ protected static final AtomicInteger CONTAINER_ID = new AtomicInteger();
+ protected static final AtomicInteger ID = new AtomicInteger();
+
+
+ private final static String UNKNOWN_EXTENSION = "unk";
+ //make this configurable
+ private final static String DIGEST_KEY = "X-TIKA:digest:MD5";
+
+ private static CommonTokenCountManager commonTokenCountManager;
+ private String lastExtractExtension = null;
+
+ final AnalyzerManager analyzerManager;
+ final TokenCounter tokenCounter;
+
+ public enum EXTRACT_ERROR_TYPE {
+ //what do you see when you look at the extract file
+ NO_EXTRACT_FILE,
+ ZERO_BYTE_EXTRACT_FILE,
+ EXTRACT_PARSE_EXCEPTION
+ }
+
+ public enum EXCEPTION_TYPE {
+ RUNTIME,
+ ENCRYPTION,
+ ACCESS_PERMISSION,
+ UNSUPPORTED_VERSION,
+ }
+
+ public enum PARSE_ERROR_TYPE {
+ //what was gathered from the log file during the batch run
+ OOM,
+ TIMEOUT
+ }
+
+ public static TableInfo MIME_TABLE = new TableInfo("mimes",
+ new ColInfo(Cols.MIME_TYPE_ID, Types.INTEGER, "PRIMARY KEY"),
+ new ColInfo(Cols.MIME_STRING, Types.VARCHAR, 256),
+ new ColInfo(Cols.FILE_EXTENSION, Types.VARCHAR, 12)
+ );
+
+ private static Pattern FILE_NAME_CLEANER = Pattern.compile("\\.(json|txt)(\\.(bz2|gz|zip))?$");
+
+
+ final static int FILE_PATH_MAX_LEN = 512;//max len for varchar for file_path
+ final static int MAX_STRING_LENGTH = 1000000;
+ final static int MAX_LEN_FOR_LANG_ID = 20000;
+
+ //these remove runtime info from the stacktraces so
+ //that actual causes can be counted.
+ private final static Pattern CAUSED_BY_SNIPPER =
+ Pattern.compile("(Caused by: [^:]+):[^\\r\\n]+");
+
+ private final static Pattern ACCESS_PERMISSION_EXCEPTION =
+ Pattern.compile("org\\.apache\\.tika\\.exception\\.AccessPermissionException");
+ private final static Pattern ENCRYPTION_EXCEPTION =
+ Pattern.compile("org\\.apache\\.tika.exception\\.EncryptedDocumentException");
+
+ private TikaConfig config = TikaConfig.getDefaultConfig();//TODO: allow configuration
+ final LanguageIDWrapper langIder;
+ protected IDBWriter writer;
+
+ public static void loadCommonTokens(Path p) throws IOException {
+ commonTokenCountManager = new CommonTokenCountManager(p);
+ }
+
+ public AbstractProfiler(ArrayBlockingQueue<FileResource> fileQueue,
+ IDBWriter writer) {
+ super(fileQueue);
+ this.writer = writer;
+ langIder = new LanguageIDWrapper();
+ try {
+ analyzerManager = AnalyzerManager.newInstance();
+ tokenCounter = new TokenCounter(analyzerManager.getGeneralAnalyzer(),
+ analyzerManager.getAlphaIdeoAnalyzer());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected void writeError(TableInfo extractErrorTable, String containerId,
+ String filePath, Path extractsA) throws IOException {
+ Map<Cols, String> data = new HashMap<>();
+ data.put(Cols.CONTAINER_ID, containerId);
+ data.put(Cols.FILE_PATH, filePath);
+ int errorCode = -1;
+ long len = -1;
+ if (extractsA != null) {
+ try {
+ len = Files.size(extractsA);
+ } catch (IOException e) {
+ //swallow
+ }
+ }
+ if (extractsA == null) {
+ errorCode = EXTRACT_ERROR_TYPE.NO_EXTRACT_FILE.ordinal();
+ } else if (len == 0) {
+ errorCode = EXTRACT_ERROR_TYPE.ZERO_BYTE_EXTRACT_FILE.ordinal();
+ } else {
+ errorCode = EXTRACT_ERROR_TYPE.EXTRACT_PARSE_EXCEPTION.ordinal();
+ }
+ data.put(Cols.EXTRACT_ERROR_TYPE_ID, Integer.toString(errorCode));
+ writer.writeRow(extractErrorTable, data);
+
+ }
+
+ protected void writeProfileData(EvalFilePaths fps, int i, Metadata m,
+ String fileId, String containerId,
+ List<Integer> numAttachments, TableInfo profileTable) {
+
+ Map<Cols, String> data = new HashMap<>();
+ data.put(Cols.ID, fileId);
+ data.put(Cols.CONTAINER_ID, containerId);
+ data.put(Cols.MD5, m.get(DIGEST_KEY));
+
+ if ( i < numAttachments.size()) {
+ data.put(Cols.NUM_ATTACHMENTS, Integer.toString(numAttachments.get(i)));
+ }
+ data.put(Cols.ELAPSED_TIME_MILLIS, getTime(m));
+ data.put(Cols.NUM_METADATA_VALUES,
+ Integer.toString(countMetadataValues(m)));
+
+ Integer nPages = m.getInt(PagedText.N_PAGES);
+ if (nPages != null) {
+ data.put(Cols.NUM_PAGES, Integer.toString(nPages));
+ }
+
+ //if the outer wrapper document
+ if (i == 0) {
+
+ data.put(Cols.IS_EMBEDDED, FALSE);
+ data.put(Cols.FILE_NAME, fps.getRelativeSourceFilePath().getFileName().toString());
+ } else {
+ data.put(Cols.IS_EMBEDDED, TRUE);
+ data.put(Cols.FILE_NAME, FilenameUtils.getName(m.get(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH)));
+ }
+ data.put(Cols.FILE_EXTENSION,
+ FilenameUtils.getExtension(fps.getRelativeSourceFilePath().getFileName().toString()));
+ long srcFileLen = getSourceFileLength(m);
+ if (srcFileLen > NON_EXISTENT_FILE_LENGTH) {
+ data.put(Cols.LENGTH, Long.toString(srcFileLen));
+ } else {
+ data.put(Cols.LENGTH, "");
+ }
+ int numMetadataValues = countMetadataValues(m);
+ data.put(Cols.NUM_METADATA_VALUES,
+ Integer.toString(numMetadataValues));
+
+ data.put(Cols.ELAPSED_TIME_MILLIS,
+ getTime(m));
+
+ String content = getContent(m, MAX_STRING_LENGTH);
+ if (content == null || content.trim().length() == 0) {
+ data.put(Cols.HAS_CONTENT, FALSE);
+ } else {
+ data.put(Cols.HAS_CONTENT, TRUE);
+ }
+ getFileTypes(m, data);
+ try {
+ writer.writeRow(profileTable, data);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected void writeExceptionData(String fileId, Metadata m, TableInfo exceptionTable) {
+ Map<Cols, String> data = new HashMap<>();
+ getExceptionStrings(m, data);
+ if (data.keySet().size() > 0) {
+ try {
+ data.put(Cols.ID, fileId);
+ writer.writeRow(exceptionTable, data);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Checks to see if metadata is null or content is empty (null or only whitespace).
+ * If any of these, then this does no processing, and the fileId is not
+ * entered into the content table.
+ *
+ * @param fileId
+ * @param m
+ * @param fieldName
+ * @param contentsTable
+ */
+ protected void writeContentData(String fileId, Metadata m,
+ String fieldName, TableInfo contentsTable) throws IOException {
+ if (m == null) {
+ return;
+ }
+
+ String content = getContent(m, MAX_STRING_LENGTH);
+ if (content == null || content.trim().length() == 0) {
+ return;
+ }
+ tokenCounter.clear(fieldName);
+ tokenCounter.add(fieldName, content);
+
+ Map<Cols, String> data = new HashMap<>();
+ data.put(Cols.ID, fileId);
+ data.put(Cols.CONTENT_LENGTH, Integer.toString(content.length()));
+ langid(m, data);
+ String langid = data.get(Cols.LANG_ID_1);
+ langid = (langid == null) ? "" : langid;
+
+ writeTokenCounts(data, fieldName, tokenCounter);
+ CommonTokenResult commonTokenResult = null;
+ try {
+ commonTokenResult = commonTokenCountManager.countTokenOverlaps(langid,
+ tokenCounter.getAlphaTokens(fieldName));
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ }
+ data.put(Cols.COMMON_TOKENS_LANG, commonTokenResult.getLangCode());
+ data.put(Cols.NUM_COMMON_TOKENS, Integer.toString(commonTokenResult.getTokens()));
+ TokenStatistics tokenStatistics = tokenCounter.getTokenStatistics(fieldName);
+ TokenStatistics alphaTokenStatistics = tokenCounter.getAlphaTokenStatistics(fieldName);
+ data.put(Cols.NUM_UNIQUE_TOKENS,
+ Integer.toString(tokenStatistics.getTotalUniqueTokens()));
+ data.put(Cols.NUM_TOKENS,
+ Integer.toString(tokenStatistics.getTotalTokens()));
+ data.put(Cols.NUM_ALPHABETIC_TOKENS,
+ Integer.toString(alphaTokenStatistics.getTotalTokens()));
+
+ data.put(Cols.TOKEN_ENTROPY_RATE,
+ Double.toString(tokenStatistics.getEntropy()));
+ SummaryStatistics summStats = tokenStatistics.getSummaryStatistics();
+ data.put(Cols.TOKEN_LENGTH_SUM,
+ Integer.toString((int) summStats.getSum()));
+
+ data.put(Cols.TOKEN_LENGTH_MEAN,
+ Double.toString(summStats.getMean()));
+
+ data.put(Cols.TOKEN_LENGTH_STD_DEV,
+ Double.toString(summStats.getStandardDeviation()));
+ unicodeBlocks(m, data);
+ try {
+ writer.writeRow(contentsTable, data);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ String getTime(Metadata m) {
+ String elapsed = "-1";
+
+ String v = m.get(RecursiveParserWrapper.PARSE_TIME_MILLIS);
+ if (v != null) {
+ return v;
+ }
+ return elapsed;
+ }
+
+ int countMetadataValues(Metadata m) {
+ if (m == null) {
+ return 0;
+ }
+ int i = 0;
+ for (String n : m.names()) {
+ i += m.getValues(n).length;
+ }
+ return i;
+ }
+
+ void getExceptionStrings(Metadata metadata, Map<Cols, String> data) {
+
+ String fullTrace = metadata.get(TikaCoreProperties.TIKA_META_EXCEPTION_PREFIX + "runtime");
+
+ if (fullTrace == null) {
+ fullTrace = metadata.get(RecursiveParserWrapper.EMBEDDED_EXCEPTION);
+ }
+
+ if (fullTrace != null) {
+ //check for "expected" exceptions...exceptions
+ //that can't be fixed.
+ //Do not store trace for "expected" exceptions
+
+ Matcher matcher = ACCESS_PERMISSION_EXCEPTION.matcher(fullTrace);
+ if (matcher.find()) {
+ data.put(Cols.PARSE_EXCEPTION_TYPE_ID,
+ Integer.toString(EXCEPTION_TYPE.ACCESS_PERMISSION.ordinal()));
+ return;
+ }
+ matcher = ENCRYPTION_EXCEPTION.matcher(fullTrace);
+ if (matcher.find()) {
+ data.put(Cols.PARSE_EXCEPTION_TYPE_ID,
+ Integer.toString(EXCEPTION_TYPE.ENCRYPTION.ordinal()));
+ return;
+ }
+
+ data.put(Cols.PARSE_EXCEPTION_TYPE_ID,
+ Integer.toString(EXCEPTION_TYPE.RUNTIME.ordinal()));
+
+ data.put(Cols.ORIG_STACK_TRACE, fullTrace);
+ //TikaExceptions can have object ids, as in the "@2b1ea6ee" in:
+ //org.apache.tika.exception.TikaException: TIKA-198: Illegal
+ //IOException from org.apache.tika.parser.microsoft.OfficeParser@2b1ea6ee
+ //For reporting purposes, let's snip off the object id so that we can more
+ //easily count exceptions.
+ String sortTrace = ExceptionUtils.trimMessage(fullTrace);
+
+ matcher = CAUSED_BY_SNIPPER.matcher(sortTrace);
+ sortTrace = matcher.replaceAll("$1");
+ sortTrace = sortTrace.replaceAll("org.apache.tika.", "o.a.t.");
+ data.put(Cols.SORT_STACK_TRACE, sortTrace);
+ }
+ }
+
+ protected static String getContent(Metadata metadata, int maxLength) {
+ if (metadata == null) {
+ return "";
+ }
+ String c = metadata.get(RecursiveParserWrapper.TIKA_CONTENT);
+ if (c == null) {
+ return "";
+ }
+ if (c.length() > maxLength) {
+ c = c.substring(0, maxLength);
+ }
+ return c;
+ }
+
+ void unicodeBlocks(Metadata metadata, Map<Cols, String> data) {
+ String content = getContent(metadata, MAX_LEN_FOR_LANG_ID);
+ if (content.length() < 200) {
+ return;
+ }
+ String s = content;
+ if (content.length() > MAX_LEN_FOR_LANG_ID) {
+ s = content.substring(0, MAX_LEN_FOR_LANG_ID);
+ }
+ Map<String, Integer> m = new HashMap<>();
+ Reader r = new StringReader(s);
+ try {
+ int c = r.read();
+ while (c != -1) {
+ Character.UnicodeBlock block = Character.UnicodeBlock.of(c);
+ String blockString = (block == null) ? "NULL" : block.toString();
+ Integer i = m.get(blockString);
+ if (i == null) {
+ i = 0;
+ }
+ i++;
+ if (block == null) {
+ blockString = "NULL";
+ }
+ m.put(blockString, i);
+ c = r.read();
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ //swallow
+ }
+
+ List<Pair<String, Integer>> pairs = new ArrayList<>();
+ for (Map.Entry<String, Integer> e : m.entrySet()) {
+ pairs.add(Pair.of(e.getKey(), e.getValue()));
+ }
+ Collections.sort(pairs, new Comparator<Pair<String, Integer>>() {
+ @Override
+ public int compare(Pair<String, Integer> o1, Pair<String, Integer> o2) {
+ return o2.getValue().compareTo(o1.getValue());
+ }
+ });
+ StringBuilder sb = new StringBuilder();
+
+ for (int i = 0; i < 20 && i < pairs.size(); i++) {
+ if (i > 0) {
+ sb.append(" | ");
+ }
+ sb.append(pairs.get(i).getKey()+": "+pairs.get(i).getValue());
+ }
+ data.put(Cols.UNICODE_CHAR_BLOCKS, sb.toString());
+ }
+
+ void langid(Metadata metadata, Map<Cols, String> data) {
+ String content = getContent(metadata, MAX_LEN_FOR_LANG_ID);
+ if (content.length() < 50) {
+ return;
+ }
+ String s = content;
+ if (content.length() > MAX_LEN_FOR_LANG_ID) {
+ s = content.substring(0, MAX_LEN_FOR_LANG_ID);
+ }
+ List<DetectedLanguage> probabilities = langIder.getProbabilities(s);
+ if (probabilities.size() > 0) {
+ data.put(Cols.LANG_ID_1, probabilities.get(0).getLocale().getLanguage());
+ data.put(Cols.LANG_ID_PROB_1,
+ Double.toString(probabilities.get(0).getProbability()));
+ }
+ if (probabilities.size() > 1) {
+ data.put(Cols.LANG_ID_2, probabilities.get(1).getLocale().getLanguage());
+ data.put(Cols.LANG_ID_PROB_2,
+ Double.toString(probabilities.get(1).getProbability()));
+ }
+
+ }
+
+ void getFileTypes(Metadata metadata, Map<Cols, String> output) {
+ if (metadata == null) {
+ return;
+ }
+ String type = metadata.get(Metadata.CONTENT_TYPE);
+ if (type == null) {
+ return;
+ }
+ int mimeId = writer.getMimeId(type);
+ output.put(Cols.MIME_TYPE_ID, Integer.toString(mimeId));
+ }
+
+ void writeTokenCounts(Map<Cols, String> data, String field,
+ TokenCounter tokenCounter) {
+
+
+ int stops = 0;
+ int i = 0;
+ StringBuilder sb = new StringBuilder();
+ TokenStatistics tokenStatistics = tokenCounter.getTokenStatistics(field);
+ for (TokenIntPair t : tokenStatistics.getTopN()) {
+ if (i++ > 0) {
+ sb.append(" | ");
+ }
+ sb.append(t.getToken() + ": " + t.getValue());
+ }
+
+ data.put(Cols.TOP_N_TOKENS, sb.toString());
+ }
+
+
+ public void closeWriter() throws IOException {
+ writer.close();
+ }
+
+
+ /**
+ *
+ * @param metadata
+ * @param extractDir
+ * @return evalfilepaths for files if crawling an extract directory
+ */
+ protected EvalFilePaths getPathsFromExtractCrawl(Metadata metadata,
+ Path extractDir) {
+ String relExtractFilePath = metadata.get(FSProperties.FS_REL_PATH);
+ Matcher m = FILE_NAME_CLEANER.matcher(relExtractFilePath);
+ Path relativeSourceFilePath = Paths.get(m.replaceAll(""));
+ //just try slapping the relextractfilepath on the extractdir
+ Path extractFile = extractDir.resolve(relExtractFilePath);
+ if (! Files.isRegularFile(extractFile)) {
+ //if that doesn't work, try to find the right extract file.
+ //This is necessary if crawling extractsA and trying to find a file in
+ //extractsB that is not in the same format: json vs txt or compressed
+ extractFile = findFile(extractDir, relativeSourceFilePath);
+ }
+ return new EvalFilePaths(relativeSourceFilePath, extractFile);
+ }
+ //call this if the crawler is crawling through the src directory
+ protected EvalFilePaths getPathsFromSrcCrawl(Metadata metadata, Path srcDir,
+ Path extractDir) {
+ Path relativeSourceFilePath = Paths.get(metadata.get(FSProperties.FS_REL_PATH));
+ Path extractFile = findFile(extractDir, relativeSourceFilePath);
+ Path inputFile = srcDir.resolve(relativeSourceFilePath);
+ long srcLen = -1l;
+ //try to get the length of the source file in case there was an error
+ //in both extracts
+ try {
+ srcLen = Files.size(inputFile);
+ } catch (IOException e) {
+ logger.warn("Couldn't get length for: "+inputFile.toAbsolutePath());
+ }
+ return new EvalFilePaths(relativeSourceFilePath, extractFile, srcLen);
+ }
+
+ /**
+ *
+ * @param extractRootDir
+ * @param relativeSourceFilePath
+ * @return extractFile or null if couldn't find one.
+ */
+ private Path findFile(Path extractRootDir, Path relativeSourceFilePath) {
+ String relSrcFilePathString = relativeSourceFilePath.toString();
+ if (lastExtractExtension != null) {
+ Path candidate = extractRootDir.resolve(relSrcFilePathString+lastExtractExtension);
+ if (Files.isRegularFile(candidate)) {
+ return candidate;
+ }
+ }
+ for (String ext : EXTRACT_EXTENSIONS) {
+ for (String compress : COMPRESSION_EXTENSIONS) {
+ Path candidate = extractRootDir.resolve(relSrcFilePathString+ext+compress);
+ if (Files.isRegularFile(candidate)) {
+ lastExtractExtension = ext+compress;
+ return candidate;
+ }
+ }
+ }
+ return null;
+ }
+
+ protected long getSourceFileLength(EvalFilePaths fps, List<Metadata> metadataList) {
+ if (fps.getSourceFileLength() > NON_EXISTENT_FILE_LENGTH) {
+ return fps.getSourceFileLength();
+ }
+ return getSourceFileLength(metadataList);
+ }
+
+ long getSourceFileLength(List<Metadata> metadataList) {
+ if (metadataList == null || metadataList.size() < 1) {
+ return NON_EXISTENT_FILE_LENGTH;
+ }
+ return getSourceFileLength(metadataList.get(0));
+ }
+
+ long getSourceFileLength(Metadata m) {
+ String lenString = m.get(Metadata.CONTENT_LENGTH);
+ if (lenString == null) {
+ return NON_EXISTENT_FILE_LENGTH;
+ }
+ try {
+ return Long.parseLong(lenString);
+ } catch (NumberFormatException e) {
+ //swallow
+ }
+ return NON_EXISTENT_FILE_LENGTH;
+ }
+
+ protected long getFileLength(Path p) {
+ if (p != null && Files.isRegularFile(p)) {
+ try {
+ return Files.size(p);
+ } catch (IOException e) {
+ //swallow
+ }
+ }
+ return NON_EXISTENT_FILE_LENGTH;
+ }
+
+ /**
+ *
+ * @param list
+ * @return empty list if input list is empty or null
+ */
+ static List<Integer> countAttachments(List<Metadata> list) {
+ List<Integer> ret = new ArrayList<>();
+ if (list == null || list.size() == 0) {
+ return ret;
+ }
+ //container document attachment count = list.size()-1
+ ret.add(list.size()-1);
+
+ Map<String, Integer> counts = new HashMap<>();
+ for (int i = 1; i < list.size(); i++) {
+ String path = list.get(i).get(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH);
+ if (path == null) {
+ //shouldn't ever happen
+ continue;
+ }
+ String[] parts = path.split("/");
+ StringBuilder parent = new StringBuilder();
+ for (int end = 1; end < parts.length-1; end++) {
+ parent.setLength(0);
+ join("/", parent, parts, 1, end);
+ String parentPath = parent.toString();
+ Integer count = counts.get(parentPath);
+ if (count == null) {
+ count = 1;
+ } else {
+ count++;
+ }
+ counts.put(parentPath, count);
+ }
+ }
+
+ for (int i = 1; i < list.size(); i++) {
+ Integer count = counts.get(list.get(i).get(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH));
+ if (count == null) {
+ count = 0;
+ }
+ ret.add(i, count);
+ }
+ return ret;
+
+
+ }
+
+ private static void join(String delimiter, StringBuilder sb, String[] parts, int start, int end) {
+ for (int i = start; i <= end; i++) {
+ sb.append(delimiter);
+ sb.append(parts[i]);
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/EvalFilePaths.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/EvalFilePaths.java b/tika-eval/src/main/java/org/apache/tika/eval/EvalFilePaths.java
new file mode 100644
index 0000000..1a3d29c
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/EvalFilePaths.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval;
+
+import static org.apache.tika.eval.AbstractProfiler.NON_EXISTENT_FILE_LENGTH;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Simple struct to keep track of relative path of source file (
+ * original binary file, e.g. /subdir/document1.doc)
+ * and the extract file (e.g. /subdir/document1.doc.json).
+ */
+class EvalFilePaths {
+
+ private final Path relativeSourceFilePath;
+ private final Path extractFile;
+
+ private long sourceFileLength = NON_EXISTENT_FILE_LENGTH;
+ private long extractFileLength = NON_EXISTENT_FILE_LENGTH;
+
+
+ public EvalFilePaths(Path relativeSourceFilePath, Path extractFile, long srcFileLen) {
+ this(relativeSourceFilePath, extractFile);
+ this.sourceFileLength = srcFileLen;
+ }
+
+ public EvalFilePaths(Path relativeSourceFilePath, Path extractFile) {
+ if (extractFile != null && Files.isRegularFile(extractFile)) {
+ try {
+ extractFileLength = Files.size(extractFile);
+ } catch (IOException e) {
+ //swallow ?
+ }
+ }
+ this.relativeSourceFilePath = relativeSourceFilePath;
+ this.extractFile = extractFile;
+ }
+
+ public Path getRelativeSourceFilePath() {
+ return relativeSourceFilePath;
+ }
+
+ //this path may or may not exist and it could be null!
+ public Path getExtractFile() {
+ return extractFile;
+ }
+
+ //if it doesn't exist, it'll be -1l.
+ public long getSourceFileLength() {
+ return sourceFileLength;
+ }
+
+ public long getExtractFileLength() {
+ return extractFileLength;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ EvalFilePaths that = (EvalFilePaths) o;
+
+ if (sourceFileLength != that.sourceFileLength) return false;
+ if (extractFileLength != that.extractFileLength) return false;
+ if (relativeSourceFilePath != null ? !relativeSourceFilePath.equals(that.relativeSourceFilePath) : that.relativeSourceFilePath != null)
+ return false;
+ return extractFile != null ? extractFile.equals(that.extractFile) : that.extractFile == null;
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = relativeSourceFilePath != null ? relativeSourceFilePath.hashCode() : 0;
+ result = 31 * result + (extractFile != null ? extractFile.hashCode() : 0);
+ result = 31 * result + (int) (sourceFileLength ^ (sourceFileLength >>> 32));
+ result = 31 * result + (int) (extractFileLength ^ (extractFileLength >>> 32));
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "EvalFilePaths{" +
+ "relativeSourceFilePath=" + relativeSourceFilePath +
+ ", extractFile=" + extractFile +
+ ", sourceFileLength=" + sourceFileLength +
+ ", extractFileLength=" + extractFileLength +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java b/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
new file mode 100644
index 0000000..8b3d266
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
@@ -0,0 +1,455 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.sql.Types;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.batch.fs.FSProperties;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.eval.db.ColInfo;
+import org.apache.tika.eval.db.Cols;
+import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.ExtractReader;
+import org.apache.tika.eval.io.IDBWriter;
+import org.apache.tika.eval.tokens.ContrastStatistics;
+import org.apache.tika.eval.tokens.TokenContraster;
+import org.apache.tika.eval.tokens.TokenIntPair;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.RecursiveParserWrapper;
+
+public class ExtractComparer extends AbstractProfiler {
+
+ static Options OPTIONS;
+ static {
+ Option extractsA = new Option("extractsA", true, "directory for extractsA files");
+ extractsA.setRequired(true);
+
+ Option extractsB = new Option("extractsB", true, "directory for extractsB files");
+ extractsB.setRequired(true);
+
+ Option db = new Option("db", true, "db file to which to write results");
+ db.setRequired(true);
+
+ Option inputDir = new Option("inputDir", true,
+ "optional: directory of original binary input files if it exists " +
+ "or can be the same as -extractsA or -extractsB. If not specified, -inputDir=-extractsA");
+ inputDir.setRequired(true);
+
+
+ OPTIONS = new Options()
+ .addOption(extractsA)
+ .addOption(extractsB)
+ .addOption(db)
+ .addOption(inputDir)
+ .addOption("bc", "optional: tika-batch config file")
+ .addOption("numConsumers", true, "optional: number of consumer threads")
+ .addOption(new Option("alterExtract", true,
+ "for json-formatted extract files, " +
+ "process full metadata list ('as_is'=default), " +
+ "take just the first/container document ('first_only'), " +
+ "concatenate all content into the first metadata item ('concatenate_content')")
+ );
+ }
+
+ public static void USAGE() {
+ HelpFormatter helpFormatter = new HelpFormatter();
+ helpFormatter.printHelp(
+ 80,
+ "java -jar tika-eval-x.y.jar Compare -extractsA extractsA -extractsB extractsB -db mydb",
+ "Tool: Compare",
+ ExtractComparer.OPTIONS,
+ "Note: for h2 db, do not include the .mv.db at the end of the db name.");
+ }
+
+ private final static String FIELD_A = "fa";
+ private final static String FIELD_B = "fb";
+
+ public static TableInfo REF_PAIR_NAMES = new TableInfo("pair_names",
+ new ColInfo(Cols.DIR_NAME_A, Types.VARCHAR, 128),
+ new ColInfo(Cols.DIR_NAME_B, Types.VARCHAR, 128)
+ );
+
+ public static TableInfo COMPARISON_CONTAINERS = new TableInfo("containers",
+ new ColInfo(Cols.CONTAINER_ID, Types.INTEGER, "PRIMARY KEY"),
+ new ColInfo(Cols.FILE_PATH, Types.VARCHAR, FILE_PATH_MAX_LEN),
+ new ColInfo(Cols.FILE_EXTENSION, Types.VARCHAR, 12),
+ new ColInfo(Cols.LENGTH, Types.BIGINT),
+ new ColInfo(Cols.EXTRACT_FILE_LENGTH_A, Types.BIGINT),
+ new ColInfo(Cols.EXTRACT_FILE_LENGTH_B, Types.BIGINT)
+ );
+
+ public static TableInfo CONTENT_COMPARISONS = new TableInfo("content_comparisons",
+ new ColInfo(Cols.ID, Types.INTEGER, "PRIMARY KEY"),
+ new ColInfo(Cols.TOP_10_UNIQUE_TOKEN_DIFFS_A, Types.VARCHAR, 1024),
+ new ColInfo(Cols.TOP_10_UNIQUE_TOKEN_DIFFS_B, Types.VARCHAR, 1024),
+ new ColInfo(Cols.TOP_10_MORE_IN_A, Types.VARCHAR, 1024),
+ new ColInfo(Cols.TOP_10_MORE_IN_B, Types.VARCHAR, 1024),
+ new ColInfo(Cols.DICE_COEFFICIENT, Types.FLOAT),
+ new ColInfo(Cols.OVERLAP, Types.FLOAT)
+ );
+
+ public static TableInfo PROFILES_A = new TableInfo( "profiles_a",
+ ExtractProfiler.PROFILE_TABLE.getColInfos());
+
+ public static TableInfo PROFILES_B = new TableInfo( "profiles_b",
+ ExtractProfiler.PROFILE_TABLE.getColInfos());
+
+ public static TableInfo EMBEDDED_FILE_PATH_TABLE_A = new TableInfo( "emb_path_a",
+ ExtractProfiler.EMBEDDED_FILE_PATH_TABLE.getColInfos());
+
+ public static TableInfo EMBEDDED_FILE_PATH_TABLE_B = new TableInfo( "emb_path_b",
+ ExtractProfiler.EMBEDDED_FILE_PATH_TABLE.getColInfos());
+
+
+ public static TableInfo CONTENTS_TABLE_A = new TableInfo( "contents_a",
+ ExtractProfiler.CONTENTS_TABLE.getColInfos());
+
+ public static TableInfo CONTENTS_TABLE_B = new TableInfo( "contents_b",
+ ExtractProfiler.CONTENTS_TABLE.getColInfos());
+
+ public static TableInfo EXCEPTION_TABLE_A = new TableInfo ("exceptions_a",
+ ExtractProfiler.EXCEPTION_TABLE.getColInfos());
+
+ public static TableInfo EXCEPTION_TABLE_B = new TableInfo ("exceptions_b",
+ ExtractProfiler.EXCEPTION_TABLE.getColInfos());
+
+ public static TableInfo ERROR_TABLE_A = new TableInfo("extract_errors_a",
+ ExtractProfiler.ERROR_TABLE.getColInfos());
+ public static TableInfo ERROR_TABLE_B = new TableInfo("extract_errors_b",
+ ExtractProfiler.ERROR_TABLE.getColInfos());
+
+
+ //need to parameterize?
+ private final TikaConfig config = TikaConfig.getDefaultConfig();
+
+ private final Path inputDir;
+ private final Path extractsA;
+ private final Path extractsB;
+
+ private final long minJsonLength;
+ private final long maxJsonLength;
+ private final ExtractReader.ALTER_METADATA_LIST alterExtractList;
+
+ private final TokenContraster tokenContraster = new TokenContraster();
+ private final ExtractReader extractReader = new ExtractReader();
+
+ public ExtractComparer(ArrayBlockingQueue<FileResource> queue,
+ Path inputDir, Path extractsA, Path extractsB,
+ IDBWriter writer, long minJsonLength,
+ long maxJsonLength, ExtractReader.ALTER_METADATA_LIST alterExtractList) {
+ super(queue, writer);
+ this.minJsonLength = minJsonLength;
+ this.maxJsonLength = maxJsonLength;
+ this.inputDir = inputDir;
+ this.extractsA = extractsA;
+ this.extractsB = extractsB;
+ this.alterExtractList = alterExtractList;
+ }
+
+ @Override
+ public boolean processFileResource(FileResource fileResource) {
+ Metadata metadata = fileResource.getMetadata();
+ EvalFilePaths fpsA = null;
+ EvalFilePaths fpsB = null;
+
+ if (inputDir != null && (inputDir.equals(extractsA) ||
+ inputDir.equals(extractsB))) {
+ //crawling an extract dir
+ fpsA = getPathsFromExtractCrawl(metadata, extractsA);
+ fpsB = getPathsFromExtractCrawl(metadata, extractsB);
+
+ } else {
+ fpsA = getPathsFromSrcCrawl(metadata, inputDir, extractsA);
+ fpsB = getPathsFromSrcCrawl(metadata, inputDir, extractsB);
+ }
+
+ if (minJsonLength > -1) {
+ //if both files exist and are < minJsonLength, skip em
+ if (fpsA.getExtractFileLength() > NON_EXISTENT_FILE_LENGTH
+ && fpsA.getExtractFileLength() < minJsonLength
+ && fpsB.getExtractFileLength() > NON_EXISTENT_FILE_LENGTH
+ && fpsB.getExtractFileLength() < minJsonLength) {
+ return false;
+ }
+ }
+ if (maxJsonLength > -1) {
+ if ((fpsA.getExtractFileLength() > maxJsonLength) ||
+ (fpsB.getExtractFileLength() > maxJsonLength)) {
+ return false;
+ }
+ }
+
+
+ try {
+ compareFiles(fpsA, fpsB);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ //this should be cataclysmic...
+ throw new RuntimeException("Exception while working on: " +
+ metadata.get(FSProperties.FS_REL_PATH), e);
+ }
+ return true;
+ }
+
+ //protected for testing, should find better way so that this can be private!
+ protected void compareFiles(EvalFilePaths fpsA, EvalFilePaths fpsB) throws IOException {
+
+ List<Metadata> metadataListA =
+ extractReader.loadExtract(fpsA.getExtractFile(), alterExtractList);
+ List<Metadata> metadataListB =
+ extractReader.loadExtract(fpsB.getExtractFile(), alterExtractList);
+
+ //array indices for those metadata items handled in
+ //"that"
+ Set<Integer> handledB = new HashSet<>();
+ String containerID = Integer.toString(CONTAINER_ID.getAndIncrement());
+ //container table
+ Map<Cols, String> contData = new HashMap<>();
+ contData.put(Cols.CONTAINER_ID, containerID);
+ contData.put(Cols.FILE_PATH, fpsA.getRelativeSourceFilePath().toString());
+ long srcFileLength = getSourceFileLength(metadataListA, metadataListB);
+ contData.put(Cols.LENGTH,
+ srcFileLength > NON_EXISTENT_FILE_LENGTH ?
+ Long.toString(srcFileLength) : "");
+ contData.put(Cols.FILE_EXTENSION,
+ FilenameUtils.getExtension(fpsA.getRelativeSourceFilePath().getFileName().toString()));
+
+ long extractFileLengthA = getFileLength(fpsA.getExtractFile());
+ contData.put(Cols.EXTRACT_FILE_LENGTH_A, extractFileLengthA > NON_EXISTENT_FILE_LENGTH ?
+ Long.toString(extractFileLengthA) : "");
+
+ long extractFileLengthB = getFileLength(fpsA.getExtractFile());
+ contData.put(Cols.EXTRACT_FILE_LENGTH_B, extractFileLengthB > NON_EXISTENT_FILE_LENGTH ?
+ Long.toString(extractFileLengthB) : "");
+
+ writer.writeRow(COMPARISON_CONTAINERS, contData);
+
+ if (metadataListA == null) {
+ writeError(ERROR_TABLE_A, containerID, fpsA.getRelativeSourceFilePath().toString(),
+ fpsA.getExtractFile());
+ }
+ if (metadataListB == null) {
+ writeError(ERROR_TABLE_B, containerID, fpsB.getRelativeSourceFilePath().toString(),
+ fpsB.getExtractFile());
+ }
+
+ if (metadataListA == null && metadataListB == null) {
+ return;
+ }
+ List<Integer> numAttachmentsA = countAttachments(metadataListA);
+ List<Integer> numAttachmentsB = countAttachments(metadataListB);
+
+ //now get that metadata
+ if (metadataListA != null) {
+ for (int i = 0; i < metadataListA.size(); i++) {
+ String fileId = Integer.toString(ID.getAndIncrement());
+ Metadata metadataA = metadataListA.get(i);
+ Metadata metadataB = null;
+ //TODO: shouldn't be fileA!!!!
+ writeProfileData(fpsA, i, metadataA, fileId, containerID, numAttachmentsA, PROFILES_A);
+ writeExceptionData(fileId, metadataA, EXCEPTION_TABLE_A);
+ int matchIndex = getMatch(i, metadataListA, metadataListB);
+
+ if (matchIndex > -1) {
+ metadataB = metadataListB.get(matchIndex);
+ handledB.add(matchIndex);
+ }
+ if (metadataB != null) {
+ writeProfileData(fpsB, i, metadataB, fileId, containerID, numAttachmentsB, PROFILES_B);
+ writeExceptionData(fileId, metadataB, EXCEPTION_TABLE_B);
+ }
+ writeEmbeddedFilePathData(i, fileId, metadataA, metadataB);
+ //prep the token counting
+ tokenCounter.clear(FIELD_A);
+ tokenCounter.clear(FIELD_B);
+ //write content
+ try {
+ writeContentData(fileId, metadataA, FIELD_A, CONTENTS_TABLE_A);
+ writeContentData(fileId, metadataB, FIELD_B, CONTENTS_TABLE_B);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ //now run comparisons
+ if (tokenCounter.getTokenStatistics(FIELD_A).getTotalTokens() > 0
+ && tokenCounter.getTokenStatistics(FIELD_B).getTotalTokens() > 0) {
+ Map<Cols, String> data = new HashMap<>();
+ data.put(Cols.ID, fileId);
+
+ ContrastStatistics contrastStatistics =
+ tokenContraster.calculateContrastStatistics(
+ tokenCounter.getTokens(FIELD_A),
+ tokenCounter.getTokenStatistics(FIELD_A),
+ tokenCounter.getTokens(FIELD_B),
+ tokenCounter.getTokenStatistics(FIELD_B));
+
+ writeContrasts(data, contrastStatistics);
+ writer.writeRow(CONTENT_COMPARISONS, data);
+ }
+ }
+ }
+ //now try to get any Metadata objects in "that"
+ //that haven't yet been handled.
+ if (metadataListB != null) {
+ for (int i = 0; i < metadataListB.size(); i++) {
+ if (handledB.contains(i)) {
+ continue;
+ }
+ Metadata metadataB = metadataListB.get(i);
+ String fileId = Integer.toString(ID.getAndIncrement());
+ writeProfileData(fpsB, i, metadataB, fileId, containerID, numAttachmentsB, PROFILES_B);
+ writeEmbeddedFilePathData(i, fileId, null, metadataB);
+ writeExceptionData(fileId, metadataB, EXCEPTION_TABLE_B);
+
+ //prep the token counting
+ tokenCounter.clear(FIELD_B);
+ //write content
+ try {
+ writeContentData(fileId, metadataB, FIELD_B, CONTENTS_TABLE_B);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+
+ private void writeEmbeddedFilePathData(int i, String fileId, Metadata mA, Metadata mB) {
+ //container file, don't write anything
+ if (i == 0) {
+ return;
+ }
+ String pathA = null;
+ String pathB = null;
+ if (mA != null) {
+ pathA = mA.get(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH);
+ }
+ if (mB != null) {
+ pathB = mB.get(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH);
+ }
+ if (pathA != null) {
+ Map<Cols, String> d = new HashMap<>();
+ d.put(Cols.ID, fileId);
+ d.put(Cols.EMBEDDED_FILE_PATH, pathA);
+ try {
+ writer.writeRow(EMBEDDED_FILE_PATH_TABLE_A, d);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ if (pathB != null &&
+ (pathA == null || ! pathA.equals(pathB))) {
+ Map<Cols, String> d = new HashMap<>();
+ d.put(Cols.ID, fileId);
+ d.put(Cols.EMBEDDED_FILE_PATH, pathB);
+ try {
+ writer.writeRow(EMBEDDED_FILE_PATH_TABLE_B, d);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private long getSourceFileLength(List<Metadata> metadataListA, List<Metadata> metadataListB) {
+ long len = getSourceFileLength(metadataListA);
+ if (len > NON_EXISTENT_FILE_LENGTH) {
+ return len;
+ }
+ return getSourceFileLength(metadataListB);
+ }
+
+
+ /**
+ * Try to find the matching metadata based on the RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH
+ * If you can't find it, return -1;
+ *
+ * @param i index for match in metadataListA
+ * @param metadataListA
+ * @param metadataListB
+ * @return
+ */
+ private int getMatch(int i,
+ List<Metadata> metadataListA,
+ List<Metadata> metadataListB) {
+ //TODO: could make this more robust
+ if (metadataListB == null || metadataListB.size() == 0) {
+ return -1;
+ }
+ if (i == 0) {
+ return 0;
+ }
+ if (metadataListA.size() == metadataListB.size()) {
+ //assume no rearrangments if lists are the same size
+ return i;
+ }
+
+ Metadata thisMetadata = metadataListA.get(i);
+ String embeddedPath = thisMetadata.get(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH);
+ if (embeddedPath == null) {
+ return -1;
+ }
+ if (i < metadataListB.size()) {
+ }
+
+ for (int j = 0; j < metadataListB.size(); j++) {
+ String thatEmbeddedPath = metadataListB.get(j).get(
+ RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH);
+ if (embeddedPath.equals(thatEmbeddedPath)) {
+ return j;
+ }
+ }
+ return -1;
+ }
+
+
+
+
+ private void writeContrasts(Map<Cols, String> data, ContrastStatistics contrastStatistics) {
+ writeContrastString(data, Cols.TOP_10_MORE_IN_A, contrastStatistics.getTopNMoreA());
+ writeContrastString(data, Cols.TOP_10_MORE_IN_B, contrastStatistics.getTopNMoreB());
+ writeContrastString(data, Cols.TOP_10_UNIQUE_TOKEN_DIFFS_A, contrastStatistics.getTopNUniqueA());
+ writeContrastString(data, Cols.TOP_10_UNIQUE_TOKEN_DIFFS_B, contrastStatistics.getTopNUniqueB());
+ data.put(Cols.OVERLAP, Double.toString(contrastStatistics.getOverlap()));
+ data.put(Cols.DICE_COEFFICIENT, Double.toString(contrastStatistics.getDiceCoefficient()));
+
+ }
+
+ private void writeContrastString(Map<Cols, String> data, Cols col, TokenIntPair[] tokenIntPairs) {
+
+ int i = 0;
+ StringBuilder sb = new StringBuilder();
+ for (TokenIntPair p : tokenIntPairs) {
+ if (i++ > 0) {
+ sb.append(" | ");
+ }
+ sb.append(p.getToken()).append(": ").append(p.getValue());
+ }
+ data.put(col, sb.toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java b/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
new file mode 100644
index 0000000..6840926
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval;
+
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.sql.Types;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.eval.db.ColInfo;
+import org.apache.tika.eval.db.Cols;
+import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.ExtractReader;
+import org.apache.tika.eval.io.IDBWriter;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.RecursiveParserWrapper;
+
+public class ExtractProfiler extends AbstractProfiler {
+
+ static Options OPTIONS;
+ static {
+ Option db = new Option("db", true, "db file to which to write results");
+ db.setRequired(true);
+
+ //By the time this commandline is parsed, there should be both an extractDir and an inputDir
+ Option extractDir = new Option("extractDir", true, "directory for extract files");
+ extractDir.setRequired(true);
+
+ Option inputDir = new Option("inputDir", true,
+ "optional: directory for original binary input documents."+
+ " If not specified, -extractDir is crawled as is.");
+ inputDir.setRequired(true);
+
+ OPTIONS = new Options()
+ .addOption(db)
+ .addOption(extractDir)
+ .addOption(inputDir)
+ .addOption("bc", "optional: tika-batch config file")
+ .addOption("numConsumers", true, "optional: number of consumer threads")
+ .addOption(new Option("alterExtract", true,
+ "for json-formatted extract files, " +
+ "process full metadata list ('as_is'=default), " +
+ "take just the first/container document ('first_only'), " +
+ "concatenate all content into the first metadata item ('concatenate_content')"));
+
+ }
+
+ public static void USAGE() {
+ HelpFormatter helpFormatter = new HelpFormatter();
+ helpFormatter.printHelp(
+ 80,
+ "java -jar tika-eval-x.y.jar Profile -extractDir extracts -db mydb [-inputDir input]",
+ "Tool: Profile",
+ ExtractProfiler.OPTIONS,
+ "Note: for h2 db, do not include the .mv.db at the end of the db name.");
+ }
+
+ private final static String FIELD = "f";
+
+ public static TableInfo ERROR_TABLE = new TableInfo("errors",
+ new ColInfo(Cols.CONTAINER_ID, Types.INTEGER),
+ new ColInfo(Cols.FILE_PATH, Types.VARCHAR, FILE_PATH_MAX_LEN),
+ new ColInfo(Cols.EXTRACT_ERROR_TYPE_ID, Types.INTEGER),
+ new ColInfo(Cols.PARSE_ERROR_TYPE_ID, Types.INTEGER)
+ );
+
+ public static TableInfo EXCEPTION_TABLE = new TableInfo("parse_exceptions",
+ new ColInfo(Cols.ID, Types.INTEGER, "PRIMARY KEY"),
+ new ColInfo(Cols.ORIG_STACK_TRACE, Types.VARCHAR, 8192),
+ new ColInfo(Cols.SORT_STACK_TRACE, Types.VARCHAR, 8192),
+ new ColInfo(Cols.PARSE_EXCEPTION_TYPE_ID, Types.INTEGER)
+ );
+
+
+ public static TableInfo CONTAINER_TABLE = new TableInfo("containers",
+ new ColInfo(Cols.CONTAINER_ID, Types.INTEGER, "PRIMARY KEY"),
+ new ColInfo(Cols.FILE_PATH, Types.VARCHAR, FILE_PATH_MAX_LEN),
+ new ColInfo(Cols.LENGTH, Types.BIGINT),
+ new ColInfo(Cols.EXTRACT_FILE_LENGTH, Types.BIGINT)
+ );
+
+ public static TableInfo PROFILE_TABLE = new TableInfo("profiles",
+ new ColInfo(Cols.ID, Types.INTEGER, "PRIMARY KEY"),
+ new ColInfo(Cols.CONTAINER_ID, Types.INTEGER),
+ new ColInfo(Cols.FILE_NAME, Types.VARCHAR, 256),
+ new ColInfo(Cols.MD5, Types.CHAR, 32),
+ new ColInfo(Cols.LENGTH, Types.BIGINT),
+ new ColInfo(Cols.IS_EMBEDDED, Types.BOOLEAN),
+ new ColInfo(Cols.FILE_EXTENSION, Types.VARCHAR, 12),
+ new ColInfo(Cols.MIME_TYPE_ID, Types.INTEGER),
+ new ColInfo(Cols.ELAPSED_TIME_MILLIS, Types.INTEGER),
+ new ColInfo(Cols.NUM_ATTACHMENTS, Types.INTEGER),
+ new ColInfo(Cols.NUM_METADATA_VALUES, Types.INTEGER),
+ new ColInfo(Cols.NUM_PAGES, Types.INTEGER),
+ new ColInfo(Cols.HAS_CONTENT, Types.BOOLEAN)
+ );
+
+ public static TableInfo EMBEDDED_FILE_PATH_TABLE = new TableInfo("emb_file_names",
+ new ColInfo(Cols.ID, Types.INTEGER, "PRIMARY KEY"),
+ new ColInfo(Cols.EMBEDDED_FILE_PATH, Types.VARCHAR, 1024)
+ );
+
+ public static TableInfo CONTENTS_TABLE = new TableInfo("contents",
+ new ColInfo(Cols.ID, Types.INTEGER, "PRIMARY KEY"),
+ new ColInfo(Cols.CONTENT_LENGTH, Types.INTEGER),
+ new ColInfo(Cols.NUM_TOKENS, Types.INTEGER),
+ new ColInfo(Cols.NUM_UNIQUE_TOKENS, Types.INTEGER),
+ new ColInfo(Cols.COMMON_TOKENS_LANG, Types.VARCHAR, 12),
+ new ColInfo(Cols.NUM_COMMON_TOKENS, Types.INTEGER),
+ new ColInfo(Cols.TOP_N_TOKENS, Types.VARCHAR, 1024),
+ new ColInfo(Cols.NUM_ALPHABETIC_TOKENS, Types.INTEGER),
+ new ColInfo(Cols.LANG_ID_1, Types.VARCHAR, 12),
+ new ColInfo(Cols.LANG_ID_PROB_1, Types.FLOAT),
+ new ColInfo(Cols.LANG_ID_2, Types.VARCHAR, 12),
+ new ColInfo(Cols.LANG_ID_PROB_2, Types.FLOAT),
+ new ColInfo(Cols.UNICODE_CHAR_BLOCKS, Types.VARCHAR, 1024),
+ new ColInfo(Cols.TOKEN_ENTROPY_RATE, Types.FLOAT),
+ new ColInfo(Cols.TOKEN_LENGTH_SUM, Types.INTEGER),
+ new ColInfo(Cols.TOKEN_LENGTH_MEAN, Types.FLOAT),
+ new ColInfo(Cols.TOKEN_LENGTH_STD_DEV, Types.FLOAT)
+ );
+
+ private final Path inputDir;
+ private final Path extractDir;
+ private final ExtractReader.ALTER_METADATA_LIST alterExtractList;
+ private final ExtractReader extractReader = new ExtractReader();
+
+ public ExtractProfiler(ArrayBlockingQueue<FileResource> queue,
+ Path inputDir, Path extractDir,
+ IDBWriter dbWriter, ExtractReader.ALTER_METADATA_LIST alterExtractList) {
+ super(queue, dbWriter);
+ this.inputDir = inputDir;
+ this.extractDir = extractDir;
+ this.alterExtractList = alterExtractList;
+ }
+
+ @Override
+ public boolean processFileResource(FileResource fileResource) {
+ Metadata metadata = fileResource.getMetadata();
+ EvalFilePaths fps = null;
+
+ if (inputDir != null && inputDir.equals(extractDir)) {
+ //crawling an extract dir
+ fps = getPathsFromExtractCrawl(metadata, extractDir);
+ } else {
+ fps = getPathsFromSrcCrawl(metadata, inputDir, extractDir);
+ }
+ List<Metadata> metadataList = extractReader.loadExtract(fps.getExtractFile(), alterExtractList);
+
+ Map<Cols, String> contOutput = new HashMap<>();
+ String containerId = Integer.toString(CONTAINER_ID.incrementAndGet());
+ Long srcFileLen = getSourceFileLength(fps, metadataList);
+ contOutput.put(Cols.LENGTH,
+ srcFileLen > NON_EXISTENT_FILE_LENGTH ?
+ Long.toString(srcFileLen): "");
+ contOutput.put(Cols.CONTAINER_ID, containerId);
+ contOutput.put(Cols.FILE_PATH, fps.getRelativeSourceFilePath().toString());
+
+ if (fps.getExtractFileLength() > 0) {
+ contOutput.put(Cols.EXTRACT_FILE_LENGTH,
+ (fps.getExtractFile() == null) ?
+ "" :
+ Long.toString(fps.getExtractFileLength()));
+ }
+ try {
+ writer.writeRow(CONTAINER_TABLE, contOutput);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+
+ if (metadataList == null) {
+ try {
+ writeError(ERROR_TABLE, containerId,
+ fps.getRelativeSourceFilePath().toString(), fps.getExtractFile());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return true;
+ }
+
+ //TODO: calculate num_attachments, add to profile table
+
+ List<Integer> numAttachments = countAttachments(metadataList);
+ int i = 0;
+ for (Metadata m : metadataList) {
+ String fileId = Integer.toString(ID.incrementAndGet());
+ writeProfileData(fps, i, m, fileId, containerId, numAttachments, PROFILE_TABLE);
+ writeEmbeddedPathData(i, fileId, m, EMBEDDED_FILE_PATH_TABLE);
+ writeExceptionData(fileId, m, EXCEPTION_TABLE);
+ try {
+ writeContentData(fileId, m, FIELD, CONTENTS_TABLE);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ i++;
+ }
+ return true;
+ }
+
+ private void writeEmbeddedPathData(int i, String fileId, Metadata m,
+ TableInfo embeddedFilePathTable) {
+ if (i == 0) {
+ return;
+ }
+ Map<Cols, String> data = new HashMap<>();
+ data.put(Cols.ID, fileId);
+ data.put(Cols.EMBEDDED_FILE_PATH,
+ m.get(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH));
+ try {
+ writer.writeRow(embeddedFilePathTable, data);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
[4/6] tika git commit: TIKA-1332 initial commit of tika-eval. More
work remains.
Posted by ta...@apache.org.
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java b/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java
new file mode 100644
index 0000000..cd90f76
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/io/ExtractReader.java
@@ -0,0 +1,161 @@
+package org.apache.tika.eval.io;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.z.ZCompressorInputStream;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.IOUtils;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.serialization.JsonMetadataList;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.RecursiveParserWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class ExtractReader {
+
+ public enum ALTER_METADATA_LIST {
+ AS_IS, //leave the metadata list as is
+ FIRST_ONLY, //take only the metadata list for the "container" document
+ CONCATENATE_CONTENT_INTO_FIRST // concatenate all of the content into the first
+ }
+ private final static Logger LOGGER = LoggerFactory.getLogger(ExtractReader.class);
+ TikaConfig tikaConfig = TikaConfig.getDefaultConfig();
+
+ public List<Metadata> loadExtract(Path thisFile, ALTER_METADATA_LIST alterExtractList) {
+ List<Metadata> metadataList = null;
+ if (thisFile == null || !Files.isRegularFile(thisFile)) {
+ return metadataList;
+ }
+ Reader reader = null;
+ InputStream is = null;
+ FileSuffixes fileSuffixes = parseSuffixes(thisFile.getFileName().toString());
+ if (fileSuffixes.txtOrJson == null) {
+ LOGGER.warn("file must end with .txt or .json: "+thisFile.getFileName().toString());
+ return metadataList;
+ }
+
+ try {
+ is = Files.newInputStream(thisFile);
+ if (fileSuffixes.compression != null) {
+ if (fileSuffixes.compression.equals("bz2")) {
+ is = new BZip2CompressorInputStream(is);
+ } else if (fileSuffixes.compression.equals("gz")) {
+ is = new GzipCompressorInputStream(is);
+ } else if (fileSuffixes.compression.equals("zip")) {
+ is = new ZCompressorInputStream(is);
+ } else {
+ LOGGER.warn("Can't yet process compression of type: "+fileSuffixes.compression);
+ }
+ }
+ reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+
+ if (fileSuffixes.txtOrJson.equals("json")) {
+ metadataList = JsonMetadataList.fromJson(reader);
+ if (alterExtractList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) {
+ while (metadataList.size() > 1) {
+ metadataList.remove(metadataList.size()-1);
+ }
+ } else if (alterExtractList.equals(ALTER_METADATA_LIST.AS_IS.CONCATENATE_CONTENT_INTO_FIRST) &&
+ metadataList.size() > 1) {
+ StringBuilder sb = new StringBuilder();
+ Metadata containerMetadata = metadataList.get(0);
+ for (int i = 0; i < metadataList.size(); i++) {
+ Metadata m = metadataList.get(i);
+ String c = m.get(RecursiveParserWrapper.TIKA_CONTENT);
+ if (c != null) {
+ sb.append(c);
+ sb.append(" ");
+ }
+ }
+ containerMetadata.set(RecursiveParserWrapper.TIKA_CONTENT, sb.toString());
+ while (metadataList.size() > 1) {
+ metadataList.remove(metadataList.size()-1);
+ }
+ }
+
+
+ } else {
+ metadataList = generateListFromTextFile(reader, fileSuffixes);
+ }
+ } catch (IOException e) {
+ LOGGER.warn("couldn't open:" + thisFile.toAbsolutePath(), e);
+ } catch (TikaException e) {
+ LOGGER.warn("couldn't open:" + thisFile.toAbsolutePath(), e);
+ } finally {
+ IOUtils.closeQuietly(reader);
+ IOUtils.closeQuietly(is);
+ }
+ return metadataList;
+ }
+
+ private List<Metadata> generateListFromTextFile(Reader reader,
+ FileSuffixes fileSuffixes) throws IOException {
+ List<Metadata> metadataList = new ArrayList<>();
+ String content = IOUtils.toString(reader);
+ Metadata m = new Metadata();
+ m.set(RecursiveParserWrapper.TIKA_CONTENT, content);
+ //Let's hope the file name has a suffix that can
+ //be used to determine the mime. Could be wrong or missing,
+ //but better than nothing.
+ m.set(Metadata.RESOURCE_NAME_KEY, fileSuffixes.originalFileName);
+
+ MediaType mimeType = tikaConfig.getMimeRepository().detect(null, m);
+ if (mimeType != null) {
+ m.set(Metadata.CONTENT_TYPE, mimeType.toString());
+ }
+ metadataList.add(m);
+ return metadataList;
+
+ }
+
+ protected static FileSuffixes parseSuffixes(String fName) {
+ FileSuffixes fileSuffixes = new FileSuffixes();
+ if (fName == null) {
+ return fileSuffixes;
+ }
+ Matcher m = Pattern.compile("^(.*?)\\.(json|txt)(?:\\.(bz2|gz(?:ip)?|zip))?$").matcher(fName);
+ if (m.find()) {
+ fileSuffixes.originalFileName = m.group(1);
+ fileSuffixes.txtOrJson = m.group(2);
+ fileSuffixes.compression = m.group(3);
+ }
+ return fileSuffixes;
+ }
+
+ private static class FileSuffixes {
+ String compression;
+ String txtOrJson;
+ String originalFileName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/io/IDBWriter.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/io/IDBWriter.java b/tika-eval/src/main/java/org/apache/tika/eval/io/IDBWriter.java
new file mode 100644
index 0000000..b2b76ab
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/io/IDBWriter.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.io;
+
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.tika.eval.db.Cols;
+import org.apache.tika.eval.db.TableInfo;
+
+public interface IDBWriter {
+ public void writeRow(TableInfo table, Map<Cols, String> data) throws IOException;
+ public void close() throws IOException;
+ public int getMimeId(String mimeString);
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/io/XMLLogMsgHandler.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/io/XMLLogMsgHandler.java b/tika-eval/src/main/java/org/apache/tika/eval/io/XMLLogMsgHandler.java
new file mode 100644
index 0000000..2d509f7
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/io/XMLLogMsgHandler.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.io;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+import org.apache.log4j.Level;
+
+public interface XMLLogMsgHandler {
+ public void handleMsg(Level level, String xml) throws IOException, SQLException;
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/io/XMLLogReader.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/io/XMLLogReader.java b/tika-eval/src/main/java/org/apache/tika/eval/io/XMLLogReader.java
new file mode 100644
index 0000000..bb47b77
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/io/XMLLogReader.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.io;
+
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.sql.SQLException;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.tika.io.IOUtils;
+import org.apache.tika.parser.ParseContext;
+
+
+public class XMLLogReader {
+
+ private final static Logger logger = Logger.getLogger(XMLLogReader.class);
+ //class that wraps a logger's xml output
+ //into a single xml parseable input stream.
+
+ public void read(InputStream xmlLogFileIs, XMLLogMsgHandler handler) throws XMLStreamException {
+ InputStream is = new LogXMLWrappingInputStream(xmlLogFileIs);
+ /* try {
+ System.out.println("WRAPPED: " + IOUtils.toString(is)+ "<<WRAPPED");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }*/
+ XMLInputFactory factory = new ParseContext().getXMLInputFactory();
+ XMLStreamReader reader = factory.createXMLStreamReader(is);
+
+ Level level = null;
+ while (reader.hasNext()) {
+ reader.next();
+ switch (reader.getEventType()) {
+ case XMLStreamConstants.START_ELEMENT :
+ if ("event".equals(reader.getLocalName())) {
+ level = Level.toLevel(reader.getAttributeValue("", "level"), Level.DEBUG);
+ } else if ("message".equals(reader.getLocalName())) {
+ try {
+ handler.handleMsg(level, reader.getElementText());
+ } catch (IOException e) {
+ e.printStackTrace();
+ logger.warn("Error parsing: "+reader.getElementText());
+ } catch (SQLException e) {
+ e.printStackTrace();
+ logger.warn("SQLException: "+e.getMessage());
+ }
+ }
+ break;
+ case XMLStreamConstants.END_ELEMENT :
+ if ("event".equals(reader.getLocalName())) {
+ level = null;
+ } else if ("message".equals(reader.getLocalName())) {
+ //sdo we care any more?
+ }
+ break;
+ };
+ }
+ }
+
+
+
+ class LogXMLWrappingInputStream extends InputStream {
+ //plagiarized from log4j's chainsaw
+ private final static String HEADER =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>"
+ + "<log4j:eventSet version=\"1.2\" "
+ + "xmlns:log4j=\"http://jakarta.apache.org/log4j/\">";
+ private static final String FOOTER = "</log4j:eventSet>";
+
+ private InputStream[] streams;
+ int currentStreamIndex = 0;
+
+ private LogXMLWrappingInputStream(InputStream xmlLogFileIs){
+ streams = new InputStream[3];
+ streams[0] = new ByteArrayInputStream(HEADER.getBytes(IOUtils.UTF_8));
+ streams[1] = xmlLogFileIs;
+ streams[2] = new ByteArrayInputStream(FOOTER.getBytes(IOUtils.UTF_8));
+
+ }
+
+ @Override
+ public int read() throws IOException {
+ int c = streams[currentStreamIndex].read();
+ if (c < 0) {
+ IOUtils.closeQuietly(streams[currentStreamIndex]);
+ while (currentStreamIndex < streams.length-1) {
+ currentStreamIndex++;
+ int tmpC = streams[currentStreamIndex].read();
+ if (tmpC < 0) {
+ IOUtils.closeQuietly(streams[currentStreamIndex]);
+ } else {
+ return tmpC;
+ }
+ }
+ return -1;
+ }
+ return c;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/reports/Report.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/reports/Report.java b/tika-eval/src/main/java/org/apache/tika/eval/reports/Report.java
new file mode 100644
index 0000000..b7e2c09
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/reports/Report.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.reports;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Types;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.log4j.Logger;
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellStyle;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.VerticalAlignment;
+import org.apache.poi.xssf.streaming.SXSSFSheet;
+import org.apache.poi.xssf.streaming.SXSSFWorkbook;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+/**
+ * This class represents a single report.
+ */
+public class Report {
+
+ static final Logger logger = Logger.getLogger(Report.class);
+
+ final String NULL_VALUE = "";//TODO: make this configurable!!!
+ Map<String, XSLXCellFormatter> cellFormatters = new HashMap<>();
+ private XLSXNumFormatter defaultDoubleFormatter = new XLSXNumFormatter("0.000");
+ private XLSXNumFormatter defaultIntegerFormatter = new XLSXNumFormatter("0");
+ private CellStyle sqlCellStyle;
+
+ String sql;
+ String reportFilename;
+ boolean includeSql = true;
+
+ String reportName;
+
+ public void writeReport(Connection c, Path reportsRoot) throws SQLException, IOException {
+ logger.info("Writing report: "+reportName + " to "+reportFilename);
+ dumpXLSX(c, reportsRoot);
+ }
+
+ private void dumpXLSX(Connection c, Path reportsRoot) throws IOException, SQLException {
+ Statement st = c.createStatement();
+ Path out = reportsRoot.resolve(reportFilename);
+ Files.createDirectories(out.getParent());
+
+ SXSSFWorkbook wb = new SXSSFWorkbook(new XSSFWorkbook(), 100, true, true);
+ wb.setCompressTempFiles(true);
+ defaultIntegerFormatter.reset(wb.getXSSFWorkbook());
+ defaultDoubleFormatter.reset(wb.getXSSFWorkbook());
+ sqlCellStyle = wb.createCellStyle();
+ sqlCellStyle.setVerticalAlignment(VerticalAlignment.TOP);
+ sqlCellStyle.setWrapText(true);
+
+
+ try {
+ dumpReportToWorkbook(st, wb);
+ } finally {
+ try (OutputStream os = Files.newOutputStream(out)) {
+ wb.write(os);
+ } finally {
+ wb.dispose();
+ }
+ }
+ }
+
+ private void dumpReportToWorkbook(Statement st, SXSSFWorkbook wb) throws IOException, SQLException {
+ ResultSet rs = st.executeQuery(sql);
+
+ SXSSFSheet sheet = wb.createSheet("tika-eval Report");
+ sheet.trackColumnForAutoSizing(0);
+
+ int rowCount = 0;
+ ResultSetMetaData meta = rs.getMetaData();
+ Set<String> colNames = new HashSet<>();
+
+ Row xssfRow = sheet.createRow(rowCount++);
+ //write headers and cache them to check against styles
+ for (int i = 1; i <= meta.getColumnCount(); i++) {
+ Cell cell = xssfRow.createCell(i-1);
+ cell.setCellValue(meta.getColumnLabel(i));
+ colNames.add(meta.getColumnLabel(i));
+ }
+
+ ResultSetMetaData resultSetMetaData = rs.getMetaData();
+ while (rs.next()) {
+ xssfRow = sheet.createRow(rowCount++);
+ for (int i = 1; i <= meta.getColumnCount(); i++) {
+ Cell cell = xssfRow.createCell(i-1);
+ XSLXCellFormatter formatter = cellFormatters.get(meta.getColumnLabel(i));
+ if (formatter == null) {
+ formatter = getDefaultFormatter(resultSetMetaData.getColumnType(i));
+ }
+ if (formatter != null) {
+ formatter.applyStyleAndValue(i, rs, cell);
+ } else {
+ writeCell(meta, i, rs, cell);
+ }
+ }
+ }
+ sheet.autoSizeColumn(0);
+
+ if (!includeSql) {
+ return;
+ }
+
+ SXSSFSheet sqlSheet = wb.createSheet("tika-eval SQL");
+ sqlSheet.setColumnWidth(0, 100*250);
+ Row sqlRow = sqlSheet.createRow(0);
+ short height = 5000;
+ sqlRow.setHeight(height);
+ Cell cell = sqlRow.createCell(0);
+ cell.setCellStyle(sqlCellStyle);
+
+ cell.setCellValue(sql.trim());//.replaceAll("[\r\n]+", "\r\n"));
+ }
+
+ private XSLXCellFormatter getDefaultFormatter(int columnType) {
+ switch (columnType) {
+ case Types.INTEGER :
+ return defaultIntegerFormatter;
+ case Types.DOUBLE:
+ case Types.FLOAT:
+ case Types.DECIMAL:
+ return defaultDoubleFormatter;
+ default:
+ return null;
+ }
+ }
+
+ private void writeCell(ResultSetMetaData meta, int colIndex, ResultSet rs,
+ Cell cell) throws SQLException {
+
+ switch(meta.getColumnType(colIndex)) {
+ //fall through on numerics
+ case Types.BIGINT:
+ case Types.SMALLINT:
+ case Types.INTEGER:
+ case Types.DOUBLE:
+ case Types.FLOAT:
+ case Types.DECIMAL:
+ case Types.NUMERIC:
+ double dbl = rs.getDouble(colIndex);
+ if (rs.wasNull()) {
+ cell.setCellValue(NULL_VALUE);
+ } else {
+ cell.setCellValue(dbl);
+ }
+ break;
+ //fall through strings
+ case Types.CHAR:
+ case Types.VARCHAR:
+ case Types.LONGNVARCHAR:
+ String val = rs.getString(colIndex);
+ if (rs.wasNull()) {
+ cell.setCellValue(NULL_VALUE);
+ } else {
+ cell.setCellValue(val);
+ }
+ break;
+ default:
+ if (rs.wasNull()) {
+ cell.setCellValue(NULL_VALUE);
+ } else {
+ cell.setCellValue(rs.getString(colIndex));
+ }
+ logger.warn("Couldn't find type for: " + meta.getColumnType(colIndex) +
+ ". Defaulting to String");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/reports/ResultsReporter.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/reports/ResultsReporter.java b/tika-eval/src/main/java/org/apache/tika/eval/reports/ResultsReporter.java
new file mode 100644
index 0000000..97b7711
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/reports/ResultsReporter.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.reports;
+
+
+import javax.xml.parsers.DocumentBuilder;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.poi.common.usermodel.Hyperlink;
+import org.apache.tika.eval.ExtractComparer;
+import org.apache.tika.eval.ExtractProfiler;
+import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.H2Util;
+import org.apache.tika.parser.ParseContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class ResultsReporter {
+
+ protected static Logger LOGGER = LoggerFactory.getLogger(ResultsReporter.class);
+
+ private static Options OPTIONS;
+
+ static {
+ OPTIONS = new Options();
+
+ Option db = new Option("db", "database");
+ db.setRequired(true);
+ db.setArgs(1);
+
+ OPTIONS.addOption(db)
+ .addOption("rd", "reportsDir", true, "directory for the reports. " +
+ "If not specified, will write to 'reports'" +
+ "BEWARE: Will overwrite existing reports without warning!"
+ )
+ .addOption("rf", "reportsFile", true, "xml specifying sql to call for the reports." +
+ "If not specified, will use default reports in resources/tika-eval-*-config.xml");
+
+ }
+
+ public static void USAGE() {
+ HelpFormatter helpFormatter = new HelpFormatter();
+ helpFormatter.printHelp(
+ 80,
+ "java -jar tika-eval-x.y.jar Report -db mydb [-rd myreports] [-rf myreports.xml]",
+ "Tool: Report",
+ ResultsReporter.OPTIONS,
+ "Note: for h2 db, do not include the .mv.db at the end of the db name.");
+
+ }
+
+
+ List<String> before = new ArrayList<>();
+ List<String> after = new ArrayList<>();
+ List<Report> reports = new ArrayList<>();
+
+
+ private void addBefore(String b) {
+ before.add(b);
+ }
+
+ private void addAfter(String a) {
+ after.add(a);
+ }
+
+ private void addReport(Report r) {
+ reports.add(r);
+ }
+
+ public static ResultsReporter build(Path p) throws Exception {
+
+ ResultsReporter r = new ResultsReporter();
+
+ DocumentBuilder docBuilder = new ParseContext().getDocumentBuilder();
+ Document doc;
+ try(InputStream is = Files.newInputStream(p)) {
+ doc = docBuilder.parse(is);
+ }
+ Node docElement = doc.getDocumentElement();
+ assert(docElement.getNodeName().equals("reports"));
+ NodeList children = docElement.getChildNodes();
+ for (int i = 0; i < children.getLength(); i++) {
+ Node n = children.item(i);
+ if ("before".equals(n.getNodeName())) {
+ for (String before : getSql(n)) {
+ r.addBefore(before);
+ }
+ } else if ("after".equals(n.getNodeName())) {
+ for (String after : getSql(n)) {
+ r.addAfter(after);
+ }
+ } else if ("report".equals(n.getNodeName())) {
+ Report report = buildReport(n);
+ r.addReport(report);
+ }
+ }
+
+ return r;
+ }
+
+ private static Report buildReport(Node n) {
+ NodeList children = n.getChildNodes();
+ Report r = new Report();
+ NamedNodeMap attrs = n.getAttributes();
+
+ r.includeSql = Boolean.parseBoolean(attrs.getNamedItem("includeSql").getNodeValue());
+ r.reportFilename = attrs.getNamedItem("reportFilename").getNodeValue();
+ r.reportName = attrs.getNamedItem("reportName").getNodeValue();
+
+ for (int i = 0; i < children.getLength(); i++) {
+ Node child = children.item(i);
+ if (child.getNodeType() != 1) {
+ continue;
+ }
+ if ("sql".equals(child.getNodeName())) {
+ if (r.sql != null) {
+ throw new IllegalArgumentException("Can only have one sql statement per report");
+ }
+ r.sql = child.getTextContent();
+ } else if ("colformats".equals(child.getNodeName())) {
+ r.cellFormatters = getCellFormatters(child);
+ } else {
+ throw new IllegalArgumentException("Not expecting to see:"+child.getNodeName());
+ }
+ }
+ return r;
+ }
+
+ private static Map<String, XSLXCellFormatter> getCellFormatters(Node n) {
+ NodeList children = n.getChildNodes();
+ Map<String, XSLXCellFormatter> ret = new HashMap<>();
+ for (int i = 0; i < children.getLength(); i++) {
+ Node child = children.item(i);
+ if (child.getNodeType() != 1) {
+ continue;
+ }
+ NamedNodeMap attrs = child.getAttributes();
+ String columnName = attrs.getNamedItem("name").getNodeValue();
+ assert(!ret.containsKey(columnName));
+ String type = attrs.getNamedItem("type").getNodeValue();
+ if ("numberFormatter".equals(type)) {
+ String format = attrs.getNamedItem("format").getNodeValue();
+ XSLXCellFormatter f = new XLSXNumFormatter(format);
+ ret.put(columnName,f);
+ } else if ("urlLink".equals(type)) {
+ String base = "";
+ Node baseNode = attrs.getNamedItem("base");
+ if (baseNode != null) {
+ base = baseNode.getNodeValue();
+ }
+ XLSXHREFFormatter f = new XLSXHREFFormatter(base, Hyperlink.LINK_URL);
+ ret.put(columnName, f);
+ } else if ("fileLink".equals(type)) {
+ String base = "";
+ Node baseNode = attrs.getNamedItem("base");
+ if (baseNode != null) {
+ base = baseNode.getNodeValue();
+ }
+ XLSXHREFFormatter f = new XLSXHREFFormatter(base, Hyperlink.LINK_FILE);
+ ret.put(columnName, f);
+ }
+ }
+ return ret;
+ }
+
+ private static List<String> getSql(Node n) {
+ List<String> ret = new ArrayList<>();
+
+ NodeList children = n.getChildNodes();
+
+ for (int i = 0; i < children.getLength(); i++) {
+ Node child = children.item(i);
+ if (child.getNodeType() != 1) {
+ continue;
+ }
+ ret.add(child.getTextContent());
+ }
+ return ret;
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ DefaultParser defaultCLIParser = new DefaultParser();
+ CommandLine commandLine = null;
+ try {
+ commandLine = defaultCLIParser.parse(OPTIONS, args);
+ } catch (ParseException e) {
+ System.out.println(e.getMessage());
+ USAGE();
+ return;
+ }
+ Path db = Paths.get(commandLine.getOptionValue("db"));
+ DBUtil dbUtil = new H2Util(db);
+
+ try (Connection c = dbUtil.getConnection(true)) {
+ Path tmpReportsFile = null;
+ try {
+ ResultsReporter resultsReporter = null;
+ String reportsFile = commandLine.getOptionValue("rf");
+ if (reportsFile == null) {
+ tmpReportsFile = getDefaultReportsConfig(c);
+ resultsReporter = ResultsReporter.build(tmpReportsFile);
+ } else {
+ resultsReporter = ResultsReporter.build(Paths.get(reportsFile));
+ }
+
+ Path reportsRootDirectory = Paths.get(commandLine.getOptionValue("rd", "reports"));
+ if (Files.isDirectory(reportsRootDirectory)) {
+ LOGGER.warn("'Reports' directory exists. Will overwrite existing reports.");
+ }
+
+ resultsReporter.execute(c, reportsRootDirectory);
+ } finally {
+ if (tmpReportsFile != null) {
+ Files.delete(tmpReportsFile);
+ }
+ }
+ }
+ }
+
+ private static Path getDefaultReportsConfig(Connection c) throws IOException, SQLException {
+ DatabaseMetaData md = c.getMetaData();
+ String internalPath = null;
+ try (ResultSet rs = md.getTables(null, null, "%", null)) {
+ while (rs.next()) {
+ String tName = rs.getString(3);
+ if (ExtractComparer.CONTENTS_TABLE_B.getName().equalsIgnoreCase(tName)) {
+ internalPath = "/comparison-reports.xml";
+ break;
+ } else if (ExtractProfiler.PROFILE_TABLE.getName().equalsIgnoreCase(tName)) {
+ internalPath = "/profile-reports.xml";
+ break;
+ }
+ }
+ }
+
+ if (internalPath == null) {
+ throw new RuntimeException("Couldn't determine if this database was a 'profiler' or 'comparison' db");
+ }
+ Path tmp = Files.createTempFile("tmp-tika-reports", ".xml");
+ Files.copy(ResultsReporter.class.getResourceAsStream(internalPath), tmp, StandardCopyOption.REPLACE_EXISTING);
+ return tmp;
+ }
+
+ public void execute(Connection c, Path reportsDirectory) throws IOException, SQLException {
+ Statement st = c.createStatement();
+ for (String sql : before) {
+ st.execute(sql);
+ }
+ for (Report r : reports) {
+ r.writeReport(c, reportsDirectory);
+ }
+ for (String sql : after) {
+ st.execute(sql);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXHREFFormatter.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXHREFFormatter.java b/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXHREFFormatter.java
new file mode 100644
index 0000000..52d55e4
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXHREFFormatter.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.reports;
+
+import java.nio.file.Paths;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellStyle;
+import org.apache.poi.ss.usermodel.Font;
+import org.apache.poi.ss.usermodel.Hyperlink;
+import org.apache.poi.ss.usermodel.IndexedColors;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+/**
+ * Created by TALLISON on 4/20/2016.
+ */
+public class XLSXHREFFormatter implements XSLXCellFormatter {
+ //xlsx files can only have this many hyperlinks
+ //if they have more Excel can't read the file
+ private static final int MAX_HYPERLINKS = 65000;
+
+
+ private final String urlBase;
+ private final int linkType;
+ private XSSFWorkbook workbook;
+ private CellStyle style;
+ private int links = 0;
+
+ public XLSXHREFFormatter(String urlBase,
+ int linkType) {
+ this.urlBase = urlBase;
+ this.linkType = linkType;
+ }
+
+ @Override
+ public void reset(XSSFWorkbook workbook) {
+ this.workbook = workbook;
+ style = workbook.createCellStyle();
+ Font hlinkFont = workbook.createFont();
+ hlinkFont.setUnderline(Font.U_SINGLE);
+ hlinkFont.setColor(IndexedColors.BLUE.getIndex());
+ style.setFont(hlinkFont);
+ links = 0;
+
+ }
+
+ @Override
+ public void applyStyleAndValue(int dbColNum, ResultSet resultSet, Cell cell) throws SQLException {
+ if (links < MAX_HYPERLINKS) {
+ Hyperlink hyperlink = workbook.getCreationHelper().createHyperlink(linkType);
+ String path = resultSet.getString(dbColNum);
+ String address = urlBase+path;
+ hyperlink.setAddress(address);
+ cell.setHyperlink(hyperlink);
+ cell.setCellStyle(style);
+ String fName = Paths.get(path).getFileName().toString();
+ cell.setCellValue(fName);
+ links++;
+ } else {
+ //silently stop adding hyperlinks
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXNumFormatter.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXNumFormatter.java b/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXNumFormatter.java
new file mode 100644
index 0000000..b2ba5b0
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/reports/XLSXNumFormatter.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.reports;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellStyle;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+class XLSXNumFormatter implements XSLXCellFormatter {
+
+ private final String formatString;
+ private CellStyle style;
+
+ XLSXNumFormatter(String formatString) {
+ this.formatString = formatString;
+ }
+
+
+ @Override
+ public void reset(XSSFWorkbook workbook) {
+ style = workbook.createCellStyle();
+ style.setDataFormat(workbook.getCreationHelper()
+ .createDataFormat().getFormat(formatString));
+ }
+
+ @Override
+ public void applyStyleAndValue(int dbColNum, ResultSet resultSet, Cell cell) throws SQLException {
+ double d = resultSet.getDouble(dbColNum);
+ if (resultSet.wasNull()) {
+
+ } else {
+ cell.setCellStyle(style);
+ }
+ cell.setCellValue(resultSet.getDouble(dbColNum));
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/reports/XSLXCellFormatter.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/reports/XSLXCellFormatter.java b/tika-eval/src/main/java/org/apache/tika/eval/reports/XSLXCellFormatter.java
new file mode 100644
index 0000000..eb0e024
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/reports/XSLXCellFormatter.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.reports;
+
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+interface XSLXCellFormatter {
+
+ public void reset(XSSFWorkbook workbook);
+ public void applyStyleAndValue(int dbColNum, ResultSet resultSet, Cell cell) throws SQLException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/tokens/AlphaIdeographFilterFactory.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/AlphaIdeographFilterFactory.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/AlphaIdeographFilterFactory.java
new file mode 100644
index 0000000..fb72e84
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/AlphaIdeographFilterFactory.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.tokens;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.lucene.analysis.FilteringTokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+
+/**
+ * Factory for filter that only allows tokens with characters that "isAlphabetic" or "isIdeographic" through.
+ */
+public class AlphaIdeographFilterFactory extends TokenFilterFactory {
+
+
+
+ public AlphaIdeographFilterFactory(Map<String, String> args) {
+ super(args);
+ }
+
+ @Override
+ public TokenStream create(TokenStream tokenStream) {
+ return new AlphaFilter(tokenStream);
+ }
+
+ /**
+ * Remove tokens tokens that do not contain an "
+ */
+ private class AlphaFilter extends FilteringTokenFilter {
+
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+ public AlphaFilter(TokenStream in) {
+ super(in);
+ }
+
+ @Override
+ protected boolean accept() throws IOException {
+ char[] buff = termAtt.buffer();
+ for (int i = 0; i < termAtt.length(); i++) {
+ int cp = buff[i];
+ if (Character.isHighSurrogate(buff[i])) {
+ if (i < termAtt.length()-1) {
+ cp = Character.toCodePoint(buff[i], buff[i + 1]);
+ i++;
+ }
+ }
+
+ if (Character.isAlphabetic(cp) ||
+ Character.isIdeographic(cp)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/tokens/AnalyzerDeserializer.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/AnalyzerDeserializer.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/AnalyzerDeserializer.java
new file mode 100644
index 0000000..cfc2d4f
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/AnalyzerDeserializer.java
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.tokens;
+
+
+import java.io.IOException;
+import java.io.Reader;
+import java.lang.reflect.Type;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.util.CharFilterFactory;
+import org.apache.lucene.analysis.util.ClasspathResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.analysis.util.TokenizerFactory;
+
+class AnalyzerDeserializer implements JsonDeserializer<Map<String, Analyzer>> {
+
+
+ private static String ANALYZERS = "analyzers";
+ private static String CHAR_FILTERS = "charfilters";
+ private static String TOKEN_FILTERS = "tokenfilters";
+ private static String TOKENIZER = "tokenizer";
+ private static String FACTORY = "factory";
+ private static String PARAMS = "params";
+
+ @Override
+ public Map<String, Analyzer> deserialize(JsonElement element, Type type,
+ JsonDeserializationContext jsonDeserializationContext) throws JsonParseException {
+ if (! element.isJsonObject()) {
+ throw new IllegalArgumentException("Expecting top level 'analyzers:{}'");
+ }
+
+ JsonElement root = element.getAsJsonObject().get(ANALYZERS);
+ if (root == null) {
+ throw new IllegalArgumentException("Expecting top level 'analyzers:{}");
+ }
+ try {
+ return buildAnalyzers(root);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ public static Map<String, Analyzer> buildAnalyzers(JsonElement value) throws IOException {
+ if (! value.isJsonObject()) {
+ throw new IllegalArgumentException("Expecting map with analyzer names/analyzer definitions");
+ }
+ Map<String, Analyzer> analyzers = new HashMap<>();
+ JsonObject root = (JsonObject)value;
+ for (Map.Entry<String, JsonElement> e : root.entrySet()) {
+ String analyzerName = e.getKey();
+ Analyzer analyzer = buildAnalyzer(analyzerName, e.getValue());
+ analyzers.put(analyzerName, analyzer);
+ }
+ return analyzers;
+ }
+
+ public static Analyzer buildAnalyzer(String analyzerName, JsonElement value) throws IOException {
+ if (! value.isJsonObject()) {
+ throw new IllegalArgumentException("Expecting map of charfilter, tokenizer, tokenfilters");
+ }
+ JsonObject aRoot = (JsonObject)value;
+ CharFilterFactory[] charFilters = new CharFilterFactory[0];
+ TokenizerFactory tokenizerFactory = null;
+ TokenFilterFactory[] tokenFilterFactories = new TokenFilterFactory[0];
+ for ( Map.Entry<String, JsonElement> e : aRoot.entrySet()) {
+ String k = e.getKey();
+ if (k.equals(CHAR_FILTERS)) {
+ charFilters = buildCharFilters(e.getValue(), analyzerName);
+ } else if (k.equals(TOKEN_FILTERS)) {
+ tokenFilterFactories = buildTokenFilterFactories(e.getValue(), analyzerName);
+ } else if (k.equals(TOKENIZER)) {
+ tokenizerFactory = buildTokenizerFactory(e.getValue(), analyzerName);
+ } else {
+ throw new IllegalArgumentException("Should have one of three values here:"+
+ CHAR_FILTERS + ", "+
+ TOKENIZER+", "+
+ TOKEN_FILTERS +
+ ". I don't recognize: "+k);
+ }
+ }
+ if (tokenizerFactory == null) {
+ throw new IllegalArgumentException("Must specify at least a tokenizer factory for an analyzer!");
+ }
+ return new MyTokenizerChain(charFilters, tokenizerFactory, tokenFilterFactories);
+ }
+
+ private static TokenizerFactory buildTokenizerFactory(JsonElement map, String analyzerName) throws IOException {
+ if (!(map instanceof JsonObject)) {
+ throw new IllegalArgumentException("Expecting a map with \"factory\" string and " +
+ "\"params\" map in tokenizer factory;"+
+ " not: "+map.toString() + " in "+analyzerName);
+ }
+ JsonElement factoryEl = ((JsonObject)map).get(FACTORY);
+ if (factoryEl == null || ! factoryEl.isJsonPrimitive()) {
+ throw new IllegalArgumentException("Expecting value for factory in char filter factory builder in:"+
+ analyzerName);
+ }
+ String factoryName = factoryEl.getAsString();
+ factoryName = factoryName.startsWith("oala.") ?
+ factoryName.replaceFirst("oala.", "org.apache.lucene.analysis.") : factoryName;
+
+ JsonElement paramsEl = ((JsonObject)map).get(PARAMS);
+ Map<String, String> params = mapify(paramsEl);
+ String spiName = "";
+ for (String s : TokenizerFactory.availableTokenizers()) {
+ Class clazz = TokenizerFactory.lookupClass(s);
+ if (clazz.getName().equals(factoryName)) {
+ spiName = s;
+ break;
+ }
+ }
+ if (spiName.equals("")) {
+ throw new IllegalArgumentException("A SPI class of type org.apache.lucene.analysis.util.TokenizerFactory with name"+
+ "'"+factoryName+"' does not exist.");
+ }
+ try {
+ TokenizerFactory tokenizerFactory = TokenizerFactory.forName(spiName, params);
+ if (tokenizerFactory instanceof ResourceLoaderAware) {
+ ((ResourceLoaderAware) tokenizerFactory).inform(new ClasspathResourceLoader(AnalyzerDeserializer.class));
+ }
+
+ return tokenizerFactory;
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("While working on "+analyzerName, e);
+ }
+ }
+
+ private static CharFilterFactory[] buildCharFilters(JsonElement el, String analyzerName) throws IOException {
+ if (el == null || el.isJsonNull()) {
+ return null;
+ }
+ if (! el.isJsonArray()) {
+ throw new IllegalArgumentException("Expecting array for charfilters, but got:"+el.toString() +
+ " for "+analyzerName);
+ }
+ JsonArray jsonArray = (JsonArray)el;
+ List<CharFilterFactory> ret = new LinkedList<CharFilterFactory>();
+ for (JsonElement filterMap : jsonArray) {
+ if (!(filterMap instanceof JsonObject)) {
+ throw new IllegalArgumentException("Expecting a map with \"factory\" string and \"params\" map in char filter factory;"+
+ " not: "+filterMap.toString() + " in "+analyzerName);
+ }
+ JsonElement factoryEl = ((JsonObject)filterMap).get(FACTORY);
+ if (factoryEl == null || ! factoryEl.isJsonPrimitive()) {
+ throw new IllegalArgumentException(
+ "Expecting value for factory in char filter factory builder in:"+analyzerName);
+ }
+ String factoryName = factoryEl.getAsString();
+ factoryName = factoryName.replaceAll("oala.", "org.apache.lucene.analysis.");
+
+ JsonElement paramsEl = ((JsonObject)filterMap).get(PARAMS);
+ Map<String, String> params = mapify(paramsEl);
+ String spiName = "";
+ for (String s : CharFilterFactory.availableCharFilters()) {
+ Class clazz = CharFilterFactory.lookupClass(s);
+ if (clazz.getName().equals(factoryName)) {
+ spiName = s;
+ break;
+ }
+ }
+ if (spiName.equals("")) {
+ throw new IllegalArgumentException("A SPI class of type org.apache.lucene.analysis.util.CharFilterFactory with name"+
+ "'"+factoryName+"' does not exist.");
+ }
+
+ try {
+ CharFilterFactory charFilterFactory = CharFilterFactory.forName(spiName, params);
+ if (charFilterFactory instanceof ResourceLoaderAware) {
+ ((ResourceLoaderAware) charFilterFactory).inform(new ClasspathResourceLoader(AnalyzerDeserializer.class));
+ }
+ ret.add(charFilterFactory);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("While trying to load "+
+ analyzerName + ": "+ e.getMessage(), e);
+ }
+ }
+ if (ret.size() == 0) {
+ return new CharFilterFactory[0];
+ }
+ return ret.toArray(new CharFilterFactory[ret.size()]);
+ }
+
+ private static TokenFilterFactory[] buildTokenFilterFactories(JsonElement el,
+ String analyzerName) throws IOException {
+ if (el == null || el.isJsonNull()) {
+ return null;
+ }
+ if (! el.isJsonArray()) {
+ throw new IllegalArgumentException(
+ "Expecting array for tokenfilters, but got:"+el.toString() + " in "+analyzerName);
+ }
+ JsonArray jsonArray = (JsonArray)el;
+ List<TokenFilterFactory> ret = new LinkedList<>();
+ for (JsonElement filterMap : jsonArray) {
+ if (!(filterMap instanceof JsonObject)) {
+ throw new IllegalArgumentException("Expecting a map with \"factory\" string and \"params\" map in token filter factory;"+
+ " not: "+filterMap.toString() + " in "+ analyzerName);
+ }
+ JsonElement factoryEl = ((JsonObject)filterMap).get(FACTORY);
+ if (factoryEl == null || ! factoryEl.isJsonPrimitive()) {
+ throw new IllegalArgumentException("Expecting value for factory in token filter factory builder in "+analyzerName);
+ }
+ String factoryName = factoryEl.getAsString();
+ factoryName = factoryName.startsWith("oala.") ?
+ factoryName.replaceFirst("oala.", "org.apache.lucene.analysis.") :
+ factoryName;
+
+ JsonElement paramsEl = ((JsonObject)filterMap).get(PARAMS);
+ Map<String, String> params = mapify(paramsEl);
+ String spiName = "";
+ for (String s : TokenFilterFactory.availableTokenFilters()) {
+ Class clazz = TokenFilterFactory.lookupClass(s);
+ if (clazz.getName().equals(factoryName)) {
+ spiName = s;
+ break;
+ }
+ }
+ if (spiName.equals("")) {
+ throw new IllegalArgumentException("A SPI class of type org.apache.lucene.analysis.util.TokenFilterFactory with name"+
+ "'"+factoryName+"' does not exist.");
+ }
+
+ try {
+ TokenFilterFactory tokenFilterFactory = TokenFilterFactory.forName(spiName, params);
+ if (tokenFilterFactory instanceof ResourceLoaderAware) {
+ ((ResourceLoaderAware) tokenFilterFactory).inform(new ClasspathResourceLoader(AnalyzerDeserializer.class));
+ }
+ ret.add(tokenFilterFactory);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("While loading "+analyzerName, e);
+ }
+ }
+ if (ret.size() == 0) {
+ return new TokenFilterFactory[0];
+ }
+ return ret.toArray(new TokenFilterFactory[ret.size()]);
+ }
+
+ private static Map<String, String> mapify(JsonElement paramsEl) {
+ if (paramsEl == null || paramsEl.isJsonNull()) {
+ return Collections.EMPTY_MAP;
+ }
+ if (! paramsEl.isJsonObject()) {
+ throw new IllegalArgumentException("Expecting map, not: "+paramsEl.toString());
+ }
+ Map<String, String> params = new HashMap<>();
+ for (Map.Entry<String,JsonElement> e : ((JsonObject)paramsEl).entrySet()) {
+ JsonElement value = e.getValue();
+ if (! value.isJsonPrimitive()) {
+ throw new IllegalArgumentException("Expecting parameter to have primitive value: "+value.toString());
+ }
+ String v = e.getValue().getAsString();
+ params.put(e.getKey(), v);
+ }
+ return params;
+ }
+
+ /**
+ * Plagiarized verbatim from Solr!
+ */
+ private static class MyTokenizerChain extends Analyzer {
+
+ final private CharFilterFactory[] charFilters;
+ final private TokenizerFactory tokenizer;
+ final private TokenFilterFactory[] filters;
+
+ public MyTokenizerChain(TokenizerFactory tokenizer, TokenFilterFactory[] filters) {
+ this(null, tokenizer, filters);
+ }
+
+ public MyTokenizerChain(CharFilterFactory[] charFilters, TokenizerFactory tokenizer, TokenFilterFactory[] filters) {
+ this.charFilters = charFilters;
+ this.tokenizer = tokenizer;
+ this.filters = filters;
+ }
+
+ public CharFilterFactory[] getCharFilterFactories() {
+ return charFilters;
+ }
+
+ public TokenizerFactory getTokenizerFactory() {
+ return tokenizer;
+ }
+
+ public TokenFilterFactory[] getTokenFilterFactories() {
+ return filters;
+ }
+
+ @Override
+ public Reader initReader(String fieldName, Reader reader) {
+
+ if (charFilters != null && charFilters.length > 0) {
+ Reader cs = reader;
+ for (CharFilterFactory charFilter : charFilters) {
+ cs = charFilter.create(cs);
+ }
+ reader = cs;
+ }
+
+ return reader;
+ }
+
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tk = tokenizer.create();
+ TokenStream ts = tk;
+ for (TokenFilterFactory filter : filters) {
+ ts = filter.create(ts);
+ }
+
+ return new TokenStreamComponents(tk, ts);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/tokens/AnalyzerManager.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/AnalyzerManager.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/AnalyzerManager.java
new file mode 100644
index 0000000..db6ae26
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/AnalyzerManager.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.tokens;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.Map;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonParseException;
+import org.apache.lucene.analysis.Analyzer;
+
+public class AnalyzerManager {
+
+ private final static String GENERAL = "general";
+ private static final String ALPHA_IDEOGRAPH = "alpha";
+ private static final String COMMON_TOKENS = "common_tokens";
+
+ private final Analyzer generalAnalyzer;
+ private final Analyzer alphaIdeoAnalyzer;
+ private final Analyzer commonTokensAnalyzer;
+
+ private AnalyzerManager(Analyzer generalAnalyzer,
+ Analyzer alphaIdeoAnalyzer,
+ Analyzer commonTokensAnalyzer) {
+ this.generalAnalyzer = generalAnalyzer;
+ this.alphaIdeoAnalyzer = alphaIdeoAnalyzer;
+ this.commonTokensAnalyzer = commonTokensAnalyzer;
+ }
+
+ public static AnalyzerManager newInstance() throws IOException {
+ InputStream is = AnalyzerManager.class.getClassLoader().getResourceAsStream("lucene-analyzers.json");
+ Reader reader = new InputStreamReader(is, "UTF-8");
+ GsonBuilder builder = new GsonBuilder();
+ builder.registerTypeHierarchyAdapter(Map.class, new AnalyzerDeserializer());
+ Gson gson = builder.create();
+ Map<String, Analyzer> map = gson.fromJson(reader, Map.class);
+ Analyzer general = map.get(GENERAL);
+ Analyzer alphaIdeo = map.get(ALPHA_IDEOGRAPH);
+ Analyzer common = map.get(COMMON_TOKENS);
+ if (general == null) {
+ throw new JsonParseException("Must specify "+GENERAL + " analyzer");
+ }
+ if (alphaIdeo == null) {
+ throw new JsonParseException("Must specify "+ ALPHA_IDEOGRAPH + " analyzer");
+ }
+ if (common == null) {
+ throw new JsonParseException("Must specify "+ COMMON_TOKENS + " analyzer");
+ }
+
+ return new AnalyzerManager(general, alphaIdeo, common);
+ }
+
+ /**
+ * This analyzer should be used to extract all tokens.
+ * @return
+ */
+ public Analyzer getGeneralAnalyzer() {
+ return generalAnalyzer;
+ }
+
+ /**
+ * This analyzer is used to extract "alphabetic" tokens.
+ * @return
+ */
+ public Analyzer getAlphaIdeoAnalyzer() {
+ return alphaIdeoAnalyzer;
+ }
+
+ /**
+ * This analyzer should be used to generate common tokens lists from
+ * large corpora. It is not used by tika-eval in profiling or comparing.
+ * @return
+ */
+ public Analyzer getCommonTokensAnalyzer() {
+ return commonTokensAnalyzer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/tokens/CJKBigramAwareLengthFilterFactory.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/CJKBigramAwareLengthFilterFactory.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/CJKBigramAwareLengthFilterFactory.java
new file mode 100644
index 0000000..31fa866
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/CJKBigramAwareLengthFilterFactory.java
@@ -0,0 +1,74 @@
+package org.apache.tika.eval.tokens;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.lucene.analysis.FilteringTokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.cjk.CJKBigramFilter;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+
+/**
+ * Creates a very narrowly focused TokenFilter that limits tokens based on length
+ * _unless_ they've been identified as <DOUBLE> or <SINGLE>
+ * by the CJKBigramFilter.
+ *
+ * This class is intended to be used when generating "common tokens" files.
+ */
+public class CJKBigramAwareLengthFilterFactory extends TokenFilterFactory {
+
+
+
+ private final int min;
+ private final int max;
+ public CJKBigramAwareLengthFilterFactory(Map<String, String> args) {
+ super(args);
+ min = Integer.parseInt(args.get("min"));
+ max = Integer.parseInt(args.get("max"));
+ }
+
+ @Override
+ public TokenStream create(TokenStream tokenStream) {
+ return new CJKAwareLengthFilter(tokenStream);
+ }
+
+ private class CJKAwareLengthFilter extends FilteringTokenFilter {
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+
+ public CJKAwareLengthFilter(TokenStream in) {
+ super(in);
+ }
+
+ @Override
+ protected boolean accept() throws IOException {
+ if ( termAtt.length() < min) {
+ String type = typeAtt.type();
+ if (type == CJKBigramFilter.DOUBLE_TYPE || type == CJKBigramFilter.SINGLE_TYPE) {
+ return true;
+ }
+ }
+ return termAtt.length() >= min && termAtt.length() <= max;
+ }
+ }
+
+ /*
+ private static boolean isCJ(int codePoint) {
+ if (
+ (codePoint >= 0x4E00 && codePoint <= 0x9FFF) ||
+ ( codePoint >= 0x3400 && codePoint <= 0x4dbf) ||
+ ( codePoint >= 0x20000 && codePoint <= 0x2a6df) ||
+ ( codePoint >= 0x2A700 && codePoint <= 0x2b73f) ||
+ ( codePoint >= 0x2B740 && codePoint <= 0x2B81F) ||
+ ( codePoint >= 0x2B820 && codePoint <- 0x2CEAF) ||
+ ( codePoint >= 0xF900 && codePoint <= 0xFAFF) ||
+ ( codePoint >= 0x2F800 && codePoint <= 0x2Fa1F)
+ ) {
+ return true;
+ }
+ return false;
+ }*/
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java
new file mode 100644
index 0000000..b74daa1
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.tokens;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CommonTokenCountManager {
+
+ private final static Charset COMMON_TOKENS_CHARSET = StandardCharsets.UTF_8;
+ private final static Logger LOGGER = LoggerFactory.getLogger(CommonTokenCountManager.class);
+
+ private final Path commonTokensDir;
+
+ Map<String, Set<String>> commonTokenMap = new ConcurrentHashMap<>();
+ Set<String> alreadyTriedToLoad = new HashSet<>();
+
+ //if we have no model or if no langid is passed in
+ //make this configurable
+ String defaultLangCode = "en";
+
+ public CommonTokenCountManager(Path commonTokensDir) throws IOException {
+ this.commonTokensDir = commonTokensDir;
+ tryToLoad(defaultLangCode);
+ //if you couldn't load it, make sure to add an empty
+ //set to prevent npes later
+ Set<String> set = commonTokenMap.get(defaultLangCode);
+ if (set == null) {
+ LOGGER.warn("No common tokens for default language: '"+defaultLangCode+"'");
+ commonTokenMap.put(defaultLangCode, new HashSet<String>());
+ }
+ }
+
+ public CommonTokenResult countTokenOverlaps(String langCode,
+ Map<String, MutableInt> tokens) throws IOException {
+ String actualLangCode = getActualLangCode(langCode);
+ int overlap = 0;
+ Set<String> commonTokens = commonTokenMap.get(actualLangCode);
+ for (Map.Entry<String, MutableInt> e : tokens.entrySet()) {
+ if (commonTokens.contains(e.getKey())) {
+ overlap += e.getValue().intValue();
+ }
+ }
+ return new CommonTokenResult(actualLangCode, overlap);
+ }
+
+ //return langcode for lang that you are actually using
+ //lazily load the appropriate model
+ private String getActualLangCode(String langCode) {
+ if (langCode == null || "".equals(langCode)) {
+ return defaultLangCode;
+ }
+ if (commonTokenMap.containsKey(langCode)) {
+ return langCode;
+ }
+ tryToLoad(langCode);
+ Set<String> set = commonTokenMap.get(langCode);
+ if (set == null) {
+ return defaultLangCode;
+ }
+ return langCode;
+
+ }
+
+ public void close() throws IOException {
+ commonTokenMap.clear();
+ }
+
+ private synchronized void tryToLoad(String langCode) {
+ if (alreadyTriedToLoad.contains(langCode)) {
+ return;
+ }
+ //check once more now that we're in a
+ //synchronized block
+ if (commonTokenMap.get(langCode) != null) {
+ return;
+ }
+ Path p = commonTokensDir.resolve(langCode);
+ if (!Files.isRegularFile(p)) {
+ LOGGER.warn("Couldn't find common tokens file for: '"+langCode+"': "+
+ p.toAbsolutePath());
+ alreadyTriedToLoad.add(langCode);
+ return;
+ }
+
+ Set<String> set = commonTokenMap.get(langCode);
+ if (set == null) {
+ set = new HashSet<>();
+ commonTokenMap.put(langCode, set);
+ }
+ try (BufferedReader reader = Files.newBufferedReader(p, COMMON_TOKENS_CHARSET)) {
+ alreadyTriedToLoad.add(langCode);
+ String line = reader.readLine();
+ while (line != null) {
+ line = line.trim();
+ if (line.startsWith("#")) {
+ line = reader.readLine();
+ continue;
+ }
+ //allow language models with, e.g. tab-delimited counts after the term
+ String[] cols = line.split("\t");
+ String t = cols[0].trim();
+ if (t.length() > 0) {
+ set.add(t);
+ }
+
+ line = reader.readLine();
+ }
+ } catch (IOException e) {
+ LOGGER.warn("IOException trying to read: '"+langCode+"'");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenResult.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenResult.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenResult.java
new file mode 100644
index 0000000..340ceca
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenResult.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.tokens;
+
+
+public class CommonTokenResult {
+
+ private final String langCode;
+ private final int tokens;
+
+ public CommonTokenResult(String langCode, int tokens) {
+ this.langCode = langCode;
+ this.tokens = tokens;
+ }
+
+ public String getLangCode() {
+ return langCode;
+ }
+
+ public int getTokens() {
+ return tokens;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/tokens/ContrastStatistics.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/ContrastStatistics.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/ContrastStatistics.java
new file mode 100644
index 0000000..dd1e49d
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/ContrastStatistics.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.tokens;
+
+public class ContrastStatistics {
+
+
+ double diceCoefficient;
+ double overlap;
+
+ TokenIntPair[] topNUniqueA;
+ TokenIntPair[] topNUniqueB;
+ TokenIntPair[] topNMoreA;
+ TokenIntPair[] topNMoreB;
+
+ void setDiceCoefficient(double diceCoefficient) {
+ this.diceCoefficient = diceCoefficient;
+ }
+
+ void setOverlap(double overlap) {
+ this.overlap = overlap;
+ }
+
+ void setTopNUniqueA(TokenIntPair[] topNUniqueA) {
+ this.topNUniqueA = topNUniqueA;
+ }
+
+ void setTopNUniqueB(TokenIntPair[] topNUniqueB) {
+ this.topNUniqueB = topNUniqueB;
+ }
+
+ void setTopNMoreA(TokenIntPair[] topNMoreA) {
+ this.topNMoreA = topNMoreA;
+ }
+
+ void setTopNMoreB(TokenIntPair[] topNMoreB) {
+ this.topNMoreB = topNMoreB;
+ }
+
+ public double getDiceCoefficient() {
+ return diceCoefficient;
+ }
+
+ public double getOverlap() {
+ return overlap;
+ }
+
+ public TokenIntPair[] getTopNUniqueA() {
+ return topNUniqueA;
+ }
+
+ public TokenIntPair[] getTopNUniqueB() {
+ return topNUniqueB;
+ }
+
+ public TokenIntPair[] getTopNMoreA() {
+ return topNMoreA;
+ }
+
+ public TokenIntPair[] getTopNMoreB() {
+ return topNMoreB;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenContraster.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenContraster.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenContraster.java
new file mode 100644
index 0000000..6cbbbc7
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenContraster.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.tokens;
+
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.lucene.util.PriorityQueue;
+
+/**
+ * Computes some corpus contrast statistics.
+ *
+ * Not thread safe.
+ */
+public class TokenContraster {
+
+ private TokenStatistics tokenStatisticsA;
+ private TokenStatistics tokenStatisticsB;
+
+ private TokenCountPriorityQueue uniqA;
+ private TokenCountPriorityQueue uniqB;
+
+ private TokenCountDiffQueue moreA;
+ private TokenCountDiffQueue moreB;
+
+
+ private int topN = 10;
+
+ private int diceCoefficientNum = 0;
+ private int overlapNum = 0;
+
+ private double diceCoefficient = 0.0d;
+ private double overlap = 0.0;
+
+
+ public ContrastStatistics calculateContrastStatistics(Map<String, MutableInt> mapA,
+ TokenStatistics tokenStatisticsA,
+ Map<String, MutableInt> mapB,
+ TokenStatistics tokenStatisticsB) {
+ reset();
+ this.tokenStatisticsA = tokenStatisticsA;
+ this.tokenStatisticsB = tokenStatisticsB;
+
+ for (Map.Entry<String, MutableInt> e : mapA.entrySet()) {
+ MutableInt bVal = mapB.get(e.getKey());
+ int b = (bVal == null) ? 0 : bVal.intValue();
+ add(e.getKey(), e.getValue().intValue(), b);
+ }
+
+ for (Map.Entry<String, MutableInt> e : mapB.entrySet()) {
+ if (mapA.containsKey(e.getKey())) {
+ continue;
+ }
+ add(e.getKey(), 0, e.getValue().intValue());
+ }
+ finishComputing();
+ ContrastStatistics contrastStatistics = new ContrastStatistics();
+ contrastStatistics.setDiceCoefficient(diceCoefficient);
+ contrastStatistics.setOverlap(overlap);
+ contrastStatistics.setTopNUniqueA(uniqA.getArray());
+ contrastStatistics.setTopNUniqueB(uniqB.getArray());
+ contrastStatistics.setTopNMoreA(moreA.getArray());
+ contrastStatistics.setTopNMoreB(moreB.getArray());
+ return contrastStatistics;
+ }
+
+ private void reset() {
+ this.uniqA = new TokenCountPriorityQueue(topN);
+ this.uniqB = new TokenCountPriorityQueue(topN);
+ this.moreA = new TokenCountDiffQueue(topN);
+ this.moreB = new TokenCountDiffQueue(topN);
+ diceCoefficientNum = 0;
+ overlapNum = 0;
+ diceCoefficient = 0.0d;
+ overlap = 0.0;
+
+ }
+ private void add(String token, int tokenCountA, int tokenCountB) {
+ if (tokenCountA > 0 && tokenCountB > 0) {
+ diceCoefficientNum += 2;
+ overlapNum += 2 * Math.min(tokenCountA, tokenCountB);
+ }
+
+
+ if (tokenCountA == 0L && tokenCountB > 0L) {
+ addToken(token, tokenCountB, uniqB);
+ }
+ if (tokenCountB == 0L && tokenCountA > 0L) {
+ addToken(token, tokenCountA, uniqA);
+ }
+
+ if (tokenCountA > tokenCountB) {
+ addTokenDiff(token, tokenCountA, tokenCountA-tokenCountB, moreA);
+ } else if (tokenCountB > tokenCountA) {
+ addTokenDiff(token, tokenCountB, tokenCountB-tokenCountA, moreB);
+
+ }
+
+ }
+
+ private void finishComputing() {
+
+ long sumUniqTokens = tokenStatisticsA.getTotalUniqueTokens()
+ +tokenStatisticsB.getTotalUniqueTokens();
+
+ diceCoefficient = (double) diceCoefficientNum / (double) sumUniqTokens;
+ overlap = (float) overlapNum / (double) (tokenStatisticsA.getTotalTokens() +
+ tokenStatisticsB.getTotalTokens());
+
+ }
+
+ private void addTokenDiff(String token, int tokenCount, int diff, TokenCountDiffQueue queue) {
+ if (queue.top() == null || queue.size() < topN ||
+ diff >= queue.top().diff) {
+ queue.insertWithOverflow(new TokenCountDiff(token, diff, tokenCount));
+ }
+
+ }
+
+ private void addToken(String token, int tokenCount, TokenCountPriorityQueue queue) {
+ if (queue.top() == null || queue.size() < topN ||
+ tokenCount >= queue.top().getValue()) {
+ queue.insertWithOverflow(new TokenIntPair(token, tokenCount));
+ }
+
+ }
+
+ class TokenCountDiffQueue extends PriorityQueue<TokenCountDiff> {
+
+ TokenCountDiffQueue(int maxSize) {
+ super(maxSize);
+ }
+
+ @Override
+ protected boolean lessThan(TokenCountDiff arg0, TokenCountDiff arg1) {
+ if (arg0.diff < arg1.diff) {
+ return true;
+ } else if (arg0.diff > arg1.diff) {
+ return false;
+ }
+ return arg1.token.compareTo(arg0.token) < 0;
+ }
+
+ public TokenIntPair[] getArray() {
+ TokenIntPair[] topN = new TokenIntPair[size()];
+ //now we reverse the queue
+ TokenCountDiff token = pop();
+ int i = topN.length-1;
+ while (token != null && i > -1) {
+ topN[i--] = new TokenIntPair(token.token, token.diff);
+ token = pop();
+ }
+ return topN;
+ }
+ }
+
+ private class TokenCountDiff {
+ private final String token;
+ private final int diff;
+ private final int count;
+
+ private TokenCountDiff(String token, int diff, int count) {
+ this.token = token;
+ this.diff = diff;
+ this.count = count;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCountPriorityQueue.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCountPriorityQueue.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCountPriorityQueue.java
new file mode 100644
index 0000000..624da0c
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCountPriorityQueue.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.tokens;
+
+import org.apache.lucene.util.PriorityQueue;
+
+public class TokenCountPriorityQueue extends PriorityQueue<TokenIntPair> {
+
+ TokenCountPriorityQueue(int maxSize) {
+ super(maxSize);
+ }
+
+ @Override
+ protected boolean lessThan(TokenIntPair arg0, TokenIntPair arg1) {
+ if (arg0.getValue() < arg1.getValue()) {
+ return true;
+ } else if (arg0.getValue() > arg1.getValue()) {
+ return false;
+ }
+ return arg1.token.compareTo(arg0.token) < 0;
+ }
+
+ public TokenIntPair[] getArray() {
+ TokenIntPair[] topN = new TokenIntPair[size()];
+ //now we reverse the queue
+ TokenIntPair term = pop();
+ int i = topN.length-1;
+ while (term != null && i > -1) {
+ topN[i--] = term;
+ term = pop();
+ }
+ return topN;
+ }
+}
\ No newline at end of file
[5/6] tika git commit: TIKA-1332 initial commit of tika-eval. More
work remains.
Posted by ta...@apache.org.
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/TikaEvalCLI.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/TikaEvalCLI.java b/tika-eval/src/main/java/org/apache/tika/eval/TikaEvalCLI.java
new file mode 100644
index 0000000..5860327
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/TikaEvalCLI.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.ParseException;
+import org.apache.tika.batch.fs.FSBatchProcessCLI;
+import org.apache.tika.eval.reports.ResultsReporter;
+import org.h2.tools.Console;
+
+public class TikaEvalCLI {
+ static final String[] tools = {"Profile", "Compare", "Report", "StartDB"};
+
+ private static String specifyTools() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Must specify one of the following tools in the first parameter:\n");
+ for (String s : tools) {
+ sb.append(s+"\n");
+ }
+ return sb.toString();
+
+ }
+
+ private void execute(String[] args) throws Exception {
+ String tool = args[0];
+ String[] subsetArgs = new String[args.length-1];
+ System.arraycopy(args, 1, subsetArgs, 0, args.length - 1);
+ if (tool.equals("Report")) {
+ handleReport(subsetArgs);
+ } else if (tool.equals("Compare")) {
+ handleCompare(subsetArgs);
+ } else if (tool.equals("Profile")) {
+ handleProfile(subsetArgs);
+ } else if (tool.equals("StartDB")) {
+ handleStartDB(subsetArgs);
+ } else {
+ System.out.println(specifyTools());
+ }
+ }
+
+ private void handleStartDB(String[] args) throws SQLException {
+ List<String> argList = new ArrayList<>();
+ argList.add("-web");
+ Console.main(argList.toArray(new String[argList.size()]));
+ while(true) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e){
+ break;
+ }
+ }
+ }
+
+ private void handleProfile(String[] subsetArgs) throws Exception {
+ List<String> argList = new ArrayList(Arrays.asList(subsetArgs));
+
+ boolean containsBC = false;
+ String inputDir = null;
+ String extractDir = null;
+ String alterExtract = null;
+ //confirm there's a batch-config file
+ for (int i = 0; i < argList.size(); i++) {
+ String arg = argList.get(i);
+ if (arg.equals("-bc")) {
+ containsBC = true;
+ } else if (arg.equals("-inputDir")) {
+ if (i+1 >= argList.size()) {
+ System.err.println("Must specify directory after -inputDir");
+ ExtractProfiler.USAGE();
+ return;
+ }
+ inputDir = argList.get(i+1);
+ i++;
+ } else if (arg.equals("-extractDir")) {
+ if (i+1 >= argList.size()) {
+ System.err.println("Must specify directory after -extractDir");
+ ExtractProfiler.USAGE();
+ return;
+ }
+ extractDir = argList.get(i+1);
+ i++;
+ } else if (arg.equals("-alterExtract")) {
+ if (i+1 >= argList.size()) {
+ System.err.println("Must specify directory after -extractsB");
+ ExtractComparer.USAGE();
+ return;
+ }
+ alterExtract = argList.get(i+1);
+ i++;
+ }
+ }
+
+ if (alterExtract != null && !alterExtract.equals("as_is") &&
+ !alterExtract.equals("concatenate_content") &&
+ !alterExtract.equals("first_only")) {
+ System.out.println("Sorry, I don't understand:"+alterExtract+
+ ". The values must be one of: as_is, first_only, concatenate_content");
+ ExtractProfiler.USAGE();
+ return;
+ }
+
+ //need to specify each in this commandline
+ //if only extractDir is passed to tika-batch,
+ //the crawler will see no inputDir and start crawling "input".
+ //this allows the user to specify either extractDir or inputDir
+ if (extractDir == null && inputDir != null) {
+ argList.add("-extractDir");
+ argList.add(inputDir);
+ } else if (inputDir == null && extractDir != null) {
+ argList.add("-inputDir");
+ argList.add(extractDir);
+ }
+
+ Path tmpBCConfig = null;
+ try {
+ tmpBCConfig = Files.createTempFile("tika-eval-profiler", ".xml");
+ if (! containsBC) {
+ Files.copy(
+ this.getClass().getResourceAsStream("/tika-eval-profiler-config.xml"),
+ tmpBCConfig, StandardCopyOption.REPLACE_EXISTING);
+ argList.add("-bc");
+ argList.add(tmpBCConfig.toAbsolutePath().toString());
+ }
+
+ String[] updatedArgs = argList.toArray(new String[argList.size()]);
+ DefaultParser defaultCLIParser = new DefaultParser();
+ try {
+ defaultCLIParser.parse(ExtractProfiler.OPTIONS, updatedArgs);
+ } catch (ParseException e) {
+ System.out.println(e.getMessage()+"\n");
+ ExtractProfiler.USAGE();
+ return;
+ }
+
+ FSBatchProcessCLI.main(updatedArgs);
+ } finally {
+ if (tmpBCConfig != null && Files.isRegularFile(tmpBCConfig)) {
+ Files.delete(tmpBCConfig);
+ }
+ }
+ }
+
+ private void handleCompare(String[] subsetArgs) throws Exception{
+ List<String> argList = new ArrayList(Arrays.asList(subsetArgs));
+
+ boolean containsBC = false;
+ String inputDir = null;
+ String extractsA = null;
+ String alterExtract = null;
+ //confirm there's a batch-config file
+ for (int i = 0; i < argList.size(); i++) {
+ String arg = argList.get(i);
+ if (arg.equals("-bc")) {
+ containsBC = true;
+ } else if (arg.equals("-inputDir")) {
+ if (i+1 >= argList.size()) {
+ System.err.println("Must specify directory after -inputDir");
+ ExtractComparer.USAGE();
+ return;
+ }
+ inputDir = argList.get(i+1);
+ i++;
+ } else if (arg.equals("-extractsA")) {
+ if (i+1 >= argList.size()) {
+ System.err.println("Must specify directory after -extractsA");
+ ExtractComparer.USAGE();
+ return;
+ }
+ extractsA = argList.get(i+1);
+ i++;
+ } else if (arg.equals("-alterExtract")) {
+ if (i+1 >= argList.size()) {
+ System.err.println("Must specify directory after -extractsB");
+ ExtractComparer.USAGE();
+ return;
+ }
+ alterExtract = argList.get(i+1);
+ i++;
+ }
+ }
+ if (alterExtract != null && !alterExtract.equals("as_is") &&
+ !alterExtract.equals("concatenate_content") &&
+ !alterExtract.equals("first_only")) {
+ System.out.println("Sorry, I don't understand:"+alterExtract+
+ ". The values must be one of: as_is, first_only, concatenate_content");
+ ExtractComparer.USAGE();
+ return;
+ }
+
+ //need to specify each in the commandline that goes into tika-batch
+ //if only extractDir is passed to tika-batch,
+ //the crawler will see no inputDir and start crawling "input".
+ //if the user doesn't specify inputDir, crawl extractsA
+ if (inputDir == null && extractsA != null) {
+ argList.add("-inputDir");
+ argList.add(extractsA);
+ }
+
+ Path tmpBCConfig = null;
+ try {
+ tmpBCConfig = Files.createTempFile("tika-eval", ".xml");
+ if (! containsBC) {
+ Files.copy(
+ this.getClass().getResourceAsStream("/tika-eval-comparison-config.xml"),
+ tmpBCConfig, StandardCopyOption.REPLACE_EXISTING);
+ argList.add("-bc");
+ argList.add(tmpBCConfig.toAbsolutePath().toString());
+
+ }
+ String[] updatedArgs = argList.toArray(new String[argList.size()]);
+ DefaultParser defaultCLIParser = new DefaultParser();
+ try {
+ defaultCLIParser.parse(ExtractComparer.OPTIONS, updatedArgs);
+ } catch (ParseException e) {
+ System.out.println(e.getMessage()+"\n");
+ ExtractComparer.USAGE();
+ return;
+ }
+
+ FSBatchProcessCLI.main(updatedArgs);
+ } finally {
+ if (tmpBCConfig != null && Files.isRegularFile(tmpBCConfig)) {
+ Files.delete(tmpBCConfig);
+ }
+ }
+ }
+
+ private void handleReport(String[] subsetArgs) throws Exception {
+ ResultsReporter.main(subsetArgs);
+ }
+
+ public static void main(String[] args) throws Exception {
+ TikaEvalCLI cli = new TikaEvalCLI();
+ if (args.length == 0) {
+ System.err.println(specifyTools());
+ return;
+ }
+ cli.execute(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java b/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java
new file mode 100644
index 0000000..9a7e7aa
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/XMLErrorLogUpdater.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval;
+
+
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.apache.log4j.Level;
+import org.apache.tika.eval.db.Cols;
+import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.H2Util;
+import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.XMLLogMsgHandler;
+import org.apache.tika.eval.io.XMLLogReader;
+import org.apache.tika.eval.reports.ResultsReporter;
+import org.apache.tika.io.IOExceptionWithCause;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is a very task specific class that reads a log file and updates
+ * the "comparisons" table. It should not be run in a multithreaded environment.
+ */
+public class XMLErrorLogUpdater {
+
+ protected static Logger LOGGER = LoggerFactory.getLogger(ResultsReporter.class);
+
+ private Statement statement;
+
+ public static void main(String[] args) throws Exception {
+
+ XMLErrorLogUpdater writer = new XMLErrorLogUpdater();
+ Path xmlLogFileA = Paths.get(args[0]);
+ Path xmlLogFileB = Paths.get(args[1]);
+ Path db = Paths.get(args[2]);
+ DBUtil dbUtil = new H2Util(db);
+ Connection connection = dbUtil.getConnection(true);
+ writer.update(connection, ExtractComparer.ERROR_TABLE_A, xmlLogFileA);
+ writer.update(connection, ExtractComparer.ERROR_TABLE_B, xmlLogFileB);
+ connection.commit();
+ connection.close();
+ }
+
+ public void update(Connection connection, TableInfo tableInfo, Path xmlLogFile) throws Exception {
+ statement = connection.createStatement();
+ XMLLogReader reader = new XMLLogReader();
+ try (InputStream is = Files.newInputStream(xmlLogFile)) {
+ reader.read(is, new ErrorMsgUpdater(tableInfo.getName()));
+ } catch (IOException e) {
+ throw new RuntimeException("Problem reading: "+xmlLogFile.toAbsolutePath().toString());
+ } finally {
+ try {
+ connection.commit();
+ statement.close();
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to close db connection!", e);
+ }
+ }
+ }
+
+ private class ErrorMsgUpdater implements XMLLogMsgHandler {
+ private final String errorTablename;
+
+ private ErrorMsgUpdater(String errorTablename) {
+ this.errorTablename = errorTablename;
+ }
+
+ @Override
+ public void handleMsg(Level level, String xml) throws SQLException, IOException {
+ if (! level.equals(Level.ERROR)) {
+ return;
+ }
+ XMLStreamReader reader = null;
+ try {
+ reader = XMLInputFactory.newInstance().createXMLStreamReader(new StringReader(xml));
+ } catch (XMLStreamException e) {
+ throw new IOExceptionWithCause(e);
+ }
+ String type = null;
+ String resourceId = null;
+ try {
+ while (reader.hasNext() && type == null && resourceId == null) {
+ reader.next();
+ switch (reader.getEventType()) {
+ case XMLStreamConstants.START_ELEMENT:
+ if ("timed_out".equals(reader.getLocalName())) {
+ resourceId = reader.getAttributeValue("", "resourceId");
+ update(errorTablename, resourceId,
+ AbstractProfiler.PARSE_ERROR_TYPE.TIMEOUT);
+
+ } else if ("oom".equals(reader.getLocalName())) {
+ resourceId = reader.getAttributeValue("", "resourceId");
+ update(errorTablename, resourceId, AbstractProfiler.PARSE_ERROR_TYPE.OOM);
+ }
+ break;
+ }
+ }
+ reader.close();
+ } catch (XMLStreamException e) {
+ throw new IOExceptionWithCause(e);
+ }
+ }
+
+ private void update(String errorTableName,
+ String filePath, AbstractProfiler.PARSE_ERROR_TYPE type) throws SQLException {
+ int containerId = getContainerId(filePath);
+ String sql = "SELECT count(1) from "+errorTableName +
+ " where "+Cols.CONTAINER_ID +
+ " = "+containerId + " or "+
+ Cols.FILE_PATH + "='"+filePath+"'";
+ ResultSet rs = statement.executeQuery(sql);
+
+ //now try to figure out if that file already exists
+ //in parse errors
+ int hitCount = 0;
+ while (rs.next()) {
+ hitCount = rs.getInt(1);
+ }
+
+ //if it does, update all records matching that path or container id
+ if (hitCount > 0) {
+ sql = "UPDATE " + errorTableName +
+ " SET " + Cols.PARSE_ERROR_TYPE_ID +
+ " = " + type.ordinal() + ","+
+ Cols.FILE_PATH + "='" +filePath+"'"+
+ " where "+Cols.CONTAINER_ID +
+ "="+containerId + " or "+
+ Cols.FILE_PATH + "='"+filePath+"'";;
+
+ } else {
+ //if not and container id > -1
+ //insert full record
+ if (containerId > -1) {
+ sql = "INSERT INTO " + errorTableName +
+ " ("+Cols.CONTAINER_ID+","+Cols.FILE_PATH +","+Cols.PARSE_ERROR_TYPE_ID+")"+
+ " values (" + containerId + ", '" + filePath + "'," +
+ type.ordinal() + ");";
+ } else {
+ //if container id == -1, insert only file path and parse error type id
+ sql = "INSERT INTO " + errorTableName +
+ " ("+Cols.FILE_PATH.name()+","+Cols.PARSE_ERROR_TYPE_ID+")"+
+ "values ('" + filePath + "'," +
+ type.ordinal() + ");";
+ }
+
+ }
+ int updated = statement.executeUpdate(sql);
+ if (updated == 0) {
+ //TODO: log
+ LOGGER.warn("made no updates in xmlerrorlogupdater!");
+ } else if (updated > 1) {
+ LOGGER.warn("made too many updates");
+ }
+ }
+
+ private int getContainerId(String resourceId) throws SQLException {
+ int containerId = -1;
+ String sql = "SELECT " + Cols.CONTAINER_ID.name() +
+ " from " + ExtractProfiler.CONTAINER_TABLE.getName()+
+ " where " + Cols.FILE_PATH +
+ " ='"+resourceId+"'";
+ ResultSet rs = statement.executeQuery(sql);
+ int resultCount = 0;
+ while (rs.next()) {
+ containerId = rs.getInt(1);
+ resultCount++;
+ }
+ rs.close();
+
+ if (resultCount == 0) {
+ LOGGER.warn("Should have found a container for: "+resourceId);
+ } else if (resultCount > 1) {
+ LOGGER.error("Records ids should be unique:"+resourceId);
+ }
+/*
+ if (containerId < 0) {
+ System.err.println("CONTAINER ID < 0!!!");
+ sql = "SELECT MAX("+ Cols.CONTAINER_ID.name() +
+ ") from "+ExtractProfiler.CONTAINER_TABLE.getName();
+ rs = statement.executeQuery(sql);
+ while (rs.next()) {
+ containerId = rs.getInt(1);
+ }
+ rs.close();
+ if (containerId < 0) {
+ //log and abort
+ //return -1?
+ } else {
+ containerId++;
+ }
+
+ }*/
+ return containerId;
+ }
+
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java
new file mode 100644
index 0000000..2c655cc
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/DBConsumersManager.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.batch;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.tika.batch.ConsumersManager;
+import org.apache.tika.batch.FileResourceConsumer;
+import org.apache.tika.eval.AbstractProfiler;
+import org.apache.tika.eval.XMLErrorLogUpdater;
+import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.TableInfo;
+
+
+public class DBConsumersManager extends ConsumersManager {
+
+ private Connection conn;
+ List<LogTablePair> errorLogs = new ArrayList<>();
+
+ public DBConsumersManager(DBUtil dbUtil, List<FileResourceConsumer> consumers)
+ throws IOException {
+ super(consumers);
+ this.conn = dbUtil.getConnection(true);
+ }
+
+
+ @Override
+ public void shutdown() {
+
+ for (FileResourceConsumer consumer : getConsumers()) {
+ if (consumer instanceof AbstractProfiler) {
+ try{
+ ((AbstractProfiler)consumer).closeWriter();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ //MUST HAPPEN AFTER consumers have closed and
+ //committed container information!!!
+ XMLErrorLogUpdater up = new XMLErrorLogUpdater();
+ for (LogTablePair p : errorLogs) {
+ try {
+ up.update(conn, p.tableInfo, p.log);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ try {
+ conn.commit();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void addErrorLogTablePair(Path log, TableInfo tableName) {
+ LogTablePair p = new LogTablePair();
+ p.log = log;
+ p.tableInfo = tableName;
+ errorLogs.add(p);
+ }
+
+ class LogTablePair {
+ Path log;
+ TableInfo tableInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
new file mode 100644
index 0000000..8135887
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumerBuilder.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.batch;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.batch.FileResourceConsumer;
+import org.apache.tika.eval.AbstractProfiler;
+import org.apache.tika.eval.db.Cols;
+import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.ExtractReader;
+import org.apache.tika.eval.io.IDBWriter;
+
+public abstract class EvalConsumerBuilder {
+ private AtomicInteger count = new AtomicInteger(0);
+ protected ArrayBlockingQueue<FileResource> queue;
+ Map<String, String> localAttrs;
+ DBUtil dbUtil;
+
+ public void init(ArrayBlockingQueue<FileResource> queue, Map<String, String> localAttrs,
+ DBUtil dbUtil) {
+ this.queue = queue;
+ this.localAttrs = localAttrs;
+ this.dbUtil = dbUtil;
+ }
+
+ public abstract FileResourceConsumer build() throws IOException, SQLException;
+
+ protected abstract List<TableInfo> getTableInfo();
+
+ protected abstract IDBWriter getDBWriter() throws IOException, SQLException;
+
+ protected abstract void addErrorLogTablePairs(DBConsumersManager manager);
+
+ public void populateRefTables(IDBWriter writer) throws IOException, SQLException {
+ //figure out cleaner way of doing this!
+ if (count.getAndIncrement() > 0) {
+ return;
+ }
+ Map<Cols, String> m = new HashMap<>();
+ for (AbstractProfiler.PARSE_ERROR_TYPE t : AbstractProfiler.PARSE_ERROR_TYPE.values()) {
+ m.clear();
+ m.put(Cols.PARSE_ERROR_TYPE_ID, Integer.toString(t.ordinal()));
+ m.put(Cols.PARSE_ERROR_DESCRIPTION, t.name());
+ writer.writeRow(AbstractProfiler.REF_PARSE_ERROR_TYPES, m);
+ }
+
+ for (AbstractProfiler.EXCEPTION_TYPE t : AbstractProfiler.EXCEPTION_TYPE.values()) {
+ m.clear();
+ m.put(Cols.PARSE_EXCEPTION_TYPE_ID, Integer.toString(t.ordinal()));
+ m.put(Cols.PARSE_EXCEPTION_DESCRIPTION, t.name());
+ writer.writeRow(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES, m);
+ }
+
+ for (AbstractProfiler.EXTRACT_ERROR_TYPE t :
+ AbstractProfiler.EXTRACT_ERROR_TYPE.values()) {
+ m.clear();
+ m.put(Cols.EXTRACT_ERROR_TYPE_ID, Integer.toString(t.ordinal()));
+ m.put(Cols.EXTRACT_ERROR_DESCRIPTION, t.name());
+ writer.writeRow(AbstractProfiler.REF_EXTRACT_ERROR_TYPES, m);
+ }
+
+ }
+
+ ExtractReader.ALTER_METADATA_LIST getAlterMetadata(Map<String, String> localAttrs) {
+
+ String alterExtractString = localAttrs.get("alterExtract");
+ ExtractReader.ALTER_METADATA_LIST alterExtractList = ExtractReader.ALTER_METADATA_LIST.AS_IS;
+ if (alterExtractString == null || alterExtractString.equalsIgnoreCase("as_is")) {
+ alterExtractList = ExtractReader.ALTER_METADATA_LIST.AS_IS;
+ } else if (alterExtractString.equalsIgnoreCase("first_only")) {
+ alterExtractList = ExtractReader.ALTER_METADATA_LIST.FIRST_ONLY;
+ } else if (alterExtractString.equalsIgnoreCase("concatenate_content")) {
+ alterExtractList = ExtractReader.ALTER_METADATA_LIST.CONCATENATE_CONTENT_INTO_FIRST;
+ } else {
+ throw new RuntimeException("options for alterExtract: as_is, first_only, concatenate_content." +
+ " I don't understand:" + alterExtractString);
+ }
+ return alterExtractList;
+ }
+
+
+/*
+ public abstract Map<String, String> getIndexInfo();
+
+ class ValueComparator implements Comparator<String> {
+
+ Map<String, ColInfo> map;
+
+ public ValueComparator(Map<String, ColInfo> base) {
+ this.map = base;
+ }
+
+ public int compare(String a, String b) {
+ Integer aVal = map.get(a).getDBColOffset();
+ Integer bVal = map.get(b).getDBColOffset();
+ if (aVal == null || bVal == null) {
+ throw new IllegalArgumentException("Column offset must be specified!");
+ }
+ if (aVal == bVal && ! map.get(a).equals(map.get(b))) {
+ throw new IllegalArgumentException("Column offsets must be unique: " + a + " and " + b + " both have: "+aVal);
+ }
+ if (aVal < bVal) {
+ return -1;
+ } else {
+ return 1;
+ }
+ }
+ }
+*/
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
new file mode 100644
index 0000000..00f4ad7
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.batch;
+
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.SQLException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+
+import org.apache.tika.batch.ConsumersManager;
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.batch.FileResourceConsumer;
+import org.apache.tika.batch.builders.AbstractConsumersBuilder;
+import org.apache.tika.batch.builders.BatchProcessBuilder;
+import org.apache.tika.eval.AbstractProfiler;
+import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.H2Util;
+import org.apache.tika.eval.util.LanguageIDWrapper;
+import org.apache.tika.util.ClassLoaderUtil;
+import org.apache.tika.util.PropsUtil;
+import org.apache.tika.util.XMLDOMUtil;
+import org.w3c.dom.Node;
+
+public class EvalConsumersBuilder extends AbstractConsumersBuilder {
+
+ @Override
+ public ConsumersManager build(Node node, Map<String, String> runtimeAttributes,
+ ArrayBlockingQueue<FileResource> queue) {
+
+ List<FileResourceConsumer> consumers = new LinkedList<>();
+ int numConsumers = BatchProcessBuilder.getNumConsumers(runtimeAttributes);
+
+ Map<String, String> localAttrs = XMLDOMUtil.mapifyAttrs(node, runtimeAttributes);
+
+
+ Path db = getPath(localAttrs, "db");
+ Path langModelDir = getPath(localAttrs, "langModelDir");
+
+ try {
+ if (langModelDir == null) {
+ LanguageIDWrapper.loadBuiltInModels();
+ } else {
+ LanguageIDWrapper.loadModels(langModelDir);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ Path commonTokens = getNonNullPath(localAttrs, "commonTokens");
+ try {
+ AbstractProfiler.loadCommonTokens(commonTokens);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ boolean append = PropsUtil.getBoolean(localAttrs.get("dbAppend"), false);
+
+ if (db == null) {
+ throw new RuntimeException("Must specify: -db");
+ }
+ //parameterize which db util to use
+ DBUtil util = new H2Util(db);
+ EvalConsumerBuilder consumerBuilder = ClassLoaderUtil.buildClass(EvalConsumerBuilder.class,
+ PropsUtil.getString(localAttrs.get("consumerBuilderClass"), null));
+ if (consumerBuilder == null) {
+ throw new RuntimeException("Must specify consumerBuilderClass in config file");
+ }
+ consumerBuilder.init(queue, localAttrs, util);
+
+ try {
+ util.createDB(consumerBuilder.getTableInfo(), append);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ for (int i = 0; i < numConsumers; i++) {
+ try {
+ consumers.add(consumerBuilder.build());
+ } catch (IOException | SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ DBConsumersManager manager;
+ try {
+ manager = new DBConsumersManager(util, consumers);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ consumerBuilder.addErrorLogTablePairs(manager);
+
+ return manager;
+ }
+
+ private Path getNonNullPath(Map<String, String> attrs, String key) {
+ Path p = getPath(attrs, key);
+ if (p == null) {
+ throw new RuntimeException("Must specify a file for this attribute: "+key);
+ }
+ return p;
+ }
+
+
+ protected Path getPath(Map<String, String> attrs, String key) {
+ String filePath = attrs.get(key);
+ if (filePath == null) {
+ return null;
+ }
+ return Paths.get(filePath);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java
new file mode 100644
index 0000000..cface16
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/FileComparerBuilder.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.batch;
+
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.tika.batch.FileResourceConsumer;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.eval.AbstractProfiler;
+import org.apache.tika.eval.ExtractComparer;
+import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.DBWriter;
+import org.apache.tika.eval.io.ExtractReader;
+import org.apache.tika.eval.io.IDBWriter;
+import org.apache.tika.util.PropsUtil;
+
+public class FileComparerBuilder extends EvalConsumerBuilder {
+ private final static String WHICH_DB = "h2";//TODO: allow flexibility
+
+
+ @Override
+ public FileResourceConsumer build() throws IOException, SQLException {
+ Path thisRootDir = PropsUtil.getPath(localAttrs.get("extractsA"), null);
+ if (thisRootDir == null) {
+ throw new RuntimeException("Must specify \"extractsA\" -- directory for 'A' extracts");
+ }
+ Path thatRootDir = PropsUtil.getPath(localAttrs.get("extractsB"), null);
+ if (thatRootDir == null) {
+ throw new RuntimeException("Must specify \"extractsB\" -- directory for 'B' extracts");
+ }
+
+ Path inputRootDir = PropsUtil.getPath(localAttrs.get("inputDir"), null);
+
+ long minJsonLength = PropsUtil.getLong(localAttrs.get("minJsonFileSizeBytes"), -1L);
+ long maxJsonLength = PropsUtil.getLong(localAttrs.get("maxJsonFileSizeBytes"), -1L);
+
+ ExtractReader.ALTER_METADATA_LIST alterExtractList = getAlterMetadata(localAttrs);
+
+
+ IDBWriter writer = getDBWriter();
+ //TODO: clean up the writing of the ref tables!!!
+ try {
+ populateRefTables(writer);
+ } catch (SQLException e) {
+ throw new RuntimeException("Can't populate ref tables", e);
+ }
+
+ if (inputRootDir == null) {
+ //this is for the sake of the crawler
+ throw new RuntimeException("Must specify an -inputDir");
+ }
+
+ return new ExtractComparer(queue, inputRootDir, thisRootDir, thatRootDir, writer,
+ minJsonLength, maxJsonLength, alterExtractList);
+ }
+
+ @Override
+ protected List<TableInfo> getTableInfo() {
+ List<TableInfo> tableInfos = new ArrayList<>();
+ tableInfos.add(ExtractComparer.COMPARISON_CONTAINERS);
+ tableInfos.add(ExtractComparer.PROFILES_A);
+ tableInfos.add(ExtractComparer.PROFILES_B);
+ tableInfos.add(ExtractComparer.ERROR_TABLE_A);
+ tableInfos.add(ExtractComparer.ERROR_TABLE_B);
+ tableInfos.add(ExtractComparer.EXCEPTION_TABLE_A);
+ tableInfos.add(ExtractComparer.EXCEPTION_TABLE_B);
+ tableInfos.add(ExtractComparer.ERROR_TABLE_A);
+ tableInfos.add(ExtractComparer.ERROR_TABLE_B);
+ tableInfos.add(ExtractComparer.CONTENTS_TABLE_A);
+ tableInfos.add(ExtractComparer.CONTENTS_TABLE_B);
+ tableInfos.add(ExtractComparer.EMBEDDED_FILE_PATH_TABLE_A);
+ tableInfos.add(ExtractComparer.EMBEDDED_FILE_PATH_TABLE_B);
+
+ tableInfos.add(ExtractComparer.CONTENT_COMPARISONS);
+ tableInfos.add(AbstractProfiler.MIME_TABLE);
+ tableInfos.add(ExtractComparer.REF_PAIR_NAMES);
+ tableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
+ tableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
+ tableInfos.add(AbstractProfiler.REF_EXTRACT_ERROR_TYPES);
+ return tableInfos;
+ }
+
+ @Override
+ protected IDBWriter getDBWriter() throws IOException, SQLException {
+ return new DBWriter(getTableInfo(), TikaConfig.getDefaultConfig(), dbUtil);
+ }
+
+ @Override
+ protected void addErrorLogTablePairs(DBConsumersManager manager) {
+ Path errorLogA = PropsUtil.getPath(localAttrs.get("errorLogFileA"), null);
+ if (errorLogA == null) {
+ return;
+ }
+ manager.addErrorLogTablePair(errorLogA, ExtractComparer.ERROR_TABLE_A);
+ Path errorLogB = PropsUtil.getPath(localAttrs.get("errorLogFileB"), null);
+ if (errorLogB == null) {
+ return;
+ }
+ manager.addErrorLogTablePair(errorLogB, ExtractComparer.ERROR_TABLE_B);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java b/tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java
new file mode 100644
index 0000000..de8be64
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/batch/SingleFileConsumerBuilder.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.batch;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.tika.batch.FileResourceConsumer;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.eval.AbstractProfiler;
+import org.apache.tika.eval.ExtractProfiler;
+import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.eval.io.DBWriter;
+import org.apache.tika.eval.io.ExtractReader;
+import org.apache.tika.eval.io.IDBWriter;
+import org.apache.tika.util.PropsUtil;
+
+
+public class SingleFileConsumerBuilder extends EvalConsumerBuilder {
+
+ @Override
+ public FileResourceConsumer build() throws IOException {
+ Path extractDir = PropsUtil.getPath(localAttrs.get("extractDir"), null);
+ if (extractDir == null) {
+ throw new RuntimeException("Must specify \"extractDir\" -- directory to crawl");
+ }
+ if (!Files.isDirectory(extractDir)) {
+ throw new RuntimeException("ROOT DIRECTORY DOES NOT EXIST: " +
+ extractDir.toAbsolutePath());
+ }
+
+ Path inputDir = PropsUtil.getPath(localAttrs.get("inputDir"), null);
+
+ ExtractReader.ALTER_METADATA_LIST alterExtractList = getAlterMetadata(localAttrs);
+
+ IDBWriter writer = null;
+ try {
+ writer = getDBWriter();
+ } catch (SQLException ex) {
+ throw new IOException(ex);
+ }
+
+ //TODO: clean up the writing of the ref tables!!!
+ try {
+ populateRefTables(writer);
+ } catch (SQLException e) {
+ throw new RuntimeException("Can't populate ref tables", e);
+ }
+ //we _could_ set this to extractDir (if not null)
+ //here, but the Crawler defaults to "input" if nothing is passed
+ //so this won't work
+ if (inputDir == null) {
+ throw new RuntimeException("Must specify -inputDir");
+ }
+ if (extractDir == null && inputDir != null) {
+ extractDir = inputDir;
+ }
+ return new ExtractProfiler(queue, inputDir, extractDir, writer, alterExtractList);
+ }
+
+ @Override
+ protected List<TableInfo> getTableInfo() {
+ List<TableInfo> tableInfos = new ArrayList<TableInfo>();
+ tableInfos.add(AbstractProfiler.MIME_TABLE);
+ tableInfos.add(AbstractProfiler.REF_PARSE_ERROR_TYPES);
+ tableInfos.add(AbstractProfiler.REF_PARSE_EXCEPTION_TYPES);
+ tableInfos.add(AbstractProfiler.REF_EXTRACT_ERROR_TYPES);
+ tableInfos.add(ExtractProfiler.CONTAINER_TABLE);
+ tableInfos.add(ExtractProfiler.PROFILE_TABLE);
+ tableInfos.add(ExtractProfiler.ERROR_TABLE);
+ tableInfos.add(ExtractProfiler.EXCEPTION_TABLE);
+ tableInfos.add(ExtractProfiler.CONTENTS_TABLE);
+ tableInfos.add(ExtractProfiler.EMBEDDED_FILE_PATH_TABLE);
+ return tableInfos;
+ }
+
+ @Override
+ protected IDBWriter getDBWriter() throws IOException, SQLException {
+ return new DBWriter(getTableInfo(), TikaConfig.getDefaultConfig(), dbUtil);
+ }
+
+ @Override
+ protected void addErrorLogTablePairs(DBConsumersManager manager) {
+ Path errorLog = PropsUtil.getPath(localAttrs.get("errorLogFile"), null);
+ if (errorLog == null) {
+ return;
+ }
+ manager.addErrorLogTablePair(errorLog, ExtractProfiler.ERROR_TABLE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/db/AbstractDBBuffer.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/AbstractDBBuffer.java b/tika-eval/src/main/java/org/apache/tika/eval/db/AbstractDBBuffer.java
new file mode 100644
index 0000000..baa7994
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/AbstractDBBuffer.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.db;
+
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+
+/**
+ * Abstract buffer for map of values and unique ids.
+ * <p>
+ * Use this for fast in memory lookups of smallish sets of values.
+ *
+ */
+abstract class AbstractDBBuffer {
+
+ private final Map<String, Integer> m = new HashMap<>();
+ private final ReentrantReadWriteLock rwl = new ReentrantReadWriteLock();
+ private final Lock r = rwl.readLock();
+ private final Lock w = rwl.writeLock();
+
+ private int numWrites = 0;
+
+ public int getId(String key) {
+ r.lock();
+ try {
+ Integer v = m.get(key);
+ if (v != null) {
+ return v;
+ }
+ } finally {
+ r.unlock();
+ }
+
+ try {
+ w.lock();
+ Integer v = m.get(key);
+ if (v != null) {
+ return v;
+ }
+ v = m.size()+1;
+ m.put(key, v);
+ write(v, key);
+ numWrites++;
+ return v;
+ } finally {
+ w.unlock();
+ }
+ }
+
+ public int getNumWrites() {
+ return numWrites;
+ }
+
+ //Odd to throw RuntimeException, I know. It should be
+ //catastrophic if this buffer can't write to the db.
+ public abstract void write(int id, String value) throws RuntimeException;
+
+ public abstract void close() throws SQLException;
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/db/ColInfo.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/ColInfo.java b/tika-eval/src/main/java/org/apache/tika/eval/db/ColInfo.java
new file mode 100644
index 0000000..a32f874
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/ColInfo.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.db;
+
+
+import java.sql.Types;
+
+public class ColInfo {
+ private final Cols name;
+ private final int type;
+ private final Integer precision;
+ private final String constraints;
+
+ public ColInfo(Cols name, int type) {
+ this(name, type, null, null);
+ }
+
+ public ColInfo(Cols name, int type, String constraints) {
+ this(name, type, null, constraints);
+ }
+
+ public ColInfo(Cols name, int type, Integer precision) {
+ this(name, type, precision, null);
+ }
+
+
+ public ColInfo(Cols name, int type, Integer precision, String constraints) {
+ this.name = name;
+ this.type = type;
+ this.precision = precision;
+ this.constraints = constraints;
+ }
+
+ public int getType() {
+ return type;
+ }
+
+ public Cols getName() {
+ return name;
+ }
+ /**
+ *
+ * @return constraints string or null
+ */
+ public String getConstraints() {
+ return constraints;
+ }
+
+ /**
+ * Gets the precision. This can be null!
+ * @return precision or null
+ */
+ public Integer getPrecision() {
+ return precision;
+ }
+
+ public String getSqlDef() {
+ if (type == Types.VARCHAR){
+ return "VARCHAR("+precision+")";
+ } else if (type == Types.CHAR) {
+ return "CHAR("+precision+")";
+ }
+ switch (type) {
+ case Types.FLOAT :
+ return "FLOAT";
+ case Types.DOUBLE :
+ return "DOUBLE";
+ case Types.BLOB :
+ return "BLOB";
+ case Types.INTEGER :
+ return "INTEGER";
+ case Types.BIGINT :
+ return "BIGINT";
+ case Types.BOOLEAN :
+ return "BOOLEAN";
+ }
+ throw new UnsupportedOperationException("Don't yet recognize a type for: "+type);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ColInfo colInfo = (ColInfo) o;
+
+ if (type != colInfo.type) return false;
+ if (name != colInfo.name) return false;
+ if (precision != null ? !precision.equals(colInfo.precision) : colInfo.precision != null) return false;
+ return !(constraints != null ? !constraints.equals(colInfo.constraints) : colInfo.constraints != null);
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name != null ? name.hashCode() : 0;
+ result = 31 * result + type;
+ result = 31 * result + (precision != null ? precision.hashCode() : 0);
+ result = 31 * result + (constraints != null ? constraints.hashCode() : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/db/Cols.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/Cols.java b/tika-eval/src/main/java/org/apache/tika/eval/db/Cols.java
new file mode 100644
index 0000000..0dff9f7
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/Cols.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.db;
+
+public enum Cols {
+ //container table
+ CONTAINER_ID,
+ FILE_PATH,
+ EXTRACT_FILE_LENGTH,
+
+ EXTRACT_FILE_LENGTH_A, //for comparisons
+ EXTRACT_FILE_LENGTH_B,
+
+ //profile table
+ ID,
+ LENGTH,
+ FILE_NAME,
+ FILE_EXTENSION,
+ ELAPSED_TIME_MILLIS,
+ NUM_METADATA_VALUES,
+ IS_EMBEDDED,
+ EMBEDDED_FILE_PATH,
+ MIME_TYPE_ID,
+ MD5,
+ NUM_ATTACHMENTS,
+ HAS_CONTENT,
+
+ //content
+ CONTENT_LENGTH,
+ NUM_UNIQUE_TOKENS,
+ NUM_TOKENS,
+ NUM_ALPHABETIC_TOKENS, //alphabetic or ideographic tokens
+ COMMON_TOKENS_LANG, //which language was used for the common tokens metric?
+ NUM_COMMON_TOKENS,
+ TOP_N_TOKENS,
+ LANG_ID_1,
+ LANG_ID_PROB_1,
+ LANG_ID_2,
+ LANG_ID_PROB_2,
+ TOKEN_ENTROPY_RATE,
+ TOKEN_LENGTH_SUM,
+ TOKEN_LENGTH_MEAN,
+ TOKEN_LENGTH_STD_DEV,
+ UNICODE_CHAR_BLOCKS,
+ NUM_PAGES, //number of pages a document alleges it has
+
+ //content comparisons
+ TOP_10_UNIQUE_TOKEN_DIFFS_A,
+ TOP_10_UNIQUE_TOKEN_DIFFS_B,
+ TOP_10_MORE_IN_A,
+ TOP_10_MORE_IN_B,
+ OVERLAP,
+ DICE_COEFFICIENT,
+
+ //errors
+ PARSE_ERROR_TYPE_ID,
+
+ PARSE_ERROR_DESCRIPTION,
+ PARSE_EXCEPTION_DESCRIPTION,
+
+ EXTRACT_ERROR_TYPE_ID,
+ EXTRACT_ERROR_DESCRIPTION,
+
+
+ //exceptions
+ ORIG_STACK_TRACE,
+ SORT_STACK_TRACE,
+ PARSE_EXCEPTION_TYPE_ID,
+
+
+ MIME_STRING,//string representation of mime type
+
+ DIR_NAME_A,//for comparisons in REF_PAIR_NAMES
+ DIR_NAME_B
+ }
+
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/db/DBBuffer.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/DBBuffer.java b/tika-eval/src/main/java/org/apache/tika/eval/db/DBBuffer.java
new file mode 100644
index 0000000..8d936c2
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/DBBuffer.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.db;
+
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+public class DBBuffer extends AbstractDBBuffer {
+
+ private final PreparedStatement st;
+
+ public DBBuffer(Connection connection, String tableName,
+ String idColumnName, String valueColumnName) throws SQLException {
+ st = connection.prepareStatement("insert into "+tableName+ "( "+
+ idColumnName + ", " + valueColumnName+") values (?,?);");
+ }
+
+ @Override
+ public void write(int id, String value) throws RuntimeException {
+ try {
+ st.clearParameters();
+ st.setInt(1, id);
+ st.setString(2, value);
+ st.execute();
+
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void close() throws SQLException {
+ st.close();
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java b/tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java
new file mode 100644
index 0000000..1efa48a
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/DBUtil.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.eval.db;
+
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Types;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.log4j.Logger;
+import org.apache.tika.io.IOExceptionWithCause;
+
+public abstract class DBUtil {
+
+ public static Logger logger = Logger.getLogger(DBUtil.class);
+ public abstract String getJDBCDriverClass();
+ public abstract boolean dropTableIfExists(Connection conn, String tableName) throws SQLException;
+ private final Path db;
+ public DBUtil(Path db) {
+ this.db = db;
+ }
+
+ /**
+ * This is intended for a file/directory based db.
+ * <p>
+ * Override this any optimizations you want to do on the db
+ * before writing/reading.
+ *
+ * @return
+ * @throws IOException
+ */
+ public Connection getConnection(boolean createIfDoesntExist) throws IOException {
+ String connectionString = getConnectionString(db, createIfDoesntExist);
+ Connection conn = null;
+ try {
+ try {
+ Class.forName(getJDBCDriverClass());
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ conn = DriverManager.getConnection(connectionString);
+ conn.setAutoCommit(false);
+ } catch (SQLException e) {
+ throw new IOExceptionWithCause(e);
+ }
+ return conn;
+ }
+
+ abstract public String getConnectionString(Path db, boolean createIfDoesntExist);
+
+ /**
+ *
+ * @param connection
+ * @return a list of uppercased table names
+ * @throws SQLException
+ */
+ abstract public Set<String> getTables(Connection connection) throws SQLException;
+
+ public static int insert(PreparedStatement insertStatement,
+ TableInfo table,
+ Map<Cols, String> data) throws SQLException {
+
+ //clear parameters before setting
+ insertStatement.clearParameters();
+ try {
+ int i = 1;
+ for (ColInfo colInfo : table.getColInfos()) {
+ updateInsertStatement(i, insertStatement, colInfo, data.get(colInfo.getName()));
+ i++;
+ }
+ for (Cols c : data.keySet()) {
+ if (! table.containsColumn(c)) {
+ throw new IllegalArgumentException("Can't add data to "+c +
+ " because it doesn't exist in the table: "+table.getName());
+ }
+ }
+ return insertStatement.executeUpdate();
+ } catch (SQLException e) {
+ logger.warn("couldn't insert data for this row: "+e.getMessage());
+ return -1;
+ }
+ }
+
+ public static void updateInsertStatement(int dbColOffset, PreparedStatement st,
+ ColInfo colInfo, String value ) throws SQLException {
+ if (value == null) {
+ st.setNull(dbColOffset, colInfo.getType());
+ return;
+ }
+ try {
+ switch (colInfo.getType()) {
+ case Types.VARCHAR:
+ if (value != null && value.length() > colInfo.getPrecision()) {
+ value = value.substring(0, colInfo.getPrecision());
+ logger.warn("truncated varchar value in " + colInfo.getName() + " : "+value);
+ }
+ st.setString(dbColOffset, value);
+ break;
+ case Types.CHAR:
+ st.setString(dbColOffset, value);
+ break;
+ case Types.DOUBLE:
+ st.setDouble(dbColOffset, Double.parseDouble(value));
+ break;
+ case Types.FLOAT:
+ st.setDouble(dbColOffset, Float.parseFloat(value));
+ break;
+ case Types.INTEGER:
+ st.setDouble(dbColOffset, Integer.parseInt(value));
+ break;
+ case Types.BIGINT:
+ st.setLong(dbColOffset, Long.parseLong(value));
+ break;
+ case Types.BOOLEAN:
+ st.setBoolean(dbColOffset, Boolean.parseBoolean(value));
+ break;
+ default:
+ throw new UnsupportedOperationException("Don't yet support type: " + colInfo.getType());
+ }
+ } catch (NumberFormatException e) {
+ if (! "".equals(value)) {
+ logger.warn("number format exception: " + colInfo.getName() + " : " + value);
+ }
+ st.setNull(dbColOffset, colInfo.getType());
+ } catch (SQLException e) {
+ logger.warn("sqlexception: "+colInfo+ " : " + value);
+ st.setNull(dbColOffset, colInfo.getType());
+ }
+ }
+
+ public void createDB(List<TableInfo> tableInfos, boolean append) throws SQLException, IOException {
+ Connection conn = getConnection(true);
+ Set<String> tables = getTables(conn);
+
+ for (TableInfo tableInfo : tableInfos) {
+
+ if (append && tables.contains(tableInfo.getName().toUpperCase(Locale.ROOT))) {
+ continue;
+ }
+ if (! append) {
+ dropTableIfExists(conn, tableInfo.getName());
+ }
+ createTable(conn, tableInfo);
+ }
+
+ conn.commit();
+ conn.close();
+ }
+
+ private void createTable(Connection conn, TableInfo tableInfo) throws SQLException {
+ StringBuilder createSql = new StringBuilder();
+ createSql.append("CREATE TABLE "+tableInfo.getName());
+ createSql.append("(");
+
+ int last = 0;
+ for (ColInfo col : tableInfo.getColInfos()) {
+ last++;
+ if (last > 1) {
+ createSql.append(", ");
+ }
+ createSql.append(col.getName());
+ createSql.append(" ");
+ createSql.append(col.getSqlDef());
+ String constraints = col.getConstraints();
+ if (constraints != null) {
+ createSql.append(" ");
+ createSql.append(constraints);
+ }
+ }
+ createSql.append(")");
+ Statement st = conn.createStatement();
+ st.execute(createSql.toString());
+
+ st.close();
+ conn.commit();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/db/H2Util.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/H2Util.java b/tika-eval/src/main/java/org/apache/tika/eval/db/H2Util.java
new file mode 100644
index 0000000..0b3ebac
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/H2Util.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.db;
+
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.io.FilenameUtils;
+
+
+public class H2Util extends DBUtil {
+
+ public H2Util(Path db) {
+ super(db);
+ }
+
+ @Override
+ public String getJDBCDriverClass() {
+ return "org.h2.Driver";
+ }
+
+ @Override
+ public boolean dropTableIfExists(Connection conn, String tableName) throws SQLException {
+ Statement st = conn.createStatement();
+ String sql = "drop table if exists "+tableName;
+ boolean success = st.execute(sql);
+ st.close();
+ return success;
+ }
+
+ @Override
+ public String getConnectionString(Path db, boolean createIfDoesntExist) {
+ String s = "jdbc:h2:"+ FilenameUtils.separatorsToUnix(db.toAbsolutePath().toString());
+ if (! createIfDoesntExist) {
+ s += ";IFEXISTS=TRUE";
+ }
+ return s;
+ }
+
+ @Override
+ public Set<String> getTables(Connection connection) throws SQLException {
+ String sql = "SHOW TABLES";
+ Statement st = connection.createStatement();
+ ResultSet rs = st.executeQuery(sql);
+ Set<String> tables = new HashSet<>();
+ while (rs.next()) {
+ String table = rs.getString(1);
+ tables.add(table);
+ }
+ return tables;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java b/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java
new file mode 100644
index 0000000..11a72f8
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/MimeBuffer.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.db;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Types;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.eval.AbstractProfiler;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.mime.MimeType;
+import org.apache.tika.mime.MimeTypeException;
+import org.apache.tika.mime.MimeTypes;
+
+
+public class MimeBuffer extends AbstractDBBuffer {
+
+ private final PreparedStatement st;
+ private final TikaConfig config;
+
+ public MimeBuffer(Connection connection, TikaConfig config) throws SQLException {
+ st = connection.prepareStatement("insert into " + AbstractProfiler.MIME_TABLE.getName() + "( " +
+ Cols.MIME_TYPE_ID.name() + ", " +
+ Cols.MIME_STRING + ", " +
+ Cols.FILE_EXTENSION + ") values (?,?,?);");
+ this.config = config;
+ }
+
+ @Override
+ public void write(int id, String value) throws RuntimeException {
+ try {
+ st.clearParameters();
+ st.setInt(1, id);
+ st.setString(2, value);
+ try {
+ String ext = MimeUtil.getExtension(value, config);
+ if (ext == null || ext.length() == 0) {
+ st.setNull(3, Types.VARCHAR);
+ } else {
+ st.setString(3, ext);
+ }
+ } catch (MimeTypeException e) {
+ st.setNull(3, Types.VARCHAR);
+ }
+ st.execute();
+
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void close() throws SQLException {
+ st.close();
+ }
+
+ private static class MimeUtil {
+ //TODO: see if MimeType now works for these
+ private static final String APPLICATION = "application";
+ private static final String TEXT = "text";
+ private static final String HTML = "html";
+ private static final String XML = "xml";
+ private static final String XHTML_XML = "xhtml+xml";
+ private static final String CSS = "css";
+ private static final String CSV = "csv";
+ private static final String PLAIN = "plain";
+ private static final String EMPTY_STRING = "";
+
+ /**
+ * Utility method to convert from a string value representing a content type
+ * (e.g. "application/pdf") into the most common extension for that file type
+ * (e.g. "pdf").
+ * <p>
+ * This will has special handling for texty filetypes whose MimeTypes
+ * don't currently return anything for {@link org.apache.tika.mime.MimeType#getExtension};
+ *
+ * @param contentType string representing a content type, for example: "application/pdf"
+ * @param config config from which to get MimeRepository
+ * @return extension or empty string
+ * @throws org.apache.tika.mime.MimeTypeException thrown if MimeTypes can't parse the contentType
+ */
+ public static String getExtension(String contentType, TikaConfig config)
+ throws MimeTypeException {
+ MimeTypes types = config.getMimeRepository();
+ MimeType mime = types.forName(contentType);
+ return getExtension(mime);
+ }
+
+ public static String getExtension(MimeType mime) {
+
+ String ext = mime.getExtension();
+ if (ext.startsWith(".")) {
+ ext = ext.substring(1);
+ }
+
+ //special handling for text/html/xml
+ if (ext.length() == 0) {
+ ext = tryTextyTypes(mime.getType());
+ }
+ return ext;
+ }
+
+ private static String tryTextyTypes(MediaType mediaType) {
+
+ String type = mediaType.getType();
+ String subtype = mediaType.getSubtype();
+ if (type.equals(TEXT)) {
+ if (subtype.equals(HTML)) {
+ return HTML;
+ } else if (subtype.equals(PLAIN)) {
+ return "txt";
+ } else if (subtype.equals(CSS)) {
+ return CSS;
+ } else if (subtype.equals(CSV)) {
+ return CSV;
+ }
+ } else if (type.equals(APPLICATION)) {
+ if (subtype.equals(XML)) {
+ return XML;
+ } else if (subtype.equals(XHTML_XML)) {
+ return "html";
+ }
+ }
+ return EMPTY_STRING;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/db/TableInfo.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/db/TableInfo.java b/tika-eval/src/main/java/org/apache/tika/eval/db/TableInfo.java
new file mode 100644
index 0000000..c6e3696
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/db/TableInfo.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.db;
+
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class TableInfo {
+
+ private final String name;
+ private final List<ColInfo> colInfos = new ArrayList<>();
+ private final Set<Cols> colNames = new HashSet<>();
+
+ public TableInfo(String name, ColInfo... cols) {
+ Collections.addAll(colInfos, cols);
+ Collections.unmodifiableList(colInfos);
+ this.name = name;
+ for (ColInfo c : colInfos) {
+ assert (!colNames.contains(c.getName()));
+ colNames.add(c.getName());
+ }
+ }
+
+ public TableInfo(String name, List<ColInfo> cols) {
+ colInfos.addAll(cols);
+ Collections.unmodifiableList(colInfos);
+ this.name = name;
+ for (ColInfo c : colInfos) {
+ assert (!colNames.contains(c.getName()));
+ colNames.add(c.getName());
+ }
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public List<ColInfo> getColInfos() {
+ return colInfos;
+ }
+
+ public boolean containsColumn(Cols cols) {
+ return colNames.contains(cols);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/tika/blob/5e49c330/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java b/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java
new file mode 100644
index 0000000..db4cd04
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/io/DBWriter.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.io;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.log4j.Logger;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.eval.db.ColInfo;
+import org.apache.tika.eval.db.Cols;
+import org.apache.tika.eval.db.DBUtil;
+import org.apache.tika.eval.db.MimeBuffer;
+import org.apache.tika.eval.db.TableInfo;
+import org.apache.tika.io.IOExceptionWithCause;
+
+/**
+ * This is still in its early stages. The idea is to
+ * get something working with h2 and then add to that
+ * as necessary.
+ *
+ * Beware, this deletes the db file with each initialization.
+ */
+public class DBWriter implements IDBWriter {
+
+ private static Logger logger = Logger.getLogger(DBWriter.class);
+ private final AtomicLong insertedRows = new AtomicLong();
+ private final Long commitEveryX = 1000L;
+
+ private final List<TableInfo> tableInfos;
+ private final Connection conn;
+ private final DBUtil dbUtil;
+ private static MimeBuffer mimeBuffer;
+
+ //<tableName, preparedStatement>
+ private final Map<String, PreparedStatement> inserts = new HashMap<>();
+
+ public DBWriter(List<TableInfo> tableInfos, TikaConfig tikaConfig, DBUtil dbUtil)
+ throws IOException, SQLException {
+
+ this.conn = dbUtil.getConnection(true);
+ if (mimeBuffer == null) {
+ mimeBuffer = new MimeBuffer(conn, tikaConfig);
+ }
+ this.tableInfos = tableInfos;
+ this.dbUtil = dbUtil;
+ for (TableInfo tableInfo : tableInfos) {
+ try {
+ PreparedStatement st = createPreparedInsert(tableInfo);
+ inserts.put(tableInfo.getName(), st);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ public int getMimeId(String mimeString) {
+ return mimeBuffer.getId(mimeString);
+ }
+
+ private PreparedStatement createPreparedInsert(TableInfo tableInfo) throws SQLException {
+ StringBuilder sb = new StringBuilder();
+ sb.append("INSERT INTO ").append(tableInfo.getName());
+ sb.append("(");
+ int i = 0;
+ for (ColInfo c : tableInfo.getColInfos()) {
+ if (i++ > 0) {
+ sb.append(", ");
+ }
+ sb.append(c.getName());
+ }
+ sb.append(") ");
+
+ sb.append("VALUES");
+ sb.append("(");
+ for (int j = 0; j < i; j++) {
+ if (j > 0) {
+ sb.append(", ");
+ }
+ sb.append("?");
+ }
+ sb.append(")");
+
+ return conn.prepareStatement(sb.toString());
+ }
+
+
+ public void writeRow(TableInfo table, Map<Cols, String> data) throws IOException {
+ try {
+ PreparedStatement p = inserts.get(table.getName());
+ if (p == null) {
+ throw new RuntimeException("Failed to create prepared statement for: "+
+ table.getName());
+ }
+ dbUtil.insert(p, table, data);
+ long rows = insertedRows.incrementAndGet();
+ if (rows % commitEveryX == 0) {
+ logger.info("writer is committing after "+ rows + " rows");
+ conn.commit();
+ }
+ } catch (SQLException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public void close() throws IOException {
+ try {
+ mimeBuffer.close();
+ conn.commit();
+ } catch (SQLException e){
+ e.printStackTrace();
+ throw new IOExceptionWithCause(e);
+ }
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ throw new IOExceptionWithCause(e);
+ }
+
+ }
+}