You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/02/16 17:34:04 UTC
tika git commit: TIKA-1332 -- fix one report for eval profiler and
clean up whitespace
Repository: tika
Updated Branches:
refs/heads/master aa7a0c353 -> 506b57256
TIKA-1332 -- fix one report for eval profiler and clean up whitespace
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/506b5725
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/506b5725
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/506b5725
Branch: refs/heads/master
Commit: 506b572560f6c7f44270b55877f110719a7d4b1f
Parents: aa7a0c3
Author: tballison <ta...@mitre.org>
Authored: Thu Feb 16 12:33:58 2017 -0500
Committer: tballison <ta...@mitre.org>
Committed: Thu Feb 16 12:33:58 2017 -0500
----------------------------------------------------------------------
.../src/main/resources/comparison-reports.xml | 2 +-
.../src/main/resources/lucene-analyzers.json | 30 +++------
.../src/main/resources/profile-reports.xml | 11 ++--
.../resources/tika-eval-comparison-config.xml | 65 ++++++++++----------
...ingle-file-profiler-crawl-extract-config.xml | 2 +-
.../single-file-profiler-crawl-input-config.xml | 2 +-
6 files changed, 52 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/506b5725/tika-eval/src/main/resources/comparison-reports.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/comparison-reports.xml b/tika-eval/src/main/resources/comparison-reports.xml
index cb7befd..d69cb2a 100644
--- a/tika-eval/src/main/resources/comparison-reports.xml
+++ b/tika-eval/src/main/resources/comparison-reports.xml
@@ -206,7 +206,7 @@
</sql>
</report>
- <report reportName="Mime Differences A -> B Details"
+ <report reportName="Mime Differences A -> B Details"
reportFilename="mimes/mime_diffs_A_to_B_details.xlsx"
format="xlsx"
includeSql="true">
http://git-wip-us.apache.org/repos/asf/tika/blob/506b5725/tika-eval/src/main/resources/lucene-analyzers.json
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/lucene-analyzers.json b/tika-eval/src/main/resources/lucene-analyzers.json
index 268494f..f7141f7 100644
--- a/tika-eval/src/main/resources/lucene-analyzers.json
+++ b/tika-eval/src/main/resources/lucene-analyzers.json
@@ -1,12 +1,11 @@
{
"analyzers": {
- "general" :
- {
+ "general": {
"charfilters": [
{
"factory": "oala.charfilter.MappingCharFilterFactory",
"params": {
- "mapping" : "/lucene-char-mapping.txt"
+ "mapping": "/lucene-char-mapping.txt"
}
}
],
@@ -22,20 +21,17 @@
{
"factory": "oala.cjk.CJKBigramFilterFactory",
"params": {
- "outputUnigrams" : "false"
+ "outputUnigrams": "false"
}
}
]
-
},
-
- "alpha" :
- {
+ "alpha": {
"charfilters": [
{
"factory": "oala.charfilter.MappingCharFilterFactory",
"params": {
- "mapping" : "/lucene-char-mapping.txt"
+ "mapping": "/lucene-char-mapping.txt"
}
}
],
@@ -67,7 +63,7 @@
{
"factory": "oala.cjk.CJKBigramFilterFactory",
"params": {
- "outputUnigrams" : "false"
+ "outputUnigrams": "false"
}
},
{
@@ -75,33 +71,27 @@
"params": {}
}
]
-
},
- "common_tokens" :
- {
+ "common_tokens": {
"tokenizer": {
"factory": "oala.standard.UAX29URLEmailTokenizerFactory",
"params": {}
},
-
"tokenfilters": [
{
"factory": "oala.cjk.CJKBigramFilterFactory",
"params": {
- "outputUnigrams" : "false"
+ "outputUnigrams": "false"
}
},
{
"factory": "org.apache.tika.eval.tokens.CJKBigramAwareLengthFilterFactory",
"params": {
- "min" : 4,
- "max" : 20
+ "min": 4,
+ "max": 20
}
}
-
]
-
}
-
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/506b5725/tika-eval/src/main/resources/profile-reports.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/profile-reports.xml b/tika-eval/src/main/resources/profile-reports.xml
index 2a94a97..1f9be6a 100644
--- a/tika-eval/src/main/resources/profile-reports.xml
+++ b/tika-eval/src/main/resources/profile-reports.xml
@@ -98,7 +98,6 @@
</report>
-
<report reportName="Token Count by Detected Language"
reportFilename="content/num_tokens_by_detected_langs.xlsx"
format="xlsx"
@@ -117,10 +116,12 @@
includeSql="true">
<sql>
- select LANG_ID_1 as DetectedLang, count(1) as cnt
- from contents
- group by LANG_ID_1
- order by cnt desc
+ select parse_exception_description, count(1) cnt
+ from parse_exceptions e
+ join profiles p on p.id = e.id
+ join ref_parse_exception_types et on et.parse_exception_type_id=e.parse_exception_type_id
+ group by parse_exception_description
+ order by cnt desc;
</sql>
</report>
http://git-wip-us.apache.org/repos/asf/tika/blob/506b5725/tika-eval/src/main/resources/tika-eval-comparison-config.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/main/resources/tika-eval-comparison-config.xml b/tika-eval/src/main/resources/tika-eval-comparison-config.xml
index b29764e..04ef658 100644
--- a/tika-eval/src/main/resources/tika-eval-comparison-config.xml
+++ b/tika-eval/src/main/resources/tika-eval-comparison-config.xml
@@ -20,34 +20,34 @@
<tika-batch-config
maxAliveTimeSeconds="-1"
- pauseOnEarlyTerminationMillis = "500"
+ pauseOnEarlyTerminationMillis="500"
timeoutCheckPulseMillis="1000"
maxQueueSize="10000"
numConsumers="5"
timeoutThresholdMillis="300000"
- >
+>
<commandline>
- <option opt="c" longOpt="tika-config" hasArg="true"
- description="TikaConfig file"/>
- <option opt="bc" longOpt="batch-config" hasArg="true"
- description="xml batch config file" required="true"/>
- <option opt="inputDir" hasArg="true"
- description="dir to start crawling"/>
- <option opt="numConsumers" hasArg="true"
- description="number of fileConsumers threads"/>
- <option opt="extractsA" hasArg="true"
- description="this dir for analysis" required="false"/>
- <option opt="extractsB" hasArg="true"
- description="thatDir for analysis"/>
- <option opt="db" hasArg="true"
- description="name of db directory or file to which to write results"/>
- <option opt="alterExtract" hasArg="true"
+ <option opt="c" longOpt="tika-config" hasArg="true"
+ description="TikaConfig file"/>
+ <option opt="bc" longOpt="batch-config" hasArg="true"
+ description="xml batch config file" required="true"/>
+ <option opt="inputDir" hasArg="true"
+ description="dir to start crawling"/>
+ <option opt="numConsumers" hasArg="true"
+ description="number of fileConsumers threads"/>
+ <option opt="extractsA" hasArg="true"
+ description="this dir for analysis" required="false"/>
+ <option opt="extractsB" hasArg="true"
+ description="thatDir for analysis"/>
+ <option opt="db" hasArg="true"
+ description="name of db directory or file to which to write results"/>
+ <option opt="alterExtract" hasArg="true"
description="for json-formatted extract files
process full metadata list ('as_is'=default),
take just the first/container document ('first_only'),
concatenate all content into the first metadata item ('concatenate_content')"/>
- <option opt="includeFilePat" hasArg="true"
+ <option opt="includeFilePat" hasArg="true"
description="regex for files to include"/>
</commandline>
@@ -56,15 +56,15 @@
Can also add startDir: this tells the crawler to start indexing a
child directory of the srcDir directory.
-->
- <crawler builderClass="org.apache.tika.batch.fs.builders.FSCrawlerBuilder"
- crawlOrder="sorted"
- maxConsecWaitMillis="30000"
- maxFilesToAdd="-1"
- maxFilesToConsider="-1"
- includeFilePat=""
- excludeFilePat=""
- maxFileSizeBytes="10000000"
- />
+ <crawler builderClass="org.apache.tika.batch.fs.builders.FSCrawlerBuilder"
+ crawlOrder="sorted"
+ maxConsecWaitMillis="30000"
+ maxFilesToAdd="-1"
+ maxFilesToConsider="-1"
+ includeFilePat=""
+ excludeFilePat=""
+ maxFileSizeBytes="10000000"
+ />
<consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
consumerBuilderClass="org.apache.tika.eval.batch.FileComparerBuilder"
@@ -73,11 +73,12 @@
minJsonFileSizeBytes="-1"
maxJsonFileSizeBytes="2000000"
commonTokens="resources/commontokens"
- />
+ />
-<!-- langModelDir="resources/langmodels" -->
+ <!-- langModelDir="resources/langmodels" -->
- <!-- reporter and interrupter are optional -->
- <reporter builderClass="org.apache.tika.batch.builders.SimpleLogReporterBuilder" sleepMillis="1000" staleThresholdMillis="500000"/>
- <interrupter builderClass="org.apache.tika.batch.builders.InterrupterBuilder"/>
+ <!-- reporter and interrupter are optional -->
+ <reporter builderClass="org.apache.tika.batch.builders.SimpleLogReporterBuilder" sleepMillis="1000"
+ staleThresholdMillis="500000"/>
+ <interrupter builderClass="org.apache.tika.batch.builders.InterrupterBuilder"/>
</tika-batch-config>
http://git-wip-us.apache.org/repos/asf/tika/blob/506b5725/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml b/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
index e8b9d6c..536f39c 100644
--- a/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
+++ b/tika-eval/src/test/resources/single-file-profiler-crawl-extract-config.xml
@@ -56,7 +56,7 @@
includeFilePat=""
excludeFilePat=""
maxFileSizeBytes="-1"
- />
+ />
<consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"
http://git-wip-us.apache.org/repos/asf/tika/blob/506b5725/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
----------------------------------------------------------------------
diff --git a/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml b/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
index da59d03..29b7f3b 100644
--- a/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
+++ b/tika-eval/src/test/resources/single-file-profiler-crawl-input-config.xml
@@ -56,7 +56,7 @@
includeFilePat=""
excludeFilePat=""
maxFileSizeBytes="-1"
- />
+ />
<consumers builderClass="org.apache.tika.eval.batch.EvalConsumersBuilder"
consumerBuilderClass="org.apache.tika.eval.batch.SingleFileConsumerBuilder"