You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2019/08/13 16:28:09 UTC
[tika] branch TIKA-2906 updated: TIKA-2906 -- finish work
decoupling text stats from tika-eval application
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch TIKA-2906
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/TIKA-2906 by this push:
new 787d305 TIKA-2906 -- finish work decoupling text stats from tika-eval application
787d305 is described below
commit 787d30585ea9618926d182b3c3787afaea33117e
Author: TALLISON <ta...@apache.org>
AuthorDate: Tue Aug 13 12:27:52 2019 -0400
TIKA-2906 -- finish work decoupling text stats from tika-eval application
---
.../org/apache/tika/eval/AbstractProfiler.java | 204 ++++++++++-----------
.../java/org/apache/tika/eval/ExtractComparer.java | 31 ++--
.../java/org/apache/tika/eval/ExtractProfiler.java | 3 +-
.../apache/tika/eval/langid/LanguageIDWrapper.java | 6 +-
...ts.java => BasicTokenCountStatsCalculator.java} | 12 +-
.../apache/tika/eval/textstats/CommonTokens.java | 2 +-
.../eval/textstats/CommonTokensBhattacharyya.java | 2 +-
.../tika/eval/textstats/CommonTokensCosine.java | 2 +-
.../tika/eval/textstats/CommonTokensHellinger.java | 2 +-
.../tika/eval/textstats/CommonTokensKLDNormed.java | 2 +-
.../eval/textstats/CommonTokensKLDivergence.java | 2 +-
...ator.java => CompositeTextStatsCalculator.java} | 36 ++--
...TextStats.java => ContentLengthCalculator.java} | 9 +-
.../textstats/LanguageAwareTokenCountStats.java | 6 +-
...StringStats.java => StringStatsCalculator.java} | 4 +-
.../tika/eval/textstats/TextStatsCalculator.java | 106 +----------
.../TokenCountPriorityQueue.java | 7 +-
...ntStats.java => TokenCountStatsCalculator.java} | 4 +-
.../apache/tika/eval/textstats/TokenEntropy.java | 4 +-
.../apache/tika/eval/textstats/TokenLengths.java | 4 +-
.../{TokenLengths.java => TopNTokens.java} | 22 ++-
.../tika/eval/textstats/UnicodeBlockCounter.java | 73 ++++++++
.../tika/eval/tokens/CommonTokenCountManager.java | 1 -
.../apache/tika/eval/tokens/TokenContraster.java | 25 ++-
.../org/apache/tika/eval/tokens/TokenCounter.java | 7 +
.../org/apache/tika/eval/tokens/TokenCounts.java | 4 +
.../eval/textstats/TestTextStatsCalculator.java | 93 ++--------
.../tika/eval/tokens/LuceneTokenCounter.java | 1 +
tika-eval/src/test/resources/common_tokens/en | 20 ++
tika-eval/src/test/resources/common_tokens/es | 20 ++
tika-eval/src/test/resources/common_tokens/zh-cn | 20 ++
tika-eval/src/test/resources/common_tokens/zh-tw | 20 ++
32 files changed, 373 insertions(+), 381 deletions(-)
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java b/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
index 298e790..7c3a436 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
@@ -19,8 +19,6 @@ package org.apache.tika.eval;
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -38,6 +36,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.tika.batch.FileResource;
@@ -50,12 +49,20 @@ import org.apache.tika.eval.db.TableInfo;
import org.apache.tika.eval.io.ExtractReaderException;
import org.apache.tika.eval.io.IDBWriter;
import org.apache.tika.eval.langid.Language;
+import org.apache.tika.eval.textstats.BasicTokenCountStatsCalculator;
+import org.apache.tika.eval.textstats.CommonTokens;
+import org.apache.tika.eval.textstats.CompositeTextStatsCalculator;
+import org.apache.tika.eval.textstats.ContentLengthCalculator;
+import org.apache.tika.eval.textstats.TextStatsCalculator;
+import org.apache.tika.eval.textstats.TokenEntropy;
+import org.apache.tika.eval.textstats.TokenLengths;
+import org.apache.tika.eval.textstats.TopNTokens;
+import org.apache.tika.eval.textstats.UnicodeBlockCounter;
import org.apache.tika.eval.tokens.AnalyzerManager;
import org.apache.tika.eval.tokens.CommonTokenCountManager;
import org.apache.tika.eval.tokens.CommonTokenResult;
-import org.apache.tika.eval.tokens.TokenCounter;
+import org.apache.tika.eval.tokens.TokenCounts;
import org.apache.tika.eval.tokens.TokenIntPair;
-import org.apache.tika.eval.tokens.TokenStatistics;
import org.apache.tika.eval.util.ContentTags;
import org.apache.tika.eval.util.ContentTagParser;
import org.apache.tika.eval.langid.LanguageIDWrapper;
@@ -137,11 +144,11 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
tmp.put("UL", Cols.TAGS_UL);
return Collections.unmodifiableMap(tmp);
}
- private static CommonTokenCountManager commonTokenCountManager;
+ private static CommonTokenCountManager COMMON_TOKEN_COUNT_MANAGER;
+
private String lastExtractExtension = null;
AnalyzerManager analyzerManager;
- TokenCounter tokenCounter;
public enum EXCEPTION_TYPE {
@@ -186,7 +193,7 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
Pattern.compile("org\\.apache\\.tika.exception\\.EncryptedDocumentException");
private TikaConfig config = TikaConfig.getDefaultConfig();//TODO: allow configuration
- final LanguageIDWrapper langIder;
+ CompositeTextStatsCalculator compositeTextStatsCalculator;
protected IDBWriter writer;
/**
@@ -195,22 +202,30 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
* @throws IOException
*/
public static void loadCommonTokens(Path p, String defaultLangCode) throws IOException {
- commonTokenCountManager = new CommonTokenCountManager(p, defaultLangCode);
+ COMMON_TOKEN_COUNT_MANAGER = new CommonTokenCountManager(p, defaultLangCode);
}
public AbstractProfiler(ArrayBlockingQueue<FileResource> fileQueue,
IDBWriter writer) {
super(fileQueue);
this.writer = writer;
- langIder = new LanguageIDWrapper();
LanguageIDWrapper.setMaxTextLength(maxContentLengthForLangId);
- initAnalyzersAndTokenCounter(maxTokens);
+ this.compositeTextStatsCalculator = initAnalyzersAndTokenCounter(maxTokens, new LanguageIDWrapper());
}
- private void initAnalyzersAndTokenCounter(int maxTokens) {
+ private CompositeTextStatsCalculator initAnalyzersAndTokenCounter(int maxTokens, LanguageIDWrapper langIder) {
try {
analyzerManager = AnalyzerManager.newInstance(maxTokens);
- tokenCounter = new TokenCounter(analyzerManager.getGeneralAnalyzer());
+ List<TextStatsCalculator> calculators = new ArrayList<>();
+ calculators.add(new CommonTokens(COMMON_TOKEN_COUNT_MANAGER));
+ calculators.add(new TokenEntropy());
+ calculators.add(new TokenLengths());
+ calculators.add(new TopNTokens(10));
+ calculators.add(new BasicTokenCountStatsCalculator());
+ calculators.add(new ContentLengthCalculator());
+ calculators.add(new UnicodeBlockCounter(maxContentLengthForLangId));
+
+ return new CompositeTextStatsCalculator(calculators, analyzerManager.getGeneralAnalyzer(), langIder);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -242,7 +257,7 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
*/
public void setMaxTokens(int maxTokens) {
this.maxTokens = maxTokens;
- initAnalyzersAndTokenCounter(maxTokens);
+ initAnalyzersAndTokenCounter(maxTokens, new LanguageIDWrapper());
}
@@ -352,68 +367,72 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
}
}
+ protected Map<Class, Object> calcTextStats(ContentTags contentTags) {
+/* if (contentTags == ContentTags.EMPTY_CONTENT_TAGS) {
+ return Collections.EMPTY_MAP;
+ }*/
+ Map<Cols, String> data = new HashMap<>();
+ String content = truncateContent(contentTags, maxContentLength, data);
+ if (content == null || content.trim().length() == 0) {
+ content = "";
+ }
+ return compositeTextStatsCalculator.calculate(content);
+ }
/**
* Checks to see if metadata is null or content is empty (null or only whitespace).
* If any of these, then this does no processing, and the fileId is not
* entered into the content table.
*
* @param fileId
- * @param contentTags
- * @param fieldName
+ * @param textStats
* @param contentsTable
*/
- protected void writeContentData(String fileId, ContentTags contentTags,
- String fieldName, TableInfo contentsTable) throws IOException {
- if (contentTags == ContentTags.EMPTY_CONTENT_TAGS) {
- return;
- }
+ protected void writeContentData(String fileId, Map<Class, Object> textStats, TableInfo contentsTable) throws IOException {
Map<Cols, String> data = new HashMap<>();
- String content = truncateContent(contentTags, maxContentLength, data);
- if (content == null || content.trim().length() == 0) {
- return;
+ data.put(Cols.ID, fileId);
+ if (textStats.containsKey(ContentLengthCalculator.class)) {
+ data.put(Cols.CONTENT_LENGTH, Integer.toString((Integer) textStats.get(ContentLengthCalculator.class)));
+ } else {
+ data.put(Cols.CONTENT_LENGTH, "0");
}
- tokenCounter.clear(fieldName);
- tokenCounter.add(fieldName, content);
+ langid(textStats, data);
- data.put(Cols.ID, fileId);
- data.put(Cols.CONTENT_LENGTH, Integer.toString(content.length()));
- langid(contentTags, data);
- String langid = data.get(Cols.LANG_ID_1);
- langid = (langid == null) ? "" : langid;
+ writeTokenCounts(textStats, data);
+ CommonTokenResult commonTokenResult = (CommonTokenResult)textStats.get(CommonTokens.class);
+ if (commonTokenResult != null) {
+ data.put(Cols.COMMON_TOKENS_LANG, commonTokenResult.getLangCode());
+ data.put(Cols.NUM_UNIQUE_COMMON_TOKENS, Integer.toString(commonTokenResult.getUniqueCommonTokens()));
+ data.put(Cols.NUM_COMMON_TOKENS, Integer.toString(commonTokenResult.getCommonTokens()));
+ data.put(Cols.NUM_UNIQUE_ALPHABETIC_TOKENS,
+ Integer.toString(commonTokenResult.getUniqueAlphabeticTokens()));
+ data.put(Cols.NUM_ALPHABETIC_TOKENS,
+ Integer.toString(commonTokenResult.getAlphabeticTokens()));
+ }
+ TokenCounts tokenCounts = (TokenCounts)textStats.get(BasicTokenCountStatsCalculator.class);
+ if (tokenCounts != null) {
- writeTokenCounts(data, fieldName, tokenCounter);
- CommonTokenResult commonTokenResult = null;
- try {
- commonTokenResult = commonTokenCountManager.countTokenOverlaps(langid,
- tokenCounter.getTokens(fieldName));
- } catch (IOException e) {
- LOG.error("{}", e.getMessage(), e);
- }
- data.put(Cols.COMMON_TOKENS_LANG, commonTokenResult.getLangCode());
- data.put(Cols.NUM_UNIQUE_COMMON_TOKENS, Integer.toString(commonTokenResult.getUniqueCommonTokens()));
- data.put(Cols.NUM_COMMON_TOKENS, Integer.toString(commonTokenResult.getCommonTokens()));
- TokenStatistics tokenStatistics = tokenCounter.getTokenStatistics(fieldName);
- data.put(Cols.NUM_UNIQUE_TOKENS,
- Integer.toString(tokenStatistics.getTotalUniqueTokens()));
- data.put(Cols.NUM_TOKENS,
- Integer.toString(tokenStatistics.getTotalTokens()));
- data.put(Cols.NUM_UNIQUE_ALPHABETIC_TOKENS,
- Integer.toString(commonTokenResult.getUniqueAlphabeticTokens()));
- data.put(Cols.NUM_ALPHABETIC_TOKENS,
- Integer.toString(commonTokenResult.getAlphabeticTokens()));
-
- data.put(Cols.TOKEN_ENTROPY_RATE,
- Double.toString(tokenStatistics.getEntropy()));
- SummaryStatistics summStats = tokenStatistics.getSummaryStatistics();
- data.put(Cols.TOKEN_LENGTH_SUM,
- Integer.toString((int) summStats.getSum()));
-
- data.put(Cols.TOKEN_LENGTH_MEAN,
- Double.toString(summStats.getMean()));
-
- data.put(Cols.TOKEN_LENGTH_STD_DEV,
- Double.toString(summStats.getStandardDeviation()));
- unicodeBlocks(contentTags, data);
+ data.put(Cols.NUM_UNIQUE_TOKENS,
+ Integer.toString(tokenCounts.getTotalUniqueTokens()));
+ data.put(Cols.NUM_TOKENS,
+ Integer.toString(tokenCounts.getTotalTokens()));
+ }
+ if (textStats.get(TokenEntropy.class) != null) {
+ data.put(Cols.TOKEN_ENTROPY_RATE,
+ Double.toString((Double)textStats.get(TokenEntropy.class)));
+ }
+
+ SummaryStatistics summStats = (SummaryStatistics)textStats.get(TokenLengths.class);
+ if (summStats != null) {
+ data.put(Cols.TOKEN_LENGTH_SUM,
+ Integer.toString((int) summStats.getSum()));
+
+ data.put(Cols.TOKEN_LENGTH_MEAN,
+ Double.toString(summStats.getMean()));
+
+ data.put(Cols.TOKEN_LENGTH_STD_DEV,
+ Double.toString(summStats.getStandardDeviation()));
+ }
+ unicodeBlocks(textStats, data);
try {
writer.writeRow(contentsTable, data);
} catch (IOException e) {
@@ -544,40 +563,12 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
return parseContentAndTags(evalFilePaths, metadata);
}
- void unicodeBlocks(ContentTags contentTags, Map<Cols, String> data) {
- String content = contentTags.getContent();
- if (content.length() < 200) {
- return;
- }
- String s = content;
- if (content.length() > maxContentLengthForLangId) {
- s = content.substring(0, maxContentLengthForLangId);
- }
- Map<String, Integer> m = new HashMap<>();
- Reader r = new StringReader(s);
- try {
- int c = r.read();
- while (c != -1) {
- Character.UnicodeBlock block = Character.UnicodeBlock.of(c);
- String blockString = (block == null) ? "NULL" : block.toString();
- Integer i = m.get(blockString);
- if (i == null) {
- i = 0;
- }
- i++;
- if (block == null) {
- blockString = "NULL";
- }
- m.put(blockString, i);
- c = r.read();
- }
- } catch (IOException e) {
- LOG.warn("IOException", e);
- }
+ void unicodeBlocks(Map<Class, Object> tokenStats, Map<Cols, String> data) {
+ Map<String, MutableInt> blocks = (Map<String, MutableInt>)tokenStats.get(UnicodeBlockCounter.class);
List<Pair<String, Integer>> pairs = new ArrayList<>();
- for (Map.Entry<String, Integer> e : m.entrySet()) {
- pairs.add(Pair.of(e.getKey(), e.getValue()));
+ for (Map.Entry<String, MutableInt> e : blocks.entrySet()) {
+ pairs.add(Pair.of(e.getKey(), e.getValue().intValue()));
}
Collections.sort(pairs, new Comparator<Pair<String, Integer>>() {
@Override
@@ -596,16 +587,9 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
data.put(Cols.UNICODE_CHAR_BLOCKS, sb.toString());
}
- void langid(ContentTags contentTags, Map<Cols, String> data) {
- String content = contentTags.getContent();
- if (content.length() < 50) {
- return;
- }
- String s = content;
- if (content.length() > maxContentLengthForLangId) {
- s = content.substring(0, maxContentLengthForLangId);
- }
- List<Language> probabilities = langIder.getProbabilities(s);
+ void langid(Map<Class, Object> stats, Map<Cols, String> data) {
+ List<Language> probabilities = (List<Language>)stats.get(LanguageIDWrapper.class);
+
if (probabilities.size() > 0) {
data.put(Cols.LANG_ID_1, probabilities.get(0).getLanguage());
data.put(Cols.LANG_ID_PROB_1,
@@ -630,15 +614,11 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
output.put(Cols.MIME_ID, Integer.toString(mimeId));
}
- void writeTokenCounts(Map<Cols, String> data, String field,
- TokenCounter tokenCounter) {
-
-
- int stops = 0;
+ void writeTokenCounts(Map<Class, Object> textStats, Map<Cols, String> data) {
+ TokenIntPair[] tokenIntPairs = (TokenIntPair[])textStats.get(TopNTokens.class);
int i = 0;
StringBuilder sb = new StringBuilder();
- TokenStatistics tokenStatistics = tokenCounter.getTokenStatistics(field);
- for (TokenIntPair t : tokenStatistics.getTopN()) {
+ for (TokenIntPair t : tokenIntPairs) {
if (i++ > 0) {
sb.append(" | ");
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java b/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
index 86d1672..39342ef 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
@@ -39,8 +39,10 @@ import org.apache.tika.eval.db.TableInfo;
import org.apache.tika.eval.io.ExtractReader;
import org.apache.tika.eval.io.ExtractReaderException;
import org.apache.tika.eval.io.IDBWriter;
+import org.apache.tika.eval.textstats.BasicTokenCountStatsCalculator;
import org.apache.tika.eval.tokens.ContrastStatistics;
import org.apache.tika.eval.tokens.TokenContraster;
+import org.apache.tika.eval.tokens.TokenCounts;
import org.apache.tika.eval.tokens.TokenIntPair;
import org.apache.tika.eval.util.ContentTags;
import org.apache.tika.metadata.Metadata;
@@ -275,7 +277,8 @@ public class ExtractComparer extends AbstractProfiler {
}
List<Integer> numAttachmentsA = countAttachments(metadataListA);
List<Integer> numAttachmentsB = countAttachments(metadataListB);
-
+ Map<Class, Object> tokenStatsA = null;
+ Map<Class, Object> tokenStatsB = null;
//now get that metadata
if (metadataListA != null) {
for (int i = 0; i < metadataListA.size(); i++) {
@@ -304,29 +307,28 @@ public class ExtractComparer extends AbstractProfiler {
writeExceptionData(fileId, metadataB, EXCEPTION_TABLE_B);
}
writeEmbeddedFilePathData(i, fileId, metadataA, metadataB);
- //prep the token counting
- tokenCounter.clear(FIELD_A);
- tokenCounter.clear(FIELD_B);
//write content
try {
- writeContentData(fileId, contentTagsA, FIELD_A, CONTENTS_TABLE_A);
- writeContentData(fileId, contentTagsB, FIELD_B, CONTENTS_TABLE_B);
+ tokenStatsA = calcTextStats(contentTagsA);
+ tokenStatsB = calcTextStats(contentTagsB);
+ writeContentData(fileId, tokenStatsA, CONTENTS_TABLE_A);
+ writeContentData(fileId, tokenStatsB, CONTENTS_TABLE_B);
} catch (IOException e) {
throw new RuntimeException(e);
}
+ TokenCounts tokenCountsA = (TokenCounts)tokenStatsA.get(BasicTokenCountStatsCalculator.class);
+ TokenCounts tokenCountsB = (TokenCounts)tokenStatsB.get(BasicTokenCountStatsCalculator.class);
//now run comparisons
- if (tokenCounter.getTokenStatistics(FIELD_A).getTotalTokens() > 0
- && tokenCounter.getTokenStatistics(FIELD_B).getTotalTokens() > 0) {
+ if (tokenCountsA.getTotalTokens() > 0
+ && tokenCountsB.getTotalTokens() > 0) {
Map<Cols, String> data = new HashMap<>();
data.put(Cols.ID, fileId);
ContrastStatistics contrastStatistics =
tokenContraster.calculateContrastStatistics(
- tokenCounter.getTokens(FIELD_A),
- tokenCounter.getTokenStatistics(FIELD_A),
- tokenCounter.getTokens(FIELD_B),
- tokenCounter.getTokenStatistics(FIELD_B));
+ tokenCountsA,
+ tokenCountsB);
writeContrasts(data, contrastStatistics);
writer.writeRow(CONTENT_COMPARISONS, data);
@@ -349,11 +351,10 @@ public class ExtractComparer extends AbstractProfiler {
writeEmbeddedFilePathData(i, fileId, null, metadataB);
writeExceptionData(fileId, metadataB, EXCEPTION_TABLE_B);
- //prep the token counting
- tokenCounter.clear(FIELD_B);
//write content
try {
- writeContentData(fileId, contentTagsB, FIELD_B, CONTENTS_TABLE_B);
+ tokenStatsB = calcTextStats(contentTagsB);
+ writeContentData(fileId, tokenStatsB, CONTENTS_TABLE_B);
} catch (IOException e) {
throw new RuntimeException(e);
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java b/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
index 1ab67ae..e6e4c14 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/ExtractProfiler.java
@@ -252,7 +252,8 @@ public class ExtractProfiler extends AbstractProfiler {
writeEmbeddedPathData(i, fileId, m, EMBEDDED_FILE_PATH_TABLE);
writeExceptionData(fileId, m, EXCEPTION_TABLE);
try {
- writeContentData(fileId, contentTags, FIELD, CONTENTS_TABLE);
+ Map<Class, Object> textStats = calcTextStats(contentTags);
+ writeContentData(fileId, textStats, CONTENTS_TABLE);
} catch (IOException e) {
throw new RuntimeException(e);
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/langid/LanguageIDWrapper.java b/tika-eval/src/main/java/org/apache/tika/eval/langid/LanguageIDWrapper.java
index 727e905..cec5037 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/langid/LanguageIDWrapper.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/langid/LanguageIDWrapper.java
@@ -29,10 +29,10 @@ import opennlp.tools.util.normalizer.EmojiCharSequenceNormalizer;
import opennlp.tools.util.normalizer.NumberCharSequenceNormalizer;
import opennlp.tools.util.normalizer.ShrinkCharSequenceNormalizer;
import opennlp.tools.util.normalizer.TwitterCharSequenceNormalizer;
-import org.apache.tika.eval.textstats.StringStats;
+import org.apache.tika.eval.textstats.StringStatsCalculator;
-public class LanguageIDWrapper implements StringStats<List<Language>> {
+public class LanguageIDWrapper implements StringStatsCalculator<List<Language>> {
static LanguageDetectorModel LANG_MODEL;
@@ -83,7 +83,7 @@ public class LanguageIDWrapper implements StringStats<List<Language>> {
}
@Override
- public List<Language> compute(String txt) {
+ public List<Language> calculate(String txt) {
return getProbabilities(txt);
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenCountStats.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/BasicTokenCountStatsCalculator.java
similarity index 81%
copy from tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenCountStats.java
copy to tika-eval/src/main/java/org/apache/tika/eval/textstats/BasicTokenCountStatsCalculator.java
index 4b3c678..89274cf 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenCountStats.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/BasicTokenCountStatsCalculator.java
@@ -16,12 +16,12 @@
*/
package org.apache.tika.eval.textstats;
+
import org.apache.tika.eval.tokens.TokenCounts;
-/**
- * Interface for calculators that require token stats
- * @param <T>
- */
-public interface TokenCountStats<T> {
- T compute(TokenCounts tokenCounts);
+public class BasicTokenCountStatsCalculator implements TokenCountStatsCalculator<TokenCounts> {
+ @Override
+ public TokenCounts calculate(TokenCounts tokenCounts) {
+ return tokenCounts;
+ }
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokens.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokens.java
index 6acd849..1b47098 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokens.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokens.java
@@ -38,7 +38,7 @@ public class CommonTokens implements LanguageAwareTokenCountStats<CommonTokenRes
}
@Override
- public CommonTokenResult compute(List<Language> languages, TokenCounts tokenCounts) {
+ public CommonTokenResult calculate(List<Language> languages, TokenCounts tokenCounts) {
Pair<String, LangModel> pair = commonTokenCountManager.getLangTokens(languages.get(0).getLanguage());
String actualLangCode = pair.getKey();
Set<String> commonTokens = pair.getValue().getTokens();
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensBhattacharyya.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensBhattacharyya.java
index 1a833db..e64374f 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensBhattacharyya.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensBhattacharyya.java
@@ -36,7 +36,7 @@ public class CommonTokensBhattacharyya implements LanguageAwareTokenCountStats<D
}
@Override
- public Double compute(List<Language> languages, TokenCounts tokenCounts) {
+ public Double calculate(List<Language> languages, TokenCounts tokenCounts) {
Pair<String, LangModel> pair = commonTokenCountManager.getLangTokens(languages.get(0).getLanguage());
LangModel model = pair.getValue();
double sum = 0.0;
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensCosine.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensCosine.java
index 60ab0fb..daa59b5 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensCosine.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensCosine.java
@@ -37,7 +37,7 @@ public class CommonTokensCosine implements LanguageAwareTokenCountStats<Double>
}
@Override
- public Double compute(List<Language> languages, TokenCounts tokenCounts) {
+ public Double calculate(List<Language> languages, TokenCounts tokenCounts) {
Pair<String, LangModel> pair = commonTokenCountManager.getLangTokens(languages.get(0).getLanguage());
LangModel model = pair.getValue();
double kl = 0.0;
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensHellinger.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensHellinger.java
index 8dcfa19..0b11f8a 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensHellinger.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensHellinger.java
@@ -36,7 +36,7 @@ public class CommonTokensHellinger implements LanguageAwareTokenCountStats<Doubl
}
@Override
- public Double compute(List<Language> languages, TokenCounts tokenCounts) {
+ public Double calculate(List<Language> languages, TokenCounts tokenCounts) {
Pair<String, LangModel> pair = commonTokenCountManager.getLangTokens(languages.get(0).getLanguage());
LangModel model = pair.getValue();
double sum = 0.0;
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensKLDNormed.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensKLDNormed.java
index 6c119a6..2bd8974 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensKLDNormed.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensKLDNormed.java
@@ -36,7 +36,7 @@ public class CommonTokensKLDNormed implements LanguageAwareTokenCountStats<Doubl
}
@Override
- public Double compute(List<Language> languages, TokenCounts tokenCounts) {
+ public Double calculate(List<Language> languages, TokenCounts tokenCounts) {
Pair<String, LangModel> pair = commonTokenCountManager.getLangTokens(languages.get(0).getLanguage());
LangModel model = pair.getValue();
double kl = 0.0;
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensKLDivergence.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensKLDivergence.java
index c880752..eb6b271 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensKLDivergence.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CommonTokensKLDivergence.java
@@ -36,7 +36,7 @@ public class CommonTokensKLDivergence implements LanguageAwareTokenCountStats<Do
}
@Override
- public Double compute(List<Language> languages, TokenCounts tokenCounts) {
+ public Double calculate(List<Language> languages, TokenCounts tokenCounts) {
Pair<String, LangModel> pair = commonTokenCountManager.getLangTokens(languages.get(0).getLanguage());
LangModel model = pair.getValue();
double kl = 0.0;
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TextStatsCalculator.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CompositeTextStatsCalculator.java
similarity index 74%
copy from tika-eval/src/main/java/org/apache/tika/eval/textstats/TextStatsCalculator.java
copy to tika-eval/src/main/java/org/apache/tika/eval/textstats/CompositeTextStatsCalculator.java
index 9c3ab45..da8cd79 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TextStatsCalculator.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/CompositeTextStatsCalculator.java
@@ -30,34 +30,34 @@ import org.apache.tika.eval.langid.LanguageIDWrapper;
import org.apache.tika.eval.tokens.TokenCounts;
-public class TextStatsCalculator {
+public class CompositeTextStatsCalculator {
private static final String FIELD = "f";
private final Analyzer analyzer;
private final LanguageIDWrapper languageIDWrapper;
private final List<LanguageAwareTokenCountStats> languageAwareTokenCountStats = new ArrayList<>();
- private final List<TokenCountStats> tokenCountStats = new ArrayList<>();
- private final List<StringStats> stringStats = new ArrayList<>();
+ private final List<TokenCountStatsCalculator> tokenCountStatCalculators = new ArrayList<>();
+ private final List<StringStatsCalculator> stringStatCalculators = new ArrayList<>();
- public TextStatsCalculator(List<TextStats> calculators) {
+ public CompositeTextStatsCalculator(List<TextStatsCalculator> calculators) {
this(calculators, null, null);
}
- public TextStatsCalculator(List<TextStats> calculators, Analyzer analyzer,
- LanguageIDWrapper languageIDWrapper) {
+ public CompositeTextStatsCalculator(List<TextStatsCalculator> calculators, Analyzer analyzer,
+ LanguageIDWrapper languageIDWrapper) {
this.analyzer = analyzer;
this.languageIDWrapper = languageIDWrapper;
- for (TextStats t : calculators) {
- if (t instanceof StringStats) {
- stringStats.add((StringStats)t);
+ for (TextStatsCalculator t : calculators) {
+ if (t instanceof StringStatsCalculator) {
+ stringStatCalculators.add((StringStatsCalculator)t);
} else if (t instanceof LanguageAwareTokenCountStats) {
languageAwareTokenCountStats.add((LanguageAwareTokenCountStats) t);
if (languageIDWrapper == null) {
throw new IllegalArgumentException("Must specify a LanguageIdWrapper "+
"if you want to calculate languageAware stats: "+t.getClass());
}
- } else if (t instanceof TokenCountStats) {
- tokenCountStats.add((TokenCountStats)t);
+ } else if (t instanceof TokenCountStatsCalculator) {
+ tokenCountStatCalculators.add((TokenCountStatsCalculator)t);
if (analyzer == null) {
throw new IllegalArgumentException(
"Analyzer must not be null if you are using "+
@@ -74,12 +74,12 @@ public class TextStatsCalculator {
public Map<Class, Object> calculate(String txt) {
Map<Class, Object> results = new HashMap<>();
- for (StringStats calc : stringStats) {
- results.put(calc.getClass(), calc.compute(txt));
+ for (StringStatsCalculator calc : stringStatCalculators) {
+ results.put(calc.getClass(), calc.calculate(txt));
}
TokenCounts tokenCounts = null;
- if (tokenCountStats.size() > 0 || languageAwareTokenCountStats.size() > 0) {
+ if (tokenCountStatCalculators.size() > 0 || languageAwareTokenCountStats.size() > 0) {
try {
tokenCounts = tokenize(txt);
} catch (IOException e) {
@@ -89,15 +89,15 @@ public class TextStatsCalculator {
if (languageAwareTokenCountStats.size() > 0) {
List<Language> langs = results.containsKey(LanguageIDWrapper.class) ?
- (List)results.get(LanguageIDWrapper.class) : languageIDWrapper.compute(txt);
+ (List)results.get(LanguageIDWrapper.class) : languageIDWrapper.calculate(txt);
results.put(LanguageIDWrapper.class, langs);
for (LanguageAwareTokenCountStats calc : languageAwareTokenCountStats) {
- results.put(calc.getClass(), calc.compute(langs, tokenCounts));
+ results.put(calc.getClass(), calc.calculate(langs, tokenCounts));
}
}
- for (TokenCountStats calc : tokenCountStats) {
- results.put(calc.getClass(), calc.compute(tokenCounts));
+ for (TokenCountStatsCalculator calc : tokenCountStatCalculators) {
+ results.put(calc.getClass(), calc.calculate(tokenCounts));
}
return results;
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TextStats.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/ContentLengthCalculator.java
similarity index 83%
rename from tika-eval/src/main/java/org/apache/tika/eval/textstats/TextStats.java
rename to tika-eval/src/main/java/org/apache/tika/eval/textstats/ContentLengthCalculator.java
index 286f9ed..3eb0f23 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TextStats.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/ContentLengthCalculator.java
@@ -16,8 +16,9 @@
*/
package org.apache.tika.eval.textstats;
-/**
- * Base text stats interface
- */
-public interface TextStats {
+public class ContentLengthCalculator implements StringStatsCalculator<Integer> {
+ @Override
+ public Integer calculate(String txt) {
+ return txt.length();
+ }
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/LanguageAwareTokenCountStats.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/LanguageAwareTokenCountStats.java
index 499cbca..12cb19a 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/LanguageAwareTokenCountStats.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/LanguageAwareTokenCountStats.java
@@ -22,9 +22,9 @@ import org.apache.tika.eval.langid.Language;
import org.apache.tika.eval.tokens.TokenCounts;
/**
- * Interface for calculators that require token stats
+ * Interface for calculators that require language probabilities and token stats
* @param <T>
*/
-public interface LanguageAwareTokenCountStats<T> extends TextStats {
- T compute(List<Language> languages, TokenCounts tokenCounts);
+public interface LanguageAwareTokenCountStats<T> extends TextStatsCalculator {
+ T calculate(List<Language> languages, TokenCounts tokenCounts);
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/StringStats.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/StringStatsCalculator.java
similarity index 90%
rename from tika-eval/src/main/java/org/apache/tika/eval/textstats/StringStats.java
rename to tika-eval/src/main/java/org/apache/tika/eval/textstats/StringStatsCalculator.java
index d19c6ec..e297bb0 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/StringStats.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/StringStatsCalculator.java
@@ -20,6 +20,6 @@ package org.apache.tika.eval.textstats;
* Interface for calculators that require a string
* @param <T>
*/
-public interface StringStats<T> extends TextStats {
- T compute(String txt);
+public interface StringStatsCalculator<T> extends TextStatsCalculator {
+ T calculate(String txt);
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TextStatsCalculator.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TextStatsCalculator.java
index 9c3ab45..b083b16 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TextStatsCalculator.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TextStatsCalculator.java
@@ -16,106 +16,8 @@
*/
package org.apache.tika.eval.textstats;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.tika.eval.langid.Language;
-import org.apache.tika.eval.langid.LanguageIDWrapper;
-import org.apache.tika.eval.tokens.TokenCounts;
-
-
-public class TextStatsCalculator {
-
- private static final String FIELD = "f";
- private final Analyzer analyzer;
- private final LanguageIDWrapper languageIDWrapper;
- private final List<LanguageAwareTokenCountStats> languageAwareTokenCountStats = new ArrayList<>();
- private final List<TokenCountStats> tokenCountStats = new ArrayList<>();
- private final List<StringStats> stringStats = new ArrayList<>();
-
- public TextStatsCalculator(List<TextStats> calculators) {
- this(calculators, null, null);
- }
-
- public TextStatsCalculator(List<TextStats> calculators, Analyzer analyzer,
- LanguageIDWrapper languageIDWrapper) {
- this.analyzer = analyzer;
- this.languageIDWrapper = languageIDWrapper;
- for (TextStats t : calculators) {
- if (t instanceof StringStats) {
- stringStats.add((StringStats)t);
- } else if (t instanceof LanguageAwareTokenCountStats) {
- languageAwareTokenCountStats.add((LanguageAwareTokenCountStats) t);
- if (languageIDWrapper == null) {
- throw new IllegalArgumentException("Must specify a LanguageIdWrapper "+
- "if you want to calculate languageAware stats: "+t.getClass());
- }
- } else if (t instanceof TokenCountStats) {
- tokenCountStats.add((TokenCountStats)t);
- if (analyzer == null) {
- throw new IllegalArgumentException(
- "Analyzer must not be null if you are using "+
- "a TokenCountStats: "+t.getClass()
- );
- }
- } else {
- throw new IllegalArgumentException(
- "I regret I don't yet handle: "+t.getClass()
- );
- }
- }
- }
-
- public Map<Class, Object> calculate(String txt) {
- Map<Class, Object> results = new HashMap<>();
- for (StringStats calc : stringStats) {
- results.put(calc.getClass(), calc.compute(txt));
- }
-
- TokenCounts tokenCounts = null;
- if (tokenCountStats.size() > 0 || languageAwareTokenCountStats.size() > 0) {
- try {
- tokenCounts = tokenize(txt);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- if (languageAwareTokenCountStats.size() > 0) {
- List<Language> langs = results.containsKey(LanguageIDWrapper.class) ?
- (List)results.get(LanguageIDWrapper.class) : languageIDWrapper.compute(txt);
- results.put(LanguageIDWrapper.class, langs);
- for (LanguageAwareTokenCountStats calc : languageAwareTokenCountStats) {
- results.put(calc.getClass(), calc.compute(langs, tokenCounts));
- }
- }
-
- for (TokenCountStats calc : tokenCountStats) {
- results.put(calc.getClass(), calc.compute(tokenCounts));
- }
- return results;
- }
-
- private TokenCounts tokenize(String txt) throws IOException {
- TokenCounts counts = new TokenCounts();
- TokenStream ts = analyzer.tokenStream(FIELD, txt);
- try {
- CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
- ts.reset();
- while (ts.incrementToken()) {
- String token = termAtt.toString();
- counts.increment(token);
- }
- } finally {
- ts.close();
- ts.end();
- }
- return counts;
- }
+/**
+ * Base text stats interface
+ */
+public interface TextStatsCalculator {
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCountPriorityQueue.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenCountPriorityQueue.java
similarity index 88%
rename from tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCountPriorityQueue.java
rename to tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenCountPriorityQueue.java
index 624da0c..69d48ee 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCountPriorityQueue.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenCountPriorityQueue.java
@@ -15,13 +15,14 @@
* limitations under the License.
*/
-package org.apache.tika.eval.tokens;
+package org.apache.tika.eval.textstats;
import org.apache.lucene.util.PriorityQueue;
+import org.apache.tika.eval.tokens.TokenIntPair;
public class TokenCountPriorityQueue extends PriorityQueue<TokenIntPair> {
- TokenCountPriorityQueue(int maxSize) {
+ public TokenCountPriorityQueue(int maxSize) {
super(maxSize);
}
@@ -32,7 +33,7 @@ public class TokenCountPriorityQueue extends PriorityQueue<TokenIntPair> {
} else if (arg0.getValue() > arg1.getValue()) {
return false;
}
- return arg1.token.compareTo(arg0.token) < 0;
+ return arg1.getToken().compareTo(arg0.getToken()) < 0;
}
public TokenIntPair[] getArray() {
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenCountStats.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenCountStatsCalculator.java
similarity index 89%
rename from tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenCountStats.java
rename to tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenCountStatsCalculator.java
index 4b3c678..325f4d5 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenCountStats.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenCountStatsCalculator.java
@@ -22,6 +22,6 @@ import org.apache.tika.eval.tokens.TokenCounts;
* Interface for calculators that require token stats
* @param <T>
*/
-public interface TokenCountStats<T> {
- T compute(TokenCounts tokenCounts);
+public interface TokenCountStatsCalculator<T> extends TextStatsCalculator {
+ T calculate(TokenCounts tokenCounts);
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenEntropy.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenEntropy.java
index 9ff87b2..e9af315 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenEntropy.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenEntropy.java
@@ -20,10 +20,10 @@ import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.commons.math3.util.FastMath;
import org.apache.tika.eval.tokens.TokenCounts;
-public class TokenEntropy implements TokenCountStats<Double> {
+public class TokenEntropy implements TokenCountStatsCalculator<Double> {
@Override
- public Double compute(TokenCounts tokenCounts) {
+ public Double calculate(TokenCounts tokenCounts) {
double ent = 0.0d;
double p = 0.0d;
double base = 2.0;
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenLengths.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenLengths.java
index a12012f..ae6e719 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenLengths.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenLengths.java
@@ -22,10 +22,10 @@ import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.tika.eval.tokens.TokenCounts;
-public class TokenLengths implements TokenCountStats<SummaryStatistics> {
+public class TokenLengths implements TokenCountStatsCalculator<SummaryStatistics> {
@Override
- public SummaryStatistics compute(TokenCounts tokenCounts) {
+ public SummaryStatistics calculate(TokenCounts tokenCounts) {
SummaryStatistics summaryStatistics = new SummaryStatistics();
for (Map.Entry<String, MutableInt> e : tokenCounts.getTokens().entrySet()) {
String token = e.getKey();
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenLengths.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TopNTokens.java
similarity index 66%
copy from tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenLengths.java
copy to tika-eval/src/main/java/org/apache/tika/eval/textstats/TopNTokens.java
index a12012f..97db630 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenLengths.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TopNTokens.java
@@ -21,19 +21,29 @@ import java.util.Map;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.tika.eval.tokens.TokenCounts;
+import org.apache.tika.eval.tokens.TokenIntPair;
-public class TokenLengths implements TokenCountStats<SummaryStatistics> {
+public class TopNTokens implements TokenCountStatsCalculator<TokenIntPair[]> {
+ private final int topN;
+ public TopNTokens(int topN) {
+ this.topN = topN;
+ }
@Override
- public SummaryStatistics compute(TokenCounts tokenCounts) {
+ public TokenIntPair[] calculate(TokenCounts tokenCounts) {
+ TokenCountPriorityQueue queue = new TokenCountPriorityQueue(topN);
+
SummaryStatistics summaryStatistics = new SummaryStatistics();
for (Map.Entry<String, MutableInt> e : tokenCounts.getTokens().entrySet()) {
String token = e.getKey();
- int cp = token.codePointCount(0, token.length());
- for (int i = 0; i < e.getValue().intValue(); i++) {
- summaryStatistics.addValue(cp);
+ int termFreq = e.getValue().intValue();
+
+ if (queue.top() == null || queue.size() < topN ||
+ termFreq >= queue.top().getValue()) {
+ queue.insertWithOverflow(new TokenIntPair(token, termFreq));
}
+
}
- return summaryStatistics;
+ return queue.getArray();
}
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/UnicodeBlockCounter.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/UnicodeBlockCounter.java
new file mode 100644
index 0000000..d6c1000
--- /dev/null
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/UnicodeBlockCounter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.textstats;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.tika.eval.AbstractProfiler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class UnicodeBlockCounter implements StringStatsCalculator<Map<String, MutableInt>> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractProfiler.class);
+
+ private final int maxContentLength;
+ public UnicodeBlockCounter(int maxContentLength) {
+ this.maxContentLength = maxContentLength;
+ }
+
+ @Override
+ public Map<String, MutableInt> calculate(String txt) {
+ if (txt.length() < 200) {
+ return Collections.EMPTY_MAP;
+ }
+ String s = txt;
+ if (s.length() > maxContentLength) {
+ s = s.substring(0, maxContentLength);
+ }
+ Map<String, MutableInt> m = new HashMap<>();
+ Reader r = new StringReader(s);
+ try {
+ int c = r.read();
+ while (c != -1) {
+ Character.UnicodeBlock block = Character.UnicodeBlock.of(c);
+ String blockString = (block == null) ? "NULL" : block.toString();
+ MutableInt i = m.get(blockString);
+ if (i == null) {
+ i = new MutableInt(0);
+ m.put(blockString, i);
+ }
+ i.increment();
+ if (block == null) {
+ blockString = "NULL";
+ }
+ m.put(blockString, i);
+ c = r.read();
+ }
+ } catch (IOException e) {
+ LOG.warn("IOException", e);
+ }
+ return m;
+ }
+}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java
index 5f02708..48dbdcd 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java
@@ -175,7 +175,6 @@ public class CommonTokenCountManager {
return;
}
-
LangModel model = null;
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(is, COMMON_TOKENS_CHARSET))) {
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenContraster.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenContraster.java
index 6cbbbc7..608b9de 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenContraster.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenContraster.java
@@ -21,6 +21,7 @@ import java.util.Map;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.lucene.util.PriorityQueue;
+import org.apache.tika.eval.textstats.TokenCountPriorityQueue;
/**
* Computes some corpus contrast statistics.
@@ -29,8 +30,8 @@ import org.apache.lucene.util.PriorityQueue;
*/
public class TokenContraster {
- private TokenStatistics tokenStatisticsA;
- private TokenStatistics tokenStatisticsB;
+ private TokenCounts tokensA;
+ private TokenCounts tokensB;
private TokenCountPriorityQueue uniqA;
private TokenCountPriorityQueue uniqB;
@@ -48,14 +49,12 @@ public class TokenContraster {
private double overlap = 0.0;
- public ContrastStatistics calculateContrastStatistics(Map<String, MutableInt> mapA,
- TokenStatistics tokenStatisticsA,
- Map<String, MutableInt> mapB,
- TokenStatistics tokenStatisticsB) {
+ public ContrastStatistics calculateContrastStatistics(TokenCounts tokensA, TokenCounts tokensB) {
reset();
- this.tokenStatisticsA = tokenStatisticsA;
- this.tokenStatisticsB = tokenStatisticsB;
-
+ this.tokensA = tokensA;
+ this.tokensB = tokensB;
+ Map<String, MutableInt> mapA = tokensA.getTokens();
+ Map<String, MutableInt> mapB = tokensB.getTokens();
for (Map.Entry<String, MutableInt> e : mapA.entrySet()) {
MutableInt bVal = mapB.get(e.getKey());
int b = (bVal == null) ? 0 : bVal.intValue();
@@ -115,12 +114,12 @@ public class TokenContraster {
private void finishComputing() {
- long sumUniqTokens = tokenStatisticsA.getTotalUniqueTokens()
- +tokenStatisticsB.getTotalUniqueTokens();
+ long sumUniqTokens = tokensA.getTotalUniqueTokens()
+ +tokensB.getTotalUniqueTokens();
diceCoefficient = (double) diceCoefficientNum / (double) sumUniqTokens;
- overlap = (float) overlapNum / (double) (tokenStatisticsA.getTotalTokens() +
- tokenStatisticsB.getTotalTokens());
+ overlap = (float) overlapNum / (double) (tokensA.getTotalTokens() +
+ tokensB.getTotalTokens());
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCounter.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCounter.java
index 890dd22..684eecc 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCounter.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCounter.java
@@ -27,7 +27,14 @@ import org.apache.commons.math3.util.FastMath;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.tika.eval.textstats.TokenCountPriorityQueue;
+/**
+ * @deprecated use {@link org.apache.tika.eval.textstats.CompositeTextStatsCalculator}
+ * with {@link org.apache.tika.eval.textstats.TokenEntropy}, {@link org.apache.tika.eval.textstats.TokenLengths}
+ * and {@link org.apache.tika.eval.textstats.TopNTokens}.
+ */
+@Deprecated
public class TokenCounter {
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCounts.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCounts.java
index f942800..9fc32f7 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCounts.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/TokenCounts.java
@@ -44,4 +44,8 @@ public class TokenCounts {
public int getTotalTokens() {
return totalTokens;
}
+
+ public int getTotalUniqueTokens() {
+ return tokens.size();
+ }
}
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/textstats/TestTextStatsCalculator.java b/tika-eval/src/test/java/org/apache/tika/eval/textstats/TestTextStatsCalculator.java
index ea75465..728d0fc 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/textstats/TestTextStatsCalculator.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/textstats/TestTextStatsCalculator.java
@@ -1,23 +1,17 @@
package org.apache.tika.eval.textstats;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.text.DecimalFormat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.lucene.analysis.Analyzer;
import org.apache.tika.eval.langid.LanguageIDWrapper;
import org.apache.tika.eval.tokens.AnalyzerManager;
import org.apache.tika.eval.tokens.CommonTokenCountManager;
-import org.apache.tika.eval.tokens.CommonTokenResult;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -34,84 +28,23 @@ public class TestTextStatsCalculator {
@Test
public void testKL() throws Exception {
- List<TextStats> stats = new ArrayList<>();
+ List<TextStatsCalculator> stats = new ArrayList<>();
stats.add(new CommonTokensKLDivergence(COMMON_TOKENS));
- TextStatsCalculator calc = new TextStatsCalculator(stats, ANALYZER,
+ CompositeTextStatsCalculator calc = new CompositeTextStatsCalculator(stats, ANALYZER,
new LanguageIDWrapper());
String txt = "qwertyuiopsdflkasjf;laskjfada asdfasd asdfa fasdf sdfasdfas ";
Map<Class, Object> results = calc.calculate(txt);
- System.out.println(results);
+ assertNotNull(results.get(CommonTokensKLDivergence.class));
+ assertEquals((double)results.get(CommonTokensKLDivergence.class), 16.611, 0.01);
}
- @Test
- public void testOneOff() throws Exception {
- Path p = Paths.get(//fill in here);
- List<String> lines = FileUtils.readLines(p.toFile(), StandardCharsets.UTF_8);
- List<TextStats> stats = new ArrayList<>();
+ @Test(expected = IllegalArgumentException.class)
+ public void testKLNoLangIdWrapper() throws Exception {
+ List<TextStatsCalculator> stats = new ArrayList<>();
stats.add(new CommonTokensKLDivergence(COMMON_TOKENS));
- stats.add(new CommonTokensKLDNormed(COMMON_TOKENS));
- stats.add(new CommonTokensCosine(COMMON_TOKENS));
- stats.add(new CommonTokens(COMMON_TOKENS));
- stats.add(new CommonTokensHellinger(COMMON_TOKENS));
- stats.add(new CommonTokensBhattacharyya(COMMON_TOKENS));
- TextStatsCalculator calc = new TextStatsCalculator(stats, ANALYZER,
- new LanguageIDWrapper());
- System.out.println("Chars\ttokens\tkld\tkld_normed\tcosine\thellinger\tbhattacharyya\toov");
- DecimalFormat df = new DecimalFormat("#.###");
- for (int i = 100; i <= 100000; i += 100) {
- Map<Object, SummaryStatistics> map = new HashMap<>();
- for (int j = 0; j < 30; j++) {
- Collections.shuffle(lines);
- Map<Class, Object> results = calc.calculate(substring(lines, i));
- CommonTokenResult commonTokenResult = ((CommonTokenResult) results.get(CommonTokens.class));
- for (TextStats textStats : stats) {
- SummaryStatistics summaryStatistics = map.get(textStats.getClass());
- if (summaryStatistics == null) {
- summaryStatistics = new SummaryStatistics();
- map.put(textStats.getClass(), summaryStatistics);
- }
- if (textStats instanceof CommonTokens) {
- summaryStatistics.addValue(
- ((CommonTokenResult)results.get(textStats.getClass())).getOOV());
- } else {
- summaryStatistics.addValue((double) results.get(textStats.getClass()));
- }
- }
- }
- StringBuilder sb = new StringBuilder();
- sb.append(i).append("\t");
- for (TextStats textStats :stats) {
- sb.append(df.format(map.get(textStats.getClass()).getMean())).append("\t");
- sb.append(
- df.format(
- map.get(textStats.getClass()).getStandardDeviation())).append("\t");
- }
- System.out.println(sb.toString());
- }
-/*
- for (int i = 2000; i <= txt.length(); i += 1000) {
- Map<Class, Object> results = calc.calculate(txt.substring(0, i));
- CommonTokenResult commonTokenResult = ((CommonTokenResult)results.get(CommonTokens.class));
- System.out.println(i + "\t" + commonTokenResult.getAlphabeticTokens() +
- "\t" + results.get(CommonTokensKLDivergence.class) +
- "\t" + results.get(CommonTokensKLDNormed.class) +
- "\t" + results.get(CommonTokensCosine.class) +
- "\t" + results.get(CommonTokensHellinger.class) +
- "\t" + results.get(CommonTokensBhattacharyya.class)
- + "\t"+commonTokenResult.getOOV());
- }*/
+ CompositeTextStatsCalculator calc = new CompositeTextStatsCalculator(stats);
+ String txt = "qwertyuiopsdflkasjf;laskjfada asdfasd asdfa fasdf sdfasdfas ";
+ Map<Class, Object> results = calc.calculate(txt);
}
- private String substring(List<String> lines, int length) {
- StringBuilder sb = new StringBuilder();
-
- for (String line : lines) {
- if (sb.length() > length) {
- sb.substring(0, length);
- return sb.toString();
- }
- sb.append(" ").append(line);
- }
- return sb.toString();
- }
}
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/tokens/LuceneTokenCounter.java b/tika-eval/src/test/java/org/apache/tika/eval/tokens/LuceneTokenCounter.java
index db37450..3124006 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/tokens/LuceneTokenCounter.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/tokens/LuceneTokenCounter.java
@@ -30,6 +30,7 @@ import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.memory.MemoryIndex;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.util.BytesRef;
+import org.apache.tika.eval.textstats.TokenCountPriorityQueue;
/**
* Experimental class uses Lucene's MemoryIndex to effectively build the
diff --git a/tika-eval/src/test/resources/common_tokens/en b/tika-eval/src/test/resources/common_tokens/en
index 8d442fe..f015096 100644
--- a/tika-eval/src/test/resources/common_tokens/en
+++ b/tika-eval/src/test/resources/common_tokens/en
@@ -1,3 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#DOC_COUNT 5789407
+#SUM_DOC_FREQS 82914840
+#SUM_TERM_FREQS 90163584
+#UNIQUE_TERMS 926669
+#TOKEN DOCFREQ TERMFREQ
the
of
and
diff --git a/tika-eval/src/test/resources/common_tokens/es b/tika-eval/src/test/resources/common_tokens/es
index b9bfd03..c8f36f6 100644
--- a/tika-eval/src/test/resources/common_tokens/es
+++ b/tika-eval/src/test/resources/common_tokens/es
@@ -1,3 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#DOC_COUNT 5789407
+#SUM_DOC_FREQS 82914840
+#SUM_TERM_FREQS 90163584
+#UNIQUE_TERMS 926669
+#TOKEN DOCFREQ TERMFREQ
la
de
y
diff --git a/tika-eval/src/test/resources/common_tokens/zh-cn b/tika-eval/src/test/resources/common_tokens/zh-cn
index bec617d..55d2671 100644
--- a/tika-eval/src/test/resources/common_tokens/zh-cn
+++ b/tika-eval/src/test/resources/common_tokens/zh-cn
@@ -1,3 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#DOC_COUNT 5789407
+#SUM_DOC_FREQS 82914840
+#SUM_TERM_FREQS 90163584
+#UNIQUE_TERMS 926669
+#TOKEN DOCFREQ TERMFREQ
的
的
和
diff --git a/tika-eval/src/test/resources/common_tokens/zh-tw b/tika-eval/src/test/resources/common_tokens/zh-tw
index bc91291..0bcc0c8 100644
--- a/tika-eval/src/test/resources/common_tokens/zh-tw
+++ b/tika-eval/src/test/resources/common_tokens/zh-tw
@@ -1,3 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#DOC_COUNT 5789407
+#SUM_DOC_FREQS 82914840
+#SUM_TERM_FREQS 90163584
+#UNIQUE_TERMS 926669
+#TOKEN DOCFREQ TERMFREQ
的
的
和