You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by to...@apache.org on 2016/05/26 14:14:27 UTC
[2/2] lucene-solr:master: LUCENE-7303 - avoid NPE in
MultiFields.getTerms(leafReader, classFieldName),
removed duplicated code in DocumentSNBC
LUCENE-7303 - avoid NPE in MultiFields.getTerms(leafReader, classFieldName), removed duplicated code in DocumentSNBC
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8808cf53
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8808cf53
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8808cf53
Branch: refs/heads/master
Commit: 8808cf5373522f37bce509729b0b3a7fc9bcbd88
Parents: 2aabed4
Author: Tommaso Teofili <to...@apache.org>
Authored: Thu May 26 15:59:12 2016 +0200
Committer: Tommaso Teofili <to...@apache.org>
Committed: Thu May 26 16:13:58 2016 +0200
----------------------------------------------------------------------
.../SimpleNaiveBayesClassifier.java | 28 +++++++++++---------
.../SimpleNaiveBayesDocumentClassifier.java | 22 ---------------
2 files changed, 16 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8808cf53/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
index 73c90de..2514ae1 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
@@ -145,18 +145,19 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
List<ClassificationResult<BytesRef>> assignedClasses = new ArrayList<>();
Terms classes = MultiFields.getTerms(leafReader, classFieldName);
- TermsEnum classesEnum = classes.iterator();
- BytesRef next;
- String[] tokenizedText = tokenize(inputDocument);
- int docsWithClassSize = countDocsWithClass();
- while ((next = classesEnum.next()) != null) {
- if (next.length > 0) {
- Term term = new Term(this.classFieldName, next);
- double clVal = calculateLogPrior(term, docsWithClassSize) + calculateLogLikelihood(tokenizedText, term, docsWithClassSize);
- assignedClasses.add(new ClassificationResult<>(term.bytes(), clVal));
+ if (classes != null) {
+ TermsEnum classesEnum = classes.iterator();
+ BytesRef next;
+ String[] tokenizedText = tokenize(inputDocument);
+ int docsWithClassSize = countDocsWithClass();
+ while ((next = classesEnum.next()) != null) {
+ if (next.length > 0) {
+ Term term = new Term(this.classFieldName, next);
+ double clVal = calculateLogPrior(term, docsWithClassSize) + calculateLogLikelihood(tokenizedText, term, docsWithClassSize);
+ assignedClasses.add(new ClassificationResult<>(term.bytes(), clVal));
+ }
}
}
-
// normalization; the values transforms to a 0-1 range
return normClassificationResults(assignedClasses);
}
@@ -168,8 +169,9 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
* @throws IOException if accessing to term vectors or search fails
*/
protected int countDocsWithClass() throws IOException {
- int docCount = MultiFields.getTerms(this.leafReader, this.classFieldName).getDocCount();
- if (docCount == -1) { // in case codec doesn't support getDocCount
+ Terms terms = MultiFields.getTerms(this.leafReader, this.classFieldName);
+ int docCount;
+ if (terms == null || terms.getDocCount() == -1) { // in case codec doesn't support getDocCount
TotalHitCountCollector classQueryCountCollector = new TotalHitCountCollector();
BooleanQuery.Builder q = new BooleanQuery.Builder();
q.add(new BooleanClause(new WildcardQuery(new Term(classFieldName, String.valueOf(WildcardQuery.WILDCARD_STRING))), BooleanClause.Occur.MUST));
@@ -179,6 +181,8 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
indexSearcher.search(q.build(),
classQueryCountCollector);
docCount = classQueryCountCollector.getTotalHits();
+ } else {
+ docCount = terms.getDocCount();
}
return docCount;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8808cf53/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
index 3dc01bb..2c5a493 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
@@ -169,28 +169,6 @@ public class SimpleNaiveBayesDocumentClassifier extends SimpleNaiveBayesClassifi
}
/**
- * Counts the number of documents in the index having at least a value for the 'class' field
- *
- * @return the no. of documents having a value for the 'class' field
- * @throws java.io.IOException If accessing to term vectors or search fails
- */
- protected int countDocsWithClass() throws IOException {
- int docCount = MultiFields.getTerms(this.leafReader, this.classFieldName).getDocCount();
- if (docCount == -1) { // in case codec doesn't support getDocCount
- TotalHitCountCollector classQueryCountCollector = new TotalHitCountCollector();
- BooleanQuery.Builder q = new BooleanQuery.Builder();
- q.add(new BooleanClause(new WildcardQuery(new Term(classFieldName, String.valueOf(WildcardQuery.WILDCARD_STRING))), BooleanClause.Occur.MUST));
- if (query != null) {
- q.add(query, BooleanClause.Occur.MUST);
- }
- indexSearcher.search(q.build(),
- classQueryCountCollector);
- docCount = classQueryCountCollector.getTotalHits();
- }
- return docCount;
- }
-
- /**
* Returns a token array from the {@link org.apache.lucene.analysis.TokenStream} in input
*
* @param tokenizedText the tokenized content of a field