You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by to...@apache.org on 2016/05/26 14:14:27 UTC

[2/2] lucene-solr:master: LUCENE-7303 - avoid NPE in MultiFields.getTerms(leafReader, classFieldName), removed duplicated code in DocumentSNBC

LUCENE-7303 - avoid NPE in MultiFields.getTerms(leafReader, classFieldName), removed duplicated code in DocumentSNBC


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8808cf53
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8808cf53
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8808cf53

Branch: refs/heads/master
Commit: 8808cf5373522f37bce509729b0b3a7fc9bcbd88
Parents: 2aabed4
Author: Tommaso Teofili <to...@apache.org>
Authored: Thu May 26 15:59:12 2016 +0200
Committer: Tommaso Teofili <to...@apache.org>
Committed: Thu May 26 16:13:58 2016 +0200

----------------------------------------------------------------------
 .../SimpleNaiveBayesClassifier.java             | 28 +++++++++++---------
 .../SimpleNaiveBayesDocumentClassifier.java     | 22 ---------------
 2 files changed, 16 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8808cf53/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
index 73c90de..2514ae1 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
@@ -145,18 +145,19 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
     List<ClassificationResult<BytesRef>> assignedClasses = new ArrayList<>();
 
     Terms classes = MultiFields.getTerms(leafReader, classFieldName);
-    TermsEnum classesEnum = classes.iterator();
-    BytesRef next;
-    String[] tokenizedText = tokenize(inputDocument);
-    int docsWithClassSize = countDocsWithClass();
-    while ((next = classesEnum.next()) != null) {
-      if (next.length > 0) {
-        Term term = new Term(this.classFieldName, next);
-        double clVal = calculateLogPrior(term, docsWithClassSize) + calculateLogLikelihood(tokenizedText, term, docsWithClassSize);
-        assignedClasses.add(new ClassificationResult<>(term.bytes(), clVal));
+    if (classes != null) {
+      TermsEnum classesEnum = classes.iterator();
+      BytesRef next;
+      String[] tokenizedText = tokenize(inputDocument);
+      int docsWithClassSize = countDocsWithClass();
+      while ((next = classesEnum.next()) != null) {
+        if (next.length > 0) {
+          Term term = new Term(this.classFieldName, next);
+          double clVal = calculateLogPrior(term, docsWithClassSize) + calculateLogLikelihood(tokenizedText, term, docsWithClassSize);
+          assignedClasses.add(new ClassificationResult<>(term.bytes(), clVal));
+        }
       }
     }
-
     // normalization; the values transforms to a 0-1 range
     return normClassificationResults(assignedClasses);
   }
@@ -168,8 +169,9 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
    * @throws IOException if accessing to term vectors or search fails
    */
   protected int countDocsWithClass() throws IOException {
-    int docCount = MultiFields.getTerms(this.leafReader, this.classFieldName).getDocCount();
-    if (docCount == -1) { // in case codec doesn't support getDocCount
+    Terms terms = MultiFields.getTerms(this.leafReader, this.classFieldName);
+    int docCount;
+    if (terms == null || terms.getDocCount() == -1) { // in case codec doesn't support getDocCount
       TotalHitCountCollector classQueryCountCollector = new TotalHitCountCollector();
       BooleanQuery.Builder q = new BooleanQuery.Builder();
       q.add(new BooleanClause(new WildcardQuery(new Term(classFieldName, String.valueOf(WildcardQuery.WILDCARD_STRING))), BooleanClause.Occur.MUST));
@@ -179,6 +181,8 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
       indexSearcher.search(q.build(),
           classQueryCountCollector);
       docCount = classQueryCountCollector.getTotalHits();
+    } else {
+      docCount = terms.getDocCount();
     }
     return docCount;
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8808cf53/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
index 3dc01bb..2c5a493 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
@@ -169,28 +169,6 @@ public class SimpleNaiveBayesDocumentClassifier extends SimpleNaiveBayesClassifi
   }
 
   /**
-   * Counts the number of documents in the index having at least a value for the 'class' field
-   *
-   * @return the no. of documents having a value for the 'class' field
-   * @throws java.io.IOException If accessing to term vectors or search fails
-   */
-  protected int countDocsWithClass() throws IOException {
-    int docCount = MultiFields.getTerms(this.leafReader, this.classFieldName).getDocCount();
-    if (docCount == -1) { // in case codec doesn't support getDocCount
-      TotalHitCountCollector classQueryCountCollector = new TotalHitCountCollector();
-      BooleanQuery.Builder q = new BooleanQuery.Builder();
-      q.add(new BooleanClause(new WildcardQuery(new Term(classFieldName, String.valueOf(WildcardQuery.WILDCARD_STRING))), BooleanClause.Occur.MUST));
-      if (query != null) {
-        q.add(query, BooleanClause.Occur.MUST);
-      }
-      indexSearcher.search(q.build(),
-          classQueryCountCollector);
-      docCount = classQueryCountCollector.getTotalHits();
-    }
-    return docCount;
-  }
-
-  /**
    * Returns a token array from the {@link org.apache.lucene.analysis.TokenStream} in input
    *
    * @param tokenizedText the tokenized content of a field