You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/06/17 20:59:31 UTC

svn commit: r1136979 - in /lucene/dev/branches/flexscoring: lucene/contrib/queries/src/java/org/apache/lucene/search/ lucene/contrib/queries/src/java/org/apache/lucene/search/similar/ lucene/src/java/org/apache/lucene/search/ lucene/src/test/org/apache...

Author: rmuir
Date: Fri Jun 17 18:59:30 2011
New Revision: 1136979

URL: http://svn.apache.org/viewvc?rev=1136979&view=rev
Log:
clean up some obselete nocommits

Modified:
    lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
    lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
    lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
    lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
    lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java
    lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java
    lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/IDFValueSource.java
    lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/TFValueSource.java

Modified: lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java Fri Jun 17 18:59:30 2011
@@ -51,7 +51,10 @@ import org.apache.lucene.util.PriorityQu
  */
 public class FuzzyLikeThisQuery extends Query
 {
-    //nocommit? this query is pretty much hardcoded at TF/IDF
+    // TODO: generalize this query (at least it should not reuse this static sim!
+    // a better way might be to convert this into multitermquery rewrite methods.
+    // the rewrite method can 'average' the TermContext's term statistics (docfreq,totalTermFreq) 
+    // provided to TermQuery, so that the general idea is agnostic to any scoring system...
     static TFIDFSimilarity sim=new DefaultSimilarity();
     Query rewrittenQuery=null;
     ArrayList<FieldVals> fieldVals=new ArrayList<FieldVals>();

Modified: lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java Fri Jun 17 18:59:30 2011
@@ -286,7 +286,6 @@ public final class MoreLikeThis {
     /**
      * For idf() calculations.
      */
-    // nocommit? this is pretty much wired to tf-idf things...
     private TFIDFSimilarity similarity;// = new DefaultSimilarity();
 
     /**

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/PhraseQuery.java Fri Jun 17 18:59:30 2011
@@ -27,8 +27,10 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.TermState;
+import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.Similarity.SloppyDocScorer;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.TermContext;
 import org.apache.lucene.util.ToStringUtils;
 import org.apache.lucene.util.ArrayUtil;
@@ -213,8 +215,10 @@ public class PhraseQuery extends Query {
       for (int i = 0; i < terms.size(); i++) {
         final Term t = terms.get(i);
         final TermState state = states[i].get(context.ord);
-        if (state == null) /* term doesnt exist in this segment */
+        if (state == null) { /* term doesnt exist in this segment */
+          assert termNotInReader(reader, field, t.bytes()) : "no termstate found but term exists in reader";
           return null;
+        }
         DocsAndPositionsEnum postingsEnum = reader.termPositionsEnum(delDocs,
                                                                      t.field(),
                                                                      t.bytes(),
@@ -222,14 +226,9 @@ public class PhraseQuery extends Query {
         // PhraseQuery on a field that did not index
         // positions.
         if (postingsEnum == null) {
-          if (reader.termDocsEnum(delDocs, t.field(), t.bytes(), state) != null) {
-            // term does exist, but has no positions
-            throw new IllegalStateException("field \"" + t.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + t.text() + ")");
-          } else {
-            // term does not exist
-            // nocommit: should be impossible, state should be null?
-            return null;
-          }
+          assert (reader.termDocsEnum(delDocs, t.field(), t.bytes(), state) != null) : "termstate found but no term exists in reader";
+          // term does exist, but has no positions
+          throw new IllegalStateException("field \"" + t.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + t.text() + ")");
         }
         // get the docFreq without seeking
         TermsEnum te = reader.fields().terms(field).getThreadTermsEnum();
@@ -254,6 +253,12 @@ public class PhraseQuery extends Query {
           new SloppyPhraseScorer(this, postingsFreqs, similarity, slop, similarity.sloppyDocScorer(stats, field, context));
       }
     }
+    
+    private boolean termNotInReader(IndexReader reader, String field, BytesRef bytes) throws IOException {
+      // only called from assert
+      final Terms terms = reader.terms(field);
+      return terms == null || terms.docFreq(bytes) == 0;
+    }
 
     @Override
     public Explanation explain(AtomicReaderContext context, int doc) throws IOException {

Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Fri Jun 17 18:59:30 2011
@@ -20,7 +20,11 @@ package org.apache.lucene.search;
 import java.io.IOException;
 
 import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.search.Similarity.ExactDocScorer;
+import org.apache.lucene.search.Similarity.SloppyDocScorer;
+import org.apache.lucene.search.Similarity.Stats;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.TermContext;
 import org.apache.lucene.index.FieldInvertState;
 import org.apache.lucene.util.PriorityQueue;
 
@@ -240,28 +244,42 @@ final class JustCompileSearch {
     }
   }
   
-  // nocommit: extend the Base Similarity here?
-  static final class JustCompileSimilarity extends TFIDFSimilarity {
+  static final class JustCompileSimilarity extends Similarity {
 
     @Override
-    public float idf(int docFreq, int numDocs) {
+    public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
 
     @Override
-    public float computeNorm(FieldInvertState state) {
+    public ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
 
     @Override
-    public float sloppyFreq(int distance) {
+    public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
 
     @Override
-    public float tf(float freq) {
+    public float decodeNormValue(byte b) {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
-    }  
+    }
+
+    @Override
+    public byte encodeNormValue(float f) {
+      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+    }
+
+    @Override
+    public float computeNorm(FieldInvertState state) {
+      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+    }
+
+    @Override
+    public float sloppyFreq(int distance) {
+      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+    }
   }
   
   static final class JustCompileSimilarityProvider implements SimilarityProvider {

Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java Fri Jun 17 18:59:30 2011
@@ -24,9 +24,4 @@ public class MockBM25SimilarityProvider 
   public Similarity get(String field) {
     return impl;
   }
-
-  //nocommit: maybe ok given how bm25 works?
-  public float queryNorm(float sumOfSquaredWeights) {
-    return 1f;
-  }
 }

Modified: lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java (original)
+++ lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java Fri Jun 17 18:59:30 2011
@@ -54,9 +54,6 @@ public class QueryDriver {
     FSDirectory dir = FSDirectory.open(new File(args[3]));
     String fieldSpec = args.length == 5 ? args[4] : "T"; // default to Title-only if not specified.
     IndexSearcher searcher = new IndexSearcher(dir, true);
-    // nocommit
-    //searcher.setSimilarityProvider(new MockLMSimilarityProvider());
-    //searcher.setSimilarityProvider(new MockBM25SimilarityProvider());
     int maxResults = 1000;
     String docNameField = "docname";
 

Modified: lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/IDFValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/IDFValueSource.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/IDFValueSource.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/IDFValueSource.java Fri Jun 17 18:59:30 2011
@@ -42,14 +42,10 @@ public class IDFValueSource extends DocF
   public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
     IndexSearcher searcher = (IndexSearcher)context.get("searcher");
     Similarity sim = searcher.getSimilarityProvider().get(field);
-    // nocommit:
-    // what to do? its an idf valuesource... we could generalize to sim.computeWeight though 
-    // (which is idf for TF/IDF and something like it elsewhere)
     if (!(sim instanceof TFIDFSimilarity)) {
-      throw new UnsupportedOperationException("only works with TF/IDF Similarity");
+      throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
     }
-    // todo: we need docFreq that takes a BytesRef
-    int docfreq = searcher.docFreq(new Term(indexedField, indexedBytes.utf8ToString()));
+    int docfreq = searcher.docFreq(new Term(indexedField, indexedBytes));
     float idf = ((TFIDFSimilarity)sim).idf(docfreq, searcher.maxDoc());
     return new ConstDoubleDocValues(idf, this);
   }

Modified: lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/TFValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/TFValueSource.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/TFValueSource.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/TFValueSource.java Fri Jun 17 18:59:30 2011
@@ -43,11 +43,9 @@ public class TFValueSource extends TermF
   public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
     Fields fields = readerContext.reader.fields();
     final Terms terms = fields.terms(field);
-    // nocommit:
-    // what to do? its a TF valuesource... 
     final Similarity sim = ((IndexSearcher)context.get("searcher")).getSimilarityProvider().get(field);
     if (!(sim instanceof TFIDFSimilarity)) {
-      throw new UnsupportedOperationException("only works with TF/IDF Similarity");
+      throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
     }
     final TFIDFSimilarity similarity = (TFIDFSimilarity) sim;