You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/06/17 20:59:31 UTC
svn commit: r1136979 - in /lucene/dev/branches/flexscoring:
lucene/contrib/queries/src/java/org/apache/lucene/search/
lucene/contrib/queries/src/java/org/apache/lucene/search/similar/
lucene/src/java/org/apache/lucene/search/ lucene/src/test/org/apache...
Author: rmuir
Date: Fri Jun 17 18:59:30 2011
New Revision: 1136979
URL: http://svn.apache.org/viewvc?rev=1136979&view=rev
Log:
clean up some obselete nocommits
Modified:
lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java
lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java
lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/IDFValueSource.java
lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/TFValueSource.java
Modified: lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java Fri Jun 17 18:59:30 2011
@@ -51,7 +51,10 @@ import org.apache.lucene.util.PriorityQu
*/
public class FuzzyLikeThisQuery extends Query
{
- //nocommit? this query is pretty much hardcoded at TF/IDF
+ // TODO: generalize this query (at least it should not reuse this static sim!
+ // a better way might be to convert this into multitermquery rewrite methods.
+ // the rewrite method can 'average' the TermContext's term statistics (docfreq,totalTermFreq)
+ // provided to TermQuery, so that the general idea is agnostic to any scoring system...
static TFIDFSimilarity sim=new DefaultSimilarity();
Query rewrittenQuery=null;
ArrayList<FieldVals> fieldVals=new ArrayList<FieldVals>();
Modified: lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java Fri Jun 17 18:59:30 2011
@@ -286,7 +286,6 @@ public final class MoreLikeThis {
/**
* For idf() calculations.
*/
- // nocommit? this is pretty much wired to tf-idf things...
private TFIDFSimilarity similarity;// = new DefaultSimilarity();
/**
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/PhraseQuery.java Fri Jun 17 18:59:30 2011
@@ -27,8 +27,10 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermState;
+import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.Similarity.SloppyDocScorer;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.util.ArrayUtil;
@@ -213,8 +215,10 @@ public class PhraseQuery extends Query {
for (int i = 0; i < terms.size(); i++) {
final Term t = terms.get(i);
final TermState state = states[i].get(context.ord);
- if (state == null) /* term doesnt exist in this segment */
+ if (state == null) { /* term doesnt exist in this segment */
+ assert termNotInReader(reader, field, t.bytes()) : "no termstate found but term exists in reader";
return null;
+ }
DocsAndPositionsEnum postingsEnum = reader.termPositionsEnum(delDocs,
t.field(),
t.bytes(),
@@ -222,14 +226,9 @@ public class PhraseQuery extends Query {
// PhraseQuery on a field that did not index
// positions.
if (postingsEnum == null) {
- if (reader.termDocsEnum(delDocs, t.field(), t.bytes(), state) != null) {
- // term does exist, but has no positions
- throw new IllegalStateException("field \"" + t.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + t.text() + ")");
- } else {
- // term does not exist
- // nocommit: should be impossible, state should be null?
- return null;
- }
+ assert (reader.termDocsEnum(delDocs, t.field(), t.bytes(), state) != null) : "termstate found but no term exists in reader";
+ // term does exist, but has no positions
+ throw new IllegalStateException("field \"" + t.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + t.text() + ")");
}
// get the docFreq without seeking
TermsEnum te = reader.fields().terms(field).getThreadTermsEnum();
@@ -254,6 +253,12 @@ public class PhraseQuery extends Query {
new SloppyPhraseScorer(this, postingsFreqs, similarity, slop, similarity.sloppyDocScorer(stats, field, context));
}
}
+
+ private boolean termNotInReader(IndexReader reader, String field, BytesRef bytes) throws IOException {
+ // only called from assert
+ final Terms terms = reader.terms(field);
+ return terms == null || terms.docFreq(bytes) == 0;
+ }
@Override
public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Fri Jun 17 18:59:30 2011
@@ -20,7 +20,11 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.search.Similarity.ExactDocScorer;
+import org.apache.lucene.search.Similarity.SloppyDocScorer;
+import org.apache.lucene.search.Similarity.Stats;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.util.PriorityQueue;
@@ -240,28 +244,42 @@ final class JustCompileSearch {
}
}
- // nocommit: extend the Base Similarity here?
- static final class JustCompileSimilarity extends TFIDFSimilarity {
+ static final class JustCompileSimilarity extends Similarity {
@Override
- public float idf(int docFreq, int numDocs) {
+ public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
@Override
- public float computeNorm(FieldInvertState state) {
+ public ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
@Override
- public float sloppyFreq(int distance) {
+ public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
@Override
- public float tf(float freq) {
+ public float decodeNormValue(byte b) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
- }
+ }
+
+ @Override
+ public byte encodeNormValue(float f) {
+ throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+ }
+
+ @Override
+ public float computeNorm(FieldInvertState state) {
+ throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+ }
+
+ @Override
+ public float sloppyFreq(int distance) {
+ throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+ }
}
static final class JustCompileSimilarityProvider implements SimilarityProvider {
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java Fri Jun 17 18:59:30 2011
@@ -24,9 +24,4 @@ public class MockBM25SimilarityProvider
public Similarity get(String field) {
return impl;
}
-
- //nocommit: maybe ok given how bm25 works?
- public float queryNorm(float sumOfSquaredWeights) {
- return 1f;
- }
}
Modified: lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java (original)
+++ lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java Fri Jun 17 18:59:30 2011
@@ -54,9 +54,6 @@ public class QueryDriver {
FSDirectory dir = FSDirectory.open(new File(args[3]));
String fieldSpec = args.length == 5 ? args[4] : "T"; // default to Title-only if not specified.
IndexSearcher searcher = new IndexSearcher(dir, true);
- // nocommit
- //searcher.setSimilarityProvider(new MockLMSimilarityProvider());
- //searcher.setSimilarityProvider(new MockBM25SimilarityProvider());
int maxResults = 1000;
String docNameField = "docname";
Modified: lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/IDFValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/IDFValueSource.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/IDFValueSource.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/IDFValueSource.java Fri Jun 17 18:59:30 2011
@@ -42,14 +42,10 @@ public class IDFValueSource extends DocF
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
IndexSearcher searcher = (IndexSearcher)context.get("searcher");
Similarity sim = searcher.getSimilarityProvider().get(field);
- // nocommit:
- // what to do? its an idf valuesource... we could generalize to sim.computeWeight though
- // (which is idf for TF/IDF and something like it elsewhere)
if (!(sim instanceof TFIDFSimilarity)) {
- throw new UnsupportedOperationException("only works with TF/IDF Similarity");
+ throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
}
- // todo: we need docFreq that takes a BytesRef
- int docfreq = searcher.docFreq(new Term(indexedField, indexedBytes.utf8ToString()));
+ int docfreq = searcher.docFreq(new Term(indexedField, indexedBytes));
float idf = ((TFIDFSimilarity)sim).idf(docfreq, searcher.maxDoc());
return new ConstDoubleDocValues(idf, this);
}
Modified: lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/TFValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/TFValueSource.java?rev=1136979&r1=1136978&r2=1136979&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/TFValueSource.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/TFValueSource.java Fri Jun 17 18:59:30 2011
@@ -43,11 +43,9 @@ public class TFValueSource extends TermF
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
Fields fields = readerContext.reader.fields();
final Terms terms = fields.terms(field);
- // nocommit:
- // what to do? its a TF valuesource...
final Similarity sim = ((IndexSearcher)context.get("searcher")).getSimilarityProvider().get(field);
if (!(sim instanceof TFIDFSimilarity)) {
- throw new UnsupportedOperationException("only works with TF/IDF Similarity");
+ throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
}
final TFIDFSimilarity similarity = (TFIDFSimilarity) sim;