You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/01/25 14:59:28 UTC
svn commit: r1235747 - in /lucene/dev/trunk/lucene/src:
java/org/apache/lucene/search/similarities/
java/org/apache/lucene/search/spans/ test/org/apache/lucene/search/spans/
Author: rmuir
Date: Wed Jan 25 13:59:28 2012
New Revision: 1235747
URL: http://svn.apache.org/viewvc?rev=1235747&view=rev
Log:
LUCENE-3723: add basic sanity checks for stats in sims
Modified:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java?rev=1235747&r1=1235746&r2=1235747&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java Wed Jan 25 13:59:28 2012
@@ -154,6 +154,7 @@ public class BM25Similarity extends Simi
@Override
public final Stats computeStats(CollectionStatistics collectionStats, float queryBoost, TermStatistics... termStats) {
+ assert collectionStats.sumTotalTermFreq() >= -1;
Explanation idf = termStats.length == 1 ? idfExplain(collectionStats, termStats[0]) : idfExplain(collectionStats, termStats);
float avgdl = avgFieldLength(collectionStats);
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java?rev=1235747&r1=1235746&r2=1235747&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java Wed Jan 25 13:59:28 2012
@@ -87,6 +87,10 @@ public abstract class SimilarityBase ext
/** Fills all member fields defined in {@code BasicStats} in {@code stats}.
* Subclasses can override this method to fill additional stats. */
protected void fillBasicStats(BasicStats stats, CollectionStatistics collectionStats, TermStatistics termStats) {
+ assert termStats.totalTermFreq() >= -1;
+ assert termStats.totalTermFreq() == -1 || termStats.totalTermFreq() >= termStats.docFreq();
+ assert collectionStats.sumTotalTermFreq() >= -1;
+ assert collectionStats.sumTotalTermFreq() == -1 || collectionStats.sumTotalTermFreq() >= termStats.totalTermFreq();
int numberOfDocuments = collectionStats.maxDoc();
int docFreq = termStats.docFreq();
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java?rev=1235747&r1=1235746&r2=1235747&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java Wed Jan 25 13:59:28 2012
@@ -70,8 +70,9 @@ import org.apache.lucene.util.ToStringUt
* and 'jones' in position 1). </p>
*
* <p>Note: as {@link #getField()} returns the masked field, scoring will be
- * done using the norms of the field name supplied. This may lead to unexpected
- * scoring behaviour.</p>
+ * done using the Similarity and collection statistics of the field name supplied,
+ * but with the term statistics of the real field. This may lead to exceptions,
+ * poor performance, and unexpected scoring behaviour.</p>
*/
public class FieldMaskingSpanQuery extends SpanQuery {
private SpanQuery maskedQuery;
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java?rev=1235747&r1=1235746&r2=1235747&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java Wed Jan 25 13:59:28 2012
@@ -31,6 +31,7 @@ import org.apache.lucene.search.CheckHit
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryUtils;
+import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.AfterClass;
@@ -240,6 +241,8 @@ public class TestFieldMaskingSpanQuery e
}
public void testSimple2() throws Exception {
+ assumeTrue("Broken scoring: LUCENE-3723",
+ searcher.getSimilarityProvider().get("id") instanceof TFIDFSimilarity);
SpanQuery q1 = new SpanTermQuery(new Term("gender", "female"));
SpanQuery q2 = new SpanTermQuery(new Term("last", "smith"));
SpanQuery q = new SpanNearQuery(new SpanQuery[]
@@ -310,6 +313,8 @@ public class TestFieldMaskingSpanQuery e
}
public void testSpans2() throws Exception {
+ assumeTrue("Broken scoring: LUCENE-3723",
+ searcher.getSimilarityProvider().get("id") instanceof TFIDFSimilarity);
SpanQuery qA1 = new SpanTermQuery(new Term("gender", "female"));
SpanQuery qA2 = new SpanTermQuery(new Term("first", "james"));
SpanQuery qA = new SpanOrQuery(qA1, new FieldMaskingSpanQuery(qA2, "gender"));