You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by tf...@apache.org on 2017/05/19 00:13:57 UTC
[36/50] [abbrv] lucene-solr:jira/solr-10233: LUCENE-7730: Better
accuracy for the length normalization factor.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
index be6a87b..2bbd0dd 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
@@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -309,7 +310,7 @@ public class TestPhraseQuery extends LuceneTestCase {
RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
newIndexWriterConfig(new MockAnalyzer(random()))
.setMergePolicy(newLogMergePolicy())
- .setSimilarity(new ClassicSimilarity()));
+ .setSimilarity(new BM25Similarity()));
Document doc = new Document();
doc.add(newTextField("field", "foo firstname lastname foo", Field.Store.YES));
@@ -335,9 +336,9 @@ public class TestPhraseQuery extends LuceneTestCase {
// each other get a higher score:
assertEquals(1.0, hits[0].score, 0.01);
assertEquals(0, hits[0].doc);
- assertEquals(0.62, hits[1].score, 0.01);
+ assertEquals(0.63, hits[1].score, 0.01);
assertEquals(1, hits[1].doc);
- assertEquals(0.43, hits[2].score, 0.01);
+ assertEquals(0.47, hits[2].score, 0.01);
assertEquals(2, hits[2].doc);
QueryUtils.check(random(), query,searcher);
reader.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java b/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java
index 8b616b5..ab44297 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java
@@ -26,6 +26,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@@ -49,9 +50,14 @@ public class TestQueryRescorer extends LuceneTestCase {
return searcher;
}
+ public static IndexWriterConfig newIndexWriterConfig() {
+ // We rely on more tokens = lower score:
+ return LuceneTestCase.newIndexWriterConfig().setSimilarity(new ClassicSimilarity());
+ }
+
public void testBasic() throws Exception {
Directory dir = newDirectory();
- RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
Document doc = new Document();
doc.add(newStringField("id", "0", Field.Store.YES));
@@ -106,7 +112,7 @@ public class TestQueryRescorer extends LuceneTestCase {
// Test LUCENE-5682
public void testNullScorerTermQuery() throws Exception {
Directory dir = newDirectory();
- RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
Document doc = new Document();
doc.add(newStringField("id", "0", Field.Store.YES));
@@ -145,7 +151,7 @@ public class TestQueryRescorer extends LuceneTestCase {
public void testCustomCombine() throws Exception {
Directory dir = newDirectory();
- RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
Document doc = new Document();
doc.add(newStringField("id", "0", Field.Store.YES));
@@ -196,7 +202,7 @@ public class TestQueryRescorer extends LuceneTestCase {
public void testExplain() throws Exception {
Directory dir = newDirectory();
- RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
Document doc = new Document();
doc.add(newStringField("id", "0", Field.Store.YES));
@@ -271,7 +277,7 @@ public class TestQueryRescorer extends LuceneTestCase {
public void testMissingSecondPassScore() throws Exception {
Directory dir = newDirectory();
- RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
Document doc = new Document();
doc.add(newStringField("id", "0", Field.Store.YES));
@@ -325,7 +331,7 @@ public class TestQueryRescorer extends LuceneTestCase {
public void testRandom() throws Exception {
Directory dir = newDirectory();
int numDocs = atLeast(1000);
- RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
final int[] idToNum = new int[numDocs];
int maxValue = TestUtil.nextInt(random(), 10, 1000000);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java
index 3faa5c2..41f8832 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java
@@ -17,20 +17,18 @@
package org.apache.lucene.search;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.util.LuceneTestCase;
-
import java.io.IOException;
-import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document;
+import org.apache.lucene.util.LuceneTestCase;
/** Similarity unit test.
*
@@ -39,7 +37,7 @@ import org.apache.lucene.document.Document;
public class TestSimilarity extends LuceneTestCase {
public static class SimpleSimilarity extends ClassicSimilarity {
- @Override public float lengthNorm(FieldInvertState state) { return 1; }
+ @Override public float lengthNorm(int length) { return 1; }
@Override public float tf(float freq) { return freq; }
@Override public float sloppyFreq(int distance) { return 2.0f; }
@Override public float idf(long docFreq, long docCount) { return 1.0f; }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
index fbc0b35..d0b33f1 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
@@ -17,19 +17,21 @@
package org.apache.lucene.search;
+import java.io.IOException;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -38,7 +40,7 @@ public class TestSimilarityProvider extends LuceneTestCase {
private Directory directory;
private DirectoryReader reader;
private IndexSearcher searcher;
-
+
@Override
public void setUp() throws Exception {
super.setUp();
@@ -51,7 +53,7 @@ public class TestSimilarityProvider extends LuceneTestCase {
doc.add(field);
Field field2 = newTextField("bar", "", Field.Store.NO);
doc.add(field2);
-
+
field.setStringValue("quick brown fox");
field2.setStringValue("quick brown fox");
iw.addDocument(doc);
@@ -63,14 +65,14 @@ public class TestSimilarityProvider extends LuceneTestCase {
searcher = newSearcher(reader);
searcher.setSimilarity(sim);
}
-
+
@Override
public void tearDown() throws Exception {
reader.close();
directory.close();
super.tearDown();
}
-
+
public void testBasics() throws Exception {
// sanity check of norms writer
// TODO: generalize
@@ -81,7 +83,7 @@ public class TestSimilarityProvider extends LuceneTestCase {
assertEquals(i, barNorms.nextDoc());
assertFalse(fooNorms.longValue() == barNorms.longValue());
}
-
+
// sanity check of searching
TopDocs foodocs = searcher.search(new TermQuery(new Term("foo", "brown")), 10);
assertTrue(foodocs.totalHits > 0);
@@ -89,11 +91,11 @@ public class TestSimilarityProvider extends LuceneTestCase {
assertTrue(bardocs.totalHits > 0);
assertTrue(foodocs.scoreDocs[0].score < bardocs.scoreDocs[0].score);
}
-
+
private static class ExampleSimilarityProvider extends PerFieldSimilarityWrapper {
private Similarity sim1 = new Sim1();
private Similarity sim2 = new Sim2();
-
+
@Override
public Similarity get(String field) {
if (field.equals("foo")) {
@@ -103,80 +105,73 @@ public class TestSimilarityProvider extends LuceneTestCase {
}
}
}
-
- private static class Sim1 extends TFIDFSimilarity {
-
- @Override
- public long encodeNormValue(float f) {
- return (long) f;
- }
-
- @Override
- public float decodeNormValue(long norm) {
- return norm;
- }
- @Override
- public float lengthNorm(FieldInvertState state) {
- return 1f;
- }
+ private static class Sim1 extends Similarity {
@Override
- public float sloppyFreq(int distance) {
- return 1f;
+ public long computeNorm(FieldInvertState state) {
+ return 1;
}
@Override
- public float tf(float freq) {
- return 1f;
+ public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ return new SimWeight() {};
}
@Override
- public float idf(long docFreq, long docCount) {
- return 1f;
- }
+ public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
+ return new SimScorer() {
- @Override
- public float scorePayload(int doc, int start, int end, BytesRef payload) {
- return 1f;
+ @Override
+ public float score(int doc, float freq) throws IOException {
+ return 1;
+ }
+
+ @Override
+ public float computeSlopFactor(int distance) {
+ return 1;
+ }
+
+ @Override
+ public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
+ return 1;
+ }
+ };
}
+
}
-
- private static class Sim2 extends TFIDFSimilarity {
-
- @Override
- public long encodeNormValue(float f) {
- return (long) f;
- }
-
- @Override
- public float decodeNormValue(long norm) {
- return norm;
- }
-
- @Override
- public float lengthNorm(FieldInvertState state) {
- return 10f;
- }
- @Override
- public float sloppyFreq(int distance) {
- return 10f;
- }
+ private static class Sim2 extends Similarity {
@Override
- public float tf(float freq) {
- return 10f;
+ public long computeNorm(FieldInvertState state) {
+ return 10;
}
@Override
- public float idf(long docFreq, long docCount) {
- return 10f;
+ public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ return new SimWeight() {};
}
@Override
- public float scorePayload(int doc, int start, int end, BytesRef payload) {
- return 1f;
+ public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
+ return new SimScorer() {
+
+ @Override
+ public float score(int doc, float freq) throws IOException {
+ return 10;
+ }
+
+ @Override
+ public float computeSlopFactor(int distance) {
+ return 1;
+ }
+
+ @Override
+ public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
+ return 1;
+ }
+ };
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/core/src/test/org/apache/lucene/search/TestSortRescorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortRescorer.java b/lucene/core/src/test/org/apache/lucene/search/TestSortRescorer.java
index b2d435d..6cce5fe 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSortRescorer.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSortRescorer.java
@@ -42,7 +42,7 @@ public class TestSortRescorer extends LuceneTestCase {
public void setUp() throws Exception {
super.setUp();
dir = newDirectory();
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, newIndexWriterConfig().setSimilarity(new ClassicSimilarity()));
Document doc = new Document();
doc.add(newStringField("id", "1", Field.Store.YES));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticSimilarity.java
index 44c7e1d..eca6edc 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticSimilarity.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticSimilarity.java
@@ -20,19 +20,6 @@ import org.apache.lucene.util.LuceneTestCase;
public class TestAxiomaticSimilarity extends LuceneTestCase {
- public void testSaneNormValues() {
- Axiomatic sim = new AxiomaticF2EXP();
- for (int i = 0; i < 256; i++) {
- float len = sim.decodeNormValue((byte) i);
- assertFalse("negative len: " + len + ", byte=" + i, len < 0.0f);
- assertFalse("inf len: " + len + ", byte=" + i, Float.isInfinite(len));
- assertFalse("nan len for byte=" + i, Float.isNaN(len));
- if (i > 0) {
- assertTrue("len is not decreasing: " + len + ",byte=" + i, len < sim.decodeNormValue((byte) (i - 1)));
- }
- }
- }
-
public void testIllegalS() {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
new AxiomaticF2EXP(Float.POSITIVE_INFINITY, 0.1f);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/core/src/test/org/apache/lucene/search/similarities/TestBM25Similarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestBM25Similarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestBM25Similarity.java
index dc43406..e54ce25 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestBM25Similarity.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestBM25Similarity.java
@@ -17,23 +17,27 @@
package org.apache.lucene.search.similarities;
+import java.io.IOException;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.Version;
public class TestBM25Similarity extends LuceneTestCase {
- public void testSaneNormValues() {
- BM25Similarity sim = new BM25Similarity();
- for (int i = 0; i < 256; i++) {
- float len = sim.decodeNormValue((byte) i);
- assertFalse("negative len: " + len + ", byte=" + i, len < 0.0f);
- assertFalse("inf len: " + len + ", byte=" + i, Float.isInfinite(len));
- assertFalse("nan len for byte=" + i, Float.isNaN(len));
- if (i > 0) {
- assertTrue("len is not decreasing: " + len + ",byte=" + i, len < sim.decodeNormValue((byte)(i-1)));
- }
- }
- }
-
public void testIllegalK1() {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
new BM25Similarity(Float.POSITIVE_INFINITY, 0.75f);
@@ -72,4 +76,44 @@ public class TestBM25Similarity extends LuceneTestCase {
});
assertTrue(expected.getMessage().contains("illegal b value"));
}
+
+ public void testLengthEncodingBackwardCompatibility() throws IOException {
+ Similarity similarity = new BM25Similarity();
+ for (int indexCreatedVersionMajor : new int[] { Version.LUCENE_6_0_0.major, Version.LATEST.major}) {
+ for (int length : new int[] {1, 2, 4}) { // these length values are encoded accurately on both cases
+ Directory dir = newDirectory();
+ // set the version on the directory
+ new SegmentInfos(indexCreatedVersionMajor).commit(dir);
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setSimilarity(similarity));
+ Document doc = new Document();
+ String value = IntStream.range(0, length).mapToObj(i -> "b").collect(Collectors.joining(" "));
+ doc.add(new TextField("foo", value, Store.NO));
+ w.addDocument(doc);
+ IndexReader reader = DirectoryReader.open(w);
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(similarity);
+ Explanation expl = searcher.explain(new TermQuery(new Term("foo", "b")), 0);
+ Explanation docLen = findExplanation(expl, "fieldLength");
+ assertNotNull(docLen);
+ assertEquals(docLen.toString(), length, (int) docLen.getValue());
+ w.close();
+ reader.close();
+ dir.close();
+ }
+ }
+ }
+
+ private static Explanation findExplanation(Explanation expl, String text) {
+ if (expl.getDescription().equals(text)) {
+ return expl;
+ } else {
+ for (Explanation sub : expl.getDetails()) {
+ Explanation match = findExplanation(sub, text);
+ if (match != null) {
+ return match;
+ }
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.java
index 23f65d3..c388514 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.java
@@ -34,6 +34,7 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.Version;
public class TestBooleanSimilarity extends LuceneTestCase {
@@ -105,8 +106,8 @@ public class TestBooleanSimilarity extends LuceneTestCase {
for (int iter = 0; iter < 100; ++iter) {
final int length = TestUtil.nextInt(random(), 1, 100);
final int position = random().nextInt(length);
- final int numOverlaps = random().nextInt(50);
- FieldInvertState state = new FieldInvertState("foo", position, length, numOverlaps, 100);
+ final int numOverlaps = random().nextInt(length);
+ FieldInvertState state = new FieldInvertState(Version.LATEST.major, "foo", position, length, numOverlaps, 100);
assertEquals(
sim2.computeNorm(state),
sim1.computeNorm(state),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java
index be07564..c7f0453 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java
@@ -19,24 +19,34 @@ package org.apache.lucene.search.similarities;
import java.io.IOException;
import java.util.Arrays;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DisjunctionMaxQuery;
+import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.TFIDFSimilarity.IDFStats;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.Version;
public class TestClassicSimilarity extends LuceneTestCase {
private Directory directory;
@@ -63,14 +73,6 @@ public class TestClassicSimilarity extends LuceneTestCase {
IOUtils.close(indexReader, directory);
super.tearDown();
}
-
- // Javadocs give this as an example so we test to make sure it's correct:
- public void testPrecisionLoss() throws Exception {
- ClassicSimilarity sim = new ClassicSimilarity();
- float v = sim.decodeNormValue(sim.encodeNormValue(.89f));
- assertEquals(0.875f, v, 0.0001f);
- }
-
public void testHit() throws IOException {
Query query = new TermQuery(new Term("test", "hit"));
@@ -159,16 +161,83 @@ public class TestClassicSimilarity extends LuceneTestCase {
assertTrue(topDocs.scoreDocs[0].score != 0);
}
- public void testSaneNormValues() {
+ public void testSaneNormValues() throws IOException {
ClassicSimilarity sim = new ClassicSimilarity();
for (int i = 0; i < 256; i++) {
- float boost = sim.decodeNormValue((byte) i);
+ float boost = TFIDFSimilarity.OLD_NORM_TABLE[i];
+ assertFalse("negative boost: " + boost + ", byte=" + i, boost < 0.0f);
+ assertFalse("inf bost: " + boost + ", byte=" + i, Float.isInfinite(boost));
+ assertFalse("nan boost for byte=" + i, Float.isNaN(boost));
+ if (i > 0) {
+ assertTrue("boost is not increasing: " + boost + ",byte=" + i, boost > TFIDFSimilarity.OLD_NORM_TABLE[i-1]);
+ }
+ }
+
+ TFIDFSimilarity.IDFStats stats = (IDFStats) sim.computeWeight(1f, new IndexSearcher(new MultiReader()).collectionStatistics("foo"));
+ for (int i = 0; i < 256; i++) {
+ float boost = stats.normTable[i];
assertFalse("negative boost: " + boost + ", byte=" + i, boost < 0.0f);
assertFalse("inf bost: " + boost + ", byte=" + i, Float.isInfinite(boost));
assertFalse("nan boost for byte=" + i, Float.isNaN(boost));
if (i > 0) {
- assertTrue("boost is not increasing: " + boost + ",byte=" + i, boost > sim.decodeNormValue((byte)(i-1)));
+ assertTrue("boost is not decreasing: " + boost + ",byte=" + i, boost < stats.normTable[i-1]);
}
}
}
+
+ public void testNormEncodingBackwardCompatibility() throws IOException {
+ Similarity similarity = new ClassicSimilarity();
+ for (int indexCreatedVersionMajor : new int[] { Version.LUCENE_6_0_0.major, Version.LATEST.major}) {
+ for (int length : new int[] {1, 4, 16 }) { // these length values are encoded accurately on both cases
+ Directory dir = newDirectory();
+ // set the version on the directory
+ new SegmentInfos(indexCreatedVersionMajor).commit(dir);
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setSimilarity(similarity));
+ Document doc = new Document();
+ String value = IntStream.range(0, length).mapToObj(i -> "b").collect(Collectors.joining(" "));
+ doc.add(new TextField("foo", value, Store.NO));
+ w.addDocument(doc);
+ IndexReader reader = DirectoryReader.open(w);
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(similarity);
+ Explanation expl = searcher.explain(new TermQuery(new Term("foo", "b")), 0);
+ Explanation fieldNorm = findExplanation(expl, "fieldNorm");
+ assertNotNull(fieldNorm);
+ assertEquals(fieldNorm.toString(), 1/Math.sqrt(length), fieldNorm.getValue(), 0f);
+ w.close();
+ reader.close();
+ dir.close();
+ }
+ }
+ }
+
+ private static Explanation findExplanation(Explanation expl, String text) {
+ if (expl.getDescription().startsWith(text)) {
+ return expl;
+ } else {
+ for (Explanation sub : expl.getDetails()) {
+ Explanation match = findExplanation(sub, text);
+ if (match != null) {
+ return match;
+ }
+ }
+ }
+ return null;
+ }
+
+ public void testSameNormsAsBM25() {
+ ClassicSimilarity sim1 = new ClassicSimilarity();
+ BM25Similarity sim2 = new BM25Similarity();
+ sim2.setDiscountOverlaps(true);
+ for (int iter = 0; iter < 100; ++iter) {
+ final int length = TestUtil.nextInt(random(), 1, 1000);
+ final int position = random().nextInt(length);
+ final int numOverlaps = random().nextInt(length);
+ FieldInvertState state = new FieldInvertState(Version.LATEST.major, "foo", position, length, numOverlaps, 100);
+ assertEquals(
+ sim2.computeNorm(state),
+ sim1.computeNorm(state),
+ 0f);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
index 373b9e6..59be73a 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
@@ -20,16 +20,23 @@ package org.apache.lucene.search.similarities;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermContext;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
@@ -37,9 +44,13 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.Similarity.SimWeight;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.Version;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
/**
* Tests the {@link SimilarityBase}-based Similarities. Contains unit tests and
@@ -586,11 +597,11 @@ public class TestSimilarityBase extends LuceneTestCase {
// LUCENE-5221
public void testDiscountOverlapsBoost() throws IOException {
- ClassicSimilarity expected = new ClassicSimilarity();
+ BM25Similarity expected = new BM25Similarity();
SimilarityBase actual = new DFRSimilarity(new BasicModelIne(), new AfterEffectB(), new NormalizationH2());
expected.setDiscountOverlaps(false);
actual.setDiscountOverlaps(false);
- FieldInvertState state = new FieldInvertState("foo");
+ FieldInvertState state = new FieldInvertState(Version.LATEST.major, "foo");
state.setLength(5);
state.setNumOverlap(2);
assertEquals(expected.computeNorm(state), actual.computeNorm(state));
@@ -598,64 +609,32 @@ public class TestSimilarityBase extends LuceneTestCase {
actual.setDiscountOverlaps(true);
assertEquals(expected.computeNorm(state), actual.computeNorm(state));
}
-
- public void testSaneNormValues() {
- for (SimilarityBase sim : sims) {
- for (int i = 0; i < 256; i++) {
- float len = sim.decodeNormValue((byte) i);
- assertFalse("negative len: " + len + ", byte=" + i + ", sim=" + sim, len < 0.0f);
- assertFalse("inf len: " + len + ", byte=" + i + ", sim=" + sim, Float.isInfinite(len));
- assertFalse("nan len for byte=" + i + ", sim=" + sim, Float.isNaN(len));
- if (i > 0) {
- assertTrue("len is not decreasing: " + len + ",byte=" + i + ",sim=" + sim, len < sim.decodeNormValue((byte)(i-1)));
- }
- }
- }
- }
-
- /**
- * make sure the similarity does not go crazy when tested against all possible norm values.
- */
- public void testCrazyIndexTimeBoosts() throws Exception {
- long avgLength = 750;
- long docCount = 500000;
- long numTokens = docCount * avgLength;
-
- CollectionStatistics collectionStats = new CollectionStatistics("body", docCount, docCount, numTokens, numTokens);
-
- long docFreq = 2000;
- long totalTermFreq = 2000 * avgLength;
-
- TermStatistics termStats = new TermStatistics(new BytesRef("term"), docFreq, totalTermFreq);
-
- for (SimilarityBase sim : sims) {
- if (sim instanceof IBSimilarity) {
- if (((IBSimilarity)sim).getDistribution() instanceof DistributionSPL) {
- // score goes infinite for tiny doc lengths and negative for huge doc lengths
- // TODO: fix this
- continue;
- }
- } else if (sim instanceof DFRSimilarity) {
- BasicModel model = ((DFRSimilarity)sim).getBasicModel();
- if (model instanceof BasicModelD || model instanceof BasicModelP) {
- // score goes NaN for tiny doc lengths
- // TODO: fix this
- continue;
- } else if (model instanceof BasicModelBE) {
- // score goes negative infinity for tiny doc lengths
- // TODO: fix this
- continue;
- }
- }
- BasicStats stats = (BasicStats) sim.computeWeight(1f, collectionStats, termStats);
- for (float tf = 1.0f; tf <= 10.0f; tf += 1.0f) {
- for (int i = 0; i < 256; i++) {
- float len = sim.decodeNormValue((byte) i);
- float score = sim.score(stats, tf, len);
- assertFalse("negative score for " + sim + ", len=" + len + ",score=" + score, score < 0.0f);
- assertFalse("inf score for " + sim + ", len=" + len, Float.isInfinite(score));
- assertFalse("nan score for " + sim + ", len=" + len, Float.isNaN(score));
- }
+
+ public void testLengthEncodingBackwardCompatibility() throws IOException {
+ Similarity similarity = RandomPicks.randomFrom(random(), sims);
+ for (int indexCreatedVersionMajor : new int[] { Version.LUCENE_6_0_0.major, Version.LATEST.major}) {
+ for (int length : new int[] {1, 2, 4}) { // these length values are encoded accurately on both cases
+ Directory dir = newDirectory();
+ // set the version on the directory
+ new SegmentInfos(indexCreatedVersionMajor).commit(dir);
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setSimilarity(similarity));
+ Document doc = new Document();
+ String value = IntStream.range(0, length).mapToObj(i -> "b").collect(Collectors.joining(" "));
+ doc.add(new TextField("foo", value, Store.NO));
+ w.addDocument(doc);
+ IndexReader reader = DirectoryReader.open(w);
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(similarity);
+ Term term = new Term("foo", "b");
+ TermContext context = TermContext.build(reader.getContext(), term);
+ SimWeight simWeight = similarity.computeWeight(1f, searcher.collectionStatistics("foo"), searcher.termStatistics(term, context));
+ SimilarityBase.BasicSimScorer simScorer = (SimilarityBase.BasicSimScorer) similarity.simScorer(simWeight, reader.leaves().get(0));
+ float docLength = simScorer.getLengthValue(0);
+ assertEquals(length, (int) docLength);
+
+ w.close();
+ reader.close();
+ dir.close();
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/core/src/test/org/apache/lucene/util/TestSmallFloat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestSmallFloat.java b/lucene/core/src/test/org/apache/lucene/util/TestSmallFloat.java
index 86c6bb8..dca5f2d 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestSmallFloat.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestSmallFloat.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.util;
+import java.util.Arrays;
+
public class TestSmallFloat extends LuceneTestCase {
// original lucene byteToFloat
@@ -87,10 +89,6 @@ public class TestSmallFloat extends LuceneTestCase {
float f3 = SmallFloat.byte315ToFloat((byte)i);
assertEquals(f1,f2,0.0);
assertEquals(f2,f3,0.0);
-
- float f4 = SmallFloat.byteToFloat((byte)i,5,2);
- float f5 = SmallFloat.byte52ToFloat((byte)i);
- assertEquals(f4,f5,0.0);
}
}
@@ -121,10 +119,51 @@ public class TestSmallFloat extends LuceneTestCase {
byte b3 = SmallFloat.floatToByte315(f);
assertEquals(b1,b2);
assertEquals(b2,b3);
+ }
+ }
+
+ public void testInt4() {
+ for (int i = 0; i <= 16; ++i) {
+ // all values in 0-16 are encoded accurately
+ assertEquals(i, SmallFloat.int4ToLong(SmallFloat.longToInt4(i)));
+ }
+ final int maxEncoded = SmallFloat.longToInt4(Long.MAX_VALUE);
+ for (int i = 1; i < maxEncoded; ++i) {
+ assertTrue(SmallFloat.int4ToLong(i) > SmallFloat.int4ToLong(i - 1));
+ }
+ final int iters = atLeast(1000);
+ for (int iter = 0; iter < iters; ++iter) {
+ final long l = TestUtil.nextLong(random(), 0, 1L << TestUtil.nextInt(random(), 5, 61));
+ int numBits = 64 - Long.numberOfLeadingZeros(l);
+ long expected = l;
+ if (numBits > 4) {
+ long mask = ~0L << (numBits - 4);
+ expected &= mask;
+ }
+ long l2 = SmallFloat.int4ToLong(SmallFloat.longToInt4(l));
+ assertEquals(expected, l2);
+ }
+ }
- byte b4 = SmallFloat.floatToByte(f,5,2);
- byte b5 = SmallFloat.floatToByte52(f);
- assertEquals(b4,b5);
+ public void testByte4() {
+ int[] decoded = new int[256];
+ for (int b = 0; b < 256; ++b) {
+ decoded[b] = SmallFloat.byte4ToInt((byte) b);
+ assertEquals((byte) b, SmallFloat.intToByte4(decoded[b]));
+ }
+ for (int i = 1; i < 256; ++i) {
+ assertTrue(decoded[i] > decoded[i-1]);
+ }
+ assertEquals((byte) 255, SmallFloat.intToByte4(Integer.MAX_VALUE));
+ final int iters = atLeast(1000);
+ for (int iter = 0; iter < iters; ++iter) {
+ final int i = random().nextInt(1 << TestUtil.nextInt(random(), 5, 30));
+ int idx = Arrays.binarySearch(decoded, i);
+ if (idx < 0) {
+ idx = -2 - idx;
+ }
+ assertTrue(decoded[idx] <= i);
+ assertEquals((byte) idx, SmallFloat.intToByte4(i));
}
}
@@ -146,5 +185,4 @@ public class TestSmallFloat extends LuceneTestCase {
}
}
***/
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionRescorer.java
----------------------------------------------------------------------
diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionRescorer.java b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionRescorer.java
index 98012a5..20a6eba 100644
--- a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionRescorer.java
+++ b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionRescorer.java
@@ -44,7 +44,7 @@ public class TestExpressionRescorer extends LuceneTestCase {
public void setUp() throws Exception {
super.setUp();
dir = newDirectory();
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, newIndexWriterConfig().setSimilarity(new ClassicSimilarity()));
Document doc = new Document();
doc.add(newStringField("id", "1", Field.Store.YES));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
index c37709b..d03432e 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
@@ -72,6 +72,8 @@ import org.apache.lucene.search.PhraseQuery.Builder;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.RegexpQuery;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SynonymQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
@@ -147,7 +149,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
CustomScoreQuery query = new CustomScoreQuery(termQuery);
searcher = newSearcher(reader);
- TopDocs hits = searcher.search(query, 10);
+ TopDocs hits = searcher.search(query, 10, new Sort(SortField.FIELD_DOC, SortField.FIELD_SCORE));
assertEquals(2, hits.totalHits);
QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
Highlighter highlighter = new Highlighter(scorer);
@@ -199,7 +201,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
query.add(new Term(FIELD_NAME, "very"));
searcher = newSearcher(reader);
- TopDocs hits = searcher.search(query, 10);
+ TopDocs hits = searcher.search(query, 10, new Sort(SortField.FIELD_DOC, SortField.FIELD_SCORE));
assertEquals(2, hits.totalHits);
QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
Highlighter highlighter = new Highlighter(scorer);
@@ -271,7 +273,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
};
searcher = newSearcher(reader);
- TopDocs hits = searcher.search(query, 10);
+ TopDocs hits = searcher.search(query, 10, new Sort(SortField.FIELD_DOC, SortField.FIELD_SCORE));
assertEquals(2, hits.totalHits);
QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
Highlighter highlighter = new Highlighter(scorer);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index a1f2b07..a409a17 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -892,7 +892,7 @@ public class MemoryIndex {
NumericDocValues getNormDocValues() {
if (norm == null) {
- FieldInvertState invertState = new FieldInvertState(fieldInfo.name, fieldInfo.number,
+ FieldInvertState invertState = new FieldInvertState(Version.LATEST.major, fieldInfo.name, fieldInfo.number,
numTokens, numOverlapTokens, 0);
final long value = normSimilarity.computeNorm(invertState);
if (DEBUG) System.err.println("MemoryIndexReader.norms: " + fieldInfo.name + ":" + value + ":" + numTokens);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
index 1e20f30..3d3a12c 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
@@ -50,6 +50,7 @@ import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SortedDocValues;
@@ -57,13 +58,16 @@ import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.ClassicSimilarity;
+import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -145,32 +149,32 @@ public class TestMemoryIndex extends LuceneTestCase {
assertEquals(reader.getTermVectors(0).size(), 1);
}
-
+
public void testReaderConsistency() throws IOException {
Analyzer analyzer = new MockPayloadAnalyzer();
-
+
// defaults
MemoryIndex mi = new MemoryIndex();
mi.addField("field", "some terms be here", analyzer);
TestUtil.checkReader(mi.createSearcher().getIndexReader());
-
+
// all combinations of offsets/payloads options
mi = new MemoryIndex(true, true);
mi.addField("field", "some terms be here", analyzer);
TestUtil.checkReader(mi.createSearcher().getIndexReader());
-
+
mi = new MemoryIndex(true, false);
mi.addField("field", "some terms be here", analyzer);
TestUtil.checkReader(mi.createSearcher().getIndexReader());
-
+
mi = new MemoryIndex(false, true);
mi.addField("field", "some terms be here", analyzer);
TestUtil.checkReader(mi.createSearcher().getIndexReader());
-
+
mi = new MemoryIndex(false, false);
mi.addField("field", "some terms be here", analyzer);
TestUtil.checkReader(mi.createSearcher().getIndexReader());
-
+
analyzer.close();
}
@@ -187,11 +191,23 @@ public class TestMemoryIndex extends LuceneTestCase {
float n1 = norms.longValue();
// Norms are re-computed when we change the Similarity
- mi.setSimilarity(new ClassicSimilarity() {
+ mi.setSimilarity(new Similarity() {
+
@Override
- public float lengthNorm(FieldInvertState state) {
+ public long computeNorm(FieldInvertState state) {
return 74;
}
+
+ @Override
+ public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
});
norms = reader.getNormValues("f1");
assertEquals(0, norms.nextDoc());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java b/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
index 9307b94..8cc962d 100644
--- a/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
+++ b/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
@@ -17,7 +17,6 @@
package org.apache.lucene.misc;
import org.apache.lucene.search.similarities.ClassicSimilarity;
-import org.apache.lucene.index.FieldInvertState;
/**
* <p>
@@ -86,7 +85,7 @@ public class SweetSpotSimilarity extends ClassicSimilarity {
* Sets the default function variables used by lengthNorm when no field
* specific variables have been set.
*
- * @see #computeLengthNorm
+ * @see #lengthNorm
*/
public void setLengthNormFactors(int min, int max, float steepness, boolean discountOverlaps) {
this.ln_min = min;
@@ -94,25 +93,6 @@ public class SweetSpotSimilarity extends ClassicSimilarity {
this.ln_steep = steepness;
this.discountOverlaps = discountOverlaps;
}
-
- /**
- * Implemented as <code> state.getBoost() *
- * computeLengthNorm(numTokens) </code> where
- * numTokens does not count overlap tokens if
- * discountOverlaps is true by default or true for this
- * specific field.
- */
- @Override
- public float lengthNorm(FieldInvertState state) {
- final int numTokens;
-
- if (discountOverlaps)
- numTokens = state.getLength() - state.getNumOverlap();
- else
- numTokens = state.getLength();
-
- return computeLengthNorm(numTokens);
- }
/**
* Implemented as:
@@ -133,7 +113,8 @@ public class SweetSpotSimilarity extends ClassicSimilarity {
* @see #setLengthNormFactors
* @see <a href="doc-files/ss.computeLengthNorm.svg">An SVG visualization of this function</a>
*/
- public float computeLengthNorm(int numTerms) {
+ @Override
+ public float lengthNorm(int numTerms) {
final int l = ln_min;
final int h = ln_max;
final float s = ln_steep;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java b/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
index cd6a819..d21a3ea 100644
--- a/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
+++ b/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
@@ -16,27 +16,62 @@
*/
package org.apache.lucene.misc;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.index.FieldInvertState;
/**
* Test of the SweetSpotSimilarity
*/
public class SweetSpotSimilarityTest extends LuceneTestCase {
- public static float computeAndDecodeNorm(SweetSpotSimilarity decode, Similarity encode, FieldInvertState state) {
- return decode.decodeNormValue(computeAndGetNorm(encode, state));
+ private static float computeNorm(Similarity sim, String field, int length) throws IOException {
+ String value = IntStream.range(0, length).mapToObj(i -> "a").collect(Collectors.joining(" "));
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setSimilarity(sim));
+ w.addDocument(Collections.singleton(newTextField(field, value, Store.NO)));
+ DirectoryReader reader = DirectoryReader.open(w);
+ w.close();
+ IndexSearcher searcher = new IndexSearcher(reader);
+ searcher.setSimilarity(sim);
+ Explanation expl = searcher.explain(new TermQuery(new Term(field, "a")), 0);
+ reader.close();
+ dir.close();
+ Explanation norm = findExplanation(expl, "fieldNorm");
+ assertNotNull(norm);
+ return norm.getValue();
}
-
- public static byte computeAndGetNorm(Similarity s, FieldInvertState state) {
- return (byte) s.computeNorm(state);
+
+ private static Explanation findExplanation(Explanation expl, String text) {
+ if (expl.getDescription().startsWith(text)) {
+ return expl;
+ } else {
+ for (Explanation sub : expl.getDetails()) {
+ Explanation match = findExplanation(sub, text);
+ if (match != null) {
+ return match;
+ }
+ }
+ }
+ return null;
}
- public void testSweetSpotComputeNorm() {
+ public void testSweetSpotComputeNorm() throws IOException {
final SweetSpotSimilarity ss = new SweetSpotSimilarity();
ss.setLengthNormFactors(1,1,0.5f,true);
@@ -46,12 +81,10 @@ public class SweetSpotSimilarityTest extends LuceneTestCase {
// base case, should degrade
- FieldInvertState invertState = new FieldInvertState("bogus");
for (int i = 1; i < 1000; i++) {
- invertState.setLength(i);
assertEquals("base case: i="+i,
- computeAndGetNorm(d, invertState),
- computeAndGetNorm(s, invertState),
+ computeNorm(d, "bogus", i),
+ computeNorm(s, "bogus", i),
0.0f);
}
@@ -60,22 +93,19 @@ public class SweetSpotSimilarityTest extends LuceneTestCase {
ss.setLengthNormFactors(3,10,0.5f,true);
for (int i = 3; i <=10; i++) {
- invertState.setLength(i);
assertEquals("3,10: spot i="+i,
1.0f,
- computeAndDecodeNorm(ss, ss, invertState),
+ computeNorm(ss, "bogus", i),
0.0f);
}
for (int i = 10; i < 1000; i++) {
- invertState.setLength(i-9);
- final byte normD = computeAndGetNorm(d, invertState);
- invertState.setLength(i);
- final byte normS = computeAndGetNorm(s, invertState);
+ final float normD = computeNorm(d, "bogus", i - 9);
+ final float normS = computeNorm(s, "bogus", i);
assertEquals("3,10: 10<x : i="+i,
normD,
normS,
- 0.0f);
+ 0.01f);
}
@@ -106,78 +136,60 @@ public class SweetSpotSimilarityTest extends LuceneTestCase {
}
};
- invertState = new FieldInvertState("foo");
for (int i = 3; i <=10; i++) {
- invertState.setLength(i);
assertEquals("f: 3,10: spot i="+i,
1.0f,
- computeAndDecodeNorm(ss, sp, invertState),
+ computeNorm(sp, "foo", i),
0.0f);
}
for (int i = 10; i < 1000; i++) {
- invertState.setLength(i-9);
- final byte normD = computeAndGetNorm(d, invertState);
- invertState.setLength(i);
- final byte normS = computeAndGetNorm(sp, invertState);
+ final float normD = computeNorm(d, "foo", i-9);
+ final float normS = computeNorm(sp, "foo", i);
assertEquals("f: 3,10: 10<x : i="+i,
normD,
normS,
- 0.0f);
+ 0.01f);
}
- invertState = new FieldInvertState("bar");
for (int i = 8; i <=13; i++) {
- invertState.setLength(i);
assertEquals("f: 8,13: spot i="+i,
1.0f,
- computeAndDecodeNorm(ss, sp, invertState),
- 0.0f);
+ computeNorm(sp, "bar", i),
+ 0.01f);
}
- invertState = new FieldInvertState("yak");
for (int i = 6; i <=9; i++) {
- invertState.setLength(i);
assertEquals("f: 6,9: spot i="+i,
1.0f,
- computeAndDecodeNorm(ss, sp, invertState),
- 0.0f);
+ computeNorm(sp, "yak", i),
+ 0.01f);
}
- invertState = new FieldInvertState("bar");
for (int i = 13; i < 1000; i++) {
- invertState.setLength(i-12);
- final byte normD = computeAndGetNorm(d, invertState);
- invertState.setLength(i);
- final byte normS = computeAndGetNorm(sp, invertState);
+ final float normD = computeNorm(d, "bar", i-12);
+ final float normS = computeNorm(sp, "bar", i);
assertEquals("f: 8,13: 13<x : i="+i,
normD,
normS,
- 0.0f);
+ 0.01f);
}
- invertState = new FieldInvertState("yak");
for (int i = 9; i < 1000; i++) {
- invertState.setLength(i-8);
- final byte normD = computeAndGetNorm(d, invertState);
- invertState.setLength(i);
- final byte normS = computeAndGetNorm(sp, invertState);
+ final float normD = computeNorm(d, "yak", i-8);
+ final float normS = computeNorm(sp, "yak", i);
assertEquals("f: 6,9: 9<x : i="+i,
normD,
normS,
- 0.0f);
+ 0.01f);
}
// steepness
for (int i = 9; i < 1000; i++) {
- invertState = new FieldInvertState("a");
- invertState.setLength(i);
- final byte normSS = computeAndGetNorm(sp, invertState);
- invertState = new FieldInvertState("b");
- invertState.setLength(i);
- final byte normS = computeAndGetNorm(sp, invertState);
+ final float normSS = computeNorm(sp, "a", i);
+ final float normS = computeNorm(sp, "b", i);
assertTrue("s: i="+i+" : a="+normSS+
" < b="+normS,
normSS < normS);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
index d556454..ea63de9 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
@@ -20,19 +20,24 @@ import java.io.IOException;
import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.FloatDocValues;
+import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.search.similarities.Similarity.SimScorer;
+import org.apache.lucene.search.similarities.Similarity.SimWeight;
/**
- * Function that returns {@link TFIDFSimilarity#decodeNormValue(long)}
- * for every document.
+ * Function that returns the decoded norm for every document.
* <p>
* Note that the configured Similarity for the field must be
- * a subclass of {@link TFIDFSimilarity}
+ * a subclass of {@link TFIDFSimilarity} and the contribution of
+ * the TF needs to be 1 when the freq is 1 and the contribution
+ * of the IDF needs to be 1 when docFreq == docCount == 1.
* @lucene.internal */
public class NormValueSource extends ValueSource {
protected final String field;
@@ -61,11 +66,12 @@ public class NormValueSource extends ValueSource {
if (similarity == null) {
throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as ClassicSimilarity)");
}
- final NumericDocValues norms = readerContext.reader().getNormValues(field);
-
- if (norms == null) {
- return new ConstDoubleDocValues(0.0, this);
- }
+ // Only works if the contribution of the tf is 1 when the freq is 1 and contribution of the idf
+ // is 1 when docCount == docFreq == 1
+ final SimWeight simWeight = similarity.computeWeight(1f,
+ new CollectionStatistics(field, 1, 1, 1, 1),
+ new TermStatistics(new BytesRef("bogus"), 1, 1));
+ final SimScorer simScorer = similarity.simScorer(simWeight, readerContext);
return new FloatDocValues(this) {
int lastDocID = -1;
@@ -74,16 +80,8 @@ public class NormValueSource extends ValueSource {
if (docID < lastDocID) {
throw new AssertionError("docs out of order: lastDocID=" + lastDocID + " docID=" + docID);
}
- if (docID > norms.docID()) {
- norms.advance(docID);
- }
- long norm;
- if (docID == norms.docID()) {
- norm = norms.longValue();
- } else {
- norm = 0;
- }
- return similarity.decodeNormValue(norm);
+ lastDocID = docID;
+ return simScorer.score(docID, 1f);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java
index 11060e5..842c117 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java
@@ -21,7 +21,6 @@ import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
@@ -33,10 +32,9 @@ import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -48,7 +46,7 @@ public class TestLongNormValueSource extends LuceneTestCase {
static IndexSearcher searcher;
static Analyzer analyzer;
- private static Similarity sim = new PreciseClassicSimilarity();
+ private static Similarity sim = new ClassicSimilarity();
@BeforeClass
public static void beforeClass() throws Exception {
@@ -116,114 +114,3 @@ public class TestLongNormValueSource extends LuceneTestCase {
CheckHits.checkExplanations(q, "", searcher);
}
}
-
-
-/** Encodes norm as 4-byte float. */
-class PreciseClassicSimilarity extends TFIDFSimilarity {
-
- /** Sole constructor: parameter-free */
- public PreciseClassicSimilarity() {}
-
- /**
- * Encodes a normalization factor for storage in an index.
- * <p>
- * The encoding uses a three-bit mantissa, a five-bit exponent, and the
- * zero-exponent point at 15, thus representing values from around 7x10^9 to
- * 2x10^-9 with about one significant decimal digit of accuracy. Zero is also
- * represented. Negative numbers are rounded up to zero. Values too large to
- * represent are rounded down to the largest representable value. Positive
- * values too small to represent are rounded up to the smallest positive
- * representable value.
- *
- * @see org.apache.lucene.util.SmallFloat
- */
- @Override
- public final long encodeNormValue(float f) {
- return Float.floatToIntBits(f);
- }
-
- /**
- * Decodes the norm value, assuming it is a single byte.
- *
- * @see #encodeNormValue(float)
- */
- @Override
- public final float decodeNormValue(long norm) {
- return Float.intBitsToFloat((int)norm);
- }
-
- /** Implemented as
- * <code>state.getBoost()*lengthNorm(numTerms)</code>, where
- * <code>numTerms</code> is {@link org.apache.lucene.index.FieldInvertState#getLength()} if {@link
- * #setDiscountOverlaps} is false, else it's {@link
- * org.apache.lucene.index.FieldInvertState#getLength()} - {@link
- * org.apache.lucene.index.FieldInvertState#getNumOverlap()}.
- *
- * @lucene.experimental */
- @Override
- public float lengthNorm(FieldInvertState state) {
- final int numTerms;
- if (discountOverlaps) {
- numTerms = state.getLength() - state.getNumOverlap();
- } else {
- numTerms = state.getLength();
- }
- return (float) (1.0 / Math.sqrt(numTerms));
- }
-
- /** Implemented as <code>sqrt(freq)</code>. */
- @Override
- public float tf(float freq) {
- return (float)Math.sqrt(freq);
- }
-
- /** Implemented as <code>1 / (distance + 1)</code>. */
- @Override
- public float sloppyFreq(int distance) {
- return 1.0f / (distance + 1);
- }
-
- /** The default implementation returns <code>1</code> */
- @Override
- public float scorePayload(int doc, int start, int end, BytesRef payload) {
- return 1;
- }
-
- /** Implemented as <code>log(docCount/(docFreq+1)) + 1</code>. */
- @Override
- public float idf(long docFreq, long docCount) {
- return (float)(Math.log(docCount/(double)(docFreq+1)) + 1.0);
- }
-
- /**
- * True if overlap tokens (tokens with a position of increment of zero) are
- * discounted from the document's length.
- */
- protected boolean discountOverlaps = true;
-
- /** Determines whether overlap tokens (Tokens with
- * 0 position increment) are ignored when computing
- * norm. By default this is true, meaning overlap
- * tokens do not count when computing norms.
- *
- * @lucene.experimental
- *
- * @see #computeNorm
- */
- public void setDiscountOverlaps(boolean v) {
- discountOverlaps = v;
- }
-
- /**
- * Returns true if overlap tokens are discounted from the document's length.
- * @see #setDiscountOverlaps
- */
- public boolean getDiscountOverlaps() {
- return discountOverlaps;
- }
-
- @Override
- public String toString() {
- return "DefaultSimilarity";
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
index 8008590..4822e03 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
@@ -367,7 +367,7 @@ public class TestValueSources extends LuceneTestCase {
// no norm field (so agnostic to indexed similarity)
searcher.setSimilarity(new ClassicSimilarity());
ValueSource vs = new NormValueSource("byte");
- assertHits(new FunctionQuery(vs), new float[] { 0f, 0f });
+ assertHits(new FunctionQuery(vs), new float[] { 1f, 1f });
// regardless of whether norms exist, value source exists == 0
assertAllExist(vs);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java
index 810e4c0..f00cc9b 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java
@@ -26,7 +26,6 @@ import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.RandomIndexWriter;
@@ -143,9 +142,9 @@ public class TestPayloadScoreQuery extends LuceneTestCase {
// check includeSpanScore makes a difference here
searcher.setSimilarity(new MultiplyingSimilarity());
try {
- checkQuery(q, new MaxPayloadFunction(), new int[]{ 122, 222 }, new float[]{ 41.802513122558594f, 34.13160705566406f });
- checkQuery(q, new MinPayloadFunction(), new int[]{ 222, 122 }, new float[]{ 34.13160705566406f, 20.901256561279297f });
- checkQuery(q, new AveragePayloadFunction(), new int[] { 122, 222 }, new float[]{ 38.3189697265625f, 34.13160705566406f });
+ checkQuery(q, new MaxPayloadFunction(), new int[]{ 122, 222 }, new float[]{ 20.901256561279297f, 17.06580352783203f });
+ checkQuery(q, new MinPayloadFunction(), new int[]{ 222, 122 }, new float[]{ 17.06580352783203f, 10.450628280639648f });
+ checkQuery(q, new AveragePayloadFunction(), new int[] { 122, 222 }, new float[]{ 19.15948486328125f, 17.06580352783203f });
checkQuery(q, new MaxPayloadFunction(), false, new int[]{122, 222}, new float[]{4.0f, 4.0f});
checkQuery(q, new MinPayloadFunction(), false, new int[]{222, 122}, new float[]{4.0f, 2.0f});
checkQuery(q, new AveragePayloadFunction(), false, new int[]{222, 122}, new float[]{4.0f, 3.666666f});
@@ -298,7 +297,7 @@ public class TestPayloadScoreQuery extends LuceneTestCase {
//Make everything else 1 so we see the effect of the payload
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@Override
- public float lengthNorm(FieldInvertState state) {
+ public float lengthNorm(int length) {
return 1;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java
index da46a50..faa8dd0 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java
@@ -25,7 +25,6 @@ import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@@ -268,7 +267,7 @@ public class TestPayloadTermQuery extends LuceneTestCase {
//Make everything else 1 so we see the effect of the payload
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@Override
- public float lengthNorm(FieldInvertState state) {
+ public float lengthNorm(int length) {
return 1;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/lucene/test-framework/src/java/org/apache/lucene/search/similarities/RandomSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/similarities/RandomSimilarity.java b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/RandomSimilarity.java
index 4bfe4b8..f880935 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/similarities/RandomSimilarity.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/RandomSimilarity.java
@@ -31,7 +31,7 @@ import java.util.Random;
* for the same field.
*/
public class RandomSimilarity extends PerFieldSimilarityWrapper {
- final ClassicSimilarity defaultSim = new ClassicSimilarity();
+ final BM25Similarity defaultSim = new BM25Similarity();
final List<Similarity> knownSims;
Map<String,Similarity> previousMappings = new HashMap<>();
final int perFieldSeed;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java
index a8070fa..8d1b758 100644
--- a/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java
@@ -86,8 +86,8 @@ public class DisMaxRequestHandlerTest extends SolrTestCaseJ4 {
req("cool stuff")
,"//*[@numFound='3']"
,"//result/doc[1]/int[@name='id'][.='42']"
- ,"//result/doc[2]/int[@name='id'][.='8675309']"
- ,"//result/doc[3]/int[@name='id'][.='666']"
+ ,"//result/doc[2]/int[@name='id'][.='666']"
+ ,"//result/doc[3]/int[@name='id'][.='8675309']"
);
assertQ("multi qf",
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
index f113004..f439e87 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
@@ -97,8 +97,8 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
CommonParams.FL, "id, score, [elevated]")
, "//*[@numFound='3']"
, "//result/doc[1]/float[@name='id'][.='7.0']"
- , "//result/doc[2]/float[@name='id'][.='8.0']"
- , "//result/doc[3]/float[@name='id'][.='9.0']",
+ , "//result/doc[2]/float[@name='id'][.='9.0']"
+ , "//result/doc[3]/float[@name='id'][.='8.0']",
"//result/doc[1]/bool[@name='[elevated]'][.='true']",
"//result/doc[2]/bool[@name='[elevated]'][.='false']",
"//result/doc[3]/bool[@name='[elevated]'][.='false']"
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/solr/core/src/test/org/apache/solr/search/TestPayloadScoreQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestPayloadScoreQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestPayloadScoreQParserPlugin.java
index 34017c1..8ac09bb 100644
--- a/solr/core/src/test/org/apache/solr/search/TestPayloadScoreQParserPlugin.java
+++ b/solr/core/src/test/org/apache/solr/search/TestPayloadScoreQParserPlugin.java
@@ -49,6 +49,6 @@ public class TestPayloadScoreQParserPlugin extends SolrTestCaseJ4 {
// TODO: fix this includeSpanScore test to be less brittle - score result is score of "A" (via BM25) multipled by 1.0 (payload value)
assertQ(req("fl","*,score", "q", "{!payload_score f=vals_dpf v=A func=min}"), "//float[@name='score']='1.0'");
- assertQ(req("fl","*,score", "q", "{!payload_score f=vals_dpf v=A func=min includeSpanScore=true}"), "//float[@name='score']='0.25811607'");
+ assertQ(req("fl","*,score", "q", "{!payload_score f=vals_dpf v=A func=min includeSpanScore=true}"), "//float[@name='score']='0.2876821'");
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java b/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java
index e28c957..58a1b53 100644
--- a/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java
+++ b/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java
@@ -65,9 +65,9 @@ public class SortByFunctionTest extends AbstractSolrTestCase {
assertQ(req("fl", "id,score", "q", "f_t:ipod", "sort", "score desc"),
"//*[@numFound='4']",
"//result/doc[1]/int[@name='id'][.='1']",
- "//result/doc[2]/int[@name='id'][.='4']",
- "//result/doc[3]/int[@name='id'][.='2']",
- "//result/doc[4]/int[@name='id'][.='3']"
+ "//result/doc[2]/int[@name='id'][.='2']",
+ "//result/doc[3]/int[@name='id'][.='3']",
+ "//result/doc[4]/int[@name='id'][.='4']"
);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
index 8aae74c..e77c4ee 100644
--- a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
+++ b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
@@ -25,7 +25,6 @@ import java.util.Arrays;
import java.util.List;
import java.util.Random;
-import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.solr.SolrTestCaseJ4;
@@ -431,12 +430,8 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
assertQ(req("fl","*,score","q", "{!func}tf(a_tfidf,cow)", "fq","id:6"),
"//float[@name='score']='" + similarity.tf(5) + "'");
- FieldInvertState state = new FieldInvertState("a_tfidf");
- state.setLength(4);
- long norm = similarity.computeNorm(state);
- float nrm = similarity.decodeNormValue((byte) norm);
assertQ(req("fl","*,score","q", "{!func}norm(a_tfidf)", "fq","id:2"),
- "//float[@name='score']='" + nrm + "'"); // sqrt(4)==2 and is exactly representable when quantized to a byte
+ "//float[@name='score']='0.5'"); // 1/sqrt(4)==1/2==0.5
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06a6034d/solr/core/src/test/org/apache/solr/search/similarities/TestSweetSpotSimilarityFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/similarities/TestSweetSpotSimilarityFactory.java b/solr/core/src/test/org/apache/solr/search/similarities/TestSweetSpotSimilarityFactory.java
index 4b5503d..52d6f8d 100644
--- a/solr/core/src/test/org/apache/solr/search/similarities/TestSweetSpotSimilarityFactory.java
+++ b/solr/core/src/test/org/apache/solr/search/similarities/TestSweetSpotSimilarityFactory.java
@@ -16,8 +16,22 @@
*/
package org.apache.solr.search.similarities;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
import org.apache.lucene.misc.SweetSpotSimilarity;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.similarities.ClassicSimilarity;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.store.Directory;
import org.junit.BeforeClass;
/**
@@ -28,7 +42,38 @@ public class TestSweetSpotSimilarityFactory extends BaseSimilarityTestCase {
public static void beforeClass() throws Exception {
initCore("solrconfig-basic.xml","schema-sweetspot.xml");
}
-
+
+ private static float computeNorm(Similarity sim, int length) throws IOException {
+ String value = IntStream.range(0, length).mapToObj(i -> "a").collect(Collectors.joining(" "));
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setSimilarity(sim));
+ w.addDocument(Collections.singleton(newTextField("foo", value, Store.NO)));
+ DirectoryReader reader = DirectoryReader.open(w);
+ w.close();
+ IndexSearcher searcher = new IndexSearcher(reader);
+ searcher.setSimilarity(sim);
+ Explanation expl = searcher.explain(new TermQuery(new Term("foo", "a")), 0);
+ reader.close();
+ dir.close();
+ Explanation norm = findExplanation(expl, "fieldNorm");
+ assertNotNull(norm);
+ return norm.getValue();
+ }
+
+ private static Explanation findExplanation(Explanation expl, String text) {
+ if (expl.getDescription().startsWith(text)) {
+ return expl;
+ } else {
+ for (Explanation sub : expl.getDetails()) {
+ Explanation match = findExplanation(sub, text);
+ if (match != null) {
+ return match;
+ }
+ }
+ }
+ return null;
+ }
+
/** default parameters */
public void testDefaults() throws Exception {
SweetSpotSimilarity sim = getSimilarity("text", SweetSpotSimilarity.class);
@@ -40,9 +85,9 @@ public class TestSweetSpotSimilarityFactory extends BaseSimilarityTestCase {
}
// default norm sanity check
- assertEquals("norm 1", 1.00F, sim.computeLengthNorm(1), 0.0F);
- assertEquals("norm 4", 0.50F, sim.computeLengthNorm(4), 0.0F);
- assertEquals("norm 16", 0.25F, sim.computeLengthNorm(16), 0.0F);
+ assertEquals("norm 1", 1.00F, computeNorm(sim, 1), 0.0F);
+ assertEquals("norm 4", 0.50F, computeNorm(sim, 4), 0.0F);
+ assertEquals("norm 16", 0.25F, computeNorm(sim, 16), 0.0F);
}
/** baseline with parameters */
@@ -65,17 +110,17 @@ public class TestSweetSpotSimilarityFactory extends BaseSimilarityTestCase {
// norms: plateau from 3-5
assertEquals("norm 1 == 7",
- sim.computeLengthNorm(1), sim.computeLengthNorm(7), 0.0F);
+ computeNorm(sim, 1), computeNorm(sim, 7), 0.0F);
assertEquals("norm 2 == 6",
- sim.computeLengthNorm(1), sim.computeLengthNorm(7), 0.0F);
- assertEquals("norm 3", 1.00F, sim.computeLengthNorm(3), 0.0F);
- assertEquals("norm 4", 1.00F, sim.computeLengthNorm(4), 0.0F);
- assertEquals("norm 5", 1.00F, sim.computeLengthNorm(5), 0.0F);
- assertTrue("norm 6 too high: " + sim.computeLengthNorm(6),
- sim.computeLengthNorm(6) < 1.0F);
+ computeNorm(sim, 1), computeNorm(sim, 7), 0.0F);
+ assertEquals("norm 3", 1.00F, computeNorm(sim, 3), 0.0F);
+ assertEquals("norm 4", 1.00F, computeNorm(sim, 4), 0.0F);
+ assertEquals("norm 5", 1.00F, computeNorm(sim, 5), 0.0F);
+ assertTrue("norm 6 too high: " + computeNorm(sim, 6),
+ computeNorm(sim, 6) < 1.0F);
assertTrue("norm 7 higher then norm 6",
- sim.computeLengthNorm(7) < sim.computeLengthNorm(6));
- assertEquals("norm 20", 0.25F, sim.computeLengthNorm(20), 0.0F);
+ computeNorm(sim, 7) < computeNorm(sim, 6));
+ assertEquals("norm 20", 0.25F, computeNorm(sim, 20), 0.0F);
}
/** hyperbolic with parameters */
@@ -92,16 +137,16 @@ public class TestSweetSpotSimilarityFactory extends BaseSimilarityTestCase {
assertEquals("MID tf", 3.3F+(7.7F - 3.3F)/2.0F, sim.tf(5), 0.00001F);
// norms: plateau from 1-5, shallow slope
- assertEquals("norm 1", 1.00F, sim.computeLengthNorm(1), 0.0F);
- assertEquals("norm 2", 1.00F, sim.computeLengthNorm(2), 0.0F);
- assertEquals("norm 3", 1.00F, sim.computeLengthNorm(3), 0.0F);
- assertEquals("norm 4", 1.00F, sim.computeLengthNorm(4), 0.0F);
- assertEquals("norm 5", 1.00F, sim.computeLengthNorm(5), 0.0F);
- assertTrue("norm 6 too high: " + sim.computeLengthNorm(6),
- sim.computeLengthNorm(6) < 1.0F);
+ assertEquals("norm 1", 1.00F, computeNorm(sim, 1), 0.0F);
+ assertEquals("norm 2", 1.00F, computeNorm(sim, 2), 0.0F);
+ assertEquals("norm 3", 1.00F, computeNorm(sim, 3), 0.0F);
+ assertEquals("norm 4", 1.00F, computeNorm(sim, 4), 0.0F);
+ assertEquals("norm 5", 1.00F, computeNorm(sim, 5), 0.0F);
+ assertTrue("norm 6 too high: " + computeNorm(sim, 6),
+ computeNorm(sim, 6) < 1.0F);
assertTrue("norm 7 higher then norm 6",
- sim.computeLengthNorm(7) < sim.computeLengthNorm(6));
- assertTrue("norm 20 not high enough: " + sim.computeLengthNorm(20),
- 0.25F < sim.computeLengthNorm(20));
+ computeNorm(sim, 7) < computeNorm(sim, 6));
+ assertTrue("norm 20 not high enough: " + computeNorm(sim, 20),
+ 0.25F < computeNorm(sim, 20));
}
}