You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2017/03/02 18:39:41 UTC
[3/4] lucene-solr:master: LUCENE-6819: Remove index-time boosts.
LUCENE-6819: Remove index-time boosts.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8ed2b764
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8ed2b764
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8ed2b764
Branch: refs/heads/master
Commit: 8ed2b764ed4d4d5203b5df1e16fdc1ffd640322c
Parents: 707d7b9
Author: Adrien Grand <jp...@gmail.com>
Authored: Thu Mar 2 19:18:16 2017 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Thu Mar 2 19:18:16 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +
lucene/MIGRATE.txt | 6 +
.../apache/lucene/legacy/TestLegacyField.java | 10 -
.../lucene/codecs/StoredFieldsWriter.java | 5 -
.../java/org/apache/lucene/document/Field.java | 32 --
.../lucene/index/DefaultIndexingChain.java | 8 -
.../apache/lucene/index/FieldInvertState.java | 20 +-
.../org/apache/lucene/index/IndexableField.java | 24 --
.../index/SortingStoredFieldsConsumer.java | 5 -
.../org/apache/lucene/search/package-info.java | 23 +-
.../search/similarities/BM25Similarity.java | 12 +-
.../search/similarities/ClassicSimilarity.java | 5 +-
.../search/similarities/SimilarityBase.java | 8 +-
.../search/similarities/TFIDFSimilarity.java | 46 +--
.../org/apache/lucene/document/TestField.java | 30 --
.../apache/lucene/index/TestCustomNorms.java | 14 +-
.../org/apache/lucene/index/TestFieldReuse.java | 5 -
.../apache/lucene/index/TestIndexSorting.java | 8 +-
.../lucene/index/TestIndexWriterExceptions.java | 66 ----
.../apache/lucene/index/TestIndexableField.java | 10 -
.../test/org/apache/lucene/index/TestNorms.java | 13 +-
.../org/apache/lucene/index/TestOmitTf.java | 2 +-
.../lucene/search/TestDisjunctionMaxQuery.java | 2 +-
.../org/apache/lucene/search/TestDocBoost.java | 98 ------
.../lucene/search/TestMatchAllDocsQuery.java | 9 +-
.../apache/lucene/search/TestSimilarity.java | 2 +-
.../similarities/TestBooleanSimilarity.java | 3 +-
.../search/similarities/TestSimilarityBase.java | 1 -
.../apache/lucene/index/memory/MemoryIndex.java | 70 +---
.../lucene/index/memory/TestMemoryIndex.java | 8 +-
.../memory/TestMemoryIndexAgainstRAMDir.java | 5 +-
.../apache/lucene/document/LazyDocument.java | 5 -
.../apache/lucene/misc/SweetSpotSimilarity.java | 2 +-
.../lucene/misc/SweetSpotSimilarityTest.java | 8 -
.../function/TestLongNormValueSource.java | 3 +-
.../queries/payloads/TestPayloadScoreQuery.java | 2 +-
.../queries/payloads/TestPayloadTermQuery.java | 2 +-
.../lucene/search/BaseExplanationTestCase.java | 1 -
solr/CHANGES.txt | 5 +
.../apache/solr/schema/ICUCollationField.java | 6 +-
.../solr/handler/dataimport/DocBuilder.java | 24 +-
.../handler/dataimport/config/EntityField.java | 6 -
.../handler/extraction/ExtractingParams.java | 13 -
.../handler/extraction/SolrContentHandler.java | 17 +-
.../LanguageIdentifierUpdateProcessor.java | 2 +-
.../solr/morphlines/solr/LoadSolrBuilder.java | 32 +-
.../solr/uima/processor/UIMAToSolrMapper.java | 2 +-
.../handler/DocumentAnalysisRequestHandler.java | 4 +-
.../org/apache/solr/handler/ExportWriter.java | 2 +-
.../solr/handler/admin/LukeRequestHandler.java | 1 -
.../solr/handler/loader/CSVLoaderBase.java | 4 +-
.../apache/solr/handler/loader/JsonLoader.java | 29 +-
.../apache/solr/handler/loader/XMLLoader.java | 23 +-
.../solr/schema/AbstractSpatialFieldType.java | 6 +-
.../java/org/apache/solr/schema/BBoxField.java | 2 +-
.../org/apache/solr/schema/BinaryField.java | 7 +-
.../java/org/apache/solr/schema/BoolField.java | 4 +-
.../org/apache/solr/schema/CollationField.java | 6 +-
.../org/apache/solr/schema/CurrencyField.java | 8 +-
.../org/apache/solr/schema/DateRangeField.java | 4 +-
.../apache/solr/schema/DoublePointField.java | 5 +-
.../java/org/apache/solr/schema/EnumField.java | 14 +-
.../java/org/apache/solr/schema/FieldType.java | 26 +-
.../org/apache/solr/schema/FloatPointField.java | 5 +-
.../org/apache/solr/schema/IntPointField.java | 5 +-
.../java/org/apache/solr/schema/LatLonType.java | 10 +-
.../org/apache/solr/schema/LongPointField.java | 5 +-
.../java/org/apache/solr/schema/PointField.java | 6 +-
.../java/org/apache/solr/schema/PointType.java | 8 +-
.../apache/solr/schema/PreAnalyzedField.java | 10 +-
.../org/apache/solr/schema/SchemaField.java | 10 +-
.../java/org/apache/solr/schema/StrField.java | 4 +-
.../java/org/apache/solr/schema/TrieField.java | 9 +-
.../java/org/apache/solr/search/Grouping.java | 2 +-
.../GroupedEndResultTransformer.java | 2 +-
.../org/apache/solr/update/DocumentBuilder.java | 41 +--
...aluesOrNoneFieldMutatingUpdateProcessor.java | 3 +-
.../processor/AtomicUpdateDocumentMerger.java | 8 +-
.../CloneFieldUpdateProcessorFactory.java | 3 +-
.../ConcatFieldUpdateProcessorFactory.java | 3 +-
.../CountFieldValuesUpdateProcessorFactory.java | 3 +-
.../FieldValueMutatingUpdateProcessor.java | 3 +-
.../FieldValueSubsetUpdateProcessorFactory.java | 3 +-
.../PreAnalyzedUpdateProcessorFactory.java | 9 +-
.../org/apache/solr/BasicFunctionalityTest.java | 12 +-
.../org/apache/solr/TestDocumentBuilder.java | 1 -
.../solr/cloud/TestCloudDeleteByQuery.java | 2 +-
.../TestStressCloudBlindAtomicUpdates.java | 6 +-
.../cloud/TestTolerantUpdateProcessorCloud.java | 2 +-
.../org/apache/solr/handler/JsonLoaderTest.java | 5 +-
.../handler/XmlUpdateRequestHandlerTest.java | 7 -
.../solr/schema/AbstractCurrencyFieldTest.java | 2 +-
.../org/apache/solr/schema/DateFieldTest.java | 6 +-
.../org/apache/solr/schema/DocValuesTest.java | 2 +-
.../org/apache/solr/schema/PolyFieldTest.java | 8 +-
.../solr/schema/PreAnalyzedFieldTest.java | 10 +-
.../apache/solr/schema/SortableBinaryField.java | 6 +-
.../schema/TestSchemalessBufferedUpdates.java | 2 +-
.../solr/search/function/TestFunctionQuery.java | 1 -
.../apache/solr/update/DocumentBuilderTest.java | 317 +------------------
.../CloneFieldUpdateProcessorFactoryTest.java | 14 +-
.../DefaultValueUpdateProcessorTest.java | 7 +-
.../FieldMutatingUpdateProcessorTest.java | 14 +-
.../processor/TolerantUpdateProcessorTest.java | 16 +-
.../UUIDUpdateProcessorFallbackTest.java | 3 +-
.../processor/UpdateProcessorTestBase.java | 7 +-
.../solrj/beans/DocumentObjectBinder.java | 4 +-
.../request/JavaBinUpdateRequestCodec.java | 29 +-
.../solr/client/solrj/util/ClientUtils.java | 32 +-
.../apache/solr/common/SolrInputDocument.java | 53 +---
.../org/apache/solr/common/SolrInputField.java | 28 +-
.../apache/solr/common/util/JavaBinCodec.java | 34 +-
.../solr/client/solrj/SolrExampleTests.java | 76 ++---
.../solr/client/solrj/SolrExampleTestsBase.java | 14 +-
.../solrj/request/TestUpdateRequestCodec.java | 20 +-
.../apache/solr/common/SolrDocumentTest.java | 6 +-
.../java/org/apache/solr/SolrTestCaseJ4.java | 7 -
117 files changed, 438 insertions(+), 1296 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 5e468bf..a339760 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -37,6 +37,10 @@ API Changes
to be less trappy. See IndexReader.getReaderCacheHelper and
LeafReader.getCoreCacheHelper. (Adrien Grand)
+* LUCENE-6819: Index-time boosts are not supported anymore. As a replacement,
+ index-time scoring factors should be indexed into a doc value field and
+ combined at query time using eg. FunctionScoreQuery. (Adrien Grand)
+
Bug Fixes
* LUCENE-7626: IndexWriter will no longer accept broken token offsets
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/MIGRATE.txt
----------------------------------------------------------------------
diff --git a/lucene/MIGRATE.txt b/lucene/MIGRATE.txt
index 51f6435..1ccf56f 100644
--- a/lucene/MIGRATE.txt
+++ b/lucene/MIGRATE.txt
@@ -55,3 +55,9 @@ order to be less trappy. You should now use IndexReader.getReaderCacheHelper()
to have manage caches that take deleted docs and doc values updates into
account, and LeafReader.getCoreCacheHelper() to manage per-segment caches that
do not take deleted docs and doc values updates into account.
+
+## Index-time boosts removal (LUCENE-6819)
+
+Index-time boosts are not supported anymore. As a replacement, index-time
+scoring factors should be indexed in a doc value field and combined with the
+score at query time using FunctionScoreQuery for instance.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyField.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyField.java b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyField.java
index 65ff096..92d1dd6 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyField.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyField.java
@@ -33,7 +33,6 @@ public class TestLegacyField extends LuceneTestCase {
};
for (Field field : fields) {
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -57,7 +56,6 @@ public class TestLegacyField extends LuceneTestCase {
};
for (Field field : fields) {
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -81,7 +79,6 @@ public class TestLegacyField extends LuceneTestCase {
};
for (Field field : fields) {
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -105,7 +102,6 @@ public class TestLegacyField extends LuceneTestCase {
};
for (Field field : fields) {
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -187,10 +183,4 @@ public class TestLegacyField extends LuceneTestCase {
f.setTokenStream(new CannedTokenStream(new Token("foo", 0, 3)));
});
}
-
- private void trySetBoost(Field f) {
- expectThrows(IllegalArgumentException.class, () -> {
- f.setBoost(5.0f);
- });
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
index 0540f4f..39ade42 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
@@ -249,11 +249,6 @@ public abstract class StoredFieldsWriter implements Closeable {
public Reader readerValue() {
return null;
}
-
- @Override
- public float boost() {
- return 1F;
- }
@Override
public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/java/org/apache/lucene/document/Field.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/Field.java b/lucene/core/src/java/org/apache/lucene/document/Field.java
index fa3f5a7..cbb559a 100644
--- a/lucene/core/src/java/org/apache/lucene/document/Field.java
+++ b/lucene/core/src/java/org/apache/lucene/document/Field.java
@@ -81,12 +81,6 @@ public class Field implements IndexableField {
protected TokenStream tokenStream;
/**
- * Field's boost
- * @see #boost()
- */
- protected float boost = 1.0f;
-
- /**
* Expert: creates a field with no initial value.
* Intended only for custom Field subclasses.
* @param name field name
@@ -432,32 +426,6 @@ public class Field implements IndexableField {
public String name() {
return name;
}
-
- /**
- * {@inheritDoc}
- * <p>
- * The default value is <code>1.0f</code> (no boost).
- * @see #setBoost(float)
- */
- @Override
- public float boost() {
- return boost;
- }
-
- /**
- * Sets the boost factor on this field.
- * @throws IllegalArgumentException if this field is not indexed,
- * or if it omits norms.
- * @see #boost()
- */
- public void setBoost(float boost) {
- if (boost != 1.0f) {
- if (type.indexOptions() == IndexOptions.NONE || type.omitNorms()) {
- throw new IllegalArgumentException("You cannot set an index-time boost on an unindexed field, or one that omits norms");
- }
- }
- this.boost = boost;
- }
@Override
public Number numericValue() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
index b118c13..ba65629 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
@@ -426,12 +426,6 @@ final class DefaultIndexingChain extends DocConsumer {
// Invert indexed fields:
if (fieldType.indexOptions() != IndexOptions.NONE) {
-
- // if the field omits norms, the boost cannot be indexed.
- if (fieldType.omitNorms() && field.boost() != 1.0f) {
- throw new UnsupportedOperationException("You cannot set an index-time boost: norms are omitted for field '" + field.name() + "'");
- }
-
fp = getOrAddField(fieldName, fieldType, true);
boolean first = fp.fieldGen != fieldGen;
fp.invert(field, first);
@@ -823,8 +817,6 @@ final class DefaultIndexingChain extends DocConsumer {
invertState.position += docState.analyzer.getPositionIncrementGap(fieldInfo.name);
invertState.offset += docState.analyzer.getOffsetGap(fieldInfo.name);
}
-
- invertState.boost *= field.boost();
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java b/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
index f7eef96..1da02b2 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
@@ -38,7 +38,6 @@ public final class FieldInvertState {
int offset;
int maxTermFrequency;
int uniqueTermCount;
- float boost;
// we must track these across field instances (multi-valued case)
int lastStartOffset = 0;
int lastPosition = 0;
@@ -57,13 +56,12 @@ public final class FieldInvertState {
/** Creates {code FieldInvertState} for the specified
* field name and values for all fields. */
- public FieldInvertState(String name, int position, int length, int numOverlap, int offset, float boost) {
+ public FieldInvertState(String name, int position, int length, int numOverlap, int offset) {
this.name = name;
this.position = position;
this.length = length;
this.numOverlap = numOverlap;
this.offset = offset;
- this.boost = boost;
}
/**
@@ -76,7 +74,6 @@ public final class FieldInvertState {
offset = 0;
maxTermFrequency = 0;
uniqueTermCount = 0;
- boost = 1.0f;
lastStartOffset = 0;
lastPosition = 0;
}
@@ -139,21 +136,6 @@ public final class FieldInvertState {
}
/**
- * Get boost value. This is the cumulative product of
- * document boost and field boost for all field instances
- * sharing the same field name.
- * @return the boost
- */
- public float getBoost() {
- return boost;
- }
-
- /** Set boost value. */
- public void setBoost(float boost) {
- this.boost = boost;
- }
-
- /**
* Get the maximum term-frequency encountered for any term in the field. A
* field containing "the quick brown fox jumps over the lazy dog" would have
* a value of 2, because "the" appears twice.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexableField.java b/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
index f6fc615..f08eab5 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
@@ -21,8 +21,6 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.search.similarities.ClassicSimilarity;
-import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.BytesRef;
// TODO: how to handle versioning here...?
@@ -57,28 +55,6 @@ public interface IndexableField {
*/
public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse);
- /**
- * Returns the field's index-time boost.
- * <p>
- * Only fields can have an index-time boost, if you want to simulate
- * a "document boost", then you must pre-multiply it across all the
- * relevant fields yourself.
- * <p>The boost is used to compute the norm factor for the field. By
- * default, in the {@link Similarity#computeNorm(FieldInvertState)} method,
- * the boost value is multiplied by the length normalization factor and then
- * rounded by {@link ClassicSimilarity#encodeNormValue(float)} before it is stored in the
- * index. One should attempt to ensure that this product does not overflow
- * the range of that encoding.
- * <p>
- * It is illegal to return a boost other than 1.0f for a field that is not
- * indexed ({@link IndexableFieldType#indexOptions()} is IndexOptions.NONE) or
- * omits normalization values ({@link IndexableFieldType#omitNorms()} returns true).
- *
- * @see Similarity#computeNorm(FieldInvertState)
- * @see ClassicSimilarity#encodeNormValue(float)
- */
- public float boost();
-
/** Non-null if this field has a binary value */
public BytesRef binaryValue();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
index b3cc1f4..e5443b2 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
@@ -183,11 +183,6 @@ final class SortingStoredFieldsConsumer extends StoredFieldsConsumer {
}
@Override
- public float boost() {
- return 1F;
- }
-
- @Override
public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) {
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/java/org/apache/lucene/search/package-info.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/package-info.java b/lucene/core/src/java/org/apache/lucene/search/package-info.java
index fc0ab2e..c9d8e26 100644
--- a/lucene/core/src/java/org/apache/lucene/search/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/search/package-info.java
@@ -269,27 +269,8 @@
* Fields and the other in one Field may return different scores for the same query due to length
* normalization.
* <h3>Score Boosting</h3>
- * <p>Lucene allows influencing search results by "boosting" at different times:
- * <ul>
- * <li><b>Index-time boost</b> by calling
- * {@link org.apache.lucene.document.Field#setBoost(float) Field.setBoost()} before a document is
- * added to the index.</li>
- * <li><b>Query-time boost</b> by applying a boost to a query by wrapping with
- * {@link org.apache.lucene.search.BoostQuery}.</li>
- * </ul>
- * <p>Indexing time boosts are pre-processed for storage efficiency and written to
- * storage for a field as follows:
- * <ul>
- * <li>All boosts of that field (i.e. all boosts under the same field name in that doc) are
- * multiplied.</li>
- * <li>The boost is then encoded into a normalization value by the Similarity
- * object at index-time: {@link org.apache.lucene.search.similarities.Similarity#computeNorm computeNorm()}.
- * The actual encoding depends upon the Similarity implementation, but note that most
- * use a lossy encoding (such as multiplying the boost with document length or similar, packed
- * into a single byte!).</li>
- * <li>Decoding of any index-time normalization values and integration into the document's score is also performed
- * at search time by the Similarity.</li>
- * </ul>
+ * <p>Lucene allows influencing the score contribution of various parts of the query by wrapping with
+ * {@link org.apache.lucene.search.BoostQuery}.</p>
*
* <a name="changingScoring"></a>
* <h2>Changing Scoring — Similarity</h2>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
index 6763118..74978fd 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
@@ -96,12 +96,12 @@ public class BM25Similarity extends Similarity {
}
}
- /** The default implementation encodes <code>boost / sqrt(length)</code>
+ /** The default implementation encodes <code>1 / sqrt(length)</code>
* with {@link SmallFloat#floatToByte315(float)}. This is compatible with
- * Lucene's default implementation. If you change this, then you should
- * change {@link #decodeNormValue(byte)} to match. */
- protected byte encodeNormValue(float boost, int fieldLength) {
- return SmallFloat.floatToByte315(boost / (float) Math.sqrt(fieldLength));
+ * Lucene's historic implementation: {@link ClassicSimilarity}. If you
+ * change this, then you should change {@link #decodeNormValue(byte)} to match. */
+ protected byte encodeNormValue(int fieldLength) {
+ return SmallFloat.floatToByte315((float) (1 / Math.sqrt(fieldLength)));
}
/** The default implementation returns <code>1 / f<sup>2</sup></code>
@@ -146,7 +146,7 @@ public class BM25Similarity extends Similarity {
@Override
public final long computeNorm(FieldInvertState state) {
final int numTerms = discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength();
- return encodeNormValue(state.getBoost(), numTerms);
+ return encodeNormValue(numTerms);
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/java/org/apache/lucene/search/similarities/ClassicSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/ClassicSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/ClassicSimilarity.java
index 5a1e237..f56575f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/ClassicSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/ClassicSimilarity.java
@@ -69,8 +69,7 @@ public class ClassicSimilarity extends TFIDFSimilarity {
* represent are rounded down to the largest representable value. Positive
* values too small to represent are rounded up to the smallest positive
* representable value.
- *
- * @see org.apache.lucene.document.Field#setBoost(float)
+ *
* @see org.apache.lucene.util.SmallFloat
*/
@Override
@@ -103,7 +102,7 @@ public class ClassicSimilarity extends TFIDFSimilarity {
numTerms = state.getLength() - state.getNumOverlap();
else
numTerms = state.getLength();
- return state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)));
+ return (float) (1.0 / Math.sqrt(numTerms));
}
/** Implemented as <code>sqrt(freq)</code>. */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
index 925dc59..dbf8d45 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
@@ -235,19 +235,19 @@ public abstract class SimilarityBase extends Similarity {
numTerms = state.getLength() - state.getNumOverlap();
else
numTerms = state.getLength();
- return encodeNormValue(state.getBoost(), numTerms);
+ return encodeNormValue(numTerms);
}
/** Decodes a normalization factor (document length) stored in an index.
- * @see #encodeNormValue(float,float)
+ * @see #encodeNormValue(float)
*/
protected float decodeNormValue(byte norm) {
return NORM_TABLE[norm & 0xFF]; // & 0xFF maps negative bytes to positive above 127
}
/** Encodes the length to a byte via SmallFloat. */
- protected byte encodeNormValue(float boost, float length) {
- return SmallFloat.floatToByte315((boost / (float) Math.sqrt(length)));
+ protected byte encodeNormValue(float length) {
+ return SmallFloat.floatToByte315((float) (1 / Math.sqrt(length)));
}
// ----------------------------- Static methods ------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
index 12ab1a2..2246561 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
@@ -369,49 +369,9 @@ import org.apache.lucene.util.BytesRef;
*
* <li>
* <A NAME="formula_norm"></A>
- * <b><i>norm(t,d)</i></b> encapsulates a few (indexing time) boost and length factors:
- *
- * <ul>
- * <li><b>Field boost</b> - set by calling
- * {@link org.apache.lucene.document.Field#setBoost(float) field.setBoost()}
- * before adding the field to a document.
- * </li>
- * <li><b>lengthNorm</b> - computed
- * when the document is added to the index in accordance with the number of tokens
- * of this field in the document, so that shorter fields contribute more to the score.
- * LengthNorm is computed by the Similarity class in effect at indexing.
- * </li>
- * </ul>
- * The {@link #computeNorm} method is responsible for
- * combining all of these factors into a single float.
- *
- * <p>
- * When a document is added to the index, all the above factors are multiplied.
- * If the document has multiple fields with the same name, all their boosts are multiplied together:
- *
- * <br> <br>
- * <table cellpadding="1" cellspacing="0" border="0" style="width:auto; margin-left:auto; margin-right:auto" summary="index-time normalization">
- * <tr>
- * <td valign="middle" align="right" rowspan="1">
- * norm(t,d) =
- * lengthNorm
- * ·
- * </td>
- * <td valign="bottom" align="center" rowspan="1" style="text-align: center">
- * <big><big><big>∏</big></big></big>
- * </td>
- * <td valign="middle" align="right" rowspan="1">
- * {@link org.apache.lucene.index.IndexableField#boost() f.boost}()
- * </td>
- * </tr>
- * <tr valign="top">
- * <td></td>
- * <td align="center" style="text-align: center"><small>field <i><b>f</b></i> in <i>d</i> named as <i><b>t</b></i></small></td>
- * <td></td>
- * </tr>
- * </table>
- * Note that search time is too late to modify this <i>norm</i> part of scoring,
- * e.g. by using a different {@link Similarity} for search.
+ * <b><i>norm(t,d)</i></b> is an index-time boost factor that solely
+ * depends on the number of tokens of this field in the document, so
+ * that shorter fields contribute more to the score.
* </li>
* </ol>
*
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/document/TestField.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/document/TestField.java b/lucene/core/src/test/org/apache/lucene/document/TestField.java
index 4ef7ffb..c087505 100644
--- a/lucene/core/src/test/org/apache/lucene/document/TestField.java
+++ b/lucene/core/src/test/org/apache/lucene/document/TestField.java
@@ -38,7 +38,6 @@ public class TestField extends LuceneTestCase {
public void testDoublePoint() throws Exception {
Field field = new DoublePoint("foo", 5d);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -58,7 +57,6 @@ public class TestField extends LuceneTestCase {
public void testDoublePoint2D() throws Exception {
DoublePoint field = new DoublePoint("foo", 5d, 4d);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -84,7 +82,6 @@ public class TestField extends LuceneTestCase {
public void testDoubleDocValuesField() throws Exception {
DoubleDocValuesField field = new DoubleDocValuesField("foo", 5d);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -103,7 +100,6 @@ public class TestField extends LuceneTestCase {
public void testFloatDocValuesField() throws Exception {
FloatDocValuesField field = new FloatDocValuesField("foo", 5f);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -122,7 +118,6 @@ public class TestField extends LuceneTestCase {
public void testFloatPoint() throws Exception {
Field field = new FloatPoint("foo", 5f);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -142,7 +137,6 @@ public class TestField extends LuceneTestCase {
public void testFloatPoint2D() throws Exception {
FloatPoint field = new FloatPoint("foo", 5f, 4f);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -166,7 +160,6 @@ public class TestField extends LuceneTestCase {
public void testIntPoint() throws Exception {
Field field = new IntPoint("foo", 5);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -186,7 +179,6 @@ public class TestField extends LuceneTestCase {
public void testIntPoint2D() throws Exception {
IntPoint field = new IntPoint("foo", 5, 4);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -210,7 +202,6 @@ public class TestField extends LuceneTestCase {
public void testNumericDocValuesField() throws Exception {
NumericDocValuesField field = new NumericDocValuesField("foo", 5L);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -229,7 +220,6 @@ public class TestField extends LuceneTestCase {
public void testLongPoint() throws Exception {
Field field = new LongPoint("foo", 5);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -249,7 +239,6 @@ public class TestField extends LuceneTestCase {
public void testLongPoint2D() throws Exception {
LongPoint field = new LongPoint("foo", 5, 4);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -273,7 +262,6 @@ public class TestField extends LuceneTestCase {
public void testSortedBytesDocValuesField() throws Exception {
SortedDocValuesField field = new SortedDocValuesField("foo", new BytesRef("bar"));
- trySetBoost(field);
trySetByteValue(field);
field.setBytesValue("fubar".getBytes(StandardCharsets.UTF_8));
field.setBytesValue(new BytesRef("baz"));
@@ -292,7 +280,6 @@ public class TestField extends LuceneTestCase {
public void testBinaryDocValuesField() throws Exception {
BinaryDocValuesField field = new BinaryDocValuesField("foo", new BytesRef("bar"));
- trySetBoost(field);
trySetByteValue(field);
field.setBytesValue("fubar".getBytes(StandardCharsets.UTF_8));
field.setBytesValue(new BytesRef("baz"));
@@ -315,7 +302,6 @@ public class TestField extends LuceneTestCase {
};
for (Field field : fields) {
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -339,7 +325,6 @@ public class TestField extends LuceneTestCase {
};
for (Field field : fields) {
- field.setBoost(5f);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -353,14 +338,12 @@ public class TestField extends LuceneTestCase {
field.setTokenStream(new CannedTokenStream(new Token("foo", 0, 3)));
assertEquals("baz", field.stringValue());
- assertEquals(5f, field.boost(), 0f);
}
}
public void testTextFieldReader() throws Exception {
Field field = new TextField("foo", new StringReader("bar"));
- field.setBoost(5f);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -374,7 +357,6 @@ public class TestField extends LuceneTestCase {
field.setTokenStream(new CannedTokenStream(new Token("foo", 0, 3)));
assertNotNull(field.readerValue());
- assertEquals(5f, field.boost(), 0f);
}
/* TODO: this is pretty expert and crazy
@@ -391,7 +373,6 @@ public class TestField extends LuceneTestCase {
};
for (Field field : fields) {
- trySetBoost(field);
trySetByteValue(field);
field.setBytesValue("baz".getBytes(StandardCharsets.UTF_8));
field.setBytesValue(new BytesRef("baz"));
@@ -410,7 +391,6 @@ public class TestField extends LuceneTestCase {
public void testStoredFieldString() throws Exception {
Field field = new StoredField("foo", "bar");
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -428,7 +408,6 @@ public class TestField extends LuceneTestCase {
public void testStoredFieldInt() throws Exception {
Field field = new StoredField("foo", 1);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -446,7 +425,6 @@ public class TestField extends LuceneTestCase {
public void testStoredFieldDouble() throws Exception {
Field field = new StoredField("foo", 1D);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -464,7 +442,6 @@ public class TestField extends LuceneTestCase {
public void testStoredFieldFloat() throws Exception {
Field field = new StoredField("foo", 1F);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -482,7 +459,6 @@ public class TestField extends LuceneTestCase {
public void testStoredFieldLong() throws Exception {
Field field = new StoredField("foo", 1L);
- trySetBoost(field);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
@@ -586,10 +562,4 @@ public class TestField extends LuceneTestCase {
});
}
- private void trySetBoost(Field f) {
- expectThrows(IllegalArgumentException.class, () -> {
- f.setBoost(5.0f);
- });
- }
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
index 65310cf..f193140 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
@@ -17,6 +17,8 @@
package org.apache.lucene.index;
import java.io.IOException;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -53,9 +55,9 @@ public class TestCustomNorms extends LuceneTestCase {
int num = atLeast(100);
for (int i = 0; i < num; i++) {
Document doc = docs.nextDoc();
- float nextFloat = random().nextFloat();
- Field f = new TextField(floatTestField, "" + nextFloat, Field.Store.YES);
- f.setBoost(nextFloat);
+ int boost = TestUtil.nextInt(random(), 1, 10);
+ String value = IntStream.range(0, boost).mapToObj(k -> Integer.toString(boost)).collect(Collectors.joining(" "));
+ Field f = new TextField(floatTestField, value, Field.Store.YES);
doc.add(f);
writer.addDocument(doc);
@@ -71,9 +73,9 @@ public class TestCustomNorms extends LuceneTestCase {
assertNotNull(norms);
for (int i = 0; i < open.maxDoc(); i++) {
Document document = open.document(i);
- float expected = Float.parseFloat(document.get(floatTestField));
+ int expected = Integer.parseInt(document.get(floatTestField).split(" ")[0]);
assertEquals(i, norms.nextDoc());
- assertEquals(expected, Float.intBitsToFloat((int)norms.longValue()), 0.0f);
+ assertEquals(expected, norms.longValue());
}
open.close();
dir.close();
@@ -97,7 +99,7 @@ public class TestCustomNorms extends LuceneTestCase {
@Override
public long computeNorm(FieldInvertState state) {
- return Float.floatToIntBits(state.getBoost());
+ return state.getLength();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java
index 977df3d..c5552a4 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java
@@ -86,11 +86,6 @@ public class TestFieldReuse extends BaseTokenStreamTestCase {
lastSeen = reuse;
return lastReturned = new CannedTokenStream(new Token("unimportant", 0, 10));
}
-
- @Override
- public float boost() {
- return 1;
- }
@Override
public BytesRef binaryValue() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index c2b180a..4275056 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -29,6 +29,8 @@ import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -1939,7 +1941,7 @@ public class TestIndexSorting extends LuceneTestCase {
@Override
public long computeNorm(FieldInvertState state) {
if (state.getName().equals("norms")) {
- return Float.floatToIntBits(state.getBoost());
+ return state.getLength();
} else {
return in.computeNorm(state);
}
@@ -2021,8 +2023,8 @@ public class TestIndexSorting extends LuceneTestCase {
positions.setId(id);
doc.add(new Field("positions", positions, POSITIONS_TYPE));
doc.add(new NumericDocValuesField("numeric", id));
- TextField norms = new TextField("norms", Integer.toString(id), Store.NO);
- norms.setBoost(Float.intBitsToFloat(id));
+ String value = IntStream.range(0, id).mapToObj(k -> Integer.toString(id)).collect(Collectors.joining(" "));
+ TextField norms = new TextField("norms", value, Store.NO);
doc.add(norms);
doc.add(new BinaryDocValuesField("binary", new BytesRef(Integer.toString(id))));
doc.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(id))));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
index 6fdfca8..987852f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.Reader;
import java.io.StringReader;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
@@ -1687,71 +1686,6 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
uoe.doFail = false;
d.close();
}
-
- public void testBoostOmitNorms() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- IndexWriter iw = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new StringField("field1", "sometext", Field.Store.YES));
- doc.add(new TextField("field2", "sometext", Field.Store.NO));
- doc.add(new StringField("foo", "bar", Field.Store.NO));
- iw.addDocument(doc); // add an 'ok' document
- expectThrows(UnsupportedOperationException.class, () -> {
- // try to boost with norms omitted
- List<IndexableField> list = new ArrayList<>();
- list.add(new IndexableField() {
- @Override
- public String name() {
- return "foo";
- }
-
- @Override
- public IndexableFieldType fieldType() {
- return StringField.TYPE_NOT_STORED;
- }
-
- @Override
- public float boost() {
- return 5f;
- }
-
- @Override
- public BytesRef binaryValue() {
- return null;
- }
-
- @Override
- public String stringValue() {
- return "baz";
- }
-
- @Override
- public Reader readerValue() {
- return null;
- }
-
- @Override
- public Number numericValue() {
- return null;
- }
-
- @Override
- public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) {
- return null;
- }
- });
- iw.addDocument(list);
- });
-
- DirectoryReader ir = DirectoryReader.open(iw);
- assertEquals(1, ir.numDocs());
- assertEquals("sometext", ir.document(0).get("field1"));
- ir.close();
- iw.close();
- dir.close();
- }
// See LUCENE-4870 TooManyOpenFiles errors are thrown as
// FNFExceptions which can trigger data loss.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
index 67edab9..e60a3e1 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
@@ -111,11 +111,6 @@ public class TestIndexableField extends LuceneTestCase {
}
@Override
- public float boost() {
- return 1.0f + random().nextFloat();
- }
-
- @Override
public BytesRef binaryValue() {
if ((counter%10) == 3) {
final byte[] bytes = new byte[10];
@@ -339,11 +334,6 @@ public class TestIndexableField extends LuceneTestCase {
}
@Override
- public float boost() {
- return 1.0f;
- }
-
- @Override
public Number numericValue() {
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
index 52038bc..363f57e 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
@@ -19,6 +19,8 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Random;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -117,7 +119,7 @@ public class TestNorms extends LuceneTestCase {
assertNotNull(normValues);
for (int i = 0; i < open.maxDoc(); i++) {
Document document = open.document(i);
- int expected = Integer.parseInt(document.get(byteTestField));
+ int expected = Integer.parseInt(document.get(byteTestField).split(" ")[0]);
assertEquals(i, normValues.nextDoc());
assertEquals(expected, normValues.longValue());
}
@@ -139,9 +141,9 @@ public class TestNorms extends LuceneTestCase {
int num = atLeast(100);
for (int i = 0; i < num; i++) {
Document doc = docs.nextDoc();
- int boost = random().nextInt(255);
- Field f = new TextField(byteTestField, "" + boost, Field.Store.YES);
- f.setBoost(boost);
+ int boost = TestUtil.nextInt(random, 1, 255);
+ String value = IntStream.range(0, boost).mapToObj(k -> Integer.toString(boost)).collect(Collectors.joining(" "));
+ Field f = new TextField(byteTestField, value, Field.Store.YES);
doc.add(f);
writer.addDocument(doc);
doc.removeField(byteTestField);
@@ -173,8 +175,7 @@ public class TestNorms extends LuceneTestCase {
@Override
public long computeNorm(FieldInvertState state) {
- int boost = (int) state.getBoost();
- return (0xFF & boost);
+ return state.getLength();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
index 3a13d9b..0deafdd 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
@@ -46,7 +46,7 @@ public class TestOmitTf extends LuceneTestCase {
public static class SimpleSimilarity extends TFIDFSimilarity {
@Override public float decodeNormValue(long norm) { return norm; }
@Override public long encodeNormValue(float f) { return (long) f; }
- @Override public float lengthNorm(FieldInvertState state) { return state.getBoost(); }
+ @Override public float lengthNorm(FieldInvertState state) { return 1; }
@Override public float tf(float freq) { return freq; }
@Override public float sloppyFreq(int distance) { return 2.0f; }
@Override public float idf(long docFreq, long docCount) { return 1.0f; }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
index 87046c7..e20163a 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
@@ -74,7 +74,7 @@ public class TestDisjunctionMaxQuery extends LuceneTestCase {
@Override
public float lengthNorm(FieldInvertState state) {
// Disable length norm
- return state.getBoost();
+ return 1;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/search/TestDocBoost.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocBoost.java b/lucene/core/src/test/org/apache/lucene/search/TestDocBoost.java
deleted file mode 100644
index ecc4645..0000000
--- a/lucene/core/src/test/org/apache/lucene/search/TestDocBoost.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search;
-
-
-import java.io.IOException;
-
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.*;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase;
-
-/** Document boost unit test.
- *
- *
- */
-public class TestDocBoost extends LuceneTestCase {
-
- public void testDocBoost() throws Exception {
- Directory store = newDirectory();
- RandomIndexWriter writer = new RandomIndexWriter(random(), store, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
-
- Field f1 = newTextField("field", "word", Field.Store.YES);
- Field f2 = newTextField("field", "word", Field.Store.YES);
- f2.setBoost(2.0f);
-
- Document d1 = new Document();
- Document d2 = new Document();
-
- d1.add(f1); // boost = 1
- d2.add(f2); // boost = 2
-
- writer.addDocument(d1);
- writer.addDocument(d2);
-
- IndexReader reader = writer.getReader();
- writer.close();
-
- final float[] scores = new float[4];
-
- IndexSearcher searcher = newSearcher(reader);
- searcher.search
- (new TermQuery(new Term("field", "word")),
- new SimpleCollector() {
- private int base = 0;
- private Scorer scorer;
- @Override
- public void setScorer(Scorer scorer) {
- this.scorer = scorer;
- }
- @Override
- public final void collect(int doc) throws IOException {
- scores[doc + base] = scorer.score();
- }
- @Override
- protected void doSetNextReader(LeafReaderContext context) throws IOException {
- base = context.docBase;
- }
- @Override
- public boolean needsScores() {
- return true;
- }
- });
-
- float lastScore = 0.0f;
-
- for (int i = 0; i < 2; i++) {
- if (VERBOSE) {
- System.out.println(searcher.explain(new TermQuery(new Term("field", "word")), i));
- }
- if (scores[i] != 0.0) {
- assertTrue("score: " + scores[i] + " should be > lastScore: " + lastScore, scores[i] > lastScore);
- }
- lastScore = scores[i];
- }
-
- reader.close();
- store.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java
index 8e9fb28..ee02eed 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java
@@ -45,9 +45,9 @@ public class TestMatchAllDocsQuery extends LuceneTestCase {
public void testQuery() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(analyzer).setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy()));
- addDoc("one", iw, 1f);
- addDoc("two", iw, 20f);
- addDoc("three four", iw, 300f);
+ addDoc("one", iw);
+ addDoc("two", iw);
+ addDoc("three four", iw);
IndexReader ir = DirectoryReader.open(iw);
IndexSearcher is = newSearcher(ir);
@@ -92,10 +92,9 @@ public class TestMatchAllDocsQuery extends LuceneTestCase {
assertTrue(q1.equals(q2));
}
- private void addDoc(String text, IndexWriter iw, float boost) throws IOException {
+ private void addDoc(String text, IndexWriter iw) throws IOException {
Document doc = new Document();
Field f = newTextField("key", text, Field.Store.YES);
- f.setBoost(boost);
doc.add(f);
iw.addDocument(doc);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java
index 966e5da..3faa5c2 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java
@@ -39,7 +39,7 @@ import org.apache.lucene.document.Document;
public class TestSimilarity extends LuceneTestCase {
public static class SimpleSimilarity extends ClassicSimilarity {
- @Override public float lengthNorm(FieldInvertState state) { return state.getBoost(); }
+ @Override public float lengthNorm(FieldInvertState state) { return 1; }
@Override public float tf(float freq) { return freq; }
@Override public float sloppyFreq(int distance) { return 2.0f; }
@Override public float idf(long docFreq, long docCount) { return 1.0f; }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.java
index 15b1448..23f65d3 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.java
@@ -106,8 +106,7 @@ public class TestBooleanSimilarity extends LuceneTestCase {
final int length = TestUtil.nextInt(random(), 1, 100);
final int position = random().nextInt(length);
final int numOverlaps = random().nextInt(50);
- final float boost = random().nextFloat() * 10;
- FieldInvertState state = new FieldInvertState("foo", position, length, numOverlaps, 100, boost);
+ FieldInvertState state = new FieldInvertState("foo", position, length, numOverlaps, 100);
assertEquals(
sim2.computeNorm(state),
sim1.computeNorm(state),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
index aec55de..373b9e6 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
@@ -593,7 +593,6 @@ public class TestSimilarityBase extends LuceneTestCase {
FieldInvertState state = new FieldInvertState("foo");
state.setLength(5);
state.setNumOverlap(2);
- state.setBoost(3);
assertEquals(expected.computeNorm(state), actual.computeNorm(state));
expected.setDiscountOverlaps(true);
actual.setDiscountOverlaps(true);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index 32b2d24..0c8ea6d 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -275,7 +275,7 @@ public class MemoryIndex {
throw new IllegalArgumentException("analyzer must not be null");
TokenStream stream = analyzer.tokenStream(fieldName, text);
- storeTerms(getInfo(fieldName, defaultFieldType), stream, 1.0f,
+ storeTerms(getInfo(fieldName, defaultFieldType), stream,
analyzer.getPositionIncrementGap(fieldName), analyzer.getOffsetGap(fieldName));
}
@@ -358,31 +358,7 @@ public class MemoryIndex {
}
};
}
-
- /**
- * Equivalent to <code>addField(fieldName, stream, 1.0f)</code>.
- *
- * @param fieldName
- * a name to be associated with the text
- * @param stream
- * the token stream to retrieve tokens from
- */
- public void addField(String fieldName, TokenStream stream) {
- addField(fieldName, stream, 1.0f);
- }
- /**
- * Adds a lucene {@link IndexableField} to the MemoryIndex using the provided analyzer.
- * Also stores doc values based on {@link IndexableFieldType#docValuesType()} if set.
- *
- * @param field the field to add
- * @param analyzer the analyzer to use for term analysis
- * @throws IllegalArgumentException if the field is a DocValues or Point field, as these
- * structures are not supported by MemoryIndex
- */
- public void addField(IndexableField field, Analyzer analyzer) {
- addField(field, analyzer, 1.0f);
- }
/**
* Adds a lucene {@link IndexableField} to the MemoryIndex using the provided analyzer.
@@ -390,9 +366,8 @@ public class MemoryIndex {
*
* @param field the field to add
* @param analyzer the analyzer to use for term analysis
- * @param boost a field boost
*/
- public void addField(IndexableField field, Analyzer analyzer, float boost) {
+ public void addField(IndexableField field, Analyzer analyzer) {
Info info = getInfo(field.name(), field.fieldType());
@@ -409,7 +384,7 @@ public class MemoryIndex {
positionIncrementGap = 0;
}
if (tokenStream != null) {
- storeTerms(info, tokenStream, boost, positionIncrementGap, offsetGap);
+ storeTerms(info, tokenStream, positionIncrementGap, offsetGap);
}
DocValuesType docValuesType = field.fieldType().docValuesType();
@@ -451,13 +426,9 @@ public class MemoryIndex {
* a name to be associated with the text
* @param stream
* the token stream to retrieve tokens from.
- * @param boost
- * the boost factor for hits for this field
- *
- * @see org.apache.lucene.document.Field#setBoost(float)
*/
- public void addField(String fieldName, TokenStream stream, float boost) {
- addField(fieldName, stream, boost, 0);
+ public void addField(String fieldName, TokenStream stream) {
+ addField(fieldName, stream, 0);
}
@@ -472,17 +443,13 @@ public class MemoryIndex {
* a name to be associated with the text
* @param stream
* the token stream to retrieve tokens from.
- * @param boost
- * the boost factor for hits for this field
*
* @param positionIncrementGap
* the position increment gap if fields with the same name are added more than once
*
- *
- * @see org.apache.lucene.document.Field#setBoost(float)
*/
- public void addField(String fieldName, TokenStream stream, float boost, int positionIncrementGap) {
- addField(fieldName, stream, boost, positionIncrementGap, 1);
+ public void addField(String fieldName, TokenStream stream, int positionIncrementGap) {
+ addField(fieldName, stream, positionIncrementGap, 1);
}
/**
@@ -497,17 +464,14 @@ public class MemoryIndex {
* a name to be associated with the text
* @param tokenStream
* the token stream to retrieve tokens from. It's guaranteed to be closed no matter what.
- * @param boost
- * the boost factor for hits for this field
* @param positionIncrementGap
* the position increment gap if fields with the same name are added more than once
* @param offsetGap
* the offset gap if fields with the same name are added more than once
- * @see org.apache.lucene.document.Field#setBoost(float)
*/
- public void addField(String fieldName, TokenStream tokenStream, float boost, int positionIncrementGap, int offsetGap) {
+ public void addField(String fieldName, TokenStream tokenStream, int positionIncrementGap, int offsetGap) {
Info info = getInfo(fieldName, defaultFieldType);
- storeTerms(info, tokenStream, boost, positionIncrementGap, offsetGap);
+ storeTerms(info, tokenStream, positionIncrementGap, offsetGap);
}
private Info getInfo(String fieldName, IndexableFieldType fieldType) {
@@ -600,20 +564,13 @@ public class MemoryIndex {
}
}
- private void storeTerms(Info info, TokenStream tokenStream, float boost, int positionIncrementGap, int offsetGap) {
-
- if (boost <= 0.0f) {
- throw new IllegalArgumentException("boost factor must be greater than 0.0");
- }
+ private void storeTerms(Info info, TokenStream tokenStream, int positionIncrementGap, int offsetGap) {
int pos = -1;
int offset = 0;
- if (info.numTokens == 0) {
- info.boost = boost;
- } else if (info.numTokens > 0) {
+ if (info.numTokens > 0) {
pos = info.lastPosition + positionIncrementGap;
offset = info.lastOffset + offsetGap;
- info.boost *= boost;
}
try (TokenStream stream = tokenStream) {
@@ -846,9 +803,6 @@ public class MemoryIndex {
/** Number of overlapping tokens for this field */
private int numOverlapTokens;
-
- /** Boost factor for hits for this field */
- private float boost;
private long sumTotalTermFreq;
@@ -939,7 +893,7 @@ public class MemoryIndex {
NumericDocValues getNormDocValues() {
if (norm == null) {
FieldInvertState invertState = new FieldInvertState(fieldInfo.name, fieldInfo.number,
- numTokens, numOverlapTokens, 0, boost);
+ numTokens, numOverlapTokens, 0);
final long value = normSimilarity.computeNorm(invertState);
if (DEBUG) System.err.println("MemoryIndexReader.norms: " + fieldInfo.name + ":" + value + ":" + numTokens);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
index 4e2189c..f34f30c 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
@@ -422,11 +422,11 @@ public class TestMemoryIndex extends LuceneTestCase {
}
}
- public void testPointValuesDoNotAffectBoostPositionsOrOffset() throws Exception {
+ public void testPointValuesDoNotAffectPositionsOrOffset() throws Exception {
MemoryIndex mi = new MemoryIndex(true, true);
- mi.addField(new TextField("text", "quick brown fox", Field.Store.NO), analyzer, 5f);
- mi.addField(new BinaryPoint("text", "quick".getBytes(StandardCharsets.UTF_8)), analyzer, 5f);
- mi.addField(new BinaryPoint("text", "brown".getBytes(StandardCharsets.UTF_8)), analyzer, 5f);
+ mi.addField(new TextField("text", "quick brown fox", Field.Store.NO), analyzer);
+ mi.addField(new BinaryPoint("text", "quick".getBytes(StandardCharsets.UTF_8)), analyzer);
+ mi.addField(new BinaryPoint("text", "brown".getBytes(StandardCharsets.UTF_8)), analyzer);
LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();
TermsEnum tenum = leafReader.terms("text").iterator();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
index 03c17a5..8a751b1 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
@@ -536,14 +536,13 @@ public class TestMemoryIndexAgainstRAMDir extends BaseTokenStreamTestCase {
MemoryIndex mi = new MemoryIndex(true, true);
MockAnalyzer mockAnalyzer = new MockAnalyzer(random());
- mi.addField(new BinaryDocValuesField("text", new BytesRef("quick brown fox")), mockAnalyzer, 5f);
- mi.addField(new TextField("text", "quick brown fox", Field.Store.NO), mockAnalyzer, 5f);
+ mi.addField(new BinaryDocValuesField("text", new BytesRef("quick brown fox")), mockAnalyzer);
+ mi.addField(new TextField("text", "quick brown fox", Field.Store.NO), mockAnalyzer);
LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();
Document doc = new Document();
doc.add(new BinaryDocValuesField("text", new BytesRef("quick brown fox")));
Field field = new TextField("text", "quick brown fox", Field.Store.NO);
- field.setBoost(5f);
doc.add(field);
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/misc/src/java/org/apache/lucene/document/LazyDocument.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/java/org/apache/lucene/document/LazyDocument.java b/lucene/misc/src/java/org/apache/lucene/document/LazyDocument.java
index c1683a7..7fd8d98 100644
--- a/lucene/misc/src/java/org/apache/lucene/document/LazyDocument.java
+++ b/lucene/misc/src/java/org/apache/lucene/document/LazyDocument.java
@@ -162,11 +162,6 @@ public class LazyDocument {
}
@Override
- public float boost() {
- return 1.0f;
- }
-
- @Override
public BytesRef binaryValue() {
return getRealValue().binaryValue();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java b/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
index 7eeeae0..9307b94 100644
--- a/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
+++ b/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
@@ -111,7 +111,7 @@ public class SweetSpotSimilarity extends ClassicSimilarity {
else
numTokens = state.getLength();
- return state.getBoost() * computeLengthNorm(numTokens);
+ return computeLengthNorm(numTokens);
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java b/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
index b618654..cd6a819 100644
--- a/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
+++ b/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
@@ -47,7 +47,6 @@ public class SweetSpotSimilarityTest extends LuceneTestCase {
// base case, should degrade
FieldInvertState invertState = new FieldInvertState("bogus");
- invertState.setBoost(1.0f);
for (int i = 1; i < 1000; i++) {
invertState.setLength(i);
assertEquals("base case: i="+i,
@@ -108,7 +107,6 @@ public class SweetSpotSimilarityTest extends LuceneTestCase {
};
invertState = new FieldInvertState("foo");
- invertState.setBoost(1.0f);
for (int i = 3; i <=10; i++) {
invertState.setLength(i);
assertEquals("f: 3,10: spot i="+i,
@@ -129,7 +127,6 @@ public class SweetSpotSimilarityTest extends LuceneTestCase {
}
invertState = new FieldInvertState("bar");
- invertState.setBoost(1.0f);
for (int i = 8; i <=13; i++) {
invertState.setLength(i);
assertEquals("f: 8,13: spot i="+i,
@@ -139,7 +136,6 @@ public class SweetSpotSimilarityTest extends LuceneTestCase {
}
invertState = new FieldInvertState("yak");
- invertState.setBoost(1.0f);
for (int i = 6; i <=9; i++) {
invertState.setLength(i);
assertEquals("f: 6,9: spot i="+i,
@@ -149,7 +145,6 @@ public class SweetSpotSimilarityTest extends LuceneTestCase {
}
invertState = new FieldInvertState("bar");
- invertState.setBoost(1.0f);
for (int i = 13; i < 1000; i++) {
invertState.setLength(i-12);
final byte normD = computeAndGetNorm(d, invertState);
@@ -162,7 +157,6 @@ public class SweetSpotSimilarityTest extends LuceneTestCase {
}
invertState = new FieldInvertState("yak");
- invertState.setBoost(1.0f);
for (int i = 9; i < 1000; i++) {
invertState.setLength(i-8);
final byte normD = computeAndGetNorm(d, invertState);
@@ -179,11 +173,9 @@ public class SweetSpotSimilarityTest extends LuceneTestCase {
for (int i = 9; i < 1000; i++) {
invertState = new FieldInvertState("a");
- invertState.setBoost(1.0f);
invertState.setLength(i);
final byte normSS = computeAndGetNorm(sp, invertState);
invertState = new FieldInvertState("b");
- invertState.setBoost(1.0f);
invertState.setLength(i);
final byte normS = computeAndGetNorm(sp, invertState);
assertTrue("s: i="+i+" : a="+normSS+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java
index cfebc47..11060e5 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java
@@ -135,7 +135,6 @@ class PreciseClassicSimilarity extends TFIDFSimilarity {
* values too small to represent are rounded up to the smallest positive
* representable value.
*
- * @see org.apache.lucene.document.Field#setBoost(float)
* @see org.apache.lucene.util.SmallFloat
*/
@Override
@@ -169,7 +168,7 @@ class PreciseClassicSimilarity extends TFIDFSimilarity {
} else {
numTerms = state.getLength();
}
- return state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)));
+ return (float) (1.0 / Math.sqrt(numTerms));
}
/** Implemented as <code>sqrt(freq)</code>. */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java
index 188ef61..2afec44 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java
@@ -267,7 +267,7 @@ public class TestPayloadScoreQuery extends LuceneTestCase {
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@Override
public float lengthNorm(FieldInvertState state) {
- return state.getBoost();
+ return 1;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java
index 9cc7067..da46a50 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java
@@ -269,7 +269,7 @@ public class TestPayloadTermQuery extends LuceneTestCase {
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@Override
public float lengthNorm(FieldInvertState state) {
- return state.getBoost();
+ return 1;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java
index 2ea87f0..24e86e1 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java
@@ -85,7 +85,6 @@ public abstract class BaseExplanationTestCase extends LuceneTestCase {
doc.add(newStringField(KEY, ""+index, Field.Store.NO));
doc.add(new SortedDocValuesField(KEY, new BytesRef(""+index)));
Field f = newTextField(FIELD, docFields[index], Field.Store.NO);
- f.setBoost(index);
doc.add(f);
doc.add(newTextField(ALTFIELD, docFields[index], Field.Store.NO));
return doc;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index fa02c39..152b344 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -45,6 +45,11 @@ Upgrading from Solr 6.x
* Deprecated collection and configset methods on MiniSolrCloudCluster have been
removed
+* Index-time boosts are not supported anymore. If any boosts are provided, they
+ will be ignored by the indexing chain. As a replacement, index-time scoring
+ factors should be indexed in a separate field and combined with the query
+ score using a function query.
+
Bug Fixes
----------------------
* SOLR-9262: Connection and read timeouts are being ignored by UpdateShardHandler after SOLR-4509.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
index 5152768..7d9e1c7 100644
--- a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
+++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
@@ -284,10 +284,10 @@ public class ICUCollationField extends FieldType {
}
@Override
- public List<IndexableField> createFields(SchemaField field, Object value, float boost) {
+ public List<IndexableField> createFields(SchemaField field, Object value) {
if (field.hasDocValues()) {
List<IndexableField> fields = new ArrayList<>();
- fields.add(createField(field, value, boost));
+ fields.add(createField(field, value));
final BytesRef bytes = getCollationKey(field.getName(), value.toString());
if (field.multiValued()) {
fields.add(new SortedSetDocValuesField(field.getName(), bytes));
@@ -296,7 +296,7 @@ public class ICUCollationField extends FieldType {
}
return fields;
} else {
- return Collections.singletonList(createField(field, value, boost));
+ return Collections.singletonList(createField(field, value));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
index c80d275..a3d4756 100644
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
+++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
@@ -51,6 +51,7 @@ import java.util.concurrent.atomic.AtomicLong;
public class DocBuilder {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final AtomicBoolean WARNED_ABOUT_INDEX_TIME_BOOSTS = new AtomicBoolean();
private static final Date EPOCH = new Date(0);
public static final String DELETE_DOC_BY_ID = "$deleteDocById";
@@ -617,13 +618,12 @@ public class DocBuilder {
}
value = arow.get(DOC_BOOST);
if (value != null) {
- float value1 = 1.0f;
- if (value instanceof Number) {
- value1 = ((Number) value).floatValue();
+ String message = "Ignoring document boost: " + value + " as index-time boosts are not supported anymore";
+ if (WARNED_ABOUT_INDEX_TIME_BOOSTS.compareAndSet(false, true)) {
+ LOG.warn(message);
} else {
- value1 = Float.parseFloat(value.toString());
+ LOG.debug(message);
}
- doc.setDocumentBoost(value1);
}
value = arow.get(SKIP_DOC);
@@ -659,7 +659,7 @@ public class DocBuilder {
sf = config.getSchemaField(key);
}
if (sf != null) {
- addFieldToDoc(entry.getValue(), sf.getName(), 1.0f, sf.multiValued(), doc);
+ addFieldToDoc(entry.getValue(), sf.getName(), sf.multiValued(), doc);
}
//else do nothing. if we add it it may fail
} else {
@@ -679,7 +679,7 @@ public class DocBuilder {
}
}
if (toWrite) {
- addFieldToDoc(entry.getValue(), name, f.getBoost(), multiValued, doc);
+ addFieldToDoc(entry.getValue(), name, multiValued, doc);
}
}
}
@@ -687,30 +687,30 @@ public class DocBuilder {
}
}
- private void addFieldToDoc(Object value, String name, float boost, boolean multiValued, DocWrapper doc) {
+ private void addFieldToDoc(Object value, String name, boolean multiValued, DocWrapper doc) {
if (value instanceof Collection) {
Collection collection = (Collection) value;
if (multiValued) {
for (Object o : collection) {
if (o != null)
- doc.addField(name, o, boost);
+ doc.addField(name, o);
}
} else {
if (doc.getField(name) == null)
for (Object o : collection) {
if (o != null) {
- doc.addField(name, o, boost);
+ doc.addField(name, o);
break;
}
}
}
} else if (multiValued) {
if (value != null) {
- doc.addField(name, value, boost);
+ doc.addField(name, value);
}
} else {
if (doc.getField(name) == null && value != null)
- doc.addField(name, value, boost);
+ doc.addField(name, value);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/EntityField.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/EntityField.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/EntityField.java
index b61198d..2b28cb7 100644
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/EntityField.java
+++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/EntityField.java
@@ -30,7 +30,6 @@ import org.w3c.dom.Element;
public class EntityField {
private final String column;
private final String name;
- private final float boost;
private final boolean toWrite;
private final boolean multiValued;
private final boolean dynamicName;
@@ -40,7 +39,6 @@ public class EntityField {
public EntityField(Builder b) {
this.column = b.column;
this.name = b.name;
- this.boost = b.boost;
this.toWrite = b.toWrite;
this.multiValued = b.multiValued;
this.dynamicName = b.dynamicName;
@@ -60,10 +58,6 @@ public class EntityField {
return column;
}
- public float getBoost() {
- return boost;
- }
-
public boolean isToWrite() {
return toWrite;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java
----------------------------------------------------------------------
diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java
index 6b587f1..f7917bb 100644
--- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java
+++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java
@@ -47,19 +47,6 @@ public interface ExtractingParams {
public static final String MAP_PREFIX = "fmap.";
/**
- * The boost value for the name of the field. The boost can be specified by a name mapping.
- * <p>
- * For example
- * <pre>
- * map.title=solr.title
- * boost.solr.title=2.5
- * </pre>
- * will boost the solr.title field for this document by 2.5
- *
- */
- public static final String BOOST_PREFIX = "boost.";
-
- /**
* Pass in literal values to be added to the document, as in
* <pre>
* literal.myField=Foo
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8ed2b764/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java
----------------------------------------------------------------------
diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java
index 7779451..9268f1c 100644
--- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java
+++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java
@@ -252,15 +252,13 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara
vals=null;
}
- float boost = getBoost(name);
-
if (fval != null) {
- document.addField(name, transformValue(fval, sf), boost);
+ document.addField(name, transformValue(fval, sf));
}
if (vals != null) {
for (String val : vals) {
- document.addField(name, transformValue(val, sf), boost);
+ document.addField(name, transformValue(val, sf));
}
}
@@ -336,17 +334,6 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara
return result;
}
-
- /**
- * Get the value of any boost factor for the mapped name.
- *
- * @param name The name of the field to see if there is a boost specified
- * @return The boost value
- */
- protected float getBoost(String name) {
- return params.getFloat(BOOST_PREFIX + name, 1.0f);
- }
-
/**
* Get the name mapping
*