You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ab...@apache.org on 2025/06/30 09:53:22 UTC
(solr) branch branch_9x updated (996a245f238 -> da35f09a5f8)
This is an automated email from the ASF dual-hosted git repository.
abenedetti pushed a change to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
from 996a245f238 Deprecation: CommonParams.QT "qt" param (SOLR-17715)
new c790b54104a [SOLR-17726] Fix CloudMLTQParser to support copyField in qf (#3328)
new da35f09a5f8 SOLR-17760: solving bug in LTR dense/sparse format (#3354)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
solr/CHANGES.txt | 4 +
.../apache/solr/search/mlt/CloudMLTQParser.java | 20 ++-
.../solr/configsets/cloud-dynamic/conf/schema.xml | 11 +-
.../solr/search/mlt/CloudMLTQParserTest.java | 108 +++++++++++++++
.../java/org/apache/solr/ltr/CSVFeatureLogger.java | 2 +-
.../java/org/apache/solr/ltr/LTRScoringQuery.java | 37 +++---
.../solr/ltr/TestSelectiveWeightCreation.java | 19 +--
.../solr/ltr/feature/TestFieldValueFeature.java | 146 +++++++++++++--------
.../query-guide/pages/learning-to-rank.adoc | 30 +++++
9 files changed, 291 insertions(+), 86 deletions(-)
(solr) 01/02: [SOLR-17726] Fix CloudMLTQParser to support copyField in qf (#3328)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abenedetti pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit c790b54104ab38fb5b297bc43ffcd223e7eaf5f9
Author: Ilaria Petreti <66...@users.noreply.github.com>
AuthorDate: Mon Jun 30 10:30:49 2025 +0200
[SOLR-17726] Fix CloudMLTQParser to support copyField in qf (#3328)
using copy field source for more like this + tests
---------
Co-authored-by: Alessandro Benedetti <a....@sease.io>
(cherry picked from commit d249593e5affaa9795bc7c9c6e2218e31203eee4)
---
solr/CHANGES.txt | 2 +
.../apache/solr/search/mlt/CloudMLTQParser.java | 20 +++-
.../solr/configsets/cloud-dynamic/conf/schema.xml | 11 ++-
.../solr/search/mlt/CloudMLTQParserTest.java | 108 +++++++++++++++++++++
4 files changed, 138 insertions(+), 3 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 4cba2dc4920..f8364b40eac 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -130,6 +130,8 @@ Bug Fixes
* SOLR-17790: Allow the -j or --jettyconfig option to start with a dash (-). (Houston Putman)
+* SOLR-17726: MoreLikeThis to support copy-fields (Ilaria Petreti via Alessandro Benedetti)
+
Dependency Upgrades
---------------------
* SOLR-17471: Upgrade Lucene to 9.12.1. (Pierre Salagnac, Christine Poerschke)
diff --git a/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java b/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java
index 90ff5949138..6cdd3f9b9a2 100644
--- a/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java
+++ b/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java
@@ -67,7 +67,8 @@ public class CloudMLTQParser extends SimpleMLTQParser {
Map<String, Collection<Object>> filteredDocument = new HashMap<>();
for (String field : moreLikeThis.getFieldNames()) {
- Collection<Object> fieldValues = doc.getFieldValues(field);
+ Collection<Object> fieldValues = getFieldValuesIncludingCopyField(doc, field);
+
if (fieldValues != null) {
Collection<Object> values = new ArrayList<>();
for (Object val : fieldValues) {
@@ -110,4 +111,21 @@ public class CloudMLTQParser extends SimpleMLTQParser {
return (SolrDocument) response.get("doc");
}
+
+ private Collection<Object> getFieldValuesIncludingCopyField(SolrDocument doc, String field) {
+ Collection<Object> fieldValues = doc.getFieldValues(field);
+ if (fieldValues != null) return fieldValues;
+ // Fields created using copyField are not included in documents returned by RealTime Get.
+ // So if a copyField destination is used in the MLT query (qf), we need to get the values
+ // from its source field instead. If there are multiple source fields, their values must be
+ // combined.
+ Collection<Object> combinedValues = new ArrayList<>();
+ for (String sourceField : req.getSchema().getCopySources(field)) {
+ Collection<Object> sourceValues = doc.getFieldValues(sourceField);
+ if (sourceValues != null) {
+ combinedValues.addAll(sourceValues);
+ }
+ }
+ return combinedValues.isEmpty() ? null : combinedValues;
+ }
}
diff --git a/solr/core/src/test-files/solr/configsets/cloud-dynamic/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-dynamic/conf/schema.xml
index 58e6405959a..5c26be38b8a 100644
--- a/solr/core/src/test-files/solr/configsets/cloud-dynamic/conf/schema.xml
+++ b/solr/core/src/test-files/solr/configsets/cloud-dynamic/conf/schema.xml
@@ -224,11 +224,18 @@
<field name="uniq3" type="string" indexed="true" stored="true"/>
<field name="nouniq" type="string" indexed="true" stored="true" multiValued="true"/>
+ <field name="payload" type="sortable_binary" indexed="false"
+ stored="true" multiValued="false"/>
+ <!-- to test copyField in MLT qf -->
<field name="copyfield_source" type="string" indexed="true" stored="true" multiValued="true"/>
+ <field name="copyfield_dest" type="nametext" indexed="true" stored="true" multiValued="true"/>
+ <copyField source="copyfield_source" dest="copyfield_dest" />
- <field name="payload" type="sortable_binary" indexed="false"
- stored="true" multiValued="false"/>
+ <field name="copyfield_source_2" type="string" indexed="true" stored="true" multiValued="true"/>
+ <field name="copyfield_dest_multiple_sources" type="nametext" indexed="true" stored="true" multiValued="true"/>
+ <copyField source="copyfield_source" dest="copyfield_dest_multiple_sources" />
+ <copyField source="copyfield_source_2" dest="copyfield_dest_multiple_sources" />
<!-- for versioning -->
<field name="_version_" type="long" indexed="true" stored="true"/>
diff --git a/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQParserTest.java b/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQParserTest.java
index 5912465f883..da87e218dbd 100644
--- a/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQParserTest.java
+++ b/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQParserTest.java
@@ -27,6 +27,7 @@ import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -50,6 +51,8 @@ public class CloudMLTQParserTest extends SolrCloudTestCase {
String id = "id";
String FIELD1 = "lowerfilt_u";
String FIELD2 = "lowerfilt1_u";
+ String FIELD3 = "copyfield_source";
+ String FIELD4 = "copyfield_source_2";
new UpdateRequest()
.add(sdoc(id, "1", FIELD1, "toyota"))
@@ -119,6 +122,9 @@ public class CloudMLTQParserTest extends SolrCloudTestCase {
"The slim red fox jumped over the lazy brown dogs.",
FIELD2,
"yellow white black"))
+ .add(sdoc(id, "33", FIELD3, "hard rock", FIELD4, "instrumental version"))
+ .add(sdoc(id, "34", FIELD3, "hard rock", FIELD4, "instrumental version"))
+ .add(sdoc(id, "35", FIELD3, "pop rock"))
.commit(client, COLLECTION);
}
@@ -340,4 +346,106 @@ public class CloudMLTQParserTest extends SolrCloudTestCase {
.getSolrClient()
.query(COLLECTION, new SolrQuery("{!mlt qf=lowerfilt_u}999999")));
}
+
+ @Test
+ public void testUsesACopyFieldInQf_shouldReturnExpectResults() throws Exception {
+ // Verifies that when a copyField destination is used in the qf parameter, the field values are
+ // correctly retrieved from the source field(s) and the MLT query returns the expected results.
+ QueryResponse queryResponse =
+ cluster
+ .getSolrClient()
+ .query(COLLECTION, new SolrQuery("{!mlt qf=copyfield_dest mindf=0 mintf=1}33"));
+ SolrDocumentList solrDocuments = queryResponse.getResults();
+ int[] expectedIds = new int[] {34, 35};
+ int[] actualIds = new int[solrDocuments.size()];
+ int i = 0;
+ for (SolrDocument solrDocument : solrDocuments) {
+ actualIds[i++] = Integer.parseInt(String.valueOf(solrDocument.getFieldValue("id")));
+ }
+
+ Arrays.sort(actualIds);
+ Arrays.sort(expectedIds);
+ assertArrayEquals(expectedIds, actualIds);
+ }
+
+ @Test
+ public void testUsesACopyFieldInQf_shouldGenerateNonEmptyQuery() throws Exception {
+ // Verifies that the MLT query correctly uses the content of the source field(s) when a
+ // copyField destination is specified in the qf parameter.
+ QueryResponse queryResponse =
+ cluster
+ .getSolrClient()
+ .query(
+ COLLECTION,
+ new SolrQuery("{!mlt qf=copyfield_dest mindf=0 mintf=1}33").setShowDebugInfo(true));
+
+ NamedList<?> debugInfo = (NamedList<?>) queryResponse.getResponse().get("debug");
+ // Extract the parsed query string
+ String parsedQuery = (String) debugInfo.get("parsedquery_toString");
+ // Assert it matches the expected query string
+ assertEquals("+(copyfield_dest:rock copyfield_dest:hard) -id:33", parsedQuery);
+ // Assert it is not the incorrect fallback
+ assertNotEquals("+() -id:33", parsedQuery);
+ }
+
+ @Test
+ public void testCopyFieldSourceMissing_shouldReturnNoResults() throws Exception {
+ // Ensures that no results are returned when the copyField source field is missing in the source
+ // document.
+ QueryResponse queryResponse =
+ cluster
+ .getSolrClient()
+ .query(COLLECTION, new SolrQuery("{!mlt qf=copyfield_dest mindf=0 mintf=1}30"));
+ SolrDocumentList solrDocuments = queryResponse.getResults();
+ assertEquals("Expected no results if source field is missing", 0, solrDocuments.size());
+ }
+
+ @Test
+ public void testCopyFieldDestinMultipleSources_shouldReturnExpectResults() throws Exception {
+ // Validates that when multiple source fields map to a single copyField destination, their
+ // values are correctly combined and expected results are returned.
+ QueryResponse queryResponse =
+ cluster
+ .getSolrClient()
+ .query(
+ COLLECTION,
+ new SolrQuery("{!mlt qf=copyfield_dest_multiple_sources mindf=0 mintf=1}33"));
+ SolrDocumentList solrDocuments = queryResponse.getResults();
+ int[] expectedIds = new int[] {34, 35};
+ int[] actualIds = new int[solrDocuments.size()];
+ int i = 0;
+ for (SolrDocument solrDocument : solrDocuments) {
+ actualIds[i++] = Integer.parseInt(String.valueOf(solrDocument.getFieldValue("id")));
+ }
+
+ Arrays.sort(actualIds);
+ Arrays.sort(expectedIds);
+ assertArrayEquals(expectedIds, actualIds);
+ }
+
+ @Test
+ public void
+ testCopyFieldDestinationMultipleSources_shouldGenerateQueryUsingMultipleSourcesValues()
+ throws Exception {
+ // Validates that when multiple source fields map to a single copyField destination, their
+ // values are
+ // correctly combined and the resulting MLT query is properly constructed.
+ QueryResponse queryResponse =
+ cluster
+ .getSolrClient()
+ .query(
+ COLLECTION,
+ new SolrQuery("{!mlt qf=copyfield_dest_multiple_sources mindf=0 mintf=1}33")
+ .setShowDebugInfo(true));
+
+ NamedList<?> debugInfo = (NamedList<?>) queryResponse.getResponse().get("debug");
+ // Extract the parsed query string
+ String parsedQuery = (String) debugInfo.get("parsedquery_toString");
+ // Assert it matches the expected query string
+ assertEquals(
+ "+(copyfield_dest_multiple_sources:rock copyfield_dest_multiple_sources:version copyfield_dest_multiple_sources:hard copyfield_dest_multiple_sources:instrumental) -id:33",
+ parsedQuery);
+ // Assert it is not the incorrect fallback
+ assertNotEquals("+() -id:33", parsedQuery);
+ }
}
(solr) 02/02: SOLR-17760: solving bug in LTR dense/sparse format (#3354)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abenedetti pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit da35f09a5f81efa7d9530e1a208d1b24c1137622
Author: aruggero <57...@users.noreply.github.com>
AuthorDate: Mon Jun 30 11:03:04 2025 +0200
SOLR-17760: solving bug in LTR dense/sparse format (#3354)
* Fixed field value feature
(cherry picked from commit ba981cd76c21df23ee518d3ee74f26e6f673ebe5)
---
solr/CHANGES.txt | 2 +
.../java/org/apache/solr/ltr/CSVFeatureLogger.java | 2 +-
.../java/org/apache/solr/ltr/LTRScoringQuery.java | 37 +++---
.../solr/ltr/TestSelectiveWeightCreation.java | 19 +--
.../solr/ltr/feature/TestFieldValueFeature.java | 146 +++++++++++++--------
.../query-guide/pages/learning-to-rank.adoc | 30 +++++
6 files changed, 153 insertions(+), 83 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f8364b40eac..37e8beff119 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -132,6 +132,8 @@ Bug Fixes
* SOLR-17726: MoreLikeThis to support copy-fields (Ilaria Petreti via Alessandro Benedetti)
+* SOLR-16667: Fixed dense/sparse representation in LTR module. (Anna Ruggero, Alessandro Benedetti)
+
Dependency Upgrades
---------------------
* SOLR-17471: Upgrade Lucene to 9.12.1. (Pierre Salagnac, Christine Poerschke)
diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/CSVFeatureLogger.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/CSVFeatureLogger.java
index 73e98b249ed..22ddcb8724a 100644
--- a/solr/modules/ltr/src/java/org/apache/solr/ltr/CSVFeatureLogger.java
+++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/CSVFeatureLogger.java
@@ -44,7 +44,7 @@ public class CSVFeatureLogger extends FeatureLogger {
StringBuilder sb = new StringBuilder(featuresInfo.length * 3);
boolean isDense = featureFormat.equals(FeatureFormat.DENSE);
for (LTRScoringQuery.FeatureInfo featInfo : featuresInfo) {
- if (featInfo != null && (isDense || featInfo.isUsed())) {
+ if (featInfo != null && (isDense || !featInfo.isDefaultValue())) {
sb.append(featInfo.getName())
.append(keyValueSep)
.append(featInfo.getValue())
diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java
index 8731cccb346..0ce2e21217d 100644
--- a/solr/modules/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java
+++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java
@@ -331,12 +331,12 @@ public class LTRScoringQuery extends Query implements Accountable {
public static class FeatureInfo {
private final String name;
private float value;
- private boolean used;
+ private boolean isDefaultValue;
- FeatureInfo(String n, float v, boolean u) {
- name = n;
- value = v;
- used = u;
+ FeatureInfo(String name, float value, boolean isDefaultValue) {
+ this.name = name;
+ this.value = value;
+ this.isDefaultValue = isDefaultValue;
}
public void setValue(float value) {
@@ -351,12 +351,12 @@ public class LTRScoringQuery extends Query implements Accountable {
return value;
}
- public boolean isUsed() {
- return used;
+ public boolean isDefaultValue() {
+ return isDefaultValue;
}
- public void setUsed(boolean used) {
- this.used = used;
+ public void setIsDefaultValue(boolean isDefaultValue) {
+ this.isDefaultValue = isDefaultValue;
}
}
@@ -408,7 +408,7 @@ public class LTRScoringQuery extends Query implements Accountable {
String featName = extractedFeatureWeights[i].getName();
int featId = extractedFeatureWeights[i].getIndex();
float value = extractedFeatureWeights[i].getDefaultValue();
- featuresInfo[featId] = new FeatureInfo(featName, value, false);
+ featuresInfo[featId] = new FeatureInfo(featName, value, true);
}
}
@@ -440,12 +440,7 @@ public class LTRScoringQuery extends Query implements Accountable {
for (final Feature.FeatureWeight feature : modelFeatureWeights) {
final int featureId = feature.getIndex();
FeatureInfo fInfo = featuresInfo[featureId];
- // not checking for finfo == null as that would be a bug we should catch
- if (fInfo.isUsed()) {
- modelFeatureValuesNormalized[pos] = fInfo.getValue();
- } else {
- modelFeatureValuesNormalized[pos] = feature.getDefaultValue();
- }
+ modelFeatureValuesNormalized[pos] = fInfo.getValue();
pos++;
}
ltrScoringModel.normalizeFeaturesInPlace(modelFeatureValuesNormalized);
@@ -480,7 +475,7 @@ public class LTRScoringQuery extends Query implements Accountable {
// need to set default value everytime as the default value is used in 'dense'
// mode even if used=false
featuresInfo[featId].setValue(value);
- featuresInfo[featId].setUsed(false);
+ featuresInfo[featId].setIsDefaultValue(true);
}
}
@@ -598,7 +593,9 @@ public class LTRScoringQuery extends Query implements Accountable {
Feature.FeatureWeight scFW = (Feature.FeatureWeight) subScorer.getWeight();
final int featureId = scFW.getIndex();
featuresInfo[featureId].setValue(subScorer.score());
- featuresInfo[featureId].setUsed(true);
+ if (featuresInfo[featureId].getValue() != scFW.getDefaultValue()) {
+ featuresInfo[featureId].setIsDefaultValue(false);
+ }
}
}
return makeNormalizedFeaturesAndScore();
@@ -683,7 +680,9 @@ public class LTRScoringQuery extends Query implements Accountable {
Feature.FeatureWeight scFW = (Feature.FeatureWeight) scorer.getWeight();
final int featureId = scFW.getIndex();
featuresInfo[featureId].setValue(scorer.score());
- featuresInfo[featureId].setUsed(true);
+ if (featuresInfo[featureId].getValue() != scFW.getDefaultValue()) {
+ featuresInfo[featureId].setIsDefaultValue(false);
+ }
}
}
}
diff --git a/solr/modules/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java b/solr/modules/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
index 6f730bf86a3..d96bcc61886 100644
--- a/solr/modules/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
+++ b/solr/modules/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
@@ -142,7 +142,10 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
assertEquals("11", searcher.storedFields().document(hits.scoreDocs[1].doc).get("id"));
List<Feature> features = makeFeatures(new int[] {0, 1, 2});
+ List<Feature> expectedNonDefaultFeatures = makeFeatures(new int[] {1, 2});
final List<Feature> allFeatures = makeFeatures(new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+ List<Feature> expectedNonDefaultAllFeatures =
+ makeFeatures(new int[] {1, 2, 3, 4, 5, 6, 7, 8, 9});
final List<Normalizer> norms = new ArrayList<>();
for (int k = 0; k < features.size(); ++k) {
norms.add(IdentityNormalizer.INSTANCE);
@@ -167,13 +170,13 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
LTRScoringQuery.FeatureInfo[] featuresInfo = modelWeight.getFeaturesInfo();
assertEquals(features.size(), modelWeight.getModelFeatureValuesNormalized().length);
- int validFeatures = 0;
+ int nonDefaultFeatures = 0;
for (int i = 0; i < featuresInfo.length; ++i) {
- if (featuresInfo[i] != null && featuresInfo[i].isUsed()) {
- validFeatures += 1;
+ if (featuresInfo[i] != null && !featuresInfo[i].isDefaultValue()) {
+ nonDefaultFeatures += 1;
}
}
- assertEquals(validFeatures, features.size());
+ assertEquals(expectedNonDefaultFeatures.size(), nonDefaultFeatures);
// when features are requested in the response, weights should be created for all features
final LTRScoringModel ltrScoringModel2 =
@@ -194,13 +197,13 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
assertEquals(features.size(), modelWeight.getModelFeatureValuesNormalized().length);
assertEquals(allFeatures.size(), modelWeight.getExtractedFeatureWeights().length);
- validFeatures = 0;
+ nonDefaultFeatures = 0;
for (int i = 0; i < featuresInfo.length; ++i) {
- if (featuresInfo[i] != null && featuresInfo[i].isUsed()) {
- validFeatures += 1;
+ if (featuresInfo[i] != null && !featuresInfo[i].isDefaultValue()) {
+ nonDefaultFeatures += 1;
}
}
- assertEquals(validFeatures, allFeatures.size());
+ assertEquals(expectedNonDefaultAllFeatures.size(), nonDefaultFeatures);
assertU(delI("10"));
assertU(delI("11"));
diff --git a/solr/modules/ltr/src/test/org/apache/solr/ltr/feature/TestFieldValueFeature.java b/solr/modules/ltr/src/test/org/apache/solr/ltr/feature/TestFieldValueFeature.java
index b10d9d7f952..d8d9d308ac1 100644
--- a/solr/modules/ltr/src/test/org/apache/solr/ltr/feature/TestFieldValueFeature.java
+++ b/solr/modules/ltr/src/test/org/apache/solr/ltr/feature/TestFieldValueFeature.java
@@ -250,6 +250,31 @@ public class TestFieldValueFeature extends TestRerankBase {
assertJQ("/query" + query.toQueryString(), "/response/numFound/==1");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='42'");
+ final String docs0fv_dense_csv =
+ FeatureLoggerTestUtils.toFeatureVector(
+ "popularity",
+ "0.0",
+ "dvIntPopularity",
+ "0.0",
+ "dvLongPopularity",
+ "0.0",
+ "dvFloatPopularity",
+ "0.0",
+ "dvDoublePopularity",
+ "0.0",
+ "dvStringPopularity",
+ "0.0",
+ "isTrendy",
+ "0.0",
+ "dvIsTrendy",
+ "0.0",
+ "storedDvIsTrendy",
+ "0.0");
+ final String docs0fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector("");
+
+ final String docs0fv_default_csv =
+ chooseDefaultFeatureVector(docs0fv_dense_csv, docs0fv_sparse_csv);
+
query = new SolrQuery();
query.setQuery("id:42");
query.add("rq", "{!ltr model=model reRankDocs=4}");
@@ -262,9 +287,7 @@ public class TestFieldValueFeature extends TestRerankBase {
assertJQ("/query" + query.toQueryString(), "/response/numFound/==1");
assertJQ(
"/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'popularity=0.0,dvIntPopularity=0.0,dvLongPopularity=0.0,"
- + "dvFloatPopularity=0.0,dvDoublePopularity=0.0,"
- + "dvStringPopularity=0.0,isTrendy=0.0,dvIsTrendy=0.0,storedDvIsTrendy=0.0'}");
+ "/response/docs/[0]/=={'[fv]':'" + docs0fv_default_csv + "'}");
}
@Test
@@ -292,16 +315,22 @@ public class TestFieldValueFeature extends TestRerankBase {
assertJQ("/query" + query.toQueryString(), "/response/numFound/==1");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='42'");
+
+ final String docs0fv_dense_csv = FeatureLoggerTestUtils.toFeatureVector(field + "42", "42.0");
+ final String docs0fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector("");
+
+ final String docs0fv_default_csv =
+ chooseDefaultFeatureVector(docs0fv_dense_csv, docs0fv_sparse_csv);
+
query = new SolrQuery();
query.setQuery("id:42");
query.add("rq", "{!ltr model=" + field + "-model42 reRankDocs=4}");
query.add("fl", "[fv]");
+
assertJQ("/query" + query.toQueryString(), "/response/numFound/==1");
assertJQ(
"/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'"
- + FeatureLoggerTestUtils.toFeatureVector(field + "42", "42.0")
- + "'}");
+ "/response/docs/[0]/=={'[fv]':'" + docs0fv_default_csv + "'}");
}
}
@@ -360,6 +389,14 @@ public class TestFieldValueFeature extends TestRerankBase {
fstore,
"{\"weights\":{\"not-existing-field\":1.0}}");
+ final String docs0fv_dense_csv =
+ FeatureLoggerTestUtils.toFeatureVector(
+ "not-existing-field", Float.toString(FIELD_VALUE_FEATURE_DEFAULT_VAL));
+ final String docs0fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector("");
+
+ final String docs0fv_default_csv =
+ chooseDefaultFeatureVector(docs0fv_dense_csv, docs0fv_sparse_csv);
+
final SolrQuery query = new SolrQuery();
query.setQuery("id:42");
query.add("rq", "{!ltr model=not-existing-field-model reRankDocs=4}");
@@ -368,10 +405,7 @@ public class TestFieldValueFeature extends TestRerankBase {
assertJQ("/query" + query.toQueryString(), "/response/numFound/==1");
assertJQ(
"/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'"
- + FeatureLoggerTestUtils.toFeatureVector(
- "not-existing-field", Float.toString(FIELD_VALUE_FEATURE_DEFAULT_VAL))
- + "'}");
+ "/response/docs/[0]/=={'[fv]':'" + docs0fv_default_csv + "'}");
assertEquals(
FieldValueFeatureScorer.class.getName(), ObservingFieldValueFeature.usedScorerClass);
}
@@ -394,6 +428,14 @@ public class TestFieldValueFeature extends TestRerankBase {
fstore,
"{\"weights\":{\"" + field + "\":1.0}}");
+ final String docs0fv_dense_csv =
+ FeatureLoggerTestUtils.toFeatureVector(
+ field, Float.toString(FIELD_VALUE_FEATURE_DEFAULT_VAL));
+ final String docs0fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector("");
+
+ final String docs0fv_default_csv =
+ chooseDefaultFeatureVector(docs0fv_dense_csv, docs0fv_sparse_csv);
+
final SolrQuery query = new SolrQuery("id:42");
query.add("rq", "{!ltr model=" + field + "-model reRankDocs=4}");
query.add("fl", "[fv]");
@@ -402,10 +444,7 @@ public class TestFieldValueFeature extends TestRerankBase {
assertJQ("/query" + query.toQueryString(), "/response/numFound/==1");
assertJQ(
"/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'"
- + FeatureLoggerTestUtils.toFeatureVector(
- field, Float.toString(FIELD_VALUE_FEATURE_DEFAULT_VAL))
- + "'}");
+ "/response/docs/[0]/=={'[fv]':'" + docs0fv_default_csv + "'}");
assertEquals(
DefaultValueFieldValueFeatureScorer.class.getName(),
ObservingFieldValueFeature.usedScorerClass);
@@ -442,15 +481,19 @@ public class TestFieldValueFeature extends TestRerankBase {
fstore,
"{\"weights\":{\"trendy\":1.0}}");
+ final String docs0fv_dense_csv = FeatureLoggerTestUtils.toFeatureVector("trendy", "0.0");
+ final String docs0fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector("");
+
+ final String docs0fv_default_csv =
+ chooseDefaultFeatureVector(docs0fv_dense_csv, docs0fv_sparse_csv);
+
SolrQuery query = new SolrQuery();
query.setQuery("id:4");
query.add("rq", "{!ltr model=trendy-model reRankDocs=4}");
query.add("fl", "[fv]");
assertJQ(
"/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'"
- + FeatureLoggerTestUtils.toFeatureVector("trendy", "0.0")
- + "'}");
+ "/response/docs/[0]/=={'[fv]':'" + docs0fv_default_csv + "'}");
query = new SolrQuery();
query.setQuery("id:5");
@@ -469,9 +512,7 @@ public class TestFieldValueFeature extends TestRerankBase {
query.add("fl", "[fv]");
assertJQ(
"/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'"
- + FeatureLoggerTestUtils.toFeatureVector("trendy", "0.0")
- + "'}");
+ "/response/docs/[0]/=={'[fv]':'" + docs0fv_default_csv + "'}");
}
@Test
@@ -583,6 +624,17 @@ public class TestFieldValueFeature extends TestRerankBase {
@Test
public void testThatStringValuesAreCorrectlyParsed() throws Exception {
for (String field : new String[] {"dvStrNumField", "noDvStrNumField"}) {
+ final String false_docs0fv_dense_csv = FeatureLoggerTestUtils.toFeatureVector(field, "0.0");
+ final String default_docs0fv_dense_csv =
+ FeatureLoggerTestUtils.toFeatureVector(
+ field, Float.toString(FIELD_VALUE_FEATURE_DEFAULT_VAL));
+ final String docs0fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector("");
+
+ final String false_docs0fv_default_csv =
+ chooseDefaultFeatureVector(false_docs0fv_dense_csv, docs0fv_sparse_csv);
+ final String default_docs0fv_default_csv =
+ chooseDefaultFeatureVector(default_docs0fv_dense_csv, docs0fv_sparse_csv);
+
final String[][] inputsAndTests = {
new String[] {
"T",
@@ -590,39 +642,17 @@ public class TestFieldValueFeature extends TestRerankBase {
+ FeatureLoggerTestUtils.toFeatureVector(field, "1.0")
+ "'}"
},
+ new String[] {"F", "/response/docs/[0]/=={'[fv]':'" + false_docs0fv_default_csv + "'}"},
new String[] {
- "F",
- "/response/docs/[0]/=={'[fv]':'"
- + FeatureLoggerTestUtils.toFeatureVector(field, "0.0")
- + "'}"
- },
- new String[] {
- "-7324.427",
- "/response/docs/[0]/=={'[fv]':'"
- + FeatureLoggerTestUtils.toFeatureVector(
- field, Float.toString(FIELD_VALUE_FEATURE_DEFAULT_VAL))
- + "'}"
- },
- new String[] {
- "532",
- "/response/docs/[0]/=={'[fv]':'"
- + FeatureLoggerTestUtils.toFeatureVector(
- field, Float.toString(FIELD_VALUE_FEATURE_DEFAULT_VAL))
- + "'}"
+ "-7324.427", "/response/docs/[0]/=={'[fv]':'" + default_docs0fv_default_csv + "'}"
},
+ new String[] {"532", "/response/docs/[0]/=={'[fv]':'" + default_docs0fv_default_csv + "'}"},
new String[] {
Float.toString(Float.NaN),
- "/response/docs/[0]/=={'[fv]':'"
- + FeatureLoggerTestUtils.toFeatureVector(
- field, Float.toString(FIELD_VALUE_FEATURE_DEFAULT_VAL))
- + "'}"
+ "/response/docs/[0]/=={'[fv]':'" + default_docs0fv_default_csv + "'}"
},
new String[] {
- "notanumber",
- "/response/docs/[0]/=={'[fv]':'"
- + FeatureLoggerTestUtils.toFeatureVector(
- field, Float.toString(FIELD_VALUE_FEATURE_DEFAULT_VAL))
- + "'}"
+ "notanumber", "/response/docs/[0]/=={'[fv]':'" + default_docs0fv_default_csv + "'}"
}
};
@@ -651,12 +681,15 @@ public class TestFieldValueFeature extends TestRerankBase {
@Test
public void testThatDateValuesAreCorrectlyParsed() throws Exception {
for (String field : new String[] {"dvDateField", "noDvDateField"}) {
+ final String docs0fv_dense_csv = FeatureLoggerTestUtils.toFeatureVector(field, "0.0");
+ final String docs0fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector("");
+
+ final String docs0fv_default_csv =
+ chooseDefaultFeatureVector(docs0fv_dense_csv, docs0fv_sparse_csv);
+
final String[][] inputsAndTests = {
new String[] {
- "1970-01-01T00:00:00.000Z",
- "/response/docs/[0]/=={'[fv]':'"
- + FeatureLoggerTestUtils.toFeatureVector(field, "0.0")
- + "'}"
+ "1970-01-01T00:00:00.000Z", "/response/docs/[0]/=={'[fv]':'" + docs0fv_default_csv + "'}"
},
new String[] {
"1970-01-01T00:00:00.001Z",
@@ -753,12 +786,15 @@ public class TestFieldValueFeature extends TestRerankBase {
public void testRelativeDateFieldValueFeature() throws Exception {
final String field = "dvDateField";
for (boolean since : new boolean[] {false, true}) {
+ final String docs0fv_dense_csv = FeatureLoggerTestUtils.toFeatureVector(field, "0.0");
+ final String docs0fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector("");
+
+ final String docs0fv_default_csv =
+ chooseDefaultFeatureVector(docs0fv_dense_csv, docs0fv_sparse_csv);
+
final String[][] inputsAndTests = {
new String[] {
- "2000-01-01T00:00:00.000Z",
- "/response/docs/[0]/=={'[fv]':'"
- + FeatureLoggerTestUtils.toFeatureVector(field, "0.0")
- + "'}"
+ "2000-01-01T00:00:00.000Z", "/response/docs/[0]/=={'[fv]':'" + docs0fv_default_csv + "'}"
},
new String[] {
"2000-01-01T00:01:02.003Z",
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/learning-to-rank.adoc b/solr/solr-ref-guide/modules/query-guide/pages/learning-to-rank.adoc
index e40deb89cc1..27895e77829 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/learning-to-rank.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/learning-to-rank.adoc
@@ -328,6 +328,36 @@ http://localhost:8983/solr/techproducts/schema/feature-store/_DEFAULT_
]
----
+==== Feature Parameters
+
+All the ltr feature types accept the parameters described below.
+
+`defaultValue`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: `0.0`
+|===
++
+This parameter specifies the default value of the feature to use for both logging and reranking.
++
+.Example: /path/myFeatures.json
+[source,json]
+----
+[
+ {
+ "name": "productReviewScore",
+ "class": "org.apache.solr.ltr.feature.FieldValueFeature",
+ "params": {
+ "field": "product_review_score",
+ "defaultValue": "5.2"
+ }
+ }
+]
+----
++
+CAUTION: When defining a `defaultValue` for a `FieldValueFeature`, check that no `default` is assigned to that field in the schema, otherwise, the feature value will be the one defined in the schema and not in the feature store.
+
=== Logging Features
To log features as part of a query, add `[features]` to the `fl` parameter, for example: