You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by tf...@apache.org on 2020/06/30 19:00:32 UTC
[lucene-solr] 01/02: SOLR-14590 : Add support for Lucene's
FeatureField in Solr (#1620)
This is an automated email from the ASF dual-hosted git repository.
tflobbe pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit e6ef6f2b36c32bdeca236b907a826477926a3464
Author: Tomas Fernandez Lobbe <tf...@apache.org>
AuthorDate: Tue Jun 30 11:15:36 2020 -0700
SOLR-14590 : Add support for Lucene's FeatureField in Solr (#1620)
Add a new RankField type that internally creates a FeatureField
Add a new RankQParser that can create queries on the FeatureField
---
solr/CHANGES.txt | 4 +
.../src/java/org/apache/solr/schema/RankField.java | 140 ++++++++++
.../java/org/apache/solr/search/QParserPlugin.java | 1 +
.../org/apache/solr/search/RankQParserPlugin.java | 158 ++++++++++++
.../solr/collection1/conf/schema-rank-fields.xml | 27 ++
.../test-files/solr/collection1/conf/schema15.xml | 3 +
.../test/org/apache/solr/schema/RankFieldTest.java | 285 +++++++++++++++++++++
.../org/apache/solr/search/QueryEqualityTest.java | 12 +
.../apache/solr/search/RankQParserPluginTest.java | 258 +++++++++++++++++++
.../solr/configsets/_default/conf/managed-schema | 7 +
10 files changed, 895 insertions(+)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 24761fe..50e1917 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -60,6 +60,10 @@ New Features
* SOLR-14599: Package manager support for cluster level plugins (see SOLR-14404) (Ishan Chattopadhyaya)
+* SOLR-14590: Add support for RankFields. RankFields allow the use of per-document scoring factors in a
+ way that lets Solr skip over non-competitive documents when ranking. See SOLR-13289.
+ (Tomás Fernández Löbbe, Varun Thacker)
+
Improvements
---------------------
* SOLR-14316: Remove unchecked type conversion warning in JavaBinCodec's readMapEntry's equals() method
diff --git a/solr/core/src/java/org/apache/solr/schema/RankField.java b/solr/core/src/java/org/apache/solr/schema/RankField.java
new file mode 100644
index 0000000..d11d5ed
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/schema/RankField.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.lucene.document.FeatureField;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.IndexableFieldType;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.response.TextResponseWriter;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.RankQParserPlugin;
+import org.apache.solr.uninverting.UninvertingReader.Type;
+
+/**
+ * <p>
+ * {@code RankField}s can be used to store scoring factors to improve document ranking. They should be used
+ * in combination with {@link RankQParserPlugin}. To use:
+ * </p>
+ * <p>
+ * Define the {@code RankField} {@code fieldType} in your schema:
+ * </p>
+ * <pre class="prettyprint">
+ * <fieldType name="rank" class="solr.RankField" />
+ * </pre>
+ * <p>
+ * Add fields to the schema, i.e.:
+ * </p>
+ * <pre class="prettyprint">
+ * <field name="pagerank" type="rank" />
+ * </pre>
+ *
+ * Query using the {@link RankQParserPlugin}, for example
+ * <pre class="prettyprint">
+ * http://localhost:8983/solr/techproducts?q=memory _query_:{!rank f='pagerank', function='log' scalingFactor='1.2'}
+ * </pre>
+ *
+ * @see RankQParserPlugin
+ * @lucene.experimental
+ * @since 8.6
+ */
+public class RankField extends FieldType {
+
+ /*
+ * While the user can create multiple RankFields, internally we use a single Lucene field,
+ * and we map the Solr field name to the "feature" in Lucene's FeatureField. This is mainly
+ * to simplify the user experience.
+ */
+ public static final String INTERNAL_RANK_FIELD_NAME = "_rank_";
+
+ @Override
+ public Type getUninversionType(SchemaField sf) {
+ throw null;
+ }
+
+ @Override
+ public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException {
+ }
+
+ @Override
+ protected void init(IndexSchema schema, Map<String,String> args) {
+ super.init(schema, args);
+ if (schema.getFieldOrNull(INTERNAL_RANK_FIELD_NAME) != null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "A field named \"" + INTERNAL_RANK_FIELD_NAME + "\" can't be defined in the schema");
+ }
+ for (int prop:new int[] {STORED, DOC_VALUES, OMIT_TF_POSITIONS, SORT_MISSING_FIRST, SORT_MISSING_LAST}) {
+ if ((trueProperties & prop) != 0) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Property \"" + getPropertyName(prop) + "\" can't be set to true in RankFields");
+ }
+ }
+ for (int prop:new int[] {UNINVERTIBLE, INDEXED, MULTIVALUED}) {
+ if ((falseProperties & prop) != 0) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Property \"" + getPropertyName(prop) + "\" can't be set to false in RankFields");
+ }
+ }
+ properties &= ~(UNINVERTIBLE | STORED | DOC_VALUES);
+
+ }
+
+ @Override
+ protected IndexableField createField(String name, String val, IndexableFieldType type) {
+ if (val == null || val.isEmpty()) {
+ return null;
+ }
+ float featureValue;
+ try {
+ featureValue = Float.parseFloat(val);
+ } catch (NumberFormatException nfe) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error while creating field '" + name + "' from value '" + val + "'. Expecting float.", nfe);
+ }
+ // Internally, we always use the same field
+ return new FeatureField(INTERNAL_RANK_FIELD_NAME, name, featureValue);
+ }
+
+ @Override
+ public Query getExistenceQuery(QParser parser, SchemaField field) {
+ return new TermQuery(new Term(INTERNAL_RANK_FIELD_NAME, field.getName()));
+ }
+
+ @Override
+ public Query getFieldQuery(QParser parser, SchemaField field, String externalVal) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Only a \"*\" term query can be done on RankFields");
+ }
+
+ @Override
+ protected Query getSpecializedRangeQuery(QParser parser, SchemaField field, String part1, String part2,
+ boolean minInclusive, boolean maxInclusive) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Range queries not supported on RankFields");
+ }
+
+ @Override
+ public SortField getSortField(SchemaField field, boolean top) {
+ // We could use FeatureField.newFeatureSort()
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "can not sort on a rank field: " + field.getName());
+ }
+
+}
diff --git a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
index d0a5b9d..d95b540 100644
--- a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
@@ -87,6 +87,7 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI
map.put(BoolQParserPlugin.NAME, new BoolQParserPlugin());
map.put(MinHashQParserPlugin.NAME, new MinHashQParserPlugin());
map.put(HashRangeQParserPlugin.NAME, new HashRangeQParserPlugin());
+ map.put(RankQParserPlugin.NAME, new RankQParserPlugin());
standardPlugins = Collections.unmodifiableMap(map);
}
diff --git a/solr/core/src/java/org/apache/solr/search/RankQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/RankQParserPlugin.java
new file mode 100644
index 0000000..962e1d2
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/RankQParserPlugin.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+import java.util.Locale;
+import java.util.Objects;
+
+import org.apache.lucene.document.FeatureField;
+import org.apache.lucene.search.Query;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.RankField;
+import org.apache.solr.schema.SchemaField;
+/**
+ * {@code RankQParserPlugin} can be used to introduce document-depending scoring factors to ranking.
+ * While this {@code QParser} delivers a (subset of) functionality already available via {@link FunctionQParser},
+ * the benefit is that {@code RankQParserPlugin} can be used in combination with the {@code minExactCount} to
+ * use BlockMax-WAND algorithm (skip non-competitive documents) to provide faster responses.
+ *
+ * @see RankField
+ *
+ * @lucene.experimental
+ * @since 8.6
+ */
+public class RankQParserPlugin extends QParserPlugin {
+
+ public static final String NAME = "rank";
+ public static final String FIELD = "f";
+ public static final String FUNCTION = "function";
+ public static final String WEIGHT = "weight";
+ public static final String PIVOT = "pivot";
+ public static final String SCALING_FACTOR = "scalingFactor";
+ public static final String EXPONENT = "exponent";
+
+ private final static FeatureFieldFunction DEFAULT_FUNCTION = FeatureFieldFunction.SATU;
+
+ private enum FeatureFieldFunction {
+ SATU {
+ @Override
+ public Query createQuery(String fieldName, SolrParams params) throws SyntaxError {
+ Float weight = params.getFloat(WEIGHT);
+ Float pivot = params.getFloat(PIVOT);
+ if (pivot == null && (weight == null || Float.compare(weight.floatValue(), 1f) == 0)) {
+ // No IAE expected in this case
+ return FeatureField.newSaturationQuery(RankField.INTERNAL_RANK_FIELD_NAME, fieldName);
+ }
+ if (pivot == null) {
+ throw new SyntaxError("A pivot value needs to be provided if the weight is not 1 on \"satu\" function");
+ }
+ if (weight == null) {
+ weight = Float.valueOf(1);
+ }
+ try {
+ return FeatureField.newSaturationQuery(RankField.INTERNAL_RANK_FIELD_NAME, fieldName, weight, pivot);
+ } catch (IllegalArgumentException iae) {
+ throw new SyntaxError(iae.getMessage());
+ }
+ }
+ },
+ LOG {
+ @Override
+ public Query createQuery(String fieldName, SolrParams params) throws SyntaxError {
+ float weight = params.getFloat(WEIGHT, 1f);
+ float scalingFactor = params.getFloat(SCALING_FACTOR, 1f);
+ try {
+ return FeatureField.newLogQuery(RankField.INTERNAL_RANK_FIELD_NAME, fieldName, weight, scalingFactor);
+ } catch (IllegalArgumentException iae) {
+ throw new SyntaxError(iae.getMessage());
+ }
+ }
+ },
+ SIGM {
+ @Override
+ public Query createQuery(String fieldName, SolrParams params) throws SyntaxError {
+ float weight = params.getFloat(WEIGHT, 1f);
+ Float pivot = params.getFloat(PIVOT);
+ if (pivot == null) {
+ throw new SyntaxError("A pivot value needs to be provided when using \"sigm\" function");
+ }
+ Float exponent = params.getFloat(EXPONENT);
+ if (exponent == null) {
+ throw new SyntaxError("An exponent value needs to be provided when using \"sigm\" function");
+ }
+ try {
+ return FeatureField.newSigmoidQuery(RankField.INTERNAL_RANK_FIELD_NAME, fieldName, weight, pivot, exponent);
+ } catch (IllegalArgumentException iae) {
+ throw new SyntaxError(iae.getMessage());
+ }
+ }
+ };
+
+ public abstract Query createQuery(String fieldName, SolrParams params) throws SyntaxError;
+
+ }
+
+ @Override
+ public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
+ Objects.requireNonNull(localParams, "LocalParams String can't be null");
+ Objects.requireNonNull(req, "SolrQueryRequest can't be null");
+ return new RankQParser(qstr, localParams, params, req);
+ }
+
+ public static class RankQParser extends QParser {
+
+ private final String field;
+
+ public RankQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
+ super(qstr, localParams, params, req);
+ this.field = localParams.get(FIELD);
+ }
+
+ @Override
+ public Query parse() throws SyntaxError {
+ if (this.field == null || this.field.isEmpty()) {
+ throw new SyntaxError("Field can't be empty in rank queries");
+ }
+ SchemaField schemaField = req.getSchema().getFieldOrNull(field);
+ if (schemaField == null) {
+ throw new SyntaxError("Field \"" + this.field + "\" not found");
+ }
+ if (!(schemaField.getType() instanceof RankField)) {
+ throw new SyntaxError("Field \"" + this.field + "\" is not a RankField");
+ }
+ return getFeatureFieldFunction(localParams.get(FUNCTION))
+ .createQuery(field, localParams);
+ }
+
+ private FeatureFieldFunction getFeatureFieldFunction(String function) throws SyntaxError {
+ FeatureFieldFunction f = null;
+ if (function == null || function.isEmpty()) {
+ f = DEFAULT_FUNCTION;
+ } else {
+ try {
+ f = FeatureFieldFunction.valueOf(function.toUpperCase(Locale.ROOT));
+ } catch (IllegalArgumentException iae) {
+ throw new SyntaxError("Unknown function in rank query: \"" + function + "\"");
+ }
+ }
+ return f;
+ }
+
+ }
+
+}
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-rank-fields.xml b/solr/core/src/test-files/solr/collection1/conf/schema-rank-fields.xml
new file mode 100644
index 0000000..2da0c0d
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-rank-fields.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="rank_fields" version="1.6">
+ <fieldType name="string" class="solr.StrField" />
+ <fieldType name="rank" class="solr.RankField" />
+ <fields>
+ <field name="id" type="string" />
+ <field name="str_field" type="string" />
+ <field name="rank_1" type="rank" />
+ <field name="rank_2" type="rank" />
+ </fields>
+</schema>
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
index 91fb092..c2a8bbb 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
@@ -41,6 +41,7 @@
<fieldType name="tlong" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
<fieldType name="tdouble" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
<fieldType name="currency" class="solr.CurrencyField" currencyConfig="currency.xml" multiValued="false"/>
+ <fieldType name="rank" class="solr.RankField"/>
<!-- Field type demonstrating an Analyzer failure -->
<fieldType name="failtype1" class="solr.TextField">
@@ -614,6 +615,8 @@
<dynamicField name="ignored_*" type="ignored" multiValued="true"/>
<dynamicField name="attr_*" type="text" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="rank_*" type="rank"/>
+
<dynamicField name="random_*" type="random"/>
<dynamicField name="*_dpf" type="delimited_payloads_float" indexed="true" stored="true"/>
diff --git a/solr/core/src/test/org/apache/solr/schema/RankFieldTest.java b/solr/core/src/test/org/apache/solr/schema/RankFieldTest.java
new file mode 100644
index 0000000..11877f0
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/schema/RankFieldTest.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import javax.xml.xpath.XPathConstants;
+
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.util.TestHarness;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+
+public class RankFieldTest extends SolrTestCaseJ4 {
+
+ private static final String RANK_1 = "rank_1";
+ private static final String RANK_2 = "rank_2";
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-minimal.xml","schema-rank-fields.xml");
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ clearIndex();
+ assertU(commit());
+ super.setUp();
+ }
+
+ public void testInternalFieldName() {
+ assertEquals("RankField.INTERNAL_RANK_FIELD_NAME changed in an incompatible way",
+ "_rank_", RankField.INTERNAL_RANK_FIELD_NAME);
+ }
+
+ public void testBasic() {
+ assertNotNull(h.getCore().getLatestSchema().getFieldOrNull(RANK_1));
+ assertEquals(RankField.class, h.getCore().getLatestSchema().getField(RANK_1).getType().getClass());
+ }
+
+ public void testBadFormat() {
+ ignoreException("Expecting float");
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, "foo"
+ ));
+
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, "1.2.3"
+ ));
+
+ unIgnoreException("Expecting float");
+
+ ignoreException("must be finite");
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, Float.toString(Float.POSITIVE_INFINITY)
+ ));
+
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, Float.toString(Float.NEGATIVE_INFINITY)
+ ));
+
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, Float.toString(Float.NaN)
+ ));
+
+ unIgnoreException("must be finite");
+
+ ignoreException("must be a positive");
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, Float.toString(-0.0f)
+ ));
+
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, Float.toString(-1f)
+ ));
+
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, Float.toString(0.0f)
+ ));
+ unIgnoreException("must be a positive");
+ }
+
+ public void testAddRandom() {
+ for (int i = 0 ; i < random().nextInt(TEST_NIGHTLY ? 10000 : 100); i++) {
+ assertU(adoc(
+ "id", String.valueOf(i),
+ RANK_1, Float.toString(random().nextFloat())
+ ));
+ }
+ assertU(commit());
+ }
+
+ public void testSkipEmpty() {
+ assertU(adoc(
+ "id", "1",
+ RANK_1, ""
+ ));
+ }
+
+ public void testBasicAdd() throws IOException {
+ assertU(adoc(
+ "id", "testBasicAdd",
+ RANK_1, "1"
+ ));
+ assertU(commit());
+ //assert that the document made it in
+ assertQ(req("q", "id:testBasicAdd"), "//*[@numFound='1']");
+ h.getCore().withSearcher((searcher) -> {
+ LeafReader reader = searcher.getIndexReader().getContext().leaves().get(0).reader();
+ // assert that the field made it in
+ assertNotNull(reader.getFieldInfos().fieldInfo(RankField.INTERNAL_RANK_FIELD_NAME));
+ // assert that the feature made it in
+ assertTrue(reader.terms(RankField.INTERNAL_RANK_FIELD_NAME).iterator().seekExact(new BytesRef(RANK_1.getBytes(StandardCharsets.UTF_8))));
+ return null;
+ });
+ }
+
+ public void testMultipleRankFields() throws IOException {
+ assertU(adoc(
+ "id", "testMultiValueAdd",
+ RANK_1, "1",
+ RANK_2, "2"
+ ));
+ assertU(commit());
+ //assert that the document made it in
+ assertQ(req("q", "id:testMultiValueAdd"), "//*[@numFound='1']");
+ h.getCore().withSearcher((searcher) -> {
+ LeafReader reader = searcher.getIndexReader().getContext().leaves().get(0).reader();
+ // assert that the field made it in
+ assertNotNull(reader.getFieldInfos().fieldInfo(RankField.INTERNAL_RANK_FIELD_NAME));
+ // assert that the features made it in
+ assertTrue(reader.terms(RankField.INTERNAL_RANK_FIELD_NAME).iterator().seekExact(new BytesRef(RANK_2.getBytes(StandardCharsets.UTF_8))));
+ assertTrue(reader.terms(RankField.INTERNAL_RANK_FIELD_NAME).iterator().seekExact(new BytesRef(RANK_1.getBytes(StandardCharsets.UTF_8))));
+ return null;
+ });
+ }
+
+ public void testSortFails() throws IOException {
+ assertU(adoc(
+ "id", "testSortFails",
+ RANK_1, "1"
+ ));
+ assertU(commit());
+ assertQEx("Can't sort on rank field", req(
+ "q", "id:testSortFails",
+ "sort", RANK_1 + " desc"), 400);
+ }
+
+ @Ignore("We currently don't fail these kinds of requests with other field types")
+ public void testFacetFails() throws IOException {
+ assertU(adoc(
+ "id", "testFacetFails",
+ RANK_1, "1"
+ ));
+ assertU(commit());
+ assertQEx("Can't facet on rank field", req(
+ "q", "id:testFacetFails",
+ "facet", "true",
+ "facet.field", RANK_1), 400);
+ }
+
+ public void testTermQuery() throws IOException {
+ assertU(adoc(
+ "id", "testTermQuery",
+ RANK_1, "1",
+ RANK_2, "1"
+ ));
+ assertU(adoc(
+ "id", "testTermQuery2",
+ RANK_1, "1"
+ ));
+ assertU(commit());
+ assertQ(req("q", RANK_1 + ":*"), "//*[@numFound='2']");
+ assertQ(req("q", RANK_1 + ":[* TO *]"), "//*[@numFound='2']");
+ assertQ(req("q", RANK_2 + ":*"), "//*[@numFound='1']");
+ assertQ(req("q", RANK_2 + ":[* TO *]"), "//*[@numFound='1']");
+
+ assertQEx("Term queries not supported", req("q", RANK_1 + ":1"), 400);
+ assertQEx("Range queries not supported", req("q", RANK_1 + ":[1 TO 10]"), 400);
+ }
+
+
+ public void testResponseQuery() throws IOException {
+ assertU(adoc(
+ "id", "testResponseQuery",
+ RANK_1, "1"
+ ));
+ assertU(commit());
+ // Ignore requests to retrieve rank
+ assertQ(req("q", RANK_1 + ":*",
+ "fl", "id," + RANK_1),
+ "//*[@numFound='1']",
+ "count(//result/doc[1]/str)=1");
+ }
+
+ public void testRankQParserQuery() throws IOException {
+ assertU(adoc(
+ "id", "1",
+ "str_field", "foo",
+ RANK_1, "1",
+ RANK_2, "2"
+ ));
+ assertU(adoc(
+ "id", "2",
+ "str_field", "foo",
+ RANK_1, "2",
+ RANK_2, "1"
+ ));
+ assertU(commit());
+ assertQ(req("q", "str_field:foo _query_:{!rank f='" + RANK_1 + "' function='log' scalingFactor='1'}"),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='2']",
+ "//result/doc[2]/str[@name='id'][.='1']");
+
+ assertQ(req("q", "str_field:foo _query_:{!rank f='" + RANK_2 + "' function='log' scalingFactor='1'}"),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='1']",
+ "//result/doc[2]/str[@name='id'][.='2']");
+
+ assertQ(req("q", "foo",
+ "defType", "dismax",
+ "qf", "str_field^10",
+ "bq", "{!rank f='" + RANK_1 + "' function='log' scalingFactor='1'}"
+ ),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='2']",
+ "//result/doc[2]/str[@name='id'][.='1']");
+
+ assertQ(req("q", "foo",
+ "defType", "dismax",
+ "qf", "str_field^10",
+ "bq", "{!rank f='" + RANK_2 + "' function='log' scalingFactor='1'}"
+ ),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='1']",
+ "//result/doc[2]/str[@name='id'][.='2']");
+ }
+
+ public void testScoreChanges() throws Exception {
+ assertU(adoc(
+ "id", "1",
+ "str_field", "foo",
+ RANK_1, "1"
+ ));
+ assertU(commit());
+ ModifiableSolrParams params = params("q", "foo",
+ "defType", "dismax",
+ "qf", "str_field^10",
+ "fl", "id,score",
+ "wt", "xml");
+
+ double scoreBefore = (Double) TestHarness.evaluateXPath(h.query(req(params)), "//result/doc[1]/float[@name='score']", XPathConstants.NUMBER);
+ params.add("bq", "{!rank f='" + RANK_1 + "' function='log' scalingFactor='1'}");
+ double scoreAfter = (Double) TestHarness.evaluateXPath(h.query(req(params)), "//result/doc[1]/float[@name='score']", XPathConstants.NUMBER);
+ assertNotEquals("Expecting score to change", scoreBefore, scoreAfter, 0f);
+
+ }
+
+}
diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
index a375e72..899abb7 100644
--- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
+++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
@@ -357,6 +357,18 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
req.close();
}
}
+
+ public void testRankQuery() throws Exception {
+ SolrQueryRequest req = req("df", "foo_s");
+ try {
+ assertQueryEquals("rank", req,
+ "{!rank f='rank_1'}",
+ "{!rank f='rank_1' function='satu'}",
+ "{!rank f='rank_1' function='satu' weight=1}");
+ } finally {
+ req.close();
+ }
+ }
public void testQueryNested() throws Exception {
SolrQueryRequest req = req("df", "foo_s");
diff --git a/solr/core/src/test/org/apache/solr/search/RankQParserPluginTest.java b/solr/core/src/test/org/apache/solr/search/RankQParserPluginTest.java
new file mode 100644
index 0000000..2e88ce2
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/RankQParserPluginTest.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+import static org.apache.solr.search.RankQParserPlugin.EXPONENT;
+import static org.apache.solr.search.RankQParserPlugin.FIELD;
+import static org.apache.solr.search.RankQParserPlugin.FUNCTION;
+import static org.apache.solr.search.RankQParserPlugin.NAME;
+import static org.apache.solr.search.RankQParserPlugin.PIVOT;
+import static org.apache.solr.search.RankQParserPlugin.SCALING_FACTOR;
+import static org.apache.solr.search.RankQParserPlugin.WEIGHT;
+
+import java.io.IOException;
+
+import org.apache.lucene.search.Query;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.RankField;
+import org.apache.solr.search.RankQParserPlugin.RankQParser;
+import org.hamcrest.CoreMatchers;
+import org.junit.BeforeClass;
+
+public class RankQParserPluginTest extends SolrTestCaseJ4 {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-minimal.xml", "schema-rank-fields.xml");
+ }
+
+ public void testParamCompatibility() {
+ assertEquals("RankQParserPlugin.NAME changed in an incompatible way", "rank", NAME);
+ assertEquals("RankQParserPlugin.FIELD changed in an incompatible way", "f", FIELD);
+ assertEquals("RankQParserPlugin.FUNCTION changed in an incompatible way", "function", FUNCTION);
+ assertEquals("RankQParserPlugin.PIVOT changed in an incompatible way", "pivot", PIVOT);
+ assertEquals("RankQParserPlugin.SCALING_FACTOR changed in an incompatible way", "scalingFactor", SCALING_FACTOR);
+ assertEquals("RankQParserPlugin.WEIGHT changed in an incompatible way", "weight", WEIGHT);
+ assertEquals("RankQParserPlugin.EXPONENT changed in an incompatible way", "exponent", EXPONENT);
+ }
+
+ public void testCreateParser() throws IOException {
+ try (RankQParserPlugin rankQPPlugin = new RankQParserPlugin()) {
+ QParser parser = rankQPPlugin.createParser("", new ModifiableSolrParams(), null, req());
+ assertNotNull(parser);
+ assertTrue(parser instanceof RankQParser);
+ }
+ }
+
+ public void testSyntaxErrors() throws IOException, SyntaxError {
+ assertSyntaxError("No Field", "Field can't be empty", () ->
+ getRankQParser(new ModifiableSolrParams(), null, req()).parse());
+ assertSyntaxError("Field empty", "Field can't be empty", () ->
+ getRankQParser(
+ params(FIELD, ""), null, req()).parse());
+ assertSyntaxError("Field doesn't exist", "Field \"foo\" not found", () ->
+ getRankQParser(
+ params(FIELD, "foo"), null, req()).parse());
+ assertSyntaxError("ID is not a feature field", "Field \"id\" is not a RankField", () ->
+ getRankQParser(
+ params(FIELD, "id"), null, req()).parse());
+ }
+
+ public void testBadLogParameters() throws IOException, SyntaxError {
+ assertSyntaxError("Expecting bad weight", "weight must be in", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "log",
+ WEIGHT, "0"), null, req()).parse());
+ assertSyntaxError("Expecting bad scaling factor", "scalingFactor must be", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "log",
+ SCALING_FACTOR, "0"), null, req()).parse());
+ }
+
+ public void testBadSaturationParameters() throws IOException, SyntaxError {
+ assertSyntaxError("Expecting a pivot value", "A pivot value", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "satu",
+ WEIGHT, "2"), null, req()).parse());
+ assertSyntaxError("Expecting bad weight", "weight must be in", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "satu",
+ PIVOT, "1",
+ WEIGHT, "-1"), null, req()).parse());
+ }
+
+ public void testBadSigmoidParameters() throws IOException, SyntaxError {
+ assertSyntaxError("Expecting missing pivot", "A pivot value", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ EXPONENT, "1"), null, req()).parse());
+ assertSyntaxError("Expecting missing exponent", "An exponent value", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ PIVOT, "1"), null, req()).parse());
+ assertSyntaxError("Expecting bad weight", "weight must be in", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ PIVOT, "1",
+ EXPONENT, "1",
+ WEIGHT, "-1"), null, req()).parse());
+ assertSyntaxError("Expecting bad pivot", "pivot must be", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ PIVOT, "0",
+ EXPONENT, "1"), null, req()).parse());
+ assertSyntaxError("Expecting bad exponent", "exp must be", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ PIVOT, "1",
+ EXPONENT, "0"), null, req()).parse());
+ }
+
+ public void testUnknownFunction() throws IOException, SyntaxError {
+ assertSyntaxError("Expecting bad function", "Unknown function in rank query: \"foo\"", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "foo"), null, req()).parse());
+ }
+
+ public void testParseLog() throws IOException, SyntaxError {
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedLogToString(1), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "log",
+ SCALING_FACTOR, "1",
+ WEIGHT, "1"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedLogToString(2.5f), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "log",
+ SCALING_FACTOR, "2.5",
+ WEIGHT, "1"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedLogToString(1), 2.5f),
+ params(FIELD, "rank_1",
+ FUNCTION, "log",
+ SCALING_FACTOR, "1",
+ WEIGHT, "2.5"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedLogToString(1), 2.5f),
+ params(FIELD, "rank_1",
+ FUNCTION, "Log", //use different case
+ SCALING_FACTOR, "1",
+ WEIGHT, "2.5"));
+ }
+
+ public void testParseSigm() throws IOException, SyntaxError {
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSigmoidToString(1.5f, 2f), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ PIVOT, "1.5",
+ EXPONENT, "2",
+ WEIGHT, "1"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSigmoidToString(1.5f, 2f), 2),
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ PIVOT, "1.5",
+ EXPONENT, "2",
+ WEIGHT, "2"));
+ }
+
+ public void testParseSatu() throws IOException, SyntaxError {
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(1.5f), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "satu",
+ PIVOT, "1.5",
+ WEIGHT, "1"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(1.5f), 2),
+ params(FIELD, "rank_1",
+ FUNCTION, "satu",
+ PIVOT, "1.5",
+ WEIGHT, "2"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(null), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "satu"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(null), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "satu",
+ WEIGHT, "1"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(1.5f), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "satu",
+ PIVOT, "1.5"));
+ }
+
+ public void testParseDefault() throws IOException, SyntaxError {
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(null), 1),
+ params(FIELD, "rank_1"));
+ }
+
+ private void assertValidRankQuery(String expctedToString, SolrParams localParams) throws IOException, SyntaxError {
+ QParser parser = getRankQParser(localParams, null, req());
+ Query q = parser.parse();
+ assertNotNull(q);
+ assertThat(q.toString(), CoreMatchers.equalTo(expctedToString));
+ }
+
+ private String expectedFeatureQueryToString(String fieldName, String function, float boost) {
+ String featureQueryStr = "FeatureQuery(field=" + RankField.INTERNAL_RANK_FIELD_NAME + ", feature=" + fieldName + ", function=" + function + ")";
+ if (boost == 1f) {
+ return featureQueryStr;
+ }
+ return "(" + featureQueryStr + ")^" + boost;
+ }
+
+ private String expectedLogToString(float scalingFactor) {
+ return "LogFunction(scalingFactor=" + scalingFactor + ")";
+ }
+
+ private String expectedSigmoidToString(float pivot, float exp) {
+ return "SigmoidFunction(pivot=" + pivot + ", a=" + exp + ")";
+ }
+
+ private String expectedSaturationToString(Float pivot) {
+ return "SaturationFunction(pivot=" + pivot + ")";
+ }
+
+ private void assertSyntaxError(String assertionMsg, String expectedExceptionMsg, ThrowingRunnable runnable) {
+ SyntaxError se = expectThrows(SyntaxError.class, assertionMsg, runnable);
+ assertThat(se.getMessage(), CoreMatchers.containsString(expectedExceptionMsg));
+ }
+
+ private RankQParser getRankQParser(SolrParams localParams, SolrParams params, SolrQueryRequest req) throws IOException {
+ try (RankQParserPlugin rankQPPlugin = new RankQParserPlugin()) {
+ return (RankQParser) rankQPPlugin.createParser("", localParams, params, req);
+ }
+ }
+
+}
diff --git a/solr/server/solr/configsets/_default/conf/managed-schema b/solr/server/solr/configsets/_default/conf/managed-schema
index d832ed1..e99e27e 100644
--- a/solr/server/solr/configsets/_default/conf/managed-schema
+++ b/solr/server/solr/configsets/_default/conf/managed-schema
@@ -250,6 +250,13 @@
<!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
<fieldType name="binary" class="solr.BinaryField"/>
+
+ <!--
+ RankFields can be used to store scoring factors to improve document ranking. They should be used
+ in combination with RankQParserPlugin.
+ (experimental)
+ -->
+ <fieldType name="rank" class="solr.RankField"/>
<!-- solr.TextField allows the specification of custom text analyzers
specified as a tokenizer and a list of token filters. Different