You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by tf...@apache.org on 2020/06/30 19:00:31 UTC
[lucene-solr] branch branch_8x updated (33672f9 -> e927cb7)
This is an automated email from the ASF dual-hosted git repository.
tflobbe pushed a change to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 33672f9 SOLR-14539: Fix import in {!bool excludeTags=...}
new e6ef6f2 SOLR-14590 : Add support for Lucene's FeatureField in Solr (#1620)
new e927cb7 SOLR-14590: Don't close QParserPlugin in 8.x
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
solr/CHANGES.txt | 4 +
.../src/java/org/apache/solr/schema/RankField.java | 140 ++++++++++
.../java/org/apache/solr/search/QParserPlugin.java | 1 +
.../org/apache/solr/search/RankQParserPlugin.java | 158 ++++++++++++
.../{schema-minimal.xml => schema-rank-fields.xml} | 12 +-
.../test-files/solr/collection1/conf/schema15.xml | 3 +
.../test/org/apache/solr/schema/RankFieldTest.java | 285 +++++++++++++++++++++
.../org/apache/solr/search/QueryEqualityTest.java | 12 +
.../apache/solr/search/RankQParserPluginTest.java | 256 ++++++++++++++++++
.../solr/configsets/_default/conf/managed-schema | 7 +
10 files changed, 875 insertions(+), 3 deletions(-)
create mode 100644 solr/core/src/java/org/apache/solr/schema/RankField.java
create mode 100644 solr/core/src/java/org/apache/solr/search/RankQParserPlugin.java
copy solr/core/src/test-files/solr/collection1/conf/{schema-minimal.xml => schema-rank-fields.xml} (69%)
create mode 100644 solr/core/src/test/org/apache/solr/schema/RankFieldTest.java
create mode 100644 solr/core/src/test/org/apache/solr/search/RankQParserPluginTest.java
[lucene-solr] 02/02: SOLR-14590: Don't close QParserPlugin in 8.x
Posted by tf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tflobbe pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit e927cb75f04a1ac1d182e6ea9a88bb122bb8d21a
Author: Tomas Fernandez Lobbe <tf...@apache.org>
AuthorDate: Tue Jun 30 11:59:58 2020 -0700
SOLR-14590: Don't close QParserPlugin in 8.x
It's not closeable
---
.../test/org/apache/solr/search/RankQParserPluginTest.java | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/solr/core/src/test/org/apache/solr/search/RankQParserPluginTest.java b/solr/core/src/test/org/apache/solr/search/RankQParserPluginTest.java
index 2e88ce2..9c255c9 100644
--- a/solr/core/src/test/org/apache/solr/search/RankQParserPluginTest.java
+++ b/solr/core/src/test/org/apache/solr/search/RankQParserPluginTest.java
@@ -54,11 +54,10 @@ public class RankQParserPluginTest extends SolrTestCaseJ4 {
}
public void testCreateParser() throws IOException {
- try (RankQParserPlugin rankQPPlugin = new RankQParserPlugin()) {
- QParser parser = rankQPPlugin.createParser("", new ModifiableSolrParams(), null, req());
- assertNotNull(parser);
- assertTrue(parser instanceof RankQParser);
- }
+ RankQParserPlugin rankQPPlugin = new RankQParserPlugin();
+ QParser parser = rankQPPlugin.createParser("", new ModifiableSolrParams(), null, req());
+ assertNotNull(parser);
+ assertTrue(parser instanceof RankQParser);
}
public void testSyntaxErrors() throws IOException, SyntaxError {
@@ -250,9 +249,8 @@ public class RankQParserPluginTest extends SolrTestCaseJ4 {
}
private RankQParser getRankQParser(SolrParams localParams, SolrParams params, SolrQueryRequest req) throws IOException {
- try (RankQParserPlugin rankQPPlugin = new RankQParserPlugin()) {
- return (RankQParser) rankQPPlugin.createParser("", localParams, params, req);
- }
+ RankQParserPlugin rankQPPlugin = new RankQParserPlugin();
+ return (RankQParser) rankQPPlugin.createParser("", localParams, params, req);
}
}
[lucene-solr] 01/02: SOLR-14590 : Add support for Lucene's
FeatureField in Solr (#1620)
Posted by tf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tflobbe pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit e6ef6f2b36c32bdeca236b907a826477926a3464
Author: Tomas Fernandez Lobbe <tf...@apache.org>
AuthorDate: Tue Jun 30 11:15:36 2020 -0700
SOLR-14590 : Add support for Lucene's FeatureField in Solr (#1620)
Add a new RankField type that internally creates a FeatureField
Add a new RankQParser that can create queries on the FeatureField
---
solr/CHANGES.txt | 4 +
.../src/java/org/apache/solr/schema/RankField.java | 140 ++++++++++
.../java/org/apache/solr/search/QParserPlugin.java | 1 +
.../org/apache/solr/search/RankQParserPlugin.java | 158 ++++++++++++
.../solr/collection1/conf/schema-rank-fields.xml | 27 ++
.../test-files/solr/collection1/conf/schema15.xml | 3 +
.../test/org/apache/solr/schema/RankFieldTest.java | 285 +++++++++++++++++++++
.../org/apache/solr/search/QueryEqualityTest.java | 12 +
.../apache/solr/search/RankQParserPluginTest.java | 258 +++++++++++++++++++
.../solr/configsets/_default/conf/managed-schema | 7 +
10 files changed, 895 insertions(+)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 24761fe..50e1917 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -60,6 +60,10 @@ New Features
* SOLR-14599: Package manager support for cluster level plugins (see SOLR-14404) (Ishan Chattopadhyaya)
+* SOLR-14590: Add support for RankFields. RankFields allow the use of per-document scoring factors in a
+ way that lets Solr skip over non-competitive documents when ranking. See SOLR-13289.
+ (Tomás Fernández Löbbe, Varun Thacker)
+
Improvements
---------------------
* SOLR-14316: Remove unchecked type conversion warning in JavaBinCodec's readMapEntry's equals() method
diff --git a/solr/core/src/java/org/apache/solr/schema/RankField.java b/solr/core/src/java/org/apache/solr/schema/RankField.java
new file mode 100644
index 0000000..d11d5ed
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/schema/RankField.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.lucene.document.FeatureField;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.IndexableFieldType;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.response.TextResponseWriter;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.RankQParserPlugin;
+import org.apache.solr.uninverting.UninvertingReader.Type;
+
+/**
+ * <p>
+ * {@code RankField}s can be used to store scoring factors to improve document ranking. They should be used
+ * in combination with {@link RankQParserPlugin}. To use:
+ * </p>
+ * <p>
+ * Define the {@code RankField} {@code fieldType} in your schema:
+ * </p>
+ * <pre class="prettyprint">
+ * <fieldType name="rank" class="solr.RankField" />
+ * </pre>
+ * <p>
+ * Add fields to the schema, i.e.:
+ * </p>
+ * <pre class="prettyprint">
+ * <field name="pagerank" type="rank" />
+ * </pre>
+ *
+ * Query using the {@link RankQParserPlugin}, for example
+ * <pre class="prettyprint">
+ * http://localhost:8983/solr/techproducts?q=memory _query_:{!rank f='pagerank', function='log' scalingFactor='1.2'}
+ * </pre>
+ *
+ * @see RankQParserPlugin
+ * @lucene.experimental
+ * @since 8.6
+ */
+public class RankField extends FieldType {
+
+ /*
+ * While the user can create multiple RankFields, internally we use a single Lucene field,
+ * and we map the Solr field name to the "feature" in Lucene's FeatureField. This is mainly
+ * to simplify the user experience.
+ */
+ public static final String INTERNAL_RANK_FIELD_NAME = "_rank_";
+
+ @Override
+ public Type getUninversionType(SchemaField sf) {
+ throw null;
+ }
+
+ @Override
+ public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException {
+ }
+
+ @Override
+ protected void init(IndexSchema schema, Map<String,String> args) {
+ super.init(schema, args);
+ if (schema.getFieldOrNull(INTERNAL_RANK_FIELD_NAME) != null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "A field named \"" + INTERNAL_RANK_FIELD_NAME + "\" can't be defined in the schema");
+ }
+ for (int prop:new int[] {STORED, DOC_VALUES, OMIT_TF_POSITIONS, SORT_MISSING_FIRST, SORT_MISSING_LAST}) {
+ if ((trueProperties & prop) != 0) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Property \"" + getPropertyName(prop) + "\" can't be set to true in RankFields");
+ }
+ }
+ for (int prop:new int[] {UNINVERTIBLE, INDEXED, MULTIVALUED}) {
+ if ((falseProperties & prop) != 0) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Property \"" + getPropertyName(prop) + "\" can't be set to false in RankFields");
+ }
+ }
+ properties &= ~(UNINVERTIBLE | STORED | DOC_VALUES);
+
+ }
+
+ @Override
+ protected IndexableField createField(String name, String val, IndexableFieldType type) {
+ if (val == null || val.isEmpty()) {
+ return null;
+ }
+ float featureValue;
+ try {
+ featureValue = Float.parseFloat(val);
+ } catch (NumberFormatException nfe) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error while creating field '" + name + "' from value '" + val + "'. Expecting float.", nfe);
+ }
+ // Internally, we always use the same field
+ return new FeatureField(INTERNAL_RANK_FIELD_NAME, name, featureValue);
+ }
+
+ @Override
+ public Query getExistenceQuery(QParser parser, SchemaField field) {
+ return new TermQuery(new Term(INTERNAL_RANK_FIELD_NAME, field.getName()));
+ }
+
+ @Override
+ public Query getFieldQuery(QParser parser, SchemaField field, String externalVal) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Only a \"*\" term query can be done on RankFields");
+ }
+
+ @Override
+ protected Query getSpecializedRangeQuery(QParser parser, SchemaField field, String part1, String part2,
+ boolean minInclusive, boolean maxInclusive) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Range queries not supported on RankFields");
+ }
+
+ @Override
+ public SortField getSortField(SchemaField field, boolean top) {
+ // We could use FeatureField.newFeatureSort()
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "can not sort on a rank field: " + field.getName());
+ }
+
+}
diff --git a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
index d0a5b9d..d95b540 100644
--- a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
@@ -87,6 +87,7 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI
map.put(BoolQParserPlugin.NAME, new BoolQParserPlugin());
map.put(MinHashQParserPlugin.NAME, new MinHashQParserPlugin());
map.put(HashRangeQParserPlugin.NAME, new HashRangeQParserPlugin());
+ map.put(RankQParserPlugin.NAME, new RankQParserPlugin());
standardPlugins = Collections.unmodifiableMap(map);
}
diff --git a/solr/core/src/java/org/apache/solr/search/RankQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/RankQParserPlugin.java
new file mode 100644
index 0000000..962e1d2
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/RankQParserPlugin.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+import java.util.Locale;
+import java.util.Objects;
+
+import org.apache.lucene.document.FeatureField;
+import org.apache.lucene.search.Query;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.RankField;
+import org.apache.solr.schema.SchemaField;
+/**
+ * {@code RankQParserPlugin} can be used to introduce document-depending scoring factors to ranking.
+ * While this {@code QParser} delivers a (subset of) functionality already available via {@link FunctionQParser},
+ * the benefit is that {@code RankQParserPlugin} can be used in combination with the {@code minExactCount} to
+ * use BlockMax-WAND algorithm (skip non-competitive documents) to provide faster responses.
+ *
+ * @see RankField
+ *
+ * @lucene.experimental
+ * @since 8.6
+ */
+public class RankQParserPlugin extends QParserPlugin {
+
+ public static final String NAME = "rank";
+ public static final String FIELD = "f";
+ public static final String FUNCTION = "function";
+ public static final String WEIGHT = "weight";
+ public static final String PIVOT = "pivot";
+ public static final String SCALING_FACTOR = "scalingFactor";
+ public static final String EXPONENT = "exponent";
+
+ private final static FeatureFieldFunction DEFAULT_FUNCTION = FeatureFieldFunction.SATU;
+
+ private enum FeatureFieldFunction {
+ SATU {
+ @Override
+ public Query createQuery(String fieldName, SolrParams params) throws SyntaxError {
+ Float weight = params.getFloat(WEIGHT);
+ Float pivot = params.getFloat(PIVOT);
+ if (pivot == null && (weight == null || Float.compare(weight.floatValue(), 1f) == 0)) {
+ // No IAE expected in this case
+ return FeatureField.newSaturationQuery(RankField.INTERNAL_RANK_FIELD_NAME, fieldName);
+ }
+ if (pivot == null) {
+ throw new SyntaxError("A pivot value needs to be provided if the weight is not 1 on \"satu\" function");
+ }
+ if (weight == null) {
+ weight = Float.valueOf(1);
+ }
+ try {
+ return FeatureField.newSaturationQuery(RankField.INTERNAL_RANK_FIELD_NAME, fieldName, weight, pivot);
+ } catch (IllegalArgumentException iae) {
+ throw new SyntaxError(iae.getMessage());
+ }
+ }
+ },
+ LOG {
+ @Override
+ public Query createQuery(String fieldName, SolrParams params) throws SyntaxError {
+ float weight = params.getFloat(WEIGHT, 1f);
+ float scalingFactor = params.getFloat(SCALING_FACTOR, 1f);
+ try {
+ return FeatureField.newLogQuery(RankField.INTERNAL_RANK_FIELD_NAME, fieldName, weight, scalingFactor);
+ } catch (IllegalArgumentException iae) {
+ throw new SyntaxError(iae.getMessage());
+ }
+ }
+ },
+ SIGM {
+ @Override
+ public Query createQuery(String fieldName, SolrParams params) throws SyntaxError {
+ float weight = params.getFloat(WEIGHT, 1f);
+ Float pivot = params.getFloat(PIVOT);
+ if (pivot == null) {
+ throw new SyntaxError("A pivot value needs to be provided when using \"sigm\" function");
+ }
+ Float exponent = params.getFloat(EXPONENT);
+ if (exponent == null) {
+ throw new SyntaxError("An exponent value needs to be provided when using \"sigm\" function");
+ }
+ try {
+ return FeatureField.newSigmoidQuery(RankField.INTERNAL_RANK_FIELD_NAME, fieldName, weight, pivot, exponent);
+ } catch (IllegalArgumentException iae) {
+ throw new SyntaxError(iae.getMessage());
+ }
+ }
+ };
+
+ public abstract Query createQuery(String fieldName, SolrParams params) throws SyntaxError;
+
+ }
+
+ @Override
+ public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
+ Objects.requireNonNull(localParams, "LocalParams String can't be null");
+ Objects.requireNonNull(req, "SolrQueryRequest can't be null");
+ return new RankQParser(qstr, localParams, params, req);
+ }
+
+ public static class RankQParser extends QParser {
+
+ private final String field;
+
+ public RankQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
+ super(qstr, localParams, params, req);
+ this.field = localParams.get(FIELD);
+ }
+
+ @Override
+ public Query parse() throws SyntaxError {
+ if (this.field == null || this.field.isEmpty()) {
+ throw new SyntaxError("Field can't be empty in rank queries");
+ }
+ SchemaField schemaField = req.getSchema().getFieldOrNull(field);
+ if (schemaField == null) {
+ throw new SyntaxError("Field \"" + this.field + "\" not found");
+ }
+ if (!(schemaField.getType() instanceof RankField)) {
+ throw new SyntaxError("Field \"" + this.field + "\" is not a RankField");
+ }
+ return getFeatureFieldFunction(localParams.get(FUNCTION))
+ .createQuery(field, localParams);
+ }
+
+ private FeatureFieldFunction getFeatureFieldFunction(String function) throws SyntaxError {
+ FeatureFieldFunction f = null;
+ if (function == null || function.isEmpty()) {
+ f = DEFAULT_FUNCTION;
+ } else {
+ try {
+ f = FeatureFieldFunction.valueOf(function.toUpperCase(Locale.ROOT));
+ } catch (IllegalArgumentException iae) {
+ throw new SyntaxError("Unknown function in rank query: \"" + function + "\"");
+ }
+ }
+ return f;
+ }
+
+ }
+
+}
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-rank-fields.xml b/solr/core/src/test-files/solr/collection1/conf/schema-rank-fields.xml
new file mode 100644
index 0000000..2da0c0d
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-rank-fields.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="rank_fields" version="1.6">
+ <fieldType name="string" class="solr.StrField" />
+ <fieldType name="rank" class="solr.RankField" />
+ <fields>
+ <field name="id" type="string" />
+ <field name="str_field" type="string" />
+ <field name="rank_1" type="rank" />
+ <field name="rank_2" type="rank" />
+ </fields>
+</schema>
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
index 91fb092..c2a8bbb 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
@@ -41,6 +41,7 @@
<fieldType name="tlong" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
<fieldType name="tdouble" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
<fieldType name="currency" class="solr.CurrencyField" currencyConfig="currency.xml" multiValued="false"/>
+ <fieldType name="rank" class="solr.RankField"/>
<!-- Field type demonstrating an Analyzer failure -->
<fieldType name="failtype1" class="solr.TextField">
@@ -614,6 +615,8 @@
<dynamicField name="ignored_*" type="ignored" multiValued="true"/>
<dynamicField name="attr_*" type="text" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="rank_*" type="rank"/>
+
<dynamicField name="random_*" type="random"/>
<dynamicField name="*_dpf" type="delimited_payloads_float" indexed="true" stored="true"/>
diff --git a/solr/core/src/test/org/apache/solr/schema/RankFieldTest.java b/solr/core/src/test/org/apache/solr/schema/RankFieldTest.java
new file mode 100644
index 0000000..11877f0
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/schema/RankFieldTest.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import javax.xml.xpath.XPathConstants;
+
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.util.TestHarness;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+
+public class RankFieldTest extends SolrTestCaseJ4 {
+
+ private static final String RANK_1 = "rank_1";
+ private static final String RANK_2 = "rank_2";
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-minimal.xml","schema-rank-fields.xml");
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ clearIndex();
+ assertU(commit());
+ super.setUp();
+ }
+
+ public void testInternalFieldName() {
+ assertEquals("RankField.INTERNAL_RANK_FIELD_NAME changed in an incompatible way",
+ "_rank_", RankField.INTERNAL_RANK_FIELD_NAME);
+ }
+
+ public void testBasic() {
+ assertNotNull(h.getCore().getLatestSchema().getFieldOrNull(RANK_1));
+ assertEquals(RankField.class, h.getCore().getLatestSchema().getField(RANK_1).getType().getClass());
+ }
+
+ public void testBadFormat() {
+ ignoreException("Expecting float");
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, "foo"
+ ));
+
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, "1.2.3"
+ ));
+
+ unIgnoreException("Expecting float");
+
+ ignoreException("must be finite");
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, Float.toString(Float.POSITIVE_INFINITY)
+ ));
+
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, Float.toString(Float.NEGATIVE_INFINITY)
+ ));
+
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, Float.toString(Float.NaN)
+ ));
+
+ unIgnoreException("must be finite");
+
+ ignoreException("must be a positive");
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, Float.toString(-0.0f)
+ ));
+
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, Float.toString(-1f)
+ ));
+
+ assertFailedU(adoc(
+ "id", "1",
+ RANK_1, Float.toString(0.0f)
+ ));
+ unIgnoreException("must be a positive");
+ }
+
+ public void testAddRandom() {
+ for (int i = 0 ; i < random().nextInt(TEST_NIGHTLY ? 10000 : 100); i++) {
+ assertU(adoc(
+ "id", String.valueOf(i),
+ RANK_1, Float.toString(random().nextFloat())
+ ));
+ }
+ assertU(commit());
+ }
+
+ public void testSkipEmpty() {
+ assertU(adoc(
+ "id", "1",
+ RANK_1, ""
+ ));
+ }
+
+ public void testBasicAdd() throws IOException {
+ assertU(adoc(
+ "id", "testBasicAdd",
+ RANK_1, "1"
+ ));
+ assertU(commit());
+ //assert that the document made it in
+ assertQ(req("q", "id:testBasicAdd"), "//*[@numFound='1']");
+ h.getCore().withSearcher((searcher) -> {
+ LeafReader reader = searcher.getIndexReader().getContext().leaves().get(0).reader();
+ // assert that the field made it in
+ assertNotNull(reader.getFieldInfos().fieldInfo(RankField.INTERNAL_RANK_FIELD_NAME));
+ // assert that the feature made it in
+ assertTrue(reader.terms(RankField.INTERNAL_RANK_FIELD_NAME).iterator().seekExact(new BytesRef(RANK_1.getBytes(StandardCharsets.UTF_8))));
+ return null;
+ });
+ }
+
+ public void testMultipleRankFields() throws IOException {
+ assertU(adoc(
+ "id", "testMultiValueAdd",
+ RANK_1, "1",
+ RANK_2, "2"
+ ));
+ assertU(commit());
+ //assert that the document made it in
+ assertQ(req("q", "id:testMultiValueAdd"), "//*[@numFound='1']");
+ h.getCore().withSearcher((searcher) -> {
+ LeafReader reader = searcher.getIndexReader().getContext().leaves().get(0).reader();
+ // assert that the field made it in
+ assertNotNull(reader.getFieldInfos().fieldInfo(RankField.INTERNAL_RANK_FIELD_NAME));
+ // assert that the features made it in
+ assertTrue(reader.terms(RankField.INTERNAL_RANK_FIELD_NAME).iterator().seekExact(new BytesRef(RANK_2.getBytes(StandardCharsets.UTF_8))));
+ assertTrue(reader.terms(RankField.INTERNAL_RANK_FIELD_NAME).iterator().seekExact(new BytesRef(RANK_1.getBytes(StandardCharsets.UTF_8))));
+ return null;
+ });
+ }
+
+ public void testSortFails() throws IOException {
+ assertU(adoc(
+ "id", "testSortFails",
+ RANK_1, "1"
+ ));
+ assertU(commit());
+ assertQEx("Can't sort on rank field", req(
+ "q", "id:testSortFails",
+ "sort", RANK_1 + " desc"), 400);
+ }
+
+ @Ignore("We currently don't fail these kinds of requests with other field types")
+ public void testFacetFails() throws IOException {
+ assertU(adoc(
+ "id", "testFacetFails",
+ RANK_1, "1"
+ ));
+ assertU(commit());
+ assertQEx("Can't facet on rank field", req(
+ "q", "id:testFacetFails",
+ "facet", "true",
+ "facet.field", RANK_1), 400);
+ }
+
+ public void testTermQuery() throws IOException {
+ assertU(adoc(
+ "id", "testTermQuery",
+ RANK_1, "1",
+ RANK_2, "1"
+ ));
+ assertU(adoc(
+ "id", "testTermQuery2",
+ RANK_1, "1"
+ ));
+ assertU(commit());
+ assertQ(req("q", RANK_1 + ":*"), "//*[@numFound='2']");
+ assertQ(req("q", RANK_1 + ":[* TO *]"), "//*[@numFound='2']");
+ assertQ(req("q", RANK_2 + ":*"), "//*[@numFound='1']");
+ assertQ(req("q", RANK_2 + ":[* TO *]"), "//*[@numFound='1']");
+
+ assertQEx("Term queries not supported", req("q", RANK_1 + ":1"), 400);
+ assertQEx("Range queries not supported", req("q", RANK_1 + ":[1 TO 10]"), 400);
+ }
+
+
+ public void testResponseQuery() throws IOException {
+ assertU(adoc(
+ "id", "testResponseQuery",
+ RANK_1, "1"
+ ));
+ assertU(commit());
+ // Ignore requests to retrieve rank
+ assertQ(req("q", RANK_1 + ":*",
+ "fl", "id," + RANK_1),
+ "//*[@numFound='1']",
+ "count(//result/doc[1]/str)=1");
+ }
+
+ public void testRankQParserQuery() throws IOException {
+ assertU(adoc(
+ "id", "1",
+ "str_field", "foo",
+ RANK_1, "1",
+ RANK_2, "2"
+ ));
+ assertU(adoc(
+ "id", "2",
+ "str_field", "foo",
+ RANK_1, "2",
+ RANK_2, "1"
+ ));
+ assertU(commit());
+ assertQ(req("q", "str_field:foo _query_:{!rank f='" + RANK_1 + "' function='log' scalingFactor='1'}"),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='2']",
+ "//result/doc[2]/str[@name='id'][.='1']");
+
+ assertQ(req("q", "str_field:foo _query_:{!rank f='" + RANK_2 + "' function='log' scalingFactor='1'}"),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='1']",
+ "//result/doc[2]/str[@name='id'][.='2']");
+
+ assertQ(req("q", "foo",
+ "defType", "dismax",
+ "qf", "str_field^10",
+ "bq", "{!rank f='" + RANK_1 + "' function='log' scalingFactor='1'}"
+ ),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='2']",
+ "//result/doc[2]/str[@name='id'][.='1']");
+
+ assertQ(req("q", "foo",
+ "defType", "dismax",
+ "qf", "str_field^10",
+ "bq", "{!rank f='" + RANK_2 + "' function='log' scalingFactor='1'}"
+ ),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='1']",
+ "//result/doc[2]/str[@name='id'][.='2']");
+ }
+
+ public void testScoreChanges() throws Exception {
+ assertU(adoc(
+ "id", "1",
+ "str_field", "foo",
+ RANK_1, "1"
+ ));
+ assertU(commit());
+ ModifiableSolrParams params = params("q", "foo",
+ "defType", "dismax",
+ "qf", "str_field^10",
+ "fl", "id,score",
+ "wt", "xml");
+
+ double scoreBefore = (Double) TestHarness.evaluateXPath(h.query(req(params)), "//result/doc[1]/float[@name='score']", XPathConstants.NUMBER);
+ params.add("bq", "{!rank f='" + RANK_1 + "' function='log' scalingFactor='1'}");
+ double scoreAfter = (Double) TestHarness.evaluateXPath(h.query(req(params)), "//result/doc[1]/float[@name='score']", XPathConstants.NUMBER);
+ assertNotEquals("Expecting score to change", scoreBefore, scoreAfter, 0f);
+
+ }
+
+}
diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
index a375e72..899abb7 100644
--- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
+++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
@@ -357,6 +357,18 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
req.close();
}
}
+
+ public void testRankQuery() throws Exception {
+ SolrQueryRequest req = req("df", "foo_s");
+ try {
+ assertQueryEquals("rank", req,
+ "{!rank f='rank_1'}",
+ "{!rank f='rank_1' function='satu'}",
+ "{!rank f='rank_1' function='satu' weight=1}");
+ } finally {
+ req.close();
+ }
+ }
public void testQueryNested() throws Exception {
SolrQueryRequest req = req("df", "foo_s");
diff --git a/solr/core/src/test/org/apache/solr/search/RankQParserPluginTest.java b/solr/core/src/test/org/apache/solr/search/RankQParserPluginTest.java
new file mode 100644
index 0000000..2e88ce2
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/RankQParserPluginTest.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+import static org.apache.solr.search.RankQParserPlugin.EXPONENT;
+import static org.apache.solr.search.RankQParserPlugin.FIELD;
+import static org.apache.solr.search.RankQParserPlugin.FUNCTION;
+import static org.apache.solr.search.RankQParserPlugin.NAME;
+import static org.apache.solr.search.RankQParserPlugin.PIVOT;
+import static org.apache.solr.search.RankQParserPlugin.SCALING_FACTOR;
+import static org.apache.solr.search.RankQParserPlugin.WEIGHT;
+
+import java.io.IOException;
+
+import org.apache.lucene.search.Query;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.RankField;
+import org.apache.solr.search.RankQParserPlugin.RankQParser;
+import org.hamcrest.CoreMatchers;
+import org.junit.BeforeClass;
+
+public class RankQParserPluginTest extends SolrTestCaseJ4 {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-minimal.xml", "schema-rank-fields.xml");
+ }
+
+ public void testParamCompatibility() {
+ assertEquals("RankQParserPlugin.NAME changed in an incompatible way", "rank", NAME);
+ assertEquals("RankQParserPlugin.FIELD changed in an incompatible way", "f", FIELD);
+ assertEquals("RankQParserPlugin.FUNCTION changed in an incompatible way", "function", FUNCTION);
+ assertEquals("RankQParserPlugin.PIVOT changed in an incompatible way", "pivot", PIVOT);
+ assertEquals("RankQParserPlugin.SCALING_FACTOR changed in an incompatible way", "scalingFactor", SCALING_FACTOR);
+ assertEquals("RankQParserPlugin.WEIGHT changed in an incompatible way", "weight", WEIGHT);
+ assertEquals("RankQParserPlugin.EXPONENT changed in an incompatible way", "exponent", EXPONENT);
+ }
+
+ public void testCreateParser() throws IOException {
+ try (RankQParserPlugin rankQPPlugin = new RankQParserPlugin()) {
+ QParser parser = rankQPPlugin.createParser("", new ModifiableSolrParams(), null, req());
+ assertNotNull(parser);
+ assertTrue(parser instanceof RankQParser);
+ }
+ }
+
+ public void testSyntaxErrors() throws IOException, SyntaxError {
+ assertSyntaxError("No Field", "Field can't be empty", () ->
+ getRankQParser(new ModifiableSolrParams(), null, req()).parse());
+ assertSyntaxError("Field empty", "Field can't be empty", () ->
+ getRankQParser(
+ params(FIELD, ""), null, req()).parse());
+ assertSyntaxError("Field doesn't exist", "Field \"foo\" not found", () ->
+ getRankQParser(
+ params(FIELD, "foo"), null, req()).parse());
+ assertSyntaxError("ID is not a feature field", "Field \"id\" is not a RankField", () ->
+ getRankQParser(
+ params(FIELD, "id"), null, req()).parse());
+ }
+
+ public void testBadLogParameters() throws IOException, SyntaxError {
+ assertSyntaxError("Expecting bad weight", "weight must be in", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "log",
+ WEIGHT, "0"), null, req()).parse());
+ assertSyntaxError("Expecting bad scaling factor", "scalingFactor must be", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "log",
+ SCALING_FACTOR, "0"), null, req()).parse());
+ }
+
+ public void testBadSaturationParameters() throws IOException, SyntaxError {
+ assertSyntaxError("Expecting a pivot value", "A pivot value", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "satu",
+ WEIGHT, "2"), null, req()).parse());
+ assertSyntaxError("Expecting bad weight", "weight must be in", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "satu",
+ PIVOT, "1",
+ WEIGHT, "-1"), null, req()).parse());
+ }
+
+ public void testBadSigmoidParameters() throws IOException, SyntaxError {
+ assertSyntaxError("Expecting missing pivot", "A pivot value", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ EXPONENT, "1"), null, req()).parse());
+ assertSyntaxError("Expecting missing exponent", "An exponent value", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ PIVOT, "1"), null, req()).parse());
+ assertSyntaxError("Expecting bad weight", "weight must be in", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ PIVOT, "1",
+ EXPONENT, "1",
+ WEIGHT, "-1"), null, req()).parse());
+ assertSyntaxError("Expecting bad pivot", "pivot must be", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ PIVOT, "0",
+ EXPONENT, "1"), null, req()).parse());
+ assertSyntaxError("Expecting bad exponent", "exp must be", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ PIVOT, "1",
+ EXPONENT, "0"), null, req()).parse());
+ }
+
+ public void testUnknownFunction() throws IOException, SyntaxError {
+ assertSyntaxError("Expecting bad function", "Unknown function in rank query: \"foo\"", () ->
+ getRankQParser(
+ params(FIELD, "rank_1",
+ FUNCTION, "foo"), null, req()).parse());
+ }
+
+ public void testParseLog() throws IOException, SyntaxError {
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedLogToString(1), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "log",
+ SCALING_FACTOR, "1",
+ WEIGHT, "1"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedLogToString(2.5f), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "log",
+ SCALING_FACTOR, "2.5",
+ WEIGHT, "1"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedLogToString(1), 2.5f),
+ params(FIELD, "rank_1",
+ FUNCTION, "log",
+ SCALING_FACTOR, "1",
+ WEIGHT, "2.5"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedLogToString(1), 2.5f),
+ params(FIELD, "rank_1",
+ FUNCTION, "Log", //use different case
+ SCALING_FACTOR, "1",
+ WEIGHT, "2.5"));
+ }
+
+ public void testParseSigm() throws IOException, SyntaxError {
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSigmoidToString(1.5f, 2f), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ PIVOT, "1.5",
+ EXPONENT, "2",
+ WEIGHT, "1"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSigmoidToString(1.5f, 2f), 2),
+ params(FIELD, "rank_1",
+ FUNCTION, "sigm",
+ PIVOT, "1.5",
+ EXPONENT, "2",
+ WEIGHT, "2"));
+ }
+
+ public void testParseSatu() throws IOException, SyntaxError {
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(1.5f), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "satu",
+ PIVOT, "1.5",
+ WEIGHT, "1"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(1.5f), 2),
+ params(FIELD, "rank_1",
+ FUNCTION, "satu",
+ PIVOT, "1.5",
+ WEIGHT, "2"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(null), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "satu"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(null), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "satu",
+ WEIGHT, "1"));
+
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(1.5f), 1),
+ params(FIELD, "rank_1",
+ FUNCTION, "satu",
+ PIVOT, "1.5"));
+ }
+
+ public void testParseDefault() throws IOException, SyntaxError {
+ assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(null), 1),
+ params(FIELD, "rank_1"));
+ }
+
+ private void assertValidRankQuery(String expctedToString, SolrParams localParams) throws IOException, SyntaxError {
+ QParser parser = getRankQParser(localParams, null, req());
+ Query q = parser.parse();
+ assertNotNull(q);
+ assertThat(q.toString(), CoreMatchers.equalTo(expctedToString));
+ }
+
+ private String expectedFeatureQueryToString(String fieldName, String function, float boost) {
+ String featureQueryStr = "FeatureQuery(field=" + RankField.INTERNAL_RANK_FIELD_NAME + ", feature=" + fieldName + ", function=" + function + ")";
+ if (boost == 1f) {
+ return featureQueryStr;
+ }
+ return "(" + featureQueryStr + ")^" + boost;
+ }
+
+ private String expectedLogToString(float scalingFactor) {
+ return "LogFunction(scalingFactor=" + scalingFactor + ")";
+ }
+
+ private String expectedSigmoidToString(float pivot, float exp) {
+ return "SigmoidFunction(pivot=" + pivot + ", a=" + exp + ")";
+ }
+
+ private String expectedSaturationToString(Float pivot) {
+ return "SaturationFunction(pivot=" + pivot + ")";
+ }
+
+ private void assertSyntaxError(String assertionMsg, String expectedExceptionMsg, ThrowingRunnable runnable) {
+ SyntaxError se = expectThrows(SyntaxError.class, assertionMsg, runnable);
+ assertThat(se.getMessage(), CoreMatchers.containsString(expectedExceptionMsg));
+ }
+
+ private RankQParser getRankQParser(SolrParams localParams, SolrParams params, SolrQueryRequest req) throws IOException {
+ try (RankQParserPlugin rankQPPlugin = new RankQParserPlugin()) {
+ return (RankQParser) rankQPPlugin.createParser("", localParams, params, req);
+ }
+ }
+
+}
diff --git a/solr/server/solr/configsets/_default/conf/managed-schema b/solr/server/solr/configsets/_default/conf/managed-schema
index d832ed1..e99e27e 100644
--- a/solr/server/solr/configsets/_default/conf/managed-schema
+++ b/solr/server/solr/configsets/_default/conf/managed-schema
@@ -250,6 +250,13 @@
<!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
<fieldType name="binary" class="solr.BinaryField"/>
+
+ <!--
+ RankFields can be used to store scoring factors to improve document ranking. They should be used
+ in combination with RankQParserPlugin.
+ (experimental)
+ -->
+ <fieldType name="rank" class="solr.RankField"/>
<!-- solr.TextField allows the specification of custom text analyzers
specified as a tokenizer and a list of token filters. Different