You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/01/11 08:00:37 UTC
[01/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11682: Add
gtestDataSet Stream Evaluator
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-11702 116fe4ad2 -> ca364c729
SOLR-11682: Add gtestDataSet Stream Evaluator
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/94a680c3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/94a680c3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/94a680c3
Branch: refs/heads/jira/solr-11702
Commit: 94a680c31115094cb229bb0134c15a65a7a074d9
Parents: 9586d12
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed Jan 3 21:31:12 2018 -0500
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed Jan 3 21:31:39 2018 -0500
----------------------------------------------------------------------
.../org/apache/solr/handler/StreamHandler.java | 1 +
.../solrj/io/eval/GTestDataSetEvaluator.java | 64 ++++++++++++++++++++
.../solrj/io/stream/StreamExpressionTest.java | 23 +++++++
3 files changed, 88 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/94a680c3/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index b8c5ae5..8f3c4d6 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -291,6 +291,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("akima", AkimaEvaluator.class)
.withFunctionName("lerp", LerpEvaluator.class)
.withFunctionName("chiSquareDataSet", ChiSquareDataSetEvaluator.class)
+ .withFunctionName("gtestDataSet", GTestDataSetEvaluator.class)
// Boolean Stream Evaluators
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/94a680c3/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GTestDataSetEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GTestDataSetEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GTestDataSetEvaluator.java
new file mode 100644
index 0000000..d553197
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GTestDataSetEvaluator.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.math3.stat.inference.GTest;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+
+public class GTestDataSetEvaluator extends RecursiveNumericListEvaluator implements TwoValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public GTestDataSetEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object value1, Object value2) throws IOException {
+
+ List<Number> listA = (List<Number>) value1;
+ List<Number> listB = (List<Number>) value2;
+
+ long[] sampleA = new long[listA.size()];
+ long[] sampleB = new long[listB.size()];
+
+ for(int i=0; i<sampleA.length; i++) {
+ sampleA[i] = listA.get(i).longValue();
+ }
+
+ for(int i=0; i<sampleB.length; i++) {
+ sampleB[i] = listB.get(i).longValue();
+ }
+
+ GTest gTest = new GTest();
+ double g = gTest.gDataSetsComparison(sampleA, sampleB);
+ double p = gTest.gTestDataSetsComparison(sampleA, sampleB);
+
+ Map<String,Number> m = new HashMap<>();
+ m.put("G-statistic", g);
+ m.put("p-value", p);
+ return new Tuple(m);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/94a680c3/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 17d67e3..9d41c54 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -7248,6 +7248,29 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertEquals(pval.doubleValue(), 0.04242018685334226, .0001);
}
+ @Test
+ public void testGtestDataSet() throws Exception {
+ String cexpr = "let(echo=true," +
+ " a=array(1,1,2,3,4,5,6,7,9,10,11,12), " +
+ " b=array(1,1,2,3,4,5,6,7,1,1,1,1), " +
+ " gtest=gtestDataSet(a, b))";
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Map testResult = (Map)tuples.get(0).get("gtest");
+ Number gstat = (Number)testResult.get("G-statistic");
+ Number pval = (Number)testResult.get("p-value");
+ assertEquals(gstat.doubleValue(), 22.41955157784917, .0001);
+ assertEquals(pval.doubleValue(), 0.021317102314826752, .0001);
+ }
+
@Test
public void testMultiVariateNormalDistribution() throws Exception {
[20/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11819: The schema
api reference manual page should not use hyphens in the example field name
Posted by da...@apache.org.
SOLR-11819: The schema api reference manual page should not use hyphens in the example field name
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7695544a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7695544a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7695544a
Branch: refs/heads/jira/solr-11702
Commit: 7695544a5f85b179cb39811c9c03230f4c4f5aff
Parents: d598517 1f09d5b
Author: Erick Erickson <er...@apache.org>
Authored: Sun Jan 7 10:42:44 2018 -0800
Committer: Erick Erickson <er...@apache.org>
Committed: Sun Jan 7 10:42:44 2018 -0800
----------------------------------------------------------------------
solr/solr-ref-guide/src/schema-api.adoc | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
[45/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11829: [Ref-Guide]
Indexing documents with existing id
Posted by da...@apache.org.
SOLR-11829: [Ref-Guide] Indexing documents with existing id
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ae1e1920
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ae1e1920
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ae1e1920
Branch: refs/heads/jira/solr-11702
Commit: ae1e1920220a17808a6451004b39ff5889d961f8
Parents: 4471c1b
Author: Erick Erickson <er...@apache.org>
Authored: Tue Jan 9 17:57:53 2018 -0800
Committer: Erick Erickson <er...@apache.org>
Committed: Tue Jan 9 17:57:53 2018 -0800
----------------------------------------------------------------------
solr/solr-ref-guide/src/documents-screen.adoc | 52 +++++++++-------------
1 file changed, 22 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae1e1920/solr/solr-ref-guide/src/documents-screen.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/documents-screen.adoc b/solr/solr-ref-guide/src/documents-screen.adoc
index 83da713..3274d40 100644
--- a/solr/solr-ref-guide/src/documents-screen.adoc
+++ b/solr/solr-ref-guide/src/documents-screen.adoc
@@ -23,11 +23,10 @@ image::images/documents-screen/documents_add_screen.png[image,height=400]
The screen allows you to:
-* Copy documents in JSON, CSV or XML and submit them to the index
-* Upload documents (in JSON, CSV or XML)
+* Submit JSON, CSV or XML documents in solr-specific format to Solr
+* Upload documents (in JSON, CSV or XML) to Solr
* Construct documents by selecting fields and field values
-
[TIP]
====
There are other ways to load data, see also these sections:
@@ -36,23 +35,23 @@ There are other ways to load data, see also these sections:
* <<uploading-data-with-solr-cell-using-apache-tika.adoc#uploading-data-with-solr-cell-using-apache-tika,Uploading Data with Solr Cell using Apache Tika>>
====
-The first step is to define the RequestHandler to use (aka, `qt`). By default `/update` will be defined. To use Solr Cell, for example, change the request handler to `/update/extract`.
-
-Then choose the Document Type to define the type of document to load. The remaining parameters will change depending on the document type selected.
-
-== JSON Documents
-
-When using the JSON document type, the functionality is similar to using a requestHandler on the command line. Instead of putting the documents in a curl command, they can instead be input into the Document entry box. The document structure should still be in proper JSON format.
+== Common Fields
+* Request-Handler: The first step is to define the RequestHandler. By default `/update` will be defined. Change the request handler to `/update/extract` to use Solr Cell.
+* Document Type: Select the Document Type to define the format of document to load. The remaining parameters may change depending on the document type selected.
+* Document(s): Enter a properly-formatted Solr document corresponding to the `Document Type` selected. XML and JSON documents must be formatted in a Solr-specific format, a small illustrative document will be shown. CSV files should have headers corresponding to fields defined in the schema. More details can be found at: <<uploading-data-with-index-handlers.adoc#uploading-data-with-index-handlers,Uploading Data with Index Handlers>>.
+* Commit Within: Specify the number of milliseconds between the time the document is submitted and when it is available for searching.
+* Overwrite: If `true` the new document will replace an existing document with the same value in the `id` field. If `false` multiple documents with the same id can be added.
-Then you can choose when documents should be added to the index (Commit Within), & whether existing documents should be overwritten with incoming documents with the same id (if this is not `true`, then the incoming documents will be dropped).
-
-This option will only add or overwrite documents to the index; for other update tasks, see the <<Solr Command>> option.
+[TIP]
+====
+Setting `Overwrite` to `false` is very rare in production situations, the default is `true`.
+====
-== CSV Documents
+== CSV, JSON and XML Documents
-When using the CSV document type, the functionality is similar to using a requestHandler on the command line. Instead of putting the documents in a curl command, they can instead be input into the Document entry box. The document structure should still be in proper CSV format, with columns delimited and one row per document.
+When using these document types the functionality is similar to submitting documents via `curl` or similar. The document structure must be in a Solr-specific format appropriate for the document type. Examples are illustrated in the Document(s) text box when you select the various types.
-Then you can choose when documents should be added to the index (Commit Within), and whether existing documents should be overwritten with incoming documents with the same id (if this is not `true`, then the incoming documents will be dropped).
+These options will only add or overwrite documents; for other update tasks, see the <<Solr Command>> option.
== Document Builder
@@ -60,22 +59,15 @@ The Document Builder provides a wizard-like interface to enter fields of a docum
== File Upload
-The File Upload option allows choosing a prepared file and uploading it. If using only `/update` for the Request-Handler option, you will be limited to XML, CSV, and JSON.
-
-However, to use the ExtractingRequestHandler (aka Solr Cell), you can modify the Request-Handler to `/update/extract`. You must have this defined in your `solrconfig.xml` file, with your desired defaults. You should also add `&literal.id` shown in the "Extracting Req. Handler Params" field so the file chosen is given a unique id.
+The File Upload option allows choosing a prepared file and uploading it. If using `/update` for the Request-Handler option, you will be limited to XML, CSV, and JSON.
-Then you can choose when documents should be added to the index (Commit Within), and whether existing documents should be overwritten with incoming documents with the same id (if this is not `true`, then the incoming documents will be dropped).
+Other document types (e.g Word, PDF etc) can be indexed using the ExtractingRequestHandler (aka Solr Cell). You must modify the Request-Handler to `/update/extract`, which must be defined in your `solrconfig.xml` file with your desired defaults. You should also add `&literal.id` shown in the "Extracting Request Handler Params" field so the file chosen is given a unique id.
+More information can be found at: <<uploading-data-with-solr-cell-using-apache-tika.adoc#uploading-data-with-solr-cell-using-apache-tika,Uploading Data with Solr Cell using Apache Tika>>
== Solr Command
-The Solr Command option allows you use XML or JSON to perform specific actions on documents, such as defining documents to be added or deleted, updating only certain fields of documents, or commit commands on the index.
-
-The documents should be structured as they would be if using `/update` on the command line.
-
-== XML Documents
-
-When using the XML document type, the functionality is similar to using a requestHandler on the command line. Instead of putting the documents in a curl command, they can instead be input into the Document entry box. The document structure should still be in proper Solr XML format, with each document separated by `<doc>` tags and each field defined.
-
-Then you can choose when documents should be added to the index (Commit Within), and whether existing documents should be overwritten with incoming documents with the same id (if this is not `true`, then the incoming documents will be dropped).
+The Solr Command option allows you use the `/update` request handler with XML or JSON formatted commands to perform specific actions. A few examples are:
-This option will only add or overwrite documents to the index; for other update tasks, see the <<Solr Command>> option.
+* Deleting documents
+* Updating only certain fields of documents
+* Issuing commit commands on the index
[26/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8115: remove
unnecessary-on-its-own {@inheritDoc} annotations.
Posted by da...@apache.org.
LUCENE-8115: remove unnecessary-on-its-own {@inheritDoc} annotations.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/07afc23d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/07afc23d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/07afc23d
Branch: refs/heads/jira/solr-11702
Commit: 07afc23dcee502d84e7684a9714fe3033bd8253a
Parents: 2828656
Author: Christine Poerschke <cp...@apache.org>
Authored: Mon Jan 8 19:45:19 2018 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Mon Jan 8 19:45:19 2018 +0000
----------------------------------------------------------------------
.../analysis/commongrams/CommonGramsFilter.java | 3 -
.../commongrams/CommonGramsQueryFilter.java | 3 -
.../miscellaneous/FingerprintFilter.java | 3 -
.../miscellaneous/HyphenatedWordsFilter.java | 6 --
.../RemoveDuplicatesTokenFilter.java | 6 --
.../ja/JapaneseIterationMarkCharFilter.java | 6 --
.../lucene/classification/BM25NBClassifier.java | 9 ---
.../BooleanPerceptronClassifier.java | 9 ---
.../classification/KNearestFuzzyClassifier.java | 9 ---
.../KNearestNeighborClassifier.java | 9 ---
.../SimpleNaiveBayesClassifier.java | 9 ---
.../KNearestNeighborDocumentClassifier.java | 9 ---
.../SimpleNaiveBayesDocumentClassifier.java | 9 ---
.../handler/DocumentAnalysisRequestHandler.java | 3 -
.../handler/FieldAnalysisRequestHandler.java | 3 -
.../handler/component/StatsValuesFactory.java | 60 --------------------
.../java/org/apache/solr/search/Grouping.java | 51 -----------------
.../SearchGroupsRequestFactory.java | 3 -
.../TopGroupsShardRequestFactory.java | 3 -
.../SearchGroupShardResponseProcessor.java | 3 -
.../StoredFieldsShardResponseProcessor.java | 3 -
.../TopGroupsShardResponseProcessor.java | 3 -
.../SearchGroupsResultTransformer.java | 6 --
.../TopGroupsResultTransformer.java | 6 --
.../GroupedEndResultTransformer.java | 3 -
.../MainEndResultTransformer.java | 3 -
.../SimpleEndResultTransformer.java | 3 -
.../solrj/request/DocumentAnalysisRequest.java | 3 -
.../solrj/request/FieldAnalysisRequest.java | 3 -
.../response/DocumentAnalysisResponse.java | 3 -
.../solrj/response/FieldAnalysisResponse.java | 3 -
.../solrj/response/schema/SchemaResponse.java | 39 -------------
32 files changed, 294 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java
index c01e263..f32b8c0 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java
@@ -123,9 +123,6 @@ public final class CommonGramsFilter extends TokenFilter {
return true;
}
- /**
- * {@inheritDoc}
- */
@Override
public void reset() throws IOException {
super.reset();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilter.java
index 9307e7b..e8c98b7 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilter.java
@@ -62,9 +62,6 @@ public final class CommonGramsQueryFilter extends TokenFilter {
super(input);
}
- /**
- * {@inheritDoc}
- */
@Override
public void reset() throws IOException {
super.reset();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/FingerprintFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/FingerprintFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/FingerprintFilter.java
index 7cbd6f8..dfe06c8 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/FingerprintFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/FingerprintFilter.java
@@ -204,9 +204,6 @@ public class FingerprintFilter extends TokenFilter {
}
}
- /**
- * {@inheritDoc}
- */
@Override
public void reset() throws IOException {
super.reset();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/HyphenatedWordsFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/HyphenatedWordsFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/HyphenatedWordsFilter.java
index 6c53aa3..ff5d311 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/HyphenatedWordsFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/HyphenatedWordsFilter.java
@@ -71,9 +71,6 @@ public final class HyphenatedWordsFilter extends TokenFilter {
super(in);
}
- /**
- * {@inheritDoc}
- */
@Override
public boolean incrementToken() throws IOException {
while (!exhausted && input.incrementToken()) {
@@ -112,9 +109,6 @@ public final class HyphenatedWordsFilter extends TokenFilter {
return false;
}
- /**
- * {@inheritDoc}
- */
@Override
public void reset() throws IOException {
super.reset();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/RemoveDuplicatesTokenFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/RemoveDuplicatesTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/RemoveDuplicatesTokenFilter.java
index 457087c..69a7759 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/RemoveDuplicatesTokenFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/RemoveDuplicatesTokenFilter.java
@@ -43,9 +43,6 @@ public final class RemoveDuplicatesTokenFilter extends TokenFilter {
super(in);
}
- /**
- * {@inheritDoc}
- */
@Override
public boolean incrementToken() throws IOException {
while (input.incrementToken()) {
@@ -71,9 +68,6 @@ public final class RemoveDuplicatesTokenFilter extends TokenFilter {
return false;
}
- /**
- * {@inheritDoc}
- */
@Override
public void reset() throws IOException {
super.reset();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilter.java
index 4f768b7..7e1d7a1 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilter.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilter.java
@@ -166,9 +166,6 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
buffer.reset(input);
}
- /**
- * {@inheritDoc}
- */
@Override
public int read(char[] buffer, int offset, int length) throws IOException {
int read = 0;
@@ -185,9 +182,6 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
return read == 0 ? -1 : read;
}
- /**
- * {@inheritDoc}
- */
@Override
public int read() throws IOException {
int ic = buffer.get(bufferPosition);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java
index 1a74416..f03fc53 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java
@@ -99,17 +99,11 @@ public class BM25NBClassifier implements Classifier<BytesRef> {
this.query = query;
}
- /**
- * {@inheritDoc}
- */
@Override
public ClassificationResult<BytesRef> assignClass(String inputDocument) throws IOException {
return assignClassNormalizedList(inputDocument).get(0);
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<BytesRef>> getClasses(String text) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignClassNormalizedList(text);
@@ -117,9 +111,6 @@ public class BM25NBClassifier implements Classifier<BytesRef> {
return assignedClasses;
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<BytesRef>> getClasses(String text, int max) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignClassNormalizedList(text);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
index 928c036..394d15f 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
@@ -195,9 +195,6 @@ public class BooleanPerceptronClassifier implements Classifier<Boolean> {
}
- /**
- * {@inheritDoc}
- */
@Override
public ClassificationResult<Boolean> assignClass(String text)
throws IOException {
@@ -220,18 +217,12 @@ public class BooleanPerceptronClassifier implements Classifier<Boolean> {
return new ClassificationResult<>(output >= bias, score);
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<Boolean>> getClasses(String text)
throws IOException {
return null;
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<Boolean>> getClasses(String text, int max)
throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
index 14f9a27..941d881 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
@@ -103,9 +103,6 @@ public class KNearestFuzzyClassifier implements Classifier<BytesRef> {
}
- /**
- * {@inheritDoc}
- */
@Override
public ClassificationResult<BytesRef> assignClass(String text) throws IOException {
TopDocs knnResults = knnSearch(text);
@@ -121,9 +118,6 @@ public class KNearestFuzzyClassifier implements Classifier<BytesRef> {
return assignedClass;
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<BytesRef>> getClasses(String text) throws IOException {
TopDocs knnResults = knnSearch(text);
@@ -132,9 +126,6 @@ public class KNearestFuzzyClassifier implements Classifier<BytesRef> {
return assignedClasses;
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<BytesRef>> getClasses(String text, int max) throws IOException {
TopDocs knnResults = knnSearch(text);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
index e6ad4a3..1bc53b0 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
@@ -119,9 +119,6 @@ public class KNearestNeighborClassifier implements Classifier<BytesRef> {
}
- /**
- * {@inheritDoc}
- */
@Override
public ClassificationResult<BytesRef> assignClass(String text) throws IOException {
return classifyFromTopDocs(knnSearch(text));
@@ -143,9 +140,6 @@ public class KNearestNeighborClassifier implements Classifier<BytesRef> {
return assignedClass;
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<BytesRef>> getClasses(String text) throws IOException {
TopDocs knnResults = knnSearch(text);
@@ -154,9 +148,6 @@ public class KNearestNeighborClassifier implements Classifier<BytesRef> {
return assignedClasses;
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<BytesRef>> getClasses(String text, int max) throws IOException {
TopDocs knnResults = knnSearch(text);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
index 3509df5..a154649 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
@@ -98,9 +98,6 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
this.query = query;
}
- /**
- * {@inheritDoc}
- */
@Override
public ClassificationResult<BytesRef> assignClass(String inputDocument) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignClassNormalizedList(inputDocument);
@@ -115,9 +112,6 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
return assignedClass;
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<BytesRef>> getClasses(String text) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignClassNormalizedList(text);
@@ -125,9 +119,6 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
return assignedClasses;
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<BytesRef>> getClasses(String text, int max) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignClassNormalizedList(text);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/lucene/classification/src/java/org/apache/lucene/classification/document/KNearestNeighborDocumentClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/document/KNearestNeighborDocumentClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/document/KNearestNeighborDocumentClassifier.java
index d687722..39684ee 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/document/KNearestNeighborDocumentClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/document/KNearestNeighborDocumentClassifier.java
@@ -72,17 +72,11 @@ public class KNearestNeighborDocumentClassifier extends KNearestNeighborClassifi
this.field2analyzer = field2analyzer;
}
- /**
- * {@inheritDoc}
- */
@Override
public ClassificationResult<BytesRef> assignClass(Document document) throws IOException {
return classifyFromTopDocs(knnSearch(document));
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<BytesRef>> getClasses(Document document) throws IOException {
TopDocs knnResults = knnSearch(document);
@@ -91,9 +85,6 @@ public class KNearestNeighborDocumentClassifier extends KNearestNeighborClassifi
return assignedClasses;
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<BytesRef>> getClasses(Document document, int max) throws IOException {
TopDocs knnResults = knnSearch(document);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
index 6bc8573..f640590 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
@@ -71,9 +71,6 @@ public class SimpleNaiveBayesDocumentClassifier extends SimpleNaiveBayesClassifi
this.field2analyzer = field2analyzer;
}
- /**
- * {@inheritDoc}
- */
@Override
public ClassificationResult<BytesRef> assignClass(Document document) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignNormClasses(document);
@@ -88,9 +85,6 @@ public class SimpleNaiveBayesDocumentClassifier extends SimpleNaiveBayesClassifi
return assignedClass;
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<BytesRef>> getClasses(Document document) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignNormClasses(document);
@@ -98,9 +92,6 @@ public class SimpleNaiveBayesDocumentClassifier extends SimpleNaiveBayesClassifi
return assignedClasses;
}
- /**
- * {@inheritDoc}
- */
@Override
public List<ClassificationResult<BytesRef>> getClasses(Document document, int max) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignNormClasses(document);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java
index fd568df..7f67981 100644
--- a/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java
@@ -108,9 +108,6 @@ public class DocumentAnalysisRequestHandler extends AnalysisRequestHandlerBase {
}
}
- /**
- * {@inheritDoc}
- */
@Override
protected NamedList doAnalysis(SolrQueryRequest req) throws Exception {
DocumentAnalysisRequest analysisRequest = resolveAnalysisRequest(req);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java
index 7c16606..a7e1ab9 100644
--- a/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java
@@ -91,9 +91,6 @@ import java.util.Set;
*/
public class FieldAnalysisRequestHandler extends AnalysisRequestHandlerBase {
- /**
- * {@inheritDoc}
- */
@Override
protected NamedList doAnalysis(SolrQueryRequest req) throws Exception {
FieldAnalysisRequest analysisRequest = resolveAnalysisRequest(req);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
index 752846c..04bd5f4 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
@@ -198,9 +198,6 @@ abstract class AbstractStatsValues<T> implements StatsValues {
}
}
- /**
- * {@inheritDoc}
- */
@Override
public void accumulate(NamedList stv) {
if (computeCount) {
@@ -260,9 +257,6 @@ abstract class AbstractStatsValues<T> implements StatsValues {
}
}
- /**
- * {@inheritDoc}
- */
@Override
public void accumulate(BytesRef value, int count) {
if (null == ft) {
@@ -298,9 +292,6 @@ abstract class AbstractStatsValues<T> implements StatsValues {
updateTypeSpecificStats(value, count);
}
- /**
- * {@inheritDoc}
- */
@Override
public void missing() {
if (computeMissing) {
@@ -308,25 +299,16 @@ abstract class AbstractStatsValues<T> implements StatsValues {
}
}
- /**
- * {@inheritDoc}
- */
@Override
public void addMissing(int count) {
missing += count;
}
- /**
- * {@inheritDoc}
- */
@Override
public void addFacet(String facetName, Map<String, StatsValues> facetValues) {
facets.put(facetName, facetValues);
}
- /**
- * {@inheritDoc}
- */
@Override
public NamedList<?> getStatsValues() {
NamedList<Object> res = new SimpleOrderedMap<>();
@@ -377,9 +359,6 @@ abstract class AbstractStatsValues<T> implements StatsValues {
return res;
}
- /**
- * {@inheritDoc}
- */
public void setNextReader(LeafReaderContext ctx) throws IOException {
if (valueSource == null) {
// first time we've collected local values, get the right ValueSource
@@ -503,9 +482,6 @@ class NumericStatsValues extends AbstractStatsValues<Number> {
}
}
- /**
- * {@inheritDoc}
- */
@Override
public void updateTypeSpecificStats(NamedList stv) {
if (computeSum) {
@@ -522,9 +498,6 @@ class NumericStatsValues extends AbstractStatsValues<Number> {
}
}
- /**
- * {@inheritDoc}
- */
@Override
public void updateTypeSpecificStats(Number v, int count) {
double value = v.doubleValue();
@@ -539,9 +512,6 @@ class NumericStatsValues extends AbstractStatsValues<Number> {
}
}
- /**
- * {@inheritDoc}
- */
@Override
protected void updateMinMax(Number min, Number max) {
// we always use the double values, because that way the response Object class is
@@ -645,9 +615,6 @@ class EnumStatsValues extends AbstractStatsValues<EnumFieldValue> {
return hasher.hashInt(v.toInt().intValue()).asLong();
}
- /**
- * {@inheritDoc}
- */
@Override
public void accumulate(int docID) throws IOException {
if (values.exists(docID)) {
@@ -660,9 +627,6 @@ class EnumStatsValues extends AbstractStatsValues<EnumFieldValue> {
}
}
- /**
- * {@inheritDoc}
- */
protected void updateMinMax(EnumFieldValue min, EnumFieldValue max) {
if (computeMin) { // nested if to encourage JIT to optimize aware final var?
if (null != min) {
@@ -680,17 +644,11 @@ class EnumStatsValues extends AbstractStatsValues<EnumFieldValue> {
}
}
- /**
- * {@inheritDoc}
- */
@Override
protected void updateTypeSpecificStats(NamedList stv) {
// No type specific stats
}
- /**
- * {@inheritDoc}
- */
@Override
protected void updateTypeSpecificStats(EnumFieldValue value, int count) {
// No type specific stats
@@ -737,9 +695,6 @@ class DateStatsValues extends AbstractStatsValues<Date> {
}
}
- /**
- * {@inheritDoc}
- */
@Override
protected void updateTypeSpecificStats(NamedList stv) {
if (computeSum) {
@@ -750,9 +705,6 @@ class DateStatsValues extends AbstractStatsValues<Date> {
}
}
- /**
- * {@inheritDoc}
- */
@Override
public void updateTypeSpecificStats(Date v, int count) {
long value = v.getTime();
@@ -764,9 +716,6 @@ class DateStatsValues extends AbstractStatsValues<Date> {
}
}
- /**
- * {@inheritDoc}
- */
@Override
protected void updateMinMax(Date min, Date max) {
if (computeMin) { // nested if to encourage JIT to optimize aware final var?
@@ -846,25 +795,16 @@ class StringStatsValues extends AbstractStatsValues<String> {
}
}
- /**
- * {@inheritDoc}
- */
@Override
protected void updateTypeSpecificStats(NamedList stv) {
// No type specific stats
}
- /**
- * {@inheritDoc}
- */
@Override
protected void updateTypeSpecificStats(String value, int count) {
// No type specific stats
}
- /**
- * {@inheritDoc}
- */
@Override
protected void updateMinMax(String min, String max) {
if (computeMin) { // nested if to encourage JIT to optimize aware final var?
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/search/Grouping.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/Grouping.java b/solr/core/src/java/org/apache/solr/search/Grouping.java
index 2214a04..4500464 100644
--- a/solr/core/src/java/org/apache/solr/search/Grouping.java
+++ b/solr/core/src/java/org/apache/solr/search/Grouping.java
@@ -706,17 +706,11 @@ public class Grouping {
TotalHitCountCollector fallBackCollector;
Collection<SearchGroup<BytesRef>> topGroups;
- /**
- * {@inheritDoc}
- */
@Override
protected void prepare() throws IOException {
actualGroupsToFind = getMax(offset, numGroups, maxDoc);
}
- /**
- * {@inheritDoc}
- */
@Override
protected Collector createFirstPassCollector() throws IOException {
// Ok we don't want groups, but do want a total count
@@ -730,9 +724,6 @@ public class Grouping {
return firstPass;
}
- /**
- * {@inheritDoc}
- */
@Override
protected Collector createSecondPassCollector() throws IOException {
if (actualGroupsToFind <= 0) {
@@ -767,18 +758,12 @@ public class Grouping {
}
}
- /**
- * {@inheritDoc}
- */
@Override
public AllGroupHeadsCollector<?> createAllGroupCollector() throws IOException {
Sort sortWithinGroup = withinGroupSort != null ? withinGroupSort : Sort.RELEVANCE;
return AllGroupHeadsCollector.newCollector(new TermGroupSelector(groupBy), sortWithinGroup);
}
- /**
- * {@inheritDoc}
- */
@Override
protected void finish() throws IOException {
result = secondPass != null ? secondPass.getTopGroups(0) : null;
@@ -826,9 +811,6 @@ public class Grouping {
}
}
- /**
- * {@inheritDoc}
- */
@Override
public int getMatches() {
if (result == null && fallBackCollector == null) {
@@ -838,9 +820,6 @@ public class Grouping {
return result != null ? result.totalHitCount : fallBackCollector.getTotalHits();
}
- /**
- * {@inheritDoc}
- */
@Override
protected Integer getNumberOfGroups() {
return allGroupsCollector == null ? null : allGroupsCollector.getGroupCount();
@@ -857,17 +836,11 @@ public class Grouping {
TopDocsCollector topCollector;
FilterCollector collector;
- /**
- * {@inheritDoc}
- */
@Override
protected void prepare() throws IOException {
actualGroupsToFind = getMax(offset, numGroups, maxDoc);
}
- /**
- * {@inheritDoc}
- */
@Override
protected Collector createFirstPassCollector() throws IOException {
DocSet groupFilt = searcher.getDocSet(query);
@@ -885,9 +858,6 @@ public class Grouping {
}
}
- /**
- * {@inheritDoc}
- */
@Override
protected void finish() throws IOException {
TopDocsCollector topDocsCollector = (TopDocsCollector) collector.getDelegate();
@@ -901,9 +871,6 @@ public class Grouping {
}
}
- /**
- * {@inheritDoc}
- */
@Override
public int getMatches() {
return collector.getMatches();
@@ -929,9 +896,6 @@ public class Grouping {
AllGroupsCollector<MutableValue> allGroupsCollector;
Collection<SearchGroup<MutableValue>> topGroups;
- /**
- * {@inheritDoc}
- */
@Override
protected void prepare() throws IOException {
context = ValueSource.newContext(searcher);
@@ -939,9 +903,6 @@ public class Grouping {
actualGroupsToFind = getMax(offset, numGroups, maxDoc);
}
- /**
- * {@inheritDoc}
- */
@Override
protected Collector createFirstPassCollector() throws IOException {
// Ok we don't want groups, but do want a total count
@@ -955,9 +916,6 @@ public class Grouping {
return firstPass;
}
- /**
- * {@inheritDoc}
- */
@Override
protected Collector createSecondPassCollector() throws IOException {
if (actualGroupsToFind <= 0) {
@@ -998,9 +956,6 @@ public class Grouping {
return AllGroupHeadsCollector.newCollector(newSelector(), sortWithinGroup);
}
- /**
- * {@inheritDoc}
- */
@Override
protected void finish() throws IOException {
result = secondPass != null ? secondPass.getTopGroups(0) : null;
@@ -1034,9 +989,6 @@ public class Grouping {
}
}
- /**
- * {@inheritDoc}
- */
@Override
public int getMatches() {
if (result == null && fallBackCollector == null) {
@@ -1046,9 +998,6 @@ public class Grouping {
return result != null ? result.totalHitCount : fallBackCollector.getTotalHits();
}
- /**
- * {@inheritDoc}
- */
@Override
protected Integer getNumberOfGroups() {
return allGroupsCollector == null ? null : allGroupsCollector.getGroupCount();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/SearchGroupsRequestFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/SearchGroupsRequestFactory.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/SearchGroupsRequestFactory.java
index 0cc5ea7..30dc5c5 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/SearchGroupsRequestFactory.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/SearchGroupsRequestFactory.java
@@ -32,9 +32,6 @@ import org.apache.solr.search.grouping.distributed.ShardRequestFactory;
*/
public class SearchGroupsRequestFactory implements ShardRequestFactory {
- /**
- * {@inheritDoc}
- */
@Override
public ShardRequest[] constructRequest(ResponseBuilder rb) {
ShardRequest sreq = new ShardRequest();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/TopGroupsShardRequestFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/TopGroupsShardRequestFactory.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/TopGroupsShardRequestFactory.java
index 5067c51..57776d9 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/TopGroupsShardRequestFactory.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/TopGroupsShardRequestFactory.java
@@ -48,9 +48,6 @@ public class TopGroupsShardRequestFactory implements ShardRequestFactory {
*/
public static final String GROUP_NULL_VALUE = "" + ReverseStringFilter.START_OF_HEADING_MARKER;
- /**
- * {@inheritDoc}
- */
@Override
public ShardRequest[] constructRequest(ResponseBuilder rb) {
// If we have a group.query we need to query all shards... Or we move this to the group first phase queries
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
index 18896e0..cc1c87d 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
@@ -47,9 +47,6 @@ import org.apache.solr.search.grouping.distributed.shardresultserializer.SearchG
*/
public class SearchGroupShardResponseProcessor implements ShardResponseProcessor {
- /**
- * {@inheritDoc}
- */
@Override
public void process(ResponseBuilder rb, ShardRequest shardRequest) {
SortSpec groupSortSpec = rb.getGroupingSpec().getGroupSortSpec();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/StoredFieldsShardResponseProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/StoredFieldsShardResponseProcessor.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/StoredFieldsShardResponseProcessor.java
index dcb3c61..734a0e0 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/StoredFieldsShardResponseProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/StoredFieldsShardResponseProcessor.java
@@ -31,9 +31,6 @@ import org.apache.solr.search.grouping.distributed.ShardResponseProcessor;
*/
public class StoredFieldsShardResponseProcessor implements ShardResponseProcessor {
- /**
- * {@inheritDoc}
- */
@Override
public void process(ResponseBuilder rb, ShardRequest shardRequest) {
boolean returnScores = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
index 231e9bd..cf02580 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
@@ -49,9 +49,6 @@ import org.apache.solr.search.grouping.distributed.shardresultserializer.TopGrou
*/
public class TopGroupsShardResponseProcessor implements ShardResponseProcessor {
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void process(ResponseBuilder rb, ShardRequest shardRequest) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java
index 77dfcef..b424670 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java
@@ -45,9 +45,6 @@ public class SearchGroupsResultTransformer implements ShardResultTransformer<Lis
this.searcher = searcher;
}
- /**
- * {@inheritDoc}
- */
@Override
public NamedList transform(List<Command> data) throws IOException {
final NamedList<NamedList> result = new NamedList<>(data.size());
@@ -73,9 +70,6 @@ public class SearchGroupsResultTransformer implements ShardResultTransformer<Lis
return result;
}
- /**
- * {@inheritDoc}
- */
@Override
public Map<String, SearchGroupsFieldCommandResult> transformToNative(NamedList<NamedList> shardResponse, Sort groupSort, Sort withinGroupSort, String shard) {
final Map<String, SearchGroupsFieldCommandResult> result = new HashMap<>(shardResponse.size());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
index 457814c..8d2b3dc 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
@@ -63,9 +63,6 @@ public class TopGroupsResultTransformer implements ShardResultTransformer<List<C
this.rb = rb;
}
- /**
- * {@inheritDoc}
- */
@Override
public NamedList transform(List<Command> data) throws IOException {
NamedList<NamedList> result = new NamedList<>();
@@ -88,9 +85,6 @@ public class TopGroupsResultTransformer implements ShardResultTransformer<List<C
return result;
}
- /**
- * {@inheritDoc}
- */
@Override
public Map<String, ?> transformToNative(NamedList<NamedList> shardResponse, Sort groupSort, Sort withinGroupSort, String shard) {
Map<String, Object> result = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/GroupedEndResultTransformer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/GroupedEndResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/GroupedEndResultTransformer.java
index de2dee4..b060590 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/GroupedEndResultTransformer.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/GroupedEndResultTransformer.java
@@ -44,9 +44,6 @@ public class GroupedEndResultTransformer implements EndResultTransformer {
this.searcher = searcher;
}
- /**
- * {@inheritDoc}
- */
@Override
public void transform(Map<String, ?> result, ResponseBuilder rb, SolrDocumentSource solrDocumentSource) {
NamedList<Object> commands = new SimpleOrderedMap<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/MainEndResultTransformer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/MainEndResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/MainEndResultTransformer.java
index 630aa63..3e11abc 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/MainEndResultTransformer.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/MainEndResultTransformer.java
@@ -31,9 +31,6 @@ import java.util.Map;
*/
public class MainEndResultTransformer implements EndResultTransformer {
- /**
- * {@inheritDoc}
- */
@Override
public void transform(Map<String, ?> result, ResponseBuilder rb, SolrDocumentSource solrDocumentSource) {
Object value = result.get(rb.getGroupingSpec().getFields()[0]);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/SimpleEndResultTransformer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/SimpleEndResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/SimpleEndResultTransformer.java
index 8d11674..593f84f 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/SimpleEndResultTransformer.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/SimpleEndResultTransformer.java
@@ -32,9 +32,6 @@ import java.util.Map;
*/
public class SimpleEndResultTransformer implements EndResultTransformer {
- /**
- * {@inheritDoc}
- */
@Override
public void transform(Map<String, ?> result, ResponseBuilder rb, SolrDocumentSource solrDocumentSource) {
NamedList<Object> commands = new SimpleOrderedMap<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/solrj/src/java/org/apache/solr/client/solrj/request/DocumentAnalysisRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/DocumentAnalysisRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/DocumentAnalysisRequest.java
index 82ef54d..be985e1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/DocumentAnalysisRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/DocumentAnalysisRequest.java
@@ -89,9 +89,6 @@ public class DocumentAnalysisRequest extends SolrRequest<DocumentAnalysisRespons
return new DocumentAnalysisResponse();
}
- /**
- * {@inheritDoc}
- */
@Override
public ModifiableSolrParams getParams() {
ModifiableSolrParams params = new ModifiableSolrParams();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/solrj/src/java/org/apache/solr/client/solrj/request/FieldAnalysisRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/FieldAnalysisRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/FieldAnalysisRequest.java
index 9c4993e..aeea9e0 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/FieldAnalysisRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/FieldAnalysisRequest.java
@@ -68,9 +68,6 @@ public class FieldAnalysisRequest extends SolrRequest<FieldAnalysisResponse> {
return new FieldAnalysisResponse();
}
- /**
- * {@inheritDoc}
- */
@Override
public SolrParams getParams() {
ModifiableSolrParams params = new ModifiableSolrParams();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java
index 42c20e9..231199d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java
@@ -34,9 +34,6 @@ public class DocumentAnalysisResponse extends AnalysisResponseBase implements It
private final Map<String, DocumentAnalysis> documentAnalysisByKey = new HashMap<>();
- /**
- * {@inheritDoc}
- */
@Override
public void setResponse(NamedList<Object> response) {
super.setResponse(response);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java
index 3a9a193..533885b 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java
@@ -34,9 +34,6 @@ public class FieldAnalysisResponse extends AnalysisResponseBase {
private Map<String, Analysis> analysisByFieldTypeName = new HashMap<>();
private Map<String, Analysis> analysisByFieldName = new HashMap<>();
- /**
- * {@inheritDoc}
- */
@Override
public void setResponse(NamedList<Object> response) {
super.setResponse(response);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07afc23d/solr/solrj/src/java/org/apache/solr/client/solrj/response/schema/SchemaResponse.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/schema/SchemaResponse.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/schema/SchemaResponse.java
index 5b24695..50a715c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/schema/SchemaResponse.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/schema/SchemaResponse.java
@@ -227,9 +227,6 @@ public class SchemaResponse extends SolrResponseBase {
return fieldTypeRepresentations;
}
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void setResponse(NamedList<Object> response) {
@@ -246,9 +243,6 @@ public class SchemaResponse extends SolrResponseBase {
public static class SchemaNameResponse extends SolrResponseBase {
private String schemaName;
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void setResponse(NamedList<Object> response) {
@@ -266,9 +260,6 @@ public class SchemaResponse extends SolrResponseBase {
public static class SchemaVersionResponse extends SolrResponseBase {
private float schemaVersion;
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void setResponse(NamedList<Object> response) {
@@ -286,9 +277,6 @@ public class SchemaResponse extends SolrResponseBase {
public static class FieldResponse extends SolrResponseBase {
Map<String, Object> field = new LinkedHashMap<>();
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void setResponse(NamedList<Object> response) {
@@ -307,9 +295,6 @@ public class SchemaResponse extends SolrResponseBase {
public static class FieldsResponse extends SolrResponseBase {
List<Map<String, Object>> fields;
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void setResponse(NamedList<Object> response) {
@@ -326,9 +311,6 @@ public class SchemaResponse extends SolrResponseBase {
public static class DynamicFieldResponse extends SolrResponseBase {
Map<String, Object> dynamicField = new LinkedHashMap<>();
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void setResponse(NamedList<Object> response) {
@@ -347,9 +329,6 @@ public class SchemaResponse extends SolrResponseBase {
public static class DynamicFieldsResponse extends SolrResponseBase {
List<Map<String, Object>> dynamicFields;
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void setResponse(NamedList<Object> response) {
@@ -366,9 +345,6 @@ public class SchemaResponse extends SolrResponseBase {
public static class UniqueKeyResponse extends SolrResponseBase {
private String uniqueKey;
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void setResponse(NamedList<Object> response) {
@@ -385,9 +361,6 @@ public class SchemaResponse extends SolrResponseBase {
public static class GlobalSimilarityResponse extends SolrResponseBase {
Map<String, Object> similarity;
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void setResponse(NamedList<Object> response) {
@@ -405,9 +378,6 @@ public class SchemaResponse extends SolrResponseBase {
public static class CopyFieldsResponse extends SolrResponseBase {
List<Map<String, Object>> copyFields;
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void setResponse(NamedList<Object> response) {
@@ -424,9 +394,6 @@ public class SchemaResponse extends SolrResponseBase {
public static class FieldTypeResponse extends SolrResponseBase {
private FieldTypeRepresentation fieldType;
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void setResponse(NamedList<Object> response) {
@@ -445,9 +412,6 @@ public class SchemaResponse extends SolrResponseBase {
public static class FieldTypesResponse extends SolrResponseBase {
List<FieldTypeRepresentation> fieldTypes;
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void setResponse(NamedList<Object> response) {
@@ -462,9 +426,6 @@ public class SchemaResponse extends SolrResponseBase {
}
public static class UpdateResponse extends SolrResponseBase {
- /**
- * {@inheritDoc}
- */
@Override
@SuppressWarnings("unchecked")
public void setResponse(NamedList<Object> response) {
[06/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11798: Formally
deprecate top-level syntax in solrconfig.xml in favour of
equivalent syntax.
Posted by da...@apache.org.
SOLR-11798: Formally deprecate top-level <highlighting> syntax in solrconfig.xml in favour of <searchComponent> equivalent syntax.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5d4f029f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5d4f029f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5d4f029f
Branch: refs/heads/jira/solr-11702
Commit: 5d4f029fdd0b916d631d2432e4fd0216c91c8703
Parents: 8fd7ead
Author: Christine Poerschke <cp...@apache.org>
Authored: Thu Jan 4 13:04:30 2018 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Thu Jan 4 14:54:47 2018 +0000
----------------------------------------------------------------------
solr/CHANGES.txt | 6 ++++++
.../org/apache/solr/handler/component/HighlightComponent.java | 7 +++++++
2 files changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d4f029f/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d30015e..7137b80 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -57,6 +57,9 @@ Upgrade Notes
Autoscaling API which is now a no-op. Use the 'triggerCooldownPeriodSeconds' instead to pause event
processing.
+* SOLR-11798: The top-level <highlighting> syntax in solrconfig.xml is now formally
+ deprecated in favour of <searchComponent> equivalent syntax. See also SOLR-1696.
+
New Features
----------------------
* SOLR-11285: Simulation framework for autoscaling. (ab)
@@ -107,6 +110,9 @@ Other Changes
* SOLR-11805: SolrJ's SolrResponse.getElaspedTime was sometimes a millisecond off. (David Smiley)
+* SOLR-11798: Formally deprecate top-level <highlighting> syntax in solrconfig.xml
+ in favour of <searchComponent> equivalent syntax. (Christine Poerschke)
+
================== 7.2.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d4f029f/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java b/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
index f1c9680..698cc05 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
@@ -25,6 +25,7 @@ import java.util.stream.Stream;
import com.google.common.base.Objects;
import org.apache.lucene.search.Query;
+import org.apache.lucene.util.Version;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.HighlightParams;
import org.apache.solr.common.params.SolrParams;
@@ -46,6 +47,7 @@ import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
import static java.util.stream.Collectors.toMap;
+import static org.apache.solr.core.Config.assertWarnOrFail;
/**
* TODO!
@@ -131,6 +133,11 @@ public class HighlightComponent extends SearchComponent implements PluginInfoIni
List<PluginInfo> children = info.getChildren("highlighting");
if(children.isEmpty()) {
PluginInfo pluginInfo = core.getSolrConfig().getPluginInfo(SolrHighlighter.class.getName()); //TODO deprecated configuration remove later
+ assertWarnOrFail("solrconfig.xml <highlighting> configuration is deprecated since SOLR-1696 "
+ + "and no longer supported from Solr 7.3 onwards. "
+ + "Please configure via <searchComponent> instead.",
+ (null == pluginInfo),
+ core.getSolrConfig().luceneMatchVersion.onOrAfter(Version.LUCENE_7_3_0));
if (pluginInfo != null) {
solrConfigHighlighter = core.createInitInstance(pluginInfo, SolrHighlighter.class, null, DefaultSolrHighlighter.class.getName());
} else {
[35/50] [abbrv] lucene-solr:jira/solr-11702: Add script to attempt to
reproduce failing tests from a Jenkins log
Posted by da...@apache.org.
Add script to attempt to reproduce failing tests from a Jenkins log
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f7e166e7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f7e166e7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f7e166e7
Branch: refs/heads/jira/solr-11702
Commit: f7e166e7c111bed82c00fbd3ed8d55cb6bc928e8
Parents: 9f22179
Author: Steve Rowe <sa...@apache.org>
Authored: Mon Jan 8 21:36:55 2018 -0500
Committer: Steve Rowe <sa...@apache.org>
Committed: Mon Jan 8 21:36:55 2018 -0500
----------------------------------------------------------------------
dev-tools/scripts/reproduceJenkinsFailures.py | 215 +++++++++++++++++++++
lucene/CHANGES.txt | 3 +
2 files changed, 218 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7e166e7/dev-tools/scripts/reproduceJenkinsFailures.py
----------------------------------------------------------------------
diff --git a/dev-tools/scripts/reproduceJenkinsFailures.py b/dev-tools/scripts/reproduceJenkinsFailures.py
new file mode 100644
index 0000000..bb32123
--- /dev/null
+++ b/dev-tools/scripts/reproduceJenkinsFailures.py
@@ -0,0 +1,215 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import subprocess
+import sys
+import urllib.error
+import urllib.request
+from textwrap import dedent
+
+# Number of iterations per test suite
+testIters = 5
+
+usage = dedent('''\
+ Usage:\n
+ python3 -u %s URL\n
+ Must be run from a Lucene/Solr git workspace. Downloads the Jenkins
+ log pointed to by the given URL, parses it for Git revision and failed
+ Lucene/Solr tests, checks out the Git revision in the local workspace,
+ groups the failed tests by module, then runs
+ 'ant test -Dtest.dups=%d -Dtests.class="*.test1[|*.test2[...]]" ...'
+ in each module of interest, failing at the end if any of the runs fails.
+ To control the maximum number of concurrent JVMs used for each module's
+ test run, set 'tests.jvms', e.g. in ~/lucene.build.properties
+ ''' % (sys.argv[0], testIters))
+
+reHelpArg = re.compile(r'-{1,2}(?:\?|h(?:elp)?)')
+
+# Example: Checking out Revision e441a99009a557f82ea17ee9f9c3e9b89c75cee6 (refs/remotes/origin/master)
+reGitRev = re.compile(r'Checking out Revision (\S+)')
+
+# Method example: NOTE: reproduce with: ant test -Dtestcase=ZkSolrClientTest -Dtests.method=testMultipleWatchesAsync -Dtests.seed=6EF5AB70F0032849 -Dtests.slow=true -Dtests.locale=he-IL -Dtests.timezone=NST -Dtests.asserts=true -Dtests.file.encoding=UTF-8
+# Suite example: NOTE: reproduce with: ant test -Dtestcase=CloudSolrClientTest -Dtests.seed=DB2DF2D8228BAF27 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es-AR -Dtests.timezone=America/Argentina/Cordoba -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
+reReproLine = re.compile(r'NOTE:\s+reproduce\s+with:(\s+ant\s+test\s+-Dtestcase=(\S+)\s+(?:-Dtests.method=\S+\s+)?(.*))')
+
+# Example: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/21108/
+reJenkinsURLWithoutConsoleText = re.compile(r'https?://.*/\d+/?\Z', re.IGNORECASE)
+
+reJavaFile = re.compile(r'(.*)\.java\Z')
+reModule = re.compile(r'\./(.*)/src/')
+reTestOutputFile = re.compile(r'TEST-(.*\.([^-.]+))(?:-\d+)?\.xml\Z')
+reErrorFailure = re.compile(r'(?:errors|failures)="[^0]')
+
+# consoleText from Policeman Jenkins's Windows jobs fails to decode as UTF-8
+encoding = 'iso-8859-1'
+
+tests = {}
+modules = {}
+
+lastFailureCode = 0
+gitCheckoutSucceeded = False
+
+def runOutput(cmd):
+ print('[repro] %s' % cmd)
+ try:
+ return subprocess.check_output(cmd.split(' '), universal_newlines=True).strip()
+ except CalledProcessError as e:
+ raise RuntimeError("ERROR: Cmd '%s' failed with exit code %d and the following output:\n%s"
+ % (cmd, e.returncode, e.output))
+
+# Remembers non-zero exit code in lastFailureCode unless rememberFailure==False
+def run(cmd, rememberFailure=True):
+ global lastFailureCode
+ print('[repro] %s' % cmd)
+ code = os.system(cmd)
+ if 0 != code and rememberFailure:
+ print('\n[repro] Setting last failure code to %d\n' % code)
+ lastFailureCode = code
+ return code
+
+def fetchAndParseJenkinsLog(url):
+ global revision
+ revision = None
+ print('[repro] Jenkins log URL: %s\n' % url)
+ try:
+ with urllib.request.urlopen(url) as consoleText:
+ for rawLine in consoleText:
+ line = rawLine.decode(encoding)
+ match = reGitRev.match(line)
+ if match is not None:
+ revision = match.group(1)
+ print('[repro] Revision: %s\n' % revision)
+ else:
+ match = reReproLine.search(line)
+ if match is not None:
+ print('[repro] Repro line: %s\n' % match.group(1))
+ testcase = match.group(2)
+ reproLineWithoutMethod = match.group(3).strip()
+ tests[testcase] = reproLineWithoutMethod
+ except urllib.error.URLError as e:
+ raise RuntimeError('ERROR: fetching %s : %s' % (url, e))
+
+ if revision == None:
+ if reJenkinsURLWithoutConsoleText.match(url):
+ print('[repro] Not a Jenkins log. Appending "/consoleText" and retrying ...\n')
+ fetchAndParseJenkinsLog(url + '/consoleText')
+ else:
+ raise RuntimeError('ERROR: %s does not appear to be a Jenkins log.' % url)
+ if 0 == len(tests):
+ print('[repro] No "reproduce with" lines found; exiting.')
+ sys.exit(0)
+
+def prepareWorkspace():
+ global gitCheckoutSucceeded
+ code = run('git checkout %s' % revision)
+ if 0 != code:
+ raise RuntimeError('ERROR: "git checkout %s" failed. See above. Maybe try "git pull"?' % revision)
+ gitCheckoutSucceeded = True
+ code = run('ant clean')
+ if 0 != code:
+ raise RuntimeError('ERROR: "ant clean" failed. See above.')
+
+def groupTestsByModule():
+ for (dir, _, files) in os.walk('.'):
+ for file in files:
+ match = reJavaFile.search(file)
+ if match is not None:
+ test = match.group(1)
+ if test in tests:
+ match = reModule.match(dir)
+ module = match.group(1)
+ if module not in modules:
+ modules[module] = set()
+ modules[module].add(test)
+ print('[repro] Test suites by module:')
+ for module in modules:
+ print('[repro] %s' % module)
+ for test in modules[module]:
+ print('[repro] %s' % test)
+
+def runTests():
+ global lastFailureCode
+ cwd = os.getcwd()
+ testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s'
+ for module in modules:
+ moduleTests = list(modules[module])
+ testList = '|'.join(map(lambda t: '*.%s' % t, moduleTests))
+ numTests = len(moduleTests)
+ params = tests[moduleTests[0]] # Assumption: all tests in this module have the same cmdline params
+ os.chdir(module)
+ code = run('ant compile-test')
+ try:
+ if (0 != code):
+ raise RuntimeError("ERROR: Compile failed in %s/ with code %d. See above." % (module, code))
+ run(testCmdline % (testIters, testIters * numTests, testList, params))
+ finally:
+ os.chdir(cwd)
+
+def printReport():
+ failures = {}
+ for start in ('lucene/build', 'solr/build'):
+ for (dir, _, files) in os.walk(start):
+ for file in files:
+ testOutputFileMatch = reTestOutputFile.search(file)
+ if testOutputFileMatch is not None:
+ testcase = testOutputFileMatch.group(1)
+ if testcase not in failures:
+ failures[testcase] = 0
+ with open(os.path.join(dir, file), encoding='UTF-8') as testOutputFile:
+ for line in testOutputFile:
+ errorFailureMatch = reErrorFailure.search(line)
+ if errorFailureMatch is not None:
+ failures[testcase] += 1
+ break
+ print("[repro] Failures:")
+ for testcase in sorted(failures):
+ print("[repro] %d/%d failed: %s" % (failures[testcase], testIters, testcase))
+
+def rememberGitBranch():
+ global origGitBranch
+ origGitBranch = runOutput('git rev-parse --abbrev-ref HEAD')
+ if (origGitBranch == 'HEAD'): # In detached HEAD state
+ origGitBranch = runOutput('git rev-parse HEAD') # Use the SHA when not on a branch
+ print('[repro] Initial local git branch/revision: %s' % origGitBranch)
+
+def main():
+ if 2 != len(sys.argv) or reHelpArg.match(sys.argv[1]):
+ print(usage)
+ sys.exit(0)
+ fetchAndParseJenkinsLog(sys.argv[1])
+ rememberGitBranch()
+
+ try:
+ prepareWorkspace()
+ groupTestsByModule()
+ runTests()
+ printReport()
+ except Exception as e:
+ print('[repro] %s' % e)
+ sys.exit(1)
+ finally:
+ if gitCheckoutSucceeded:
+ run('git checkout %s' % origGitBranch, rememberFailure=False) # Restore original git branch/sha
+
+ print('[repro] Exiting with code %d' % lastFailureCode)
+ sys.exit(lastFailureCode)
+
+if __name__ == '__main__':
+ try:
+ main()
+ except KeyboardInterrupt:
+ print('[repro] Keyboard interrupt...exiting')
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7e166e7/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 78cd095..f3335ac 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -134,6 +134,9 @@ Other
* LUCENE-8122: Upgrade analysis/icu to ICU 60.2. (Robert Muir)
+* LUCENE-8106: Add script (reproduceJenkinsFailures.py) to attempt to reproduce
+ failing tests from a Jenkins log. (Steve Rowe)
+
======================= Lucene 7.2.0 =======================
API Changes
[11/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8133: Rename
TermContext to TermStates,
and load TermState lazily if term stats are not required
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
index f65c54e..dd85c62 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
@@ -29,7 +29,7 @@ import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -49,7 +49,7 @@ public class TestTermQuery extends LuceneTestCase {
new TermQuery(new Term("foo", "baz")));
QueryUtils.checkEqual(
new TermQuery(new Term("foo", "bar")),
- new TermQuery(new Term("foo", "bar"), TermContext.build(new MultiReader().getContext(), new Term("foo", "bar"))));
+ new TermQuery(new Term("foo", "bar"), TermStates.build(new MultiReader().getContext(), new Term("foo", "bar"), true)));
}
public void testCreateWeightDoesNotSeekIfScoresAreNotNeeded() throws IOException {
@@ -84,7 +84,7 @@ public class TestTermQuery extends LuceneTestCase {
searcher.search(query, collector);
assertEquals(1, collector.getTotalHits());
TermQuery queryWithContext = new TermQuery(new Term("foo", "bar"),
- TermContext.build(reader.getContext(), new Term("foo", "bar")));
+ TermStates.build(reader.getContext(), new Term("foo", "bar"), true));
collector = new TotalHitCountCollector();
searcher.search(queryWithContext, collector);
assertEquals(1, collector.getTotalHits());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
index 004c06e..fbb59e3 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
@@ -148,7 +148,7 @@ public class WeightedSpanTermExtractor {
}
} else if (query instanceof CommonTermsQuery) {
// specialized since rewriting would change the result query
- // this query is TermContext sensitive.
+ // this query is index sensitive.
extractWeightedTerms(terms, query, boost);
} else if (query instanceof DisjunctionMaxQuery) {
for (Query clause : ((DisjunctionMaxQuery) query)) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java
index 2fdeaa7..10c232e 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java
@@ -25,7 +25,7 @@ import java.util.Objects;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause.Occur;
@@ -124,9 +124,9 @@ public class CommonTermsQuery extends Query {
}
final List<LeafReaderContext> leaves = reader.leaves();
final int maxDoc = reader.maxDoc();
- final TermContext[] contextArray = new TermContext[terms.size()];
+ final TermStates[] contextArray = new TermStates[terms.size()];
final Term[] queryTerms = this.terms.toArray(new Term[0]);
- collectTermContext(reader, leaves, contextArray, queryTerms);
+ collectTermStates(reader, leaves, contextArray, queryTerms);
return buildQuery(maxDoc, contextArray, queryTerms);
}
@@ -146,21 +146,21 @@ public class CommonTermsQuery extends Query {
}
protected Query buildQuery(final int maxDoc,
- final TermContext[] contextArray, final Term[] queryTerms) {
+ final TermStates[] contextArray, final Term[] queryTerms) {
List<Query> lowFreqQueries = new ArrayList<>();
List<Query> highFreqQueries = new ArrayList<>();
for (int i = 0; i < queryTerms.length; i++) {
- TermContext termContext = contextArray[i];
- if (termContext == null) {
+ TermStates termStates = contextArray[i];
+ if (termStates == null) {
lowFreqQueries.add(newTermQuery(queryTerms[i], null));
} else {
- if ((maxTermFrequency >= 1f && termContext.docFreq() > maxTermFrequency)
- || (termContext.docFreq() > (int) Math.ceil(maxTermFrequency
+ if ((maxTermFrequency >= 1f && termStates.docFreq() > maxTermFrequency)
+ || (termStates.docFreq() > (int) Math.ceil(maxTermFrequency
* (float) maxDoc))) {
highFreqQueries
- .add(newTermQuery(queryTerms[i], termContext));
+ .add(newTermQuery(queryTerms[i], termStates));
} else {
- lowFreqQueries.add(newTermQuery(queryTerms[i], termContext));
+ lowFreqQueries.add(newTermQuery(queryTerms[i], termStates));
}
}
}
@@ -208,14 +208,14 @@ public class CommonTermsQuery extends Query {
return builder.build();
}
- public void collectTermContext(IndexReader reader,
- List<LeafReaderContext> leaves, TermContext[] contextArray,
- Term[] queryTerms) throws IOException {
+ public void collectTermStates(IndexReader reader,
+ List<LeafReaderContext> leaves, TermStates[] contextArray,
+ Term[] queryTerms) throws IOException {
TermsEnum termsEnum = null;
for (LeafReaderContext context : leaves) {
for (int i = 0; i < queryTerms.length; i++) {
Term term = queryTerms[i];
- TermContext termContext = contextArray[i];
+ TermStates termStates = contextArray[i];
final Terms terms = context.reader().terms(term.field());
if (terms == null) {
// field does not exist
@@ -226,12 +226,12 @@ public class CommonTermsQuery extends Query {
if (termsEnum == TermsEnum.EMPTY) continue;
if (termsEnum.seekExact(term.bytes())) {
- if (termContext == null) {
- contextArray[i] = new TermContext(reader.getContext(),
+ if (termStates == null) {
+ contextArray[i] = new TermStates(reader.getContext(),
termsEnum.termState(), context.ord, termsEnum.docFreq(),
termsEnum.totalTermFreq());
} else {
- termContext.register(termsEnum.termState(), context.ord,
+ termStates.register(termsEnum.termState(), context.ord,
termsEnum.docFreq(), termsEnum.totalTermFreq());
}
@@ -402,10 +402,10 @@ public class CommonTermsQuery extends Query {
* Builds a new TermQuery instance.
* <p>This is intended for subclasses that wish to customize the generated queries.</p>
* @param term term
- * @param context the TermContext to be used to create the low level term query. Can be <code>null</code>.
+ * @param termStates the TermStates to be used to create the low level term query. Can be <code>null</code>.
* @return new TermQuery instance
*/
- protected Query newTermQuery(Term term, TermContext context) {
- return context == null ? new TermQuery(term) : new TermQuery(term, context);
+ protected Query newTermQuery(Term term, TermStates termStates) {
+ return termStates == null ? new TermQuery(term) : new TermQuery(term, termStates);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadScoreQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadScoreQuery.java
index c618886..bd5d927 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadScoreQuery.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadScoreQuery.java
@@ -25,7 +25,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LeafSimScorer;
@@ -135,8 +135,8 @@ public class PayloadScoreQuery extends SpanQuery {
}
@Override
- public void extractTermContexts(Map<Term, TermContext> contexts) {
- innerWeight.extractTermContexts(contexts);
+ public void extractTermStates(Map<Term, TermStates> contexts) {
+ innerWeight.extractTermStates(contexts);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/queries/src/java/org/apache/lucene/queries/payloads/SpanPayloadCheckQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/payloads/SpanPayloadCheckQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/payloads/SpanPayloadCheckQuery.java
index 05d6682..a9d3bfb 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/payloads/SpanPayloadCheckQuery.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/payloads/SpanPayloadCheckQuery.java
@@ -25,7 +25,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.Terms;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LeafSimScorer;
@@ -64,8 +64,8 @@ public class SpanPayloadCheckQuery extends SpanQuery {
@Override
public SpanWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- SpanWeight matchWeight = match.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);
- return new SpanPayloadCheckWeight(searcher, scoreMode.needsScores() ? getTermContexts(matchWeight) : null, matchWeight, boost);
+ SpanWeight matchWeight = match.createWeight(searcher, scoreMode, boost);
+ return new SpanPayloadCheckWeight(searcher, scoreMode.needsScores() ? getTermStates(matchWeight) : null, matchWeight, boost);
}
@Override
@@ -84,8 +84,8 @@ public class SpanPayloadCheckQuery extends SpanQuery {
final SpanWeight matchWeight;
- public SpanPayloadCheckWeight(IndexSearcher searcher, Map<Term, TermContext> termContexts, SpanWeight matchWeight, float boost) throws IOException {
- super(SpanPayloadCheckQuery.this, searcher, termContexts, boost);
+ public SpanPayloadCheckWeight(IndexSearcher searcher, Map<Term, TermStates> termStates, SpanWeight matchWeight, float boost) throws IOException {
+ super(SpanPayloadCheckQuery.this, searcher, termStates, boost);
this.matchWeight = matchWeight;
}
@@ -95,8 +95,8 @@ public class SpanPayloadCheckQuery extends SpanQuery {
}
@Override
- public void extractTermContexts(Map<Term, TermContext> contexts) {
- matchWeight.extractTermContexts(contexts);
+ public void extractTermStates(Map<Term, TermStates> contexts) {
+ matchWeight.extractTermStates(contexts);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java b/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java
index 716e2fb..684344e 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java
@@ -33,7 +33,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause.Occur;
@@ -541,8 +541,8 @@ public class CommonTermsQueryTest extends LuceneTestCase {
}
@Override
- protected Query newTermQuery(Term term, TermContext context) {
- Query query = super.newTermQuery(term, context);
+ protected Query newTermQuery(Term term, TermStates termStates) {
+ Query query = super.newTermQuery(term, termStates);
if (term.text().equals("universe")) {
query = new BoostQuery(query, 100f);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java
index 840ade3..6885008 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java
@@ -30,7 +30,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
@@ -68,7 +68,7 @@ public class FuzzyLikeThisQuery extends Query
{
// TODO: generalize this query (at least it should not reuse this static sim!
// a better way might be to convert this into multitermquery rewrite methods.
- // the rewrite method can 'average' the TermContext's term statistics (docfreq,totalTermFreq)
+ // the rewrite method can 'average' the TermStates's term statistics (docfreq,totalTermFreq)
// provided to TermQuery, so that the general idea is agnostic to any scoring system...
static TFIDFSimilarity sim=new ClassicSimilarity();
ArrayList<FieldVals> fieldVals=new ArrayList<>();
@@ -255,9 +255,9 @@ public class FuzzyLikeThisQuery extends Query
if (ignoreTF) {
return new ConstantScoreQuery(new TermQuery(term));
} else {
- // we build an artificial TermContext that will give an overall df and ttf
+ // we build an artificial TermStates that will give an overall df and ttf
// equal to 1
- TermContext context = new TermContext(reader.getContext());
+ TermStates context = new TermStates(reader.getContext());
for (LeafReaderContext leafContext : reader.leaves()) {
Terms terms = leafContext.reader().terms(term.field());
if (terms != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
index 92c8d59..b877976 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
@@ -29,7 +29,7 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.similarities.Similarity;
@@ -194,11 +194,11 @@ public class TermAutomatonQuery extends Query {
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
IndexReaderContext context = searcher.getTopReaderContext();
- Map<Integer,TermContext> termStates = new HashMap<>();
+ Map<Integer,TermStates> termStates = new HashMap<>();
for (Map.Entry<BytesRef,Integer> ent : termToID.entrySet()) {
if (ent.getKey() != null) {
- termStates.put(ent.getValue(), TermContext.build(context, new Term(field, ent.getKey())));
+ termStates.put(ent.getValue(), TermStates.build(context, new Term(field, ent.getKey()), scoreMode.needsScores()));
}
}
@@ -334,11 +334,11 @@ public class TermAutomatonQuery extends Query {
final class TermAutomatonWeight extends Weight {
final Automaton automaton;
- private final Map<Integer,TermContext> termStates;
+ private final Map<Integer,TermStates> termStates;
private final Similarity.SimScorer stats;
private final Similarity similarity;
- public TermAutomatonWeight(Automaton automaton, IndexSearcher searcher, Map<Integer,TermContext> termStates, float boost) throws IOException {
+ public TermAutomatonWeight(Automaton automaton, IndexSearcher searcher, Map<Integer,TermStates> termStates, float boost) throws IOException {
super(TermAutomatonQuery.this);
this.automaton = automaton;
this.termStates = termStates;
@@ -383,11 +383,11 @@ public class TermAutomatonQuery extends Query {
EnumAndScorer[] enums = new EnumAndScorer[idToTerm.size()];
boolean any = false;
- for(Map.Entry<Integer,TermContext> ent : termStates.entrySet()) {
- TermContext termContext = ent.getValue();
- assert termContext.wasBuiltFor(ReaderUtil.getTopLevelContext(context)) : "The top-reader used to create Weight is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
+ for(Map.Entry<Integer,TermStates> ent : termStates.entrySet()) {
+ TermStates termStates = ent.getValue();
+ assert termStates.wasBuiltFor(ReaderUtil.getTopLevelContext(context)) : "The top-reader used to create Weight is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
BytesRef term = idToTerm.get(ent.getKey());
- TermState state = termContext.get(context.ord);
+ TermState state = termStates.get(context);
if (state != null) {
TermsEnum termsEnum = context.reader().terms(field).iterator();
termsEnum.seekExact(term, state);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java b/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java
index 2fdef99..b92ed75 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java
@@ -30,7 +30,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
@@ -186,8 +186,8 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
}
try {
for(Term term : terms) {
- final TermContext termContext = TermContext.build(s.getIndexReader().getContext(), term);
- stats.put(term, s.termStatistics(term, termContext));
+ final TermStates termStates = TermStates.build(s.getIndexReader().getContext(), term, true);
+ stats.put(term, s.termStatistics(term, termStates));
}
} finally {
node.searchers.release(s);
@@ -262,7 +262,7 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
}
@Override
- public TermStatistics termStatistics(Term term, TermContext context) throws IOException {
+ public TermStatistics termStatistics(Term term, TermStates context) throws IOException {
assert term != null;
long docFreq = 0;
long totalTermFreq = 0;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpanWeight.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpanWeight.java b/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpanWeight.java
index 65f9599..be04e00 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpanWeight.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpanWeight.java
@@ -22,7 +22,7 @@ import java.util.Set;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LeafSimScorer;
@@ -45,8 +45,8 @@ public class AssertingSpanWeight extends SpanWeight {
}
@Override
- public void extractTermContexts(Map<Term, TermContext> contexts) {
- in.extractTermContexts(contexts);
+ public void extractTermStates(Map<Term, TermStates> contexts) {
+ in.extractTermStates(contexts);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java b/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java
index b7a1f56..e094986 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java
@@ -27,7 +27,7 @@ import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.Query;
@@ -621,18 +621,18 @@ public class TermsComponent extends SearchComponent {
terms[i] = new Term(field, fieldType.readableToIndexed(splitTerms[i]));
}
- TermContext[] termContexts = new TermContext[terms.length];
- collectTermContext(topReaderContext, termContexts, terms);
+ TermStates[] termStates = new TermStates[terms.length];
+ collectTermStates(topReaderContext, termStates, terms);
NamedList<Object> termsMap = new SimpleOrderedMap<>();
for (int i = 0; i < terms.length; i++) {
- if (termContexts[i] != null) {
+ if (termStates[i] != null) {
String outTerm = fieldType.indexedToReadable(terms[i].bytes().utf8ToString());
- int docFreq = termContexts[i].docFreq();
+ int docFreq = termStates[i].docFreq();
if (!includeTotalTermFreq) {
termsMap.add(outTerm, docFreq);
} else {
- long totalTermFreq = termContexts[i].totalTermFreq();
+ long totalTermFreq = termStates[i].totalTermFreq();
NamedList<Long> termStats = new SimpleOrderedMap<>();
termStats.add("df", (long) docFreq);
termStats.add("ttf", totalTermFreq);
@@ -645,8 +645,8 @@ public class TermsComponent extends SearchComponent {
}
}
- private static void collectTermContext(IndexReaderContext topReaderContext, TermContext[] contextArray,
- Term[] queryTerms) throws IOException {
+ private static void collectTermStates(IndexReaderContext topReaderContext, TermStates[] contextArray,
+ Term[] queryTerms) throws IOException {
TermsEnum termsEnum = null;
for (LeafReaderContext context : topReaderContext.leaves()) {
for (int i = 0; i < queryTerms.length; i++) {
@@ -661,13 +661,13 @@ public class TermsComponent extends SearchComponent {
if (termsEnum == TermsEnum.EMPTY) continue;
- TermContext termContext = contextArray[i];
+ TermStates termStates = contextArray[i];
if (termsEnum.seekExact(term.bytes())) {
- if (termContext == null) {
- termContext = new TermContext(topReaderContext);
- contextArray[i] = termContext;
+ if (termStates == null) {
+ termStates = new TermStates(topReaderContext);
+ contextArray[i] = termStates;
}
- termContext.accumulateStatistics(termsEnum.docFreq(), termsEnum.totalTermFreq());
+ termStates.accumulateStatistics(termsEnum.docFreq(), termsEnum.totalTermFreq());
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java
index 2b0d08a..7e54f8d 100644
--- a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java
+++ b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java
@@ -24,7 +24,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -396,9 +396,9 @@ public final class SolrRangeQuery extends ExtendedQueryBase implements DocSetPro
if (count < 0) {
BooleanQuery.Builder bq = new BooleanQuery.Builder();
for (TermAndState t : collectedTerms) {
- final TermContext termContext = new TermContext(searcher.getTopReaderContext());
- termContext.register(t.state, context.ord, t.docFreq, t.totalTermFreq);
- bq.add(new TermQuery(new Term( SolrRangeQuery.this.getField(), t.term), termContext), BooleanClause.Occur.SHOULD);
+ final TermStates termStates = new TermStates(searcher.getTopReaderContext());
+ termStates.register(t.state, context.ord, t.docFreq, t.totalTermFreq);
+ bq.add(new TermQuery(new Term( SolrRangeQuery.this.getField(), t.term), termStates), BooleanClause.Occur.SHOULD);
}
Query q = new ConstantScoreQuery(bq.build());
final Weight weight = searcher.rewrite(q).createWeight(searcher, needScores ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES, score());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java
index 01b0ef8..d1f7ff2 100644
--- a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java
@@ -37,7 +37,7 @@ import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.PrefixCodedTerms;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -252,15 +252,15 @@ public class GraphTermsQParserPlugin extends QParserPlugin {
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- List<TermContext> finalContexts = new ArrayList();
+ List<TermStates> finalContexts = new ArrayList();
List<Term> finalTerms = new ArrayList();
List<LeafReaderContext> contexts = searcher.getTopReaderContext().leaves();
- TermContext[] termContexts = new TermContext[this.queryTerms.length];
- collectTermContext(searcher.getIndexReader(), contexts, termContexts, this.queryTerms);
- for(int i=0; i<termContexts.length; i++) {
- TermContext termContext = termContexts[i];
- if(termContext != null && termContext.docFreq() <= this.maxDocFreq) {
- finalContexts.add(termContext);
+ TermStates[] termStates = new TermStates[this.queryTerms.length];
+ collectTermStates(searcher.getIndexReader(), contexts, termStates, this.queryTerms);
+ for(int i=0; i<termStates.length; i++) {
+ TermStates ts = termStates[i];
+ if(ts != null && ts.docFreq() <= this.maxDocFreq) {
+ finalContexts.add(ts);
finalTerms.add(queryTerms[i]);
}
}
@@ -285,11 +285,11 @@ public class GraphTermsQParserPlugin extends QParserPlugin {
PostingsEnum docs = null;
DocIdSetBuilder builder = new DocIdSetBuilder(reader.maxDoc(), terms);
for (int i=0; i<finalContexts.size(); i++) {
- TermContext termContext = finalContexts.get(i);
- TermState termState = termContext.get(context.ord);
+ TermStates ts = finalContexts.get(i);
+ TermState termState = ts.get(context);
if(termState != null) {
Term term = finalTerms.get(i);
- termsEnum.seekExact(term.bytes(), termContext.get(context.ord));
+ termsEnum.seekExact(term.bytes(), ts.get(context));
docs = termsEnum.postings(docs, PostingsEnum.NONE);
builder.add(docs);
}
@@ -340,10 +340,10 @@ public class GraphTermsQParserPlugin extends QParserPlugin {
};
}
- private void collectTermContext(IndexReader reader,
- List<LeafReaderContext> leaves,
- TermContext[] contextArray,
- Term[] queryTerms) throws IOException {
+ private void collectTermStates(IndexReader reader,
+ List<LeafReaderContext> leaves,
+ TermStates[] contextArray,
+ Term[] queryTerms) throws IOException {
TermsEnum termsEnum = null;
for (LeafReaderContext context : leaves) {
@@ -359,15 +359,15 @@ public class GraphTermsQParserPlugin extends QParserPlugin {
for (int i = 0; i < queryTerms.length; i++) {
Term term = queryTerms[i];
- TermContext termContext = contextArray[i];
+ TermStates termStates = contextArray[i];
if (termsEnum.seekExact(term.bytes())) {
- if (termContext == null) {
- contextArray[i] = new TermContext(reader.getContext(),
+ if (termStates == null) {
+ contextArray[i] = new TermStates(reader.getContext(),
termsEnum.termState(), context.ord, termsEnum.docFreq(),
termsEnum.totalTermFreq());
} else {
- termContext.register(termsEnum.termState(), context.ord,
+ termStates.register(termsEnum.termState(), context.ord,
termsEnum.docFreq(), termsEnum.totalTermFreq());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index 9ee5199..3437476 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -47,7 +47,7 @@ import org.apache.lucene.index.MultiPostingsEnum;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.*;
@@ -339,7 +339,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
* Override these two methods to provide a way to use global collection stats.
*/
@Override
- public TermStatistics termStatistics(Term term, TermContext context) throws IOException {
+ public TermStatistics termStatistics(Term term, TermStates context) throws IOException {
final SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo();
if (reqInfo != null) {
final StatsSource statsSrc = (StatsSource) reqInfo.getReq().getContext().get(STATS_SOURCE);
@@ -362,7 +362,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
return localCollectionStatistics(field);
}
- public TermStatistics localTermStatistics(Term term, TermContext context) throws IOException {
+ public TermStatistics localTermStatistics(Term term, TermStates context) throws IOException {
return super.termStatistics(term, context);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java b/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java
index 35b1b38..f09ddf7 100644
--- a/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java
+++ b/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java
@@ -19,7 +19,7 @@ package org.apache.solr.search.stats;
import com.google.common.collect.Lists;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
@@ -162,7 +162,7 @@ public class ExactStatsCache extends StatsCache {
HashMap<String,TermStats> statsMap = new HashMap<>();
HashMap<String,CollectionStats> colMap = new HashMap<>();
for (Term t : terms) {
- TermContext termContext = TermContext.build(context, t);
+ TermStates termStates = TermStates.build(context, t, true);
if (!colMap.containsKey(t.field())) { // collection stats for this field
CollectionStatistics collectionStatistics = searcher.localCollectionStatistics(t.field());
@@ -171,7 +171,7 @@ public class ExactStatsCache extends StatsCache {
}
}
- TermStatistics tst = searcher.localTermStatistics(t, termContext);
+ TermStatistics tst = searcher.localTermStatistics(t, termStates);
if (tst == null) { // skip terms that are not present here
continue;
}
@@ -322,7 +322,7 @@ public class ExactStatsCache extends StatsCache {
this.colStatsCache = colStatsCache;
}
- public TermStatistics termStatistics(SolrIndexSearcher localSearcher, Term term, TermContext context)
+ public TermStatistics termStatistics(SolrIndexSearcher localSearcher, Term term, TermStates context)
throws IOException {
TermStats termStats = termStatsCache.get(term.toString());
// TermStats == null is also true if term has no docFreq anyway,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/solr/core/src/java/org/apache/solr/search/stats/LRUStatsCache.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/stats/LRUStatsCache.java b/solr/core/src/java/org/apache/solr/search/stats/LRUStatsCache.java
index 99efb8d..94e3a5f 100644
--- a/solr/core/src/java/org/apache/solr/search/stats/LRUStatsCache.java
+++ b/solr/core/src/java/org/apache/solr/search/stats/LRUStatsCache.java
@@ -24,7 +24,7 @@ import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.TermStatistics;
import org.apache.solr.core.PluginInfo;
@@ -132,7 +132,7 @@ public class LRUStatsCache extends ExactStatsCache {
this.colStatsCache = colStatsCache;
}
@Override
- public TermStatistics termStatistics(SolrIndexSearcher localSearcher, Term term, TermContext context)
+ public TermStatistics termStatistics(SolrIndexSearcher localSearcher, Term term, TermStates context)
throws IOException {
TermStats termStats = termStatsCache.get(term.toString());
if (termStats == null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/solr/core/src/java/org/apache/solr/search/stats/LocalStatsSource.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/stats/LocalStatsSource.java b/solr/core/src/java/org/apache/solr/search/stats/LocalStatsSource.java
index 989f3ad..3a08a61 100644
--- a/solr/core/src/java/org/apache/solr/search/stats/LocalStatsSource.java
+++ b/solr/core/src/java/org/apache/solr/search/stats/LocalStatsSource.java
@@ -19,7 +19,7 @@ package org.apache.solr.search.stats;
import java.io.IOException;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.TermStatistics;
import org.apache.solr.search.SolrIndexSearcher;
@@ -34,7 +34,7 @@ public final class LocalStatsSource extends StatsSource {
}
@Override
- public TermStatistics termStatistics(SolrIndexSearcher localSearcher, Term term, TermContext context)
+ public TermStatistics termStatistics(SolrIndexSearcher localSearcher, Term term, TermStates context)
throws IOException {
return localSearcher.localTermStatistics(term, context);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/solr/core/src/java/org/apache/solr/search/stats/StatsSource.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/stats/StatsSource.java b/solr/core/src/java/org/apache/solr/search/stats/StatsSource.java
index 4daaa48..c187fef 100644
--- a/solr/core/src/java/org/apache/solr/search/stats/StatsSource.java
+++ b/solr/core/src/java/org/apache/solr/search/stats/StatsSource.java
@@ -19,7 +19,7 @@ package org.apache.solr.search.stats;
import java.io.IOException;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermStatistics;
@@ -34,7 +34,7 @@ import org.apache.solr.search.SolrIndexSearcher;
*/
public abstract class StatsSource {
- public abstract TermStatistics termStatistics(SolrIndexSearcher localSearcher, Term term, TermContext context)
+ public abstract TermStatistics termStatistics(SolrIndexSearcher localSearcher, Term term, TermStates context)
throws IOException;
public abstract CollectionStatistics collectionStatistics(SolrIndexSearcher localSearcher, String field)
[31/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8122: Upgrade
analysis/icu to ICU 60.2
Posted by da...@apache.org.
LUCENE-8122: Upgrade analysis/icu to ICU 60.2
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/07407a5b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/07407a5b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/07407a5b
Branch: refs/heads/jira/solr-11702
Commit: 07407a5b53bf4d790c316ecf3b71046242f1e2da
Parents: a3a0e0b
Author: Robert Muir <rm...@apache.org>
Authored: Mon Jan 8 16:33:38 2018 -0500
Committer: Robert Muir <rm...@apache.org>
Committed: Mon Jan 8 16:33:38 2018 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 2 +
lucene/analysis/icu/src/data/uax29/Default.rbbi | 96 +++++++++++++++----
.../icu/src/data/utr30/DiacriticFolding.txt | 11 ++-
.../icu/src/data/utr30/NativeDigitFolding.txt | 10 ++
lucene/analysis/icu/src/data/utr30/nfc.txt | 13 ++-
lucene/analysis/icu/src/data/utr30/nfkc.txt | 4 +-
lucene/analysis/icu/src/data/utr30/nfkc_cf.txt | 10 +-
.../analysis/icu/segmentation/ICUTokenizer.java | 8 +-
lucene/analysis/icu/src/java/overview.html | 2 +-
.../analysis/icu/segmentation/Default.brk | Bin 36768 -> 54488 bytes
.../icu/segmentation/MyanmarSyllable.brk | Bin 20744 -> 21976 bytes
.../org/apache/lucene/analysis/icu/utr30.nrm | Bin 55184 -> 59056 bytes
.../analysis/icu/GenerateUTR30DataFiles.java | 6 +-
lucene/ivy-versions.properties | 2 +-
lucene/licenses/icu4j-59.1.jar.sha1 | 1 -
lucene/licenses/icu4j-60.2.jar.sha1 | 1 +
solr/licenses/icu4j-59.1.jar.sha1 | 1 -
solr/licenses/icu4j-60.2.jar.sha1 | 1 +
18 files changed, 128 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 2b02b87..78cd095 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -132,6 +132,8 @@ Other
* LUCENE-8111: IndexOrDocValuesQuery Javadoc references outdated method name.
(Kai Chan via Adrien Grand)
+* LUCENE-8122: Upgrade analysis/icu to ICU 60.2. (Robert Muir)
+
======================= Lucene 7.2.0 =======================
API Changes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/analysis/icu/src/data/uax29/Default.rbbi
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/data/uax29/Default.rbbi b/lucene/analysis/icu/src/data/uax29/Default.rbbi
index 6c6d1f9..afda68f 100644
--- a/lucene/analysis/icu/src/data/uax29/Default.rbbi
+++ b/lucene/analysis/icu/src/data/uax29/Default.rbbi
@@ -14,16 +14,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-# This file is from ICU (with some small modifications, to avoid CJK dictionary break)
+# This file is from ICU (with some small modifications, to avoid CJK dictionary break,
+# and status code change related to that)
#
-# Copyright (C) 2002-2013, International Business Machines Corporation
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# License & terms of use: http://www.unicode.org/copyright.html
+# Copyright (C) 2002-2016, International Business Machines Corporation
# and others. All Rights Reserved.
#
# file: word.txt
#
# ICU Word Break Rules
# See Unicode Standard Annex #29.
-# These rules are based on UAX #29 Revision 22 for Unicode Version 6.3
+# These rules are based on UAX #29 Revision 29 for Unicode Version 9.0
+# with additions for Emoji Sequences from https://goo.gl/cluFCn
+# Plus additional characters introduces with Emoji 5, http://www.unicode.org/reports/tr51/proposed.html
#
# Note: Updates to word.txt will usually need to be merged into
# word_POSIX.txt also.
@@ -35,6 +40,7 @@
##############################################################################
!!chain;
+!!quoted_literals_only;
#
@@ -43,8 +49,9 @@
$CR = [\p{Word_Break = CR}];
$LF = [\p{Word_Break = LF}];
-$Newline = [\p{Word_Break = Newline}];
+$Newline = [\p{Word_Break = Newline} ];
$Extend = [\p{Word_Break = Extend}];
+$ZWJ = [\p{Word_Break = ZWJ}];
$Regional_Indicator = [\p{Word_Break = Regional_Indicator}];
$Format = [\p{Word_Break = Format}];
$Katakana = [\p{Word_Break = Katakana}];
@@ -57,6 +64,13 @@ $MidLetter = [\p{Word_Break = MidLetter}];
$MidNum = [\p{Word_Break = MidNum}];
$Numeric = [\p{Word_Break = Numeric}[[:Decomposition_Type=Wide:]&[:General_Category=Decimal_Number:]]];
$ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
+$E_Base = [\p{Word_Break = EB}];
+$E_Modifier = [\p{Word_Break = EM}];
+
+# Data for Extended Pictographic scraped from CLDR common/properties/ExtendedPictographic.txt, r13267
+$Extended_Pict = [\U0001F774-\U0001F77F\U00002700-\U00002701\U00002703-\U00002704\U0000270E\U00002710-\U00002711\U00002765-\U00002767\U0001F030-\U0001F093\U0001F094-\U0001F09F\U0001F10D-\U0001F10F\U0001F12F\U0001F16C-\U0001F16F\U0001F1AD-\U0001F1E5\U0001F260-\U0001F265\U0001F203-\U0001F20F\U0001F23C-\U0001F23F\U0001F249-\U0001F24F\U0001F252-\U0001F25F\U0001F266-\U0001F2FF\U0001F7D5-\U0001F7FF\U0001F000-\U0001F003\U0001F005-\U0001F02B\U0001F02C-\U0001F02F\U0001F322-\U0001F323\U0001F394-\U0001F395\U0001F398\U0001F39C-\U0001F39D\U0001F3F1-\U0001F3F2\U0001F3F6\U0001F4FE\U0001F53E-\U0001F548\U0001F54F\U0001F568-\U0001F56E\U0001F571-\U0001F572\U0001F57B-\U0001F586\U0001F588-\U0001F589\U0001F58E-\U0001F58F\U0001F591-\U0001F594\U0001F597-\U0001F5A3\U0001F5A6-\U0001F5A7\U0001F5A9-\U0001F5B0\U0001F5B3-\U0001F5BB\U0001F5BD-\U0001F5C1\U0001F5C5-\U0001F5D0\U0001F5D4-\U0001F5DB\U0001F5DF-\U0001F5E0\U0001F5E2\U0001F5E4-\U0001F5E7\U0001F5E9-\U0001F5EE\U0001F5F0-\U0001F5F2\U0001F5F4-\U0001F5F9\U0000
2605\U00002607-\U0000260D\U0000260F-\U00002610\U00002612\U00002616-\U00002617\U00002619-\U0000261C\U0000261E-\U0000261F\U00002621\U00002624-\U00002625\U00002627-\U00002629\U0000262B-\U0000262D\U00002630-\U00002637\U0000263B-\U00002647\U00002654-\U0000265F\U00002661-\U00002662\U00002664\U00002667\U00002669-\U0000267A\U0000267C-\U0000267E\U00002680-\U00002691\U00002695\U00002698\U0000269A\U0000269D-\U0000269F\U000026A2-\U000026A9\U000026AC-\U000026AF\U000026B2-\U000026BC\U000026BF-\U000026C3\U000026C6-\U000026C7\U000026C9-\U000026CD\U000026D0\U000026D2\U000026D5-\U000026E8\U000026EB-\U000026EF\U000026F6\U000026FB-\U000026FC\U000026FE-\U000026FF\U00002388\U0001FA00-\U0001FFFD\U0001F0A0-\U0001F0AE\U0001F0B1-\U0001F0BF\U0001F0C1-\U0001F0CF\U0001F0D1-\U0001F0F5\U0001F0AF-\U0001F0B0\U0001F0C0\U0001F0D0\U0001F0F6-\U0001F0FF\U0001F80C-\U0001F80F\U0001F848-\U0001F84F\U0001F85A-\U0001F85F\U0001F888-\U0001F88F\U0001F8AE-\U0001F8FF\U0001F900-\U0001F90B\U0001F91F\U0001F928-\U0001F92F\U0001F931-\U
0001F932\U0001F94C\U0001F95F-\U0001F96B\U0001F992-\U0001F997\U0001F9D0-\U0001F9E6\U0001F90C-\U0001F90F\U0001F93F\U0001F94D-\U0001F94F\U0001F96C-\U0001F97F\U0001F998-\U0001F9BF\U0001F9C1-\U0001F9CF\U0001F9E7-\U0001F9FF\U0001F6C6-\U0001F6CA\U0001F6D3-\U0001F6D4\U0001F6E6-\U0001F6E8\U0001F6EA\U0001F6F1-\U0001F6F2\U0001F6F7-\U0001F6F8\U0001F6D5-\U0001F6DF\U0001F6ED-\U0001F6EF\U0001F6F9-\U0001F6FF];
+$EBG = [\p{Word_Break = EBG}];
+$EmojiNRK = [[\p{Emoji}] - [\p{Word_Break = Regional_Indicator}\u002a\u00230-9©®™〰〽]];
$Han = [:Han:];
$Hiragana = [:Hiragana:];
@@ -83,21 +97,21 @@ $ALetterPlus = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];
# except when they appear at the beginning of a region of text.
#
# TODO: check if handling of katakana in dictionary makes rules incorrect/void
-$KatakanaEx = $Katakana ($Extend | $Format)*;
-$Hebrew_LetterEx = $Hebrew_Letter ($Extend | $Format)*;
-$ALetterEx = $ALetterPlus ($Extend | $Format)*;
-$Single_QuoteEx = $Single_Quote ($Extend | $Format)*;
-$Double_QuoteEx = $Double_Quote ($Extend | $Format)*;
-$MidNumLetEx = $MidNumLet ($Extend | $Format)*;
-$MidLetterEx = $MidLetter ($Extend | $Format)*;
-$MidNumEx = $MidNum ($Extend | $Format)*;
-$NumericEx = $Numeric ($Extend | $Format)*;
-$ExtendNumLetEx = $ExtendNumLet ($Extend | $Format)*;
-$Regional_IndicatorEx = $Regional_Indicator ($Extend | $Format)*;
+$KatakanaEx = $Katakana ($Extend | $Format | $ZWJ)*;
+$Hebrew_LetterEx = $Hebrew_Letter ($Extend | $Format | $ZWJ)*;
+$ALetterEx = $ALetterPlus ($Extend | $Format | $ZWJ)*;
+$Single_QuoteEx = $Single_Quote ($Extend | $Format | $ZWJ)*;
+$Double_QuoteEx = $Double_Quote ($Extend | $Format | $ZWJ)*;
+$MidNumLetEx = $MidNumLet ($Extend | $Format | $ZWJ)*;
+$MidLetterEx = $MidLetter ($Extend | $Format | $ZWJ)*;
+$MidNumEx = $MidNum ($Extend | $Format | $ZWJ)*;
+$NumericEx = $Numeric ($Extend | $Format | $ZWJ)*;
+$ExtendNumLetEx = $ExtendNumLet ($Extend | $Format | $ZWJ)*;
+$Regional_IndicatorEx = $Regional_Indicator ($Extend | $Format | $ZWJ)*;
$Ideographic = [\p{Ideographic}];
-$HiraganaEx = $Hiragana ($Extend | $Format)*;
-$IdeographicEx = $Ideographic ($Extend | $Format)*;
+$HiraganaEx = $Hiragana ($Extend | $Format | $ZWJ)*;
+$IdeographicEx = $Ideographic ($Extend | $Format | $ZWJ)*;
## -------------------------------------------------
@@ -108,12 +122,17 @@ $IdeographicEx = $Ideographic ($Extend | $Format)*;
#
$CR $LF;
+# Rule 3c ZWJ x (Extended_Pict | EmojiNRK). Precedes WB4, so no intervening Extend chars allowed.
+#
+$ZWJ ($Extended_Pict | $EmojiNRK);
+
+
# Rule 4 - ignore Format and Extend characters, except when they appear at the beginning
# of a region of Text. The rule here comes into play when the start of text
# begins with a group of Format chars, or with a "word" consisting of a single
# char that is not in any of the listed word break categories followed by
# format char(s), or is not a CJK dictionary character.
-[^$CR $LF $Newline]? ($Extend | $Format)+;
+[^$CR $LF $Newline]? ($Extend | $Format | $ZWJ)+;
$NumericEx {100};
$ALetterEx {200};
@@ -123,6 +142,10 @@ $KatakanaEx {300}; # note: these status values override those from rule 5
$HiraganaEx {300}; # by virtue of being numerically larger.
$IdeographicEx {400}; #
+$E_Base ($Extend | $Format | $ZWJ)*;
+$E_Modifier ($Extend | $Format | $ZWJ)*;
+$Extended_Pict ($Extend | $Format | $ZWJ)*;
+
#
# rule 5
# Do not break between most letters.
@@ -170,9 +193,42 @@ $ExtendNumLetEx $Hebrew_Letter {200}; # (13b)
$ExtendNumLetEx $NumericEx {100}; # (13b)
$ExtendNumLetEx $KatakanaEx {300}; # (13b)
-# rule 13c
+# rule 14
+# Do not break within emoji modifier sequences
+
+($E_Base | $EBG) ($Format | $Extend | $ZWJ)* $E_Modifier;
-$Regional_IndicatorEx $Regional_IndicatorEx;
+# rules 15 - 17
+# Pairs of Regional Indicators stay together.
+# With rule chaining disabled by ^, this rule will match exactly two of them.
+# No other rule begins with a Regional_Indicator, so chaining cannot extend the match.
+#
+^$Regional_IndicatorEx $Regional_IndicatorEx;
# special handling for CJK characters: chain for later dictionary segmentation
$HangulSyllable $HangulSyllable {200};
+
+# Rule 999
+# Match a single code point if no other rule applies.
+.;
+
+
+## -------------------------------------------------
+
+!!safe_reverse;
+
+# rule 3
+($Extend | $Format | $ZWJ)+ .?;
+
+# rule 6
+($MidLetter | $MidNumLet | $Single_Quote) ($Format | $Extend | $ZWJ)* ($Hebrew_Letter | $ALetterPlus);
+
+# rule 7b
+$Double_Quote ($Format | $Extend | $ZWJ)* $Hebrew_Letter;
+
+
+# rule 11
+($MidNum | $MidNumLet | $Single_Quote) ($Format | $Extend | $ZWJ)* $Numeric;
+
+# rule 13c
+$Regional_Indicator ($Format | $Extend | $ZWJ)* $Regional_Indicator;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/analysis/icu/src/data/utr30/DiacriticFolding.txt
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/data/utr30/DiacriticFolding.txt b/lucene/analysis/icu/src/data/utr30/DiacriticFolding.txt
index eb5b78e..806a4f9 100644
--- a/lucene/analysis/icu/src/data/utr30/DiacriticFolding.txt
+++ b/lucene/analysis/icu/src/data/utr30/DiacriticFolding.txt
@@ -73,12 +73,14 @@
0A4D>
0ABC>
0ACD>
+0AFD..0AFF>
0B3C>
0B4D>
0BCD>
0C4D>
0CBC>
0CCD>
+0D3B..0D3C>
0D4D>
0DCA>
0E47..0E4C>
@@ -112,10 +114,10 @@
1CD0..1CE8>
1CED>
1CF4>
-1CF8..1CF9>
+1CF7..1CF9>
1D2C..1D6A>
1DC4..1DCF>
-1DF5>
+1DF5..1DF9>
1DFD..1DFF>
1FBD>
1FBF..1FC1>
@@ -175,7 +177,12 @@ FFE3>
1163F>
116B6..116B7>
1172B>
+11A34>
+11A47>
+11A99>
11C3F>
+11D42>
+11D44..11D45>
16AF0..16AF4>
16F8F..16F9F>
1D167..1D169>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/analysis/icu/src/data/utr30/NativeDigitFolding.txt
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/data/utr30/NativeDigitFolding.txt b/lucene/analysis/icu/src/data/utr30/NativeDigitFolding.txt
index fb8cf1a..707674e 100644
--- a/lucene/analysis/icu/src/data/utr30/NativeDigitFolding.txt
+++ b/lucene/analysis/icu/src/data/utr30/NativeDigitFolding.txt
@@ -580,6 +580,16 @@ ABF9>0039 # MEETEI MAYEK DIGIT NINE
11C57>0037 # BHAIKSUKI DIGIT SEVEN
11C58>0038 # BHAIKSUKI DIGIT EIGHT
11C59>0039 # BHAIKSUKI DIGIT NINE
+11D50>0030 # MASARAM GONDI DIGIT ZERO
+11D51>0031 # MASARAM GONDI DIGIT ONE
+11D52>0032 # MASARAM GONDI DIGIT TWO
+11D53>0033 # MASARAM GONDI DIGIT THREE
+11D54>0034 # MASARAM GONDI DIGIT FOUR
+11D55>0035 # MASARAM GONDI DIGIT FIVE
+11D56>0036 # MASARAM GONDI DIGIT SIX
+11D57>0037 # MASARAM GONDI DIGIT SEVEN
+11D58>0038 # MASARAM GONDI DIGIT EIGHT
+11D59>0039 # MASARAM GONDI DIGIT NINE
16A60>0030 # MRO DIGIT ZERO
16A61>0031 # MRO DIGIT ONE
16A62>0032 # MRO DIGIT TWO
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/analysis/icu/src/data/utr30/nfc.txt
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/data/utr30/nfc.txt b/lucene/analysis/icu/src/data/utr30/nfc.txt
index 5f9b182..b41056d 100644
--- a/lucene/analysis/icu/src/data/utr30/nfc.txt
+++ b/lucene/analysis/icu/src/data/utr30/nfc.txt
@@ -1,3 +1,5 @@
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (C) 1999-2016, International Business Machines
# Corporation and others. All Rights Reserved.
#
@@ -7,7 +9,7 @@
#
# Complete data for Unicode NFC normalization.
-* Unicode 9.0.0
+* Unicode 10.0.0
# Canonical_Combining_Class (ccc) values
0300..0314:230
@@ -164,6 +166,7 @@
0C56:91
0CBC:7
0CCD:9
+0D3B..0D3C:9
0D4D:9
0DCA:9
0E38..0E39:103
@@ -234,6 +237,9 @@
1DCF:220
1DD0:202
1DD1..1DF5:230
+1DF6:232
+1DF7..1DF8:228
+1DF9:220
1DFB:230
1DFC:233
1DFD:220
@@ -322,7 +328,12 @@ FE2E..FE2F:230
116B6:9
116B7:7
1172B:9
+11A34:9
+11A47:9
+11A99:9
11C3F:9
+11D42:7
+11D44..11D45:9
16AF0..16AF4:1
16B30..16B36:230
1BC9E:1
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/analysis/icu/src/data/utr30/nfkc.txt
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/data/utr30/nfkc.txt b/lucene/analysis/icu/src/data/utr30/nfkc.txt
index f51fa5d..8b71727 100644
--- a/lucene/analysis/icu/src/data/utr30/nfkc.txt
+++ b/lucene/analysis/icu/src/data/utr30/nfkc.txt
@@ -1,3 +1,5 @@
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (C) 1999-2016, International Business Machines
# Corporation and others. All Rights Reserved.
#
@@ -11,7 +13,7 @@
# to NFKC one-way mappings.
# Use this file as the second gennorm2 input file after nfc.txt.
-* Unicode 9.0.0
+* Unicode 10.0.0
00A0>0020
00A8>0020 0308
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/analysis/icu/src/data/utr30/nfkc_cf.txt
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/data/utr30/nfkc_cf.txt b/lucene/analysis/icu/src/data/utr30/nfkc_cf.txt
index 7f33df5..726c5b5 100644
--- a/lucene/analysis/icu/src/data/utr30/nfkc_cf.txt
+++ b/lucene/analysis/icu/src/data/utr30/nfkc_cf.txt
@@ -1,7 +1,7 @@
-# Unicode Character Database
-# Copyright (c) 1991-2016 Unicode, Inc.
-# For terms of use, see http://www.unicode.org/terms_of_use.html
-# For documentation, see http://www.unicode.org/reports/tr44/
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# License & terms of use: http://www.unicode.org/copyright.html
+# Copyright (C) 1999-2016, International Business Machines
+# Corporation and others. All Rights Reserved.
#
# file name: nfkc_cf.txt
#
@@ -12,7 +12,7 @@
# and reformatted into syntax for the gennorm2 Normalizer2 data generator tool.
# Use this file as the third gennorm2 input file after nfc.txt and nfkc.txt.
-* Unicode 9.0.0
+* Unicode 10.0.0
0041>0061
0042>0062
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java
index 0941551..8b62ddb 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java
@@ -200,18 +200,18 @@ public final class ICUTokenizer extends Tokenizer {
*/
private boolean incrementTokenBuffer() {
int start = breaker.current();
- if (start == BreakIterator.DONE)
- return false; // BreakIterator exhausted
+ assert start != BreakIterator.DONE;
// find the next set of boundaries, skipping over non-tokens (rule status 0)
int end = breaker.next();
- while (start != BreakIterator.DONE && breaker.getRuleStatus() == 0) {
+ while (end != BreakIterator.DONE && breaker.getRuleStatus() == 0) {
start = end;
end = breaker.next();
}
- if (start == BreakIterator.DONE)
+ if (end == BreakIterator.DONE) {
return false; // BreakIterator exhausted
+ }
termAtt.copyBuffer(buffer, start, end - start);
offsetAtt.setOffset(correctOffset(offset + start), correctOffset(offset + end));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/analysis/icu/src/java/overview.html
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/java/overview.html b/lucene/analysis/icu/src/java/overview.html
index bdace97..6fa5821 100644
--- a/lucene/analysis/icu/src/java/overview.html
+++ b/lucene/analysis/icu/src/java/overview.html
@@ -353,7 +353,7 @@ and
<h1><a name="backcompat">Backwards Compatibility</a></h1>
<p>
This module exists to provide up-to-date Unicode functionality that supports
-the most recent version of Unicode (currently 8.0). However, some users who wish
+the most recent version of Unicode (currently 10.0). However, some users who wish
for stronger backwards compatibility can restrict
{@link org.apache.lucene.analysis.icu.ICUNormalizer2Filter} to operate on only
a specific Unicode Version by using a {@link com.ibm.icu.text.FilteredNormalizer2}.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Default.brk
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Default.brk b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Default.brk
index c94a023..4a9df15 100644
Binary files a/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Default.brk and b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Default.brk differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk
index c3357ef..a9d0673 100644
Binary files a/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk and b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm
index 1a16f3e..1c3de12 100644
Binary files a/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm and b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java b/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
index 0f2bffe..042fa37 100644
--- a/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
+++ b/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
@@ -62,9 +62,9 @@ import java.util.regex.Pattern;
*/
public class GenerateUTR30DataFiles {
private static final String ICU_SVN_TAG_URL
- = "http://source.icu-project.org/repos/icu/icu/tags";
- private static final String ICU_RELEASE_TAG = "release-58-1";
- private static final String ICU_DATA_NORM2_PATH = "source/data/unidata/norm2";
+ = "http://source.icu-project.org/repos/icu/tags";
+ private static final String ICU_RELEASE_TAG = "release-60-2";
+ private static final String ICU_DATA_NORM2_PATH = "icu4c/source/data/unidata/norm2";
private static final String NFC_TXT = "nfc.txt";
private static final String NFKC_TXT = "nfkc.txt";
private static final String NFKC_CF_TXT = "nfkc_cf.txt";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 72265a7..4021d3e 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -31,7 +31,7 @@ com.fasterxml.jackson.core.version = 2.5.4
/com.googlecode.mp4parser/isoparser = 1.1.18
/com.healthmarketscience.jackcess/jackcess = 2.1.8
/com.healthmarketscience.jackcess/jackcess-encrypt = 2.1.4
-/com.ibm.icu/icu4j = 59.1
+/com.ibm.icu/icu4j = 60.2
/com.pff/java-libpst = 0.8.1
com.rometools.version = 1.5.1
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/licenses/icu4j-59.1.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/icu4j-59.1.jar.sha1 b/lucene/licenses/icu4j-59.1.jar.sha1
deleted file mode 100644
index f3f0018..0000000
--- a/lucene/licenses/icu4j-59.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6f06e820cf4c8968bbbaae66ae0b33f6a256b57f
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/lucene/licenses/icu4j-60.2.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/icu4j-60.2.jar.sha1 b/lucene/licenses/icu4j-60.2.jar.sha1
new file mode 100644
index 0000000..e613111
--- /dev/null
+++ b/lucene/licenses/icu4j-60.2.jar.sha1
@@ -0,0 +1 @@
+e452cba3caaf93b997ff543c7246a6da74ed70f1
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/solr/licenses/icu4j-59.1.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/icu4j-59.1.jar.sha1 b/solr/licenses/icu4j-59.1.jar.sha1
deleted file mode 100644
index f3f0018..0000000
--- a/solr/licenses/icu4j-59.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6f06e820cf4c8968bbbaae66ae0b33f6a256b57f
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07407a5b/solr/licenses/icu4j-60.2.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/icu4j-60.2.jar.sha1 b/solr/licenses/icu4j-60.2.jar.sha1
new file mode 100644
index 0000000..e613111
--- /dev/null
+++ b/solr/licenses/icu4j-60.2.jar.sha1
@@ -0,0 +1 @@
+e452cba3caaf93b997ff543c7246a6da74ed70f1
[38/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11062: new tag
"diskType" in autoscaling policy
Posted by da...@apache.org.
SOLR-11062: new tag "diskType" in autoscaling policy
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6336ed46
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6336ed46
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6336ed46
Branch: refs/heads/jira/solr-11702
Commit: 6336ed46f9b200baf9a89d1a3b42f3608e391847
Parents: 74128cf
Author: Noble Paul <no...@apache.org>
Authored: Tue Jan 9 22:58:10 2018 +1100
Committer: Noble Paul <no...@apache.org>
Committed: Tue Jan 9 22:58:10 2018 +1100
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../solr/cloud/autoscaling/TestPolicyCloud.java | 8 ++++--
.../solrj/cloud/autoscaling/Suggestion.java | 11 +++++++-
.../solrj/impl/SolrClientNodeStateProvider.java | 27 +++++++++++++++++---
.../solr/common/cloud/rule/ImplicitSnitch.java | 1 +
5 files changed, 43 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6336ed46/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d7eb033..14f80a3 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -80,6 +80,8 @@ New Features
rules that sets the time interval for each collection. An internal Overseer command "ROUTEDALIAS_CREATECOLL"
was created to facilitate this. (David Smiley)
+* SOLR-11062: new tag "diskType" in autoscaling policy (noble)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6336ed46/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
index f53dde1..9637a32 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
@@ -20,11 +20,14 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
+import com.google.common.collect.ImmutableSet;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrRequest;
@@ -227,8 +230,9 @@ public class TestPolicyCloud extends SolrCloudTestCase {
for (String tag : tags) {
assertNotNull( "missing : "+ tag , val.get(tag));
}
-
-
+ val = provider.getNodeStateProvider().getNodeValues(collection.getReplicas().get(0).getNodeName(), Collections.singleton("diskType"));
+ Set<String> diskTypes = ImmutableSet.of("rotational", "ssd");
+ assertTrue(diskTypes.contains(val.get("diskType")));
}
public void testCreateCollectionAddShardWithReplicaTypeUsingPolicy() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6336ed46/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
index 070869a..51b5046 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
@@ -18,9 +18,11 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -34,6 +36,7 @@ import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.StrUtils;
+import static java.util.Collections.unmodifiableSet;
import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
@@ -224,7 +227,13 @@ public class Suggestion {
public void getSuggestions(SuggestionCtx ctx) {
perNodeSuggestions(ctx);
}
- },;
+ },
+ DISKTYPE(ImplicitSnitch.DISKTYPE, String.class, unmodifiableSet(new HashSet(Arrays.asList("ssd", "rotational"))), null, null, null) {
+ @Override
+ public void getSuggestions(SuggestionCtx ctx) {
+ perNodeSuggestions(ctx);
+ }
+ };
final Class type;
final Set<String> vals;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6336ed46/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 66f54dd..4bece2c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -27,6 +27,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.solr.client.solrj.SolrRequest;
@@ -142,14 +143,19 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
return result;
}
- static void fetchMetrics(String solrNode, ClientSnitchCtx ctx, Map<String, String> metricsKeyVsTag) {
+ static void fetchMetrics(String solrNode, ClientSnitchCtx ctx, Map<String, Object> metricsKeyVsTag) {
ModifiableSolrParams params = new ModifiableSolrParams();
params.add("key", metricsKeyVsTag.keySet().toArray(new String[metricsKeyVsTag.size()]));
try {
SimpleSolrResponse rsp = ctx.invoke(solrNode, CommonParams.METRICS_PATH, params);
metricsKeyVsTag.forEach((key, tag) -> {
Object v = Utils.getObjectByPath(rsp.nl, true, Arrays.asList("metrics", key));
- if (v != null) ctx.getTags().put(tag, v);
+ if (tag instanceof Function) {
+ Pair<String, Object> p = (Pair<String, Object>) ((Function) tag).apply(v);
+ ctx.getTags().put(p.first(), p.second());
+ } else {
+ if (v != null) ctx.getTags().put(tag.toString(), v);
+ }
});
} catch (Exception e) {
log.warn("could not get tags from node " + solrNode, e);
@@ -166,7 +172,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
@Override
protected void getRemoteInfo(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
ClientSnitchCtx snitchContext = (ClientSnitchCtx) ctx;
- Map<String, String> metricsKeyVsTag = new HashMap<>();
+ Map<String, Object> metricsKeyVsTag = new HashMap<>();
for (String tag : requestedTags) {
if (tag.startsWith(SYSPROP)) {
metricsKeyVsTag.put("solr.jvm:system.properties:" + tag.substring(SYSPROP.length()), tag);
@@ -174,6 +180,21 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
metricsKeyVsTag.put(tag.substring(METRICS_PREFIX.length()), tag);
}
}
+ if (requestedTags.contains(ImplicitSnitch.DISKTYPE)) {
+ metricsKeyVsTag.put("solr.node:CONTAINER.fs.coreRoot.spins", new Function<Object, Pair<String,Object>>() {
+ @Override
+ public Pair<String, Object> apply(Object o) {
+ if("true".equals(String.valueOf(o))){
+ return new Pair<>(ImplicitSnitch.DISKTYPE, "rotational");
+ }
+ if("false".equals(String.valueOf(o))){
+ return new Pair<>(ImplicitSnitch.DISKTYPE, "ssd");
+ }
+ return new Pair<>(ImplicitSnitch.DISKTYPE,null);
+
+ }
+ });
+ }
if (!metricsKeyVsTag.isEmpty()) {
fetchMetrics(solrNode, snitchContext, metricsKeyVsTag);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6336ed46/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java b/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
index e087342..ec55ccb 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
@@ -52,6 +52,7 @@ public class ImplicitSnitch extends Snitch {
public static final String SYSPROP = "sysprop.";
public static final String SYSLOADAVG = "sysLoadAvg";
public static final String HEAPUSAGE = "heapUsage";
+ public static final String DISKTYPE = "diskType";
public static final List<String> IP_SNITCHES = Collections.unmodifiableList(Arrays.asList("ip_1", "ip_2", "ip_3", "ip_4"));
public static final Set<String> tags = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(NODE, PORT, HOST, CORES, DISK, ROLE, "ip_1", "ip_2", "ip_3", "ip_4")));
[34/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11631: The Schema
API should return non-zero status when there are failures
Posted by da...@apache.org.
SOLR-11631: The Schema API should return non-zero status when there are failures
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9f221796
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9f221796
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9f221796
Branch: refs/heads/jira/solr-11702
Commit: 9f221796fe1b79ead6509efdcaa0a17c5a382c65
Parents: d189b58
Author: Steve Rowe <sa...@apache.org>
Authored: Mon Jan 8 21:25:14 2018 -0500
Committer: Steve Rowe <sa...@apache.org>
Committed: Mon Jan 8 21:25:14 2018 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../org/apache/solr/handler/SchemaHandler.java | 12 ++-
.../solr/core/TestConfigSetImmutable.java | 6 +-
.../solr/rest/schema/TestBulkSchemaAPI.java | 91 ++++++++++++--------
.../solr/schema/SchemaApiFailureTest.java | 66 ++++++++++++++
5 files changed, 131 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f221796/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 31197a0..533748f 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -92,6 +92,9 @@ Bug Fixes
* SOLR-11821: ConcurrentModificationException in SimSolrCloudTestCase.tearDown (shalin)
+* SOLR-11631: The Schema API should return non-zero status when there are failures.
+ (Noble Paul, Steve Rowe)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f221796/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
index e3e292b..fb84e84 100644
--- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
@@ -80,20 +80,18 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware,
String httpMethod = (String) req.getContext().get("httpMethod");
if ("POST".equals(httpMethod)) {
if (isImmutableConfigSet) {
- rsp.add("errors", "ConfigSet is immutable");
- return;
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "ConfigSet is immutable");
}
if (req.getContentStreams() == null) {
- rsp.add("errors", "no stream");
- return;
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no stream");
}
try {
List errs = new SchemaManager(req).performOperations();
- if (!errs.isEmpty()) rsp.add("errors", errs);
+ if (!errs.isEmpty())
+ throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST,"error processing commands", errs);
} catch (IOException e) {
- rsp.add("errors", Collections.singletonList("Error reading input String " + e.getMessage()));
- rsp.setException(e);
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error reading input String " + e.getMessage(), e);
}
} else {
handleGET(req, rsp);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f221796/solr/core/src/test/org/apache/solr/core/TestConfigSetImmutable.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestConfigSetImmutable.java b/solr/core/src/test/org/apache/solr/core/TestConfigSetImmutable.java
index 2388b3d..b9f456c 100644
--- a/solr/core/src/test/org/apache/solr/core/TestConfigSetImmutable.java
+++ b/solr/core/src/test/org/apache/solr/core/TestConfigSetImmutable.java
@@ -93,8 +93,10 @@ public class TestConfigSetImmutable extends RestTestBase {
String response = restTestHarness.post("/schema", json(payload));
Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNotNull(map.get("errors"));
- assertTrue(map.get("errors").toString().contains("immutable"));
+ Map error = (Map)map.get("error");
+ assertNotNull("No errors", error);
+ String msg = (String)error.get("msg");
+ assertTrue(msg.contains("immutable"));
}
@Test
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f221796/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java b/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
index 4dcfc66..4bebeca 100644
--- a/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
+++ b/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
@@ -109,14 +109,17 @@ public class TestBulkSchemaAPI extends RestTestBase {
String response = restTestHarness.post("/schema", json(payload));
Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- List l = (List) map.get("errors");
- assertNotNull("No errors", l);
- List errorList = (List) ((Map) l.get(0)).get("errorMessages");
- assertEquals(1, errorList.size());
- assertTrue (((String)errorList.get(0)).contains("Field 'a1': Field type 'string1' not found.\n"));
- errorList = (List) ((Map) l.get(1)).get("errorMessages");
- assertEquals(1, errorList.size());
- assertTrue (((String)errorList.get(0)).contains("is a required field"));
+ Map error = (Map)map.get("error");
+ assertNotNull("No errors", error);
+ List details = (List)error.get("details");
+ assertNotNull("No details", details);
+ assertEquals("Wrong number of details", 2, details.size());
+ List firstErrorList = (List)((Map)details.get(0)).get("errorMessages");
+ assertEquals(1, firstErrorList.size());
+ assertTrue (((String)firstErrorList.get(0)).contains("Field 'a1': Field type 'string1' not found.\n"));
+ List secondErrorList = (List)((Map)details.get(1)).get("errorMessages");
+ assertEquals(1, secondErrorList.size());
+ assertTrue (((String)secondErrorList.get(0)).contains("is a required field"));
}
public void testAnalyzerClass() throws Exception {
@@ -145,8 +148,12 @@ public class TestBulkSchemaAPI extends RestTestBase {
String response = restTestHarness.post("/schema",
json(addFieldTypeAnalyzerWithClass + ',' + charFilters + tokenizer + filters + suffix));
Map map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- List list = (List)map.get("errors");
- List errorList = (List)((Map)list.get(0)).get("errorMessages");
+ Map error = (Map)map.get("error");
+ assertNotNull("No errors", error);
+ List details = (List)error.get("details");
+ assertNotNull("No details", details);
+ assertEquals("Wrong number of details", 1, details.size());
+ List errorList = (List)((Map)details.get(0)).get("errorMessages");
assertEquals(1, errorList.size());
assertTrue (((String)errorList.get(0)).contains
("An analyzer with a class property may not define any char filters!"));
@@ -154,8 +161,12 @@ public class TestBulkSchemaAPI extends RestTestBase {
response = restTestHarness.post("/schema",
json(addFieldTypeAnalyzerWithClass + ',' + tokenizer + filters + suffix));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- list = (List)map.get("errors");
- errorList = (List)((Map)list.get(0)).get("errorMessages");
+ error = (Map)map.get("error");
+ assertNotNull("No errors", error);
+ details = (List)error.get("details");
+ assertNotNull("No details", details);
+ assertEquals("Wrong number of details", 1, details.size());
+ errorList = (List)((Map)details.get(0)).get("errorMessages");
assertEquals(1, errorList.size());
assertTrue (((String)errorList.get(0)).contains
("An analyzer with a class property may not define a tokenizer!"));
@@ -163,15 +174,19 @@ public class TestBulkSchemaAPI extends RestTestBase {
response = restTestHarness.post("/schema",
json(addFieldTypeAnalyzerWithClass + ',' + filters + suffix));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- list = (List)map.get("errors");
- errorList = (List)((Map)list.get(0)).get("errorMessages");
+ error = (Map)map.get("error");
+ assertNotNull("No errors", error);
+ details = (List)error.get("details");
+ assertNotNull("No details", details);
+ assertEquals("Wrong number of details", 1, details.size());
+ errorList = (List)((Map)details.get(0)).get("errorMessages");
assertEquals(1, errorList.size());
assertTrue (((String)errorList.get(0)).contains
("An analyzer with a class property may not define any filters!"));
response = restTestHarness.post("/schema", json(addFieldTypeAnalyzerWithClass + suffix));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(response, map.get("errors"));
+ assertNull(response, map.get("error"));
map = getObj(restTestHarness, "myNewTextFieldWithAnalyzerClass", "fieldTypes");
assertNotNull(map);
@@ -206,7 +221,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
String response = harness.post("/schema", json(payload));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(response, map.get("errors"));
+ assertNull(response, map.get("error"));
map = getObj(harness, newFieldName, "fields");
assertNotNull("Field '" + newFieldName + "' is not in the schema", map);
@@ -228,7 +243,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
String response = harness.post("/schema", json(payload));
Map map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNotNull(response, map.get("errors"));
+ assertNotNull(response, map.get("error"));
map = getObj(harness, newFieldName, "dynamicFields");
assertNull(newFieldName + " illegal dynamic field should not have been added to schema", map);
@@ -251,7 +266,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
String response = harness.post("/schema", json(payload));
Map map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNotNull(response, map.get("errors"));
+ assertNotNull(response, map.get("error"));
map = getObj(harness, newFieldName, "fields");
assertNull(newFieldName + " illegal dynamic field should not have been added to schema", map);
@@ -273,7 +288,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
response = harness.post("/schema", json(payload));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNotNull(response, map.get("errors"));
+ assertNotNull(response, map.get("error"));
}
public void testAddFieldWithExistingCatchallDynamicField() throws Exception {
@@ -305,7 +320,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
String response = harness.post("/schema", json(payload));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(response, map.get("errors"));
+ assertNull(response, map.get("error"));
map = getObj(harness, "*", "dynamicFields");
assertNotNull("Dynamic field '*' is not in the schema", map);
@@ -322,7 +337,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
response = harness.post("/schema", json(payload));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(response, map.get("errors"));
+ assertNull(response, map.get("error"));
map = getObj(harness, newFieldName, "fields");
assertNotNull("Field '" + newFieldName + "' is not in the schema", map);
@@ -502,7 +517,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
String response = harness.post("/schema", json(payload));
Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(response, map.get("errors"));
+ assertNull(response, map.get("error"));
m = getObj(harness, "a1", "fields");
assertNotNull("field a1 not created", m);
@@ -639,7 +654,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
String response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(response, map.get("errors"));
+ assertNull(response, map.get("error"));
map = getObj(harness, "NewFieldType", "fieldTypes");
assertNotNull("'NewFieldType' is not in the schema", map);
@@ -693,14 +708,14 @@ public class TestBulkSchemaAPI extends RestTestBase {
cmds = "{'delete-field-type' : {'name':'NewFieldType'}}";
response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- Object errors = map.get("errors");
+ Object errors = map.get("error");
assertNotNull(errors);
assertTrue(errors.toString().contains("Can't delete 'NewFieldType' because it's the field type of "));
cmds = "{'delete-field' : {'name':'NewField1'}}";
response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- errors = map.get("errors");
+ errors = map.get("error");
assertNotNull(errors);
assertTrue(errors.toString().contains
("Can't delete field 'NewField1' because it's referred to by at least one copy field directive"));
@@ -708,7 +723,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
cmds = "{'delete-field' : {'name':'NewField2'}}";
response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- errors = map.get("errors");
+ errors = map.get("error");
assertNotNull(errors);
assertTrue(errors.toString().contains
("Can't delete field 'NewField2' because it's referred to by at least one copy field directive"));
@@ -716,7 +731,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
cmds = "{'replace-field' : {'name':'NewField1', 'type':'string'}}";
response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(map.get("errors"));
+ assertNull(map.get("error"));
// Make sure the copy field directives with source NewField1 are preserved
list = getSourceCopyFields(harness, "NewField1");
set = new HashSet();
@@ -730,7 +745,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
cmds = "{'delete-dynamic-field' : {'name':'NewDynamicField1*'}}";
response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- errors = map.get("errors");
+ errors = map.get("error");
assertNotNull(errors);
assertTrue(errors.toString().contains
("copyField dest :'NewDynamicField1A' is not an explicit field and doesn't match a dynamicField."));
@@ -738,7 +753,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
cmds = "{'replace-field' : {'name':'NewField2', 'type':'string'}}";
response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- errors = map.get("errors");
+ errors = map.get("error");
assertNull(errors);
// Make sure the copy field directives with destination NewField2 are preserved
list = getDestCopyFields(harness, "NewField2");
@@ -755,7 +770,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
cmds = "{'replace-dynamic-field' : {'name':'NewDynamicField2*', 'type':'string'}}";
response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- errors = map.get("errors");
+ errors = map.get("error");
assertNull(errors);
// Make sure the copy field directives with source NewDynamicField2* are preserved
list = getSourceCopyFields(harness, "NewDynamicField2*");
@@ -765,7 +780,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
cmds = "{'replace-dynamic-field' : {'name':'NewDynamicField1*', 'type':'string'}}";
response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- errors = map.get("errors");
+ errors = map.get("error");
assertNull(errors);
// Make sure the copy field directives with destinations matching NewDynamicField1* are preserved
list = getDestCopyFields(harness, "NewDynamicField1A");
@@ -775,7 +790,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
cmds = "{'replace-field-type': {'name':'NewFieldType', 'class':'solr.BinaryField'}}";
response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(map.get("errors"));
+ assertNull(map.get("error"));
// Make sure the copy field directives with sources and destinations of type NewFieldType are preserved
list = getDestCopyFields(harness, "NewField3");
assertEquals(2, list.size());
@@ -795,7 +810,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
"}\n";
response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(map.get("errors"));
+ assertNull(map.get("error"));
list = getSourceCopyFields(harness, "NewField1");
assertEquals(0, list.size());
list = getSourceCopyFields(harness, "NewDynamicField1*");
@@ -810,7 +825,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
cmds = "{'delete-field': [{'name':'NewField1'},{'name':'NewField2'},{'name':'NewField3'},{'name':'NewField4'}]}";
response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(map.get("errors"));
+ assertNull(map.get("error"));
cmds = "{'delete-dynamic-field': [{'name':'NewDynamicField1*'}," +
" {'name':'NewDynamicField2*'},\n" +
@@ -818,12 +833,12 @@ public class TestBulkSchemaAPI extends RestTestBase {
"}\n";
response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(map.get("errors"));
+ assertNull(map.get("error"));
cmds = "{'delete-field-type':{'name':'NewFieldType'}}";
response = harness.post("/schema", json(cmds));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(map.get("errors"));
+ assertNull(map.get("error"));
}
public void testSimilarityParser() throws Exception {
@@ -852,7 +867,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
String response = harness.post("/schema", json(payload));
Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(response, map.get("errors"));
+ assertNull(response, map.get("error"));
Map fields = getObj(harness, fieldName, "fields");
assertNotNull("field " + fieldName + " not created", fields);
@@ -879,7 +894,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
response = harness.post("/schema", json(payload));
map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(response, map.get("errors"));
+ assertNull(response, map.get("error"));
fields = getObj(harness, fieldName, "fields");
assertNotNull("field " + fieldName + " not created", fields);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f221796/solr/core/src/test/org/apache/solr/schema/SchemaApiFailureTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/SchemaApiFailureTest.java b/solr/core/src/test/org/apache/solr/schema/SchemaApiFailureTest.java
new file mode 100644
index 0000000..8456787
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/schema/SchemaApiFailureTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.schema;
+
+import java.lang.invoke.MethodHandles;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.schema.SchemaRequest;
+import org.apache.solr.client.solrj.response.schema.SchemaResponse;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.util.Utils;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SchemaApiFailureTest extends SolrCloudTestCase {
+ private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final String COLLECTION = "schema-api-failure";
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(1).configure();
+ CollectionAdminRequest.createCollection(COLLECTION, 2, 1) // _default configset
+ .setMaxShardsPerNode(2)
+ .process(cluster.getSolrClient());
+ cluster.getSolrClient().waitForState(COLLECTION, DEFAULT_TIMEOUT, TimeUnit.SECONDS,
+ (n, c) -> DocCollection.isFullyActive(n, c, 2, 1));
+ }
+
+ @Test
+ public void testAddTheSameFieldTwice() throws Exception {
+ CloudSolrClient client = cluster.getSolrClient();
+ SchemaRequest.Update fieldAddition = new SchemaRequest.AddField
+ (Utils.makeMap("name","myfield", "type","string"));
+ SchemaResponse.UpdateResponse updateResponse = fieldAddition.process(client, COLLECTION);
+
+ HttpSolrClient.RemoteExecutionException ex = expectThrows(HttpSolrClient.RemoteExecutionException.class,
+ () -> fieldAddition.process(client, COLLECTION));
+
+ assertTrue("expected error message 'Field 'myfield' already exists'.",Utils.getObjectByPath(ex.getMetaData(), false, "error/details[0]/errorMessages[0]").toString().contains("Field 'myfield' already exists.") );
+
+ }
+
+
+}
[07/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11801: Support
customisation of the highlighting query response element. (Ramsey Haddad,
Pranav Murugappan, David Smiley, Christine Poerschke)
Posted by da...@apache.org.
SOLR-11801: Support customisation of the highlighting query response element.
(Ramsey Haddad, Pranav Murugappan, David Smiley, Christine Poerschke)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/65c842f9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/65c842f9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/65c842f9
Branch: refs/heads/jira/solr-11702
Commit: 65c842f9faed1c9de9dce38a247a43c36b82873f
Parents: 5d4f029
Author: Christine Poerschke <cp...@apache.org>
Authored: Thu Jan 4 13:17:25 2018 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Thu Jan 4 14:54:48 2018 +0000
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../handler/component/HighlightComponent.java | 41 ++-
.../component/CustomHighlightComponentTest.java | 310 +++++++++++++++++++
3 files changed, 348 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65c842f9/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 7137b80..61551e0 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -113,6 +113,9 @@ Other Changes
* SOLR-11798: Formally deprecate top-level <highlighting> syntax in solrconfig.xml
in favour of <searchComponent> equivalent syntax. (Christine Poerschke)
+* SOLR-11801: Support customisation of the "highlighting" query response element.
+ (Ramsey Haddad, Pranav Murugappan, David Smiley, Christine Poerschke)
+
================== 7.2.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65c842f9/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java b/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
index 698cc05..1602502 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
@@ -187,7 +187,7 @@ public class HighlightComponent extends SearchComponent implements PluginInfoIni
if(sumData != null) {
// TODO ???? add this directly to the response?
- rb.rsp.add("highlighting", sumData);
+ rb.rsp.add(highlightingResponseField(), convertHighlights(sumData));
}
}
}
@@ -245,7 +245,8 @@ public class HighlightComponent extends SearchComponent implements PluginInfoIni
public void finishStage(ResponseBuilder rb) {
if (rb.doHighlights && rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
- NamedList.NamedListEntry[] arr = new NamedList.NamedListEntry[rb.resultIds.size()];
+ final Object[] objArr = newHighlightsArray(rb.resultIds.size());
+ final String highlightingResponseField = highlightingResponseField();
// TODO: make a generic routine to do automatic merging of id keyed data
for (ShardRequest sreq : rb.finished) {
@@ -256,13 +257,12 @@ public class HighlightComponent extends SearchComponent implements PluginInfoIni
// this should only happen when using shards.tolerant=true
continue;
}
- NamedList hl = (NamedList)srsp.getSolrResponse().getResponse().get("highlighting");
- SolrPluginUtils.copyNamedListIntoArrayByDocPosInResponse(hl, rb.resultIds, arr);
+ Object hl = srsp.getSolrResponse().getResponse().get(highlightingResponseField);
+ addHighlights(objArr, hl, rb.resultIds);
}
}
- // remove nulls in case not all docs were able to be retrieved
- rb.rsp.add("highlighting", SolrPluginUtils.removeNulls(arr, new SimpleOrderedMap<>()));
+ rb.rsp.add(highlightingResponseField, getAllHighlights(objArr));
}
}
@@ -279,4 +279,33 @@ public class HighlightComponent extends SearchComponent implements PluginInfoIni
public Category getCategory() {
return Category.HIGHLIGHTER;
}
+
+ ////////////////////////////////////////////
+ /// highlighting response collation
+ ////////////////////////////////////////////
+
+ protected String highlightingResponseField() {
+ return "highlighting";
+ }
+
+ protected Object convertHighlights(NamedList hl) {
+ return hl;
+ }
+
+ protected Object[] newHighlightsArray(int size) {
+ return new NamedList.NamedListEntry[size];
+ }
+
+ protected void addHighlights(Object[] objArr, Object obj, Map<Object, ShardDoc> resultIds) {
+ Map.Entry<String, Object>[] arr = (Map.Entry<String, Object>[])objArr;
+ NamedList hl = (NamedList)obj;
+ SolrPluginUtils.copyNamedListIntoArrayByDocPosInResponse(hl, resultIds, arr);
+ }
+
+ protected Object getAllHighlights(Object[] objArr) {
+ final Map.Entry<String, Object>[] arr = (Map.Entry<String, Object>[])objArr;
+ // remove nulls in case not all docs were able to be retrieved
+ return SolrPluginUtils.removeNulls(arr, new SimpleOrderedMap<>());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65c842f9/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java
new file mode 100644
index 0000000..5f74064
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.cloud.TestCloudSearcherWarming.ConfigRequest;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.highlight.SolrFragmentsBuilder;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class CustomHighlightComponentTest extends SolrCloudTestCase {
+
+ public static class CustomHighlightComponent extends HighlightComponent {
+
+ protected String id_key = "id";
+ protected String snippets_key = "snippets";
+
+ @Override
+ protected String highlightingResponseField() {
+ return "custom_highlighting";
+ }
+
+ @Override
+ protected Object convertHighlights(NamedList hl) {
+ final ArrayList<SimpleOrderedMap> hlMaps = new ArrayList<>();
+ for (int i=0; i<hl.size(); ++i) {
+ SimpleOrderedMap hlMap = new SimpleOrderedMap<Object>();
+ hlMap.add(id_key, hl.getName(i));
+ hlMap.add(snippets_key, hl.getVal(i));
+ hlMaps.add(hlMap);
+ }
+ return hlMaps;
+ }
+
+ @Override
+ protected Object[] newHighlightsArray(int size) {
+ return new SimpleOrderedMap[size];
+ }
+
+ @Override
+ protected void addHighlights(Object[] objArr, Object obj, Map<Object, ShardDoc> resultIds) {
+ SimpleOrderedMap[] mapArr = (SimpleOrderedMap[])objArr;
+ final ArrayList<SimpleOrderedMap> hlMaps = (ArrayList<SimpleOrderedMap>)obj;
+ for (SimpleOrderedMap hlMap : hlMaps) {
+ String id = (String)hlMap.get(id_key);
+ ShardDoc sdoc = resultIds.get(id);
+ int idx = sdoc.positionInResponse;
+ mapArr[idx] = hlMap;
+ }
+ }
+
+ @Override
+ protected Object getAllHighlights(Object[] objArr) {
+ final SimpleOrderedMap[] mapArr = (SimpleOrderedMap[])objArr;
+ // remove nulls in case not all docs were able to be retrieved
+ ArrayList<SimpleOrderedMap> mapList = new ArrayList<>();
+ for (SimpleOrderedMap map : mapArr) {
+ if (map != null) {
+ mapList.add((SimpleOrderedMap)map);
+ }
+ }
+ return mapList;
+ }
+
+ }
+
+ protected String customHighlightComponentClassName() {
+ return CustomHighlightComponent.class.getName();
+ }
+
+ protected String id_key = "id";
+ protected String snippets_key = "snippets";
+
+ private static String COLLECTION;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+
+ // decide collection name ...
+ COLLECTION = "collection"+(1+random().nextInt(100)) ;
+ // ... and shard/replica/node numbers
+ final int numShards = 3;
+ final int numReplicas = 2;
+ final int maxShardsPerNode = 2;
+ final int nodeCount = (numShards*numReplicas + (maxShardsPerNode-1))/maxShardsPerNode;
+
+ // create and configure cluster
+ configureCluster(nodeCount)
+ .addConfig("conf", configset("cloud-dynamic"))
+ .configure();
+
+ // create an empty collection
+ CollectionAdminRequest
+ .createCollection(COLLECTION, "conf", numShards, numReplicas)
+ .setMaxShardsPerNode(maxShardsPerNode)
+ .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(), false, true, DEFAULT_TIMEOUT);
+ }
+
+ @Test
+ public void test() throws Exception {
+
+ // determine custom search handler name (the exact name should not matter)
+ final String customSearchHandlerName = "/custom_select"+random().nextInt();
+
+ final String defaultHighlightComponentName = HighlightComponent.COMPONENT_NAME;
+ final String highlightComponentName;
+
+ // add custom component (if needed) and handler
+ {
+ if (random().nextBoolean()) {
+ // default component
+ highlightComponentName = defaultHighlightComponentName;
+ } else {
+ // custom component
+ highlightComponentName = "customhighlight"+random().nextInt();
+ cluster.getSolrClient().request(
+ new ConfigRequest(
+ SolrRequest.METHOD.POST,
+ "/config",
+ "{\n" +
+ " 'add-searchcomponent': {\n" +
+ " 'name': '"+highlightComponentName+"',\n" +
+ " 'class': '"+customHighlightComponentClassName()+"'\n" +
+ " }\n" +
+ "}"),
+ COLLECTION);
+ }
+ // handler
+ cluster.getSolrClient().request(
+ new ConfigRequest(
+ SolrRequest.METHOD.POST,
+ "/config",
+ "{\n" +
+ " 'add-requesthandler': {\n" +
+ " 'name' : '"+customSearchHandlerName+"',\n" +
+ " 'class' : 'org.apache.solr.handler.component.SearchHandler',\n" +
+ " 'components' : [ '"+QueryComponent.COMPONENT_NAME+"', '"+highlightComponentName+"' ]\n" +
+ " }\n" +
+ "}"),
+ COLLECTION);
+ }
+
+ // add some documents
+ final String id = "id";
+ final String t1 = "a_t";
+ final String t2 = "b_t";
+ {
+ new UpdateRequest()
+ .add(sdoc(id, 1, t1, "bumble bee", t2, "bumble bee"))
+ .add(sdoc(id, 2, t1, "honey bee", t2, "honey bee"))
+ .add(sdoc(id, 3, t1, "solitary bee", t2, "solitary bee"))
+ .commit(cluster.getSolrClient(), COLLECTION);
+ }
+
+ // search for the documents
+ {
+ // compose the query
+ final SolrQuery solrQuery = new SolrQuery(t1+":bee");
+ solrQuery.setRequestHandler(customSearchHandlerName);
+ solrQuery.setHighlight(true);
+ final boolean t1Highlights = random().nextBoolean();
+ if (t1Highlights) {
+ solrQuery.addHighlightField(t1);
+ }
+ final boolean t2Highlights = random().nextBoolean();
+ if (t2Highlights) {
+ solrQuery.addHighlightField(t2);
+ }
+
+ // make the query
+ final QueryResponse queryResponse = new QueryRequest(solrQuery)
+ .process(cluster.getSolrClient(), COLLECTION);
+
+ // analyse the response
+ final Map<String, Map<String, List<String>>> highlighting = queryResponse.getHighlighting();
+ final ArrayList<SimpleOrderedMap<Object>> custom_highlighting =
+ (ArrayList<SimpleOrderedMap<Object>>)queryResponse.getResponse().get("custom_highlighting");
+
+ if (defaultHighlightComponentName.equals(highlightComponentName)) {
+ // regular 'highlighting' ...
+ if (t1Highlights) {
+ checkHighlightingResponseMap(highlighting, t1);
+ }
+ if (t2Highlights) {
+ checkHighlightingResponseMap(highlighting, t2);
+ }
+ if (!t1Highlights && !t2Highlights) {
+ checkHighlightingResponseMap(highlighting, null);
+ }
+ // ... and no 'custom_highlighting'
+ assertNull(custom_highlighting);
+ } else {
+ // no regular 'highlighting' ...
+ assertNull(highlighting);
+ // ... but 'custom_highlighting'
+ assertNotNull(custom_highlighting);
+ if (t1Highlights) {
+ checkHighlightingResponseList(custom_highlighting, t1);
+ }
+ if (t2Highlights) {
+ checkHighlightingResponseList(custom_highlighting, t2);
+ }
+ if (!t1Highlights && !t2Highlights) {
+ checkHighlightingResponseList(custom_highlighting, null);
+ }
+ }
+ }
+ }
+
+ protected void checkHighlightingResponseMap(Map<String, Map<String, List<String>>> highlightingMap,
+ String highlightedField) throws Exception {
+ assertEquals("too few or too many keys: "+highlightingMap.keySet(),
+ 3, highlightingMap.size());
+ checkHighlightingResponseMapElement(highlightingMap.get("1"), highlightedField, "bumble ", "bee");
+ checkHighlightingResponseMapElement(highlightingMap.get("2"), highlightedField, "honey ", "bee");
+ checkHighlightingResponseMapElement(highlightingMap.get("3"), highlightedField, "solitary ", "bee");
+ }
+
+ protected void checkHighlightingResponseMapElement(Map<String, List<String>> docHighlights,
+ String highlightedField, String preHighlightText, String highlightedText) throws Exception {
+ if (highlightedField == null) {
+ assertEquals(0, docHighlights.size());
+ } else {
+ List<String> docHighlightsList = docHighlights.get(highlightedField);
+ assertEquals(1, docHighlightsList.size());
+ assertEquals(preHighlightText
+ + SolrFragmentsBuilder.DEFAULT_PRE_TAGS
+ + highlightedText
+ + SolrFragmentsBuilder.DEFAULT_POST_TAGS, docHighlightsList.get(0));
+ }
+ }
+
+ protected void checkHighlightingResponseList(ArrayList<SimpleOrderedMap<Object>> highlightingList,
+ String highlightedField) throws Exception {
+ assertEquals("too few or too many elements: "+highlightingList.size(),
+ 3, highlightingList.size());
+ final Set<String> seenDocIds = new HashSet<>();
+ for (SimpleOrderedMap<Object> highlightingListElementMap : highlightingList) {
+ final String expectedHighlightText;
+ final String actualHighlightText;
+ // two elements in total: id and snippets
+ assertEquals(highlightingList.toString(), 2, highlightingListElementMap.size());
+ // id element
+ {
+ final String docId = (String)highlightingListElementMap.get(id_key);
+ seenDocIds.add(docId);
+ final String preHighlightText;
+ final String highlightedText = "bee";
+ if ("1".equals(docId)) {
+ preHighlightText = "bumble ";
+ } else if ("2".equals(docId)) {
+ preHighlightText = "honey ";
+ } else if ("3".equals(docId)) {
+ preHighlightText = "solitary ";
+ } else {
+ preHighlightText = null;
+ fail("unknown docId "+docId);
+ }
+ expectedHighlightText = preHighlightText
+ + SolrFragmentsBuilder.DEFAULT_PRE_TAGS
+ + highlightedText
+ + SolrFragmentsBuilder.DEFAULT_POST_TAGS;
+ }
+ // snippets element
+ {
+ SimpleOrderedMap<Object> snippets = (SimpleOrderedMap<Object>)highlightingListElementMap.get(snippets_key);
+ if (highlightedField == null) {
+ assertEquals(0, snippets.size());
+ } else {
+ ArrayList<String> docHighlights = (ArrayList<String>)(snippets).get(highlightedField);
+ assertEquals(1, docHighlights.size());
+ actualHighlightText = (String)docHighlights.get(0);
+ assertEquals(expectedHighlightText, actualHighlightText);
+ }
+ }
+ }
+ assertEquals(3, seenDocIds.size());
+ }
+
+}
[19/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11819: The schema
api reference manual page should not use hyphens in the example field name
Posted by da...@apache.org.
SOLR-11819: The schema api reference manual page should not use hyphens in the example field name
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1f09d5b3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1f09d5b3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1f09d5b3
Branch: refs/heads/jira/solr-11702
Commit: 1f09d5b350e08008429b8b1fe4def9960fc0e30f
Parents: 5a08fa8
Author: Erick Erickson <er...@apache.org>
Authored: Sat Jan 6 20:25:14 2018 -0800
Committer: Erick Erickson <er...@apache.org>
Committed: Sat Jan 6 20:25:14 2018 -0800
----------------------------------------------------------------------
solr/solr-ref-guide/src/schema-api.adoc | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f09d5b3/solr/solr-ref-guide/src/schema-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/schema-api.adoc b/solr/solr-ref-guide/src/schema-api.adoc
index f1ef1a4..7a61857 100644
--- a/solr/solr-ref-guide/src/schema-api.adoc
+++ b/solr/solr-ref-guide/src/schema-api.adoc
@@ -97,7 +97,7 @@ The `add-field` command adds a new field definition to your schema. If a field w
All of the properties available when defining a field with manual `schema.xml` edits can be passed via the API. These request attributes are described in detail in the section <<defining-fields.adoc#defining-fields,Defining Fields>>.
-For example, to define a new stored field named "sell-by", of type "pdate", you would POST the following request:
+For example, to define a new stored field named "sell_by", of type "pdate", you would POST the following request:
[.dynamic-tabs]
--
@@ -108,7 +108,7 @@ For example, to define a new stored field named "sell-by", of type "pdate", you
----
curl -X POST -H 'Content-type:application/json' --data-binary '{
"add-field":{
- "name":"sell-by",
+ "name":"sell_by",
"type":"pdate",
"stored":true }
}' http://localhost:8983/solr/gettingstarted/schema
@@ -122,7 +122,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
----
curl -X POST -H 'Content-type:application/json' --data-binary '{
"add-field":{
- "name":"sell-by",
+ "name":"sell_by",
"type":"pdate",
"stored":true }
}' http://localhost:8983/api/cores/gettingstarted/schema
@@ -134,7 +134,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
The `delete-field` command removes a field definition from your schema. If the field does not exist in the schema, or if the field is the source or destination of a copy field rule, an error is thrown.
-For example, to delete a field named "sell-by", you would POST the following request:
+For example, to delete a field named "sell_by", you would POST the following request:
[.dynamic-tabs]
--
@@ -144,7 +144,7 @@ For example, to delete a field named "sell-by", you would POST the following req
[source,bash]
----
curl -X POST -H 'Content-type:application/json' --data-binary '{
- "delete-field" : { "name":"sell-by" }
+ "delete-field" : { "name":"sell_by" }
}' http://localhost:8983/solr/gettingstarted/schema
----
====
@@ -155,7 +155,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
[source,bash]
----
curl -X POST -H 'Content-type:application/json' --data-binary '{
- "delete-field" : { "name":"sell-by" }
+ "delete-field" : { "name":"sell_by" }
}' http://localhost:8983/api/cores/gettingstarted/schema
----
====
@@ -167,7 +167,7 @@ The `replace-field` command replaces a field's definition. Note that you must su
All of the properties available when defining a field with manual `schema.xml` edits can be passed via the API. These request attributes are described in detail in the section <<defining-fields.adoc#defining-fields,Defining Fields>>.
-For example, to replace the definition of an existing field "sell-by", to make it be of type "date" and to not be stored, you would POST the following request:
+For example, to replace the definition of an existing field "sell_by", to make it be of type "date" and to not be stored, you would POST the following request:
[.dynamic-tabs]
--
@@ -178,7 +178,7 @@ For example, to replace the definition of an existing field "sell-by", to make i
----
curl -X POST -H 'Content-type:application/json' --data-binary '{
"replace-field":{
- "name":"sell-by",
+ "name":"sell_by",
"type":"date",
"stored":false }
}' http://localhost:8983/solr/gettingstarted/schema
@@ -192,7 +192,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
----
curl -X POST -H 'Content-type:application/json' --data-binary '{
"replace-field":{
- "name":"sell-by",
+ "name":"sell_by",
"type":"date",
"stored":false }
}' http://localhost:8983/api/cores/gettingstarted/schema
@@ -572,7 +572,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
"class":"solr.WordDelimiterFilterFactory",
"preserveOriginal":"0" }]}},
"add-field" : {
- "name":"sell-by",
+ "name":"sell_by",
"type":"myNewTxtField",
"stored":true }
}' http://localhost:8983/solr/gettingstarted/schema
[12/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8133: Rename
TermContext to TermStates,
and load TermState lazily if term stats are not required
Posted by da...@apache.org.
LUCENE-8133: Rename TermContext to TermStates, and load TermState lazily if term stats are not required
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d250a146
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d250a146
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d250a146
Branch: refs/heads/jira/solr-11702
Commit: d250a1463d87380420afc90e381623c0dc470695
Parents: 0c18acb
Author: Alan Woodward <ro...@apache.org>
Authored: Tue Jan 2 09:52:21 2018 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Fri Jan 5 14:17:15 2018 +0000
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../classification/utils/NearestFuzzyQuery.java | 12 +-
.../org/apache/lucene/index/TermContext.java | 197 ----------------
.../org/apache/lucene/index/TermStates.java | 234 +++++++++++++++++++
.../apache/lucene/search/BlendedTermQuery.java | 28 +--
.../org/apache/lucene/search/IndexSearcher.java | 4 +-
.../org/apache/lucene/search/LeafSimScorer.java | 2 +-
.../apache/lucene/search/MultiPhraseQuery.java | 22 +-
.../apache/lucene/search/MultiTermQuery.java | 8 +-
.../MultiTermQueryConstantScoreWrapper.java | 8 +-
.../org/apache/lucene/search/PhraseQuery.java | 18 +-
.../apache/lucene/search/ScoringRewrite.java | 14 +-
.../org/apache/lucene/search/SynonymQuery.java | 14 +-
.../lucene/search/TermCollectingRewrite.java | 4 +-
.../apache/lucene/search/TermInSetQuery.java | 8 +-
.../org/apache/lucene/search/TermQuery.java | 56 ++---
.../apache/lucene/search/TopTermsRewrite.java | 10 +-
.../lucene/search/spans/SpanContainQuery.java | 10 +-
.../search/spans/SpanContainingQuery.java | 10 +-
.../search/spans/SpanMultiTermQueryWrapper.java | 6 +-
.../lucene/search/spans/SpanNearQuery.java | 14 +-
.../lucene/search/spans/SpanNotQuery.java | 12 +-
.../apache/lucene/search/spans/SpanOrQuery.java | 12 +-
.../search/spans/SpanPositionCheckQuery.java | 12 +-
.../apache/lucene/search/spans/SpanQuery.java | 18 +-
.../lucene/search/spans/SpanTermQuery.java | 36 +--
.../apache/lucene/search/spans/SpanWeight.java | 24 +-
.../lucene/search/spans/SpanWithinQuery.java | 10 +-
.../lucene/search/TestMinShouldMatch2.java | 4 +-
.../org/apache/lucene/search/TestTermQuery.java | 6 +-
.../highlight/WeightedSpanTermExtractor.java | 2 +-
.../apache/lucene/queries/CommonTermsQuery.java | 40 ++--
.../queries/payloads/PayloadScoreQuery.java | 6 +-
.../queries/payloads/SpanPayloadCheckQuery.java | 14 +-
.../lucene/queries/CommonTermsQueryTest.java | 6 +-
.../sandbox/queries/FuzzyLikeThisQuery.java | 8 +-
.../lucene/search/TermAutomatonQuery.java | 18 +-
.../lucene/search/ShardSearchingTestBase.java | 8 +-
.../search/spans/AssertingSpanWeight.java | 6 +-
.../solr/handler/component/TermsComponent.java | 26 +--
.../org/apache/solr/query/SolrRangeQuery.java | 8 +-
.../solr/search/GraphTermsQParserPlugin.java | 38 +--
.../apache/solr/search/SolrIndexSearcher.java | 6 +-
.../solr/search/stats/ExactStatsCache.java | 8 +-
.../apache/solr/search/stats/LRUStatsCache.java | 4 +-
.../solr/search/stats/LocalStatsSource.java | 4 +-
.../apache/solr/search/stats/StatsSource.java | 4 +-
47 files changed, 522 insertions(+), 500 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 16050d1..89afdfd 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -35,6 +35,9 @@ API Changes
* LUCENE-8116: SimScorer now only takes a frequency and a norm as per-document
scoring factors. (Adrien Grand)
+* LUCENE-8113: TermContext has been renamed to TermStates, and can now be
+ constructed lazily if term statistics are not required (Alan Woodward)
+
Changes in Runtime Behavior
* LUCENE-7837: Indices that were created before the previous major version
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/classification/src/java/org/apache/lucene/classification/utils/NearestFuzzyQuery.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/utils/NearestFuzzyQuery.java b/lucene/classification/src/java/org/apache/lucene/classification/utils/NearestFuzzyQuery.java
index d4a2634..308dcdc 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/utils/NearestFuzzyQuery.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/utils/NearestFuzzyQuery.java
@@ -29,7 +29,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
@@ -210,20 +210,20 @@ public class NearestFuzzyQuery extends Query {
}
private Query newTermQuery(IndexReader reader, Term term) throws IOException {
- // we build an artificial TermContext that will give an overall df and ttf
+ // we build an artificial TermStates that will give an overall df and ttf
// equal to 1
- TermContext context = new TermContext(reader.getContext());
+ TermStates termStates = new TermStates(reader.getContext());
for (LeafReaderContext leafContext : reader.leaves()) {
Terms terms = leafContext.reader().terms(term.field());
if (terms != null) {
TermsEnum termsEnum = terms.iterator();
if (termsEnum.seekExact(term.bytes())) {
- int freq = 1 - context.docFreq(); // we want the total df and ttf to be 1
- context.register(termsEnum.termState(), leafContext.ord, freq, freq);
+ int freq = 1 - termStates.docFreq(); // we want the total df and ttf to be 1
+ termStates.register(termsEnum.termState(), leafContext.ord, freq, freq);
}
}
}
- return new TermQuery(term, context);
+ return new TermQuery(term, termStates);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/index/TermContext.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/TermContext.java b/lucene/core/src/java/org/apache/lucene/index/TermContext.java
deleted file mode 100644
index 3ba8dd9..0000000
--- a/lucene/core/src/java/org/apache/lucene/index/TermContext.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.index;
-
-
-import org.apache.lucene.util.BytesRef;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-/**
- * Maintains a {@link IndexReader} {@link TermState} view over
- * {@link IndexReader} instances containing a single term. The
- * {@link TermContext} doesn't track if the given {@link TermState}
- * objects are valid, neither if the {@link TermState} instances refer to the
- * same terms in the associated readers.
- *
- * @lucene.experimental
- */
-public final class TermContext {
-
- // Important: do NOT keep hard references to index readers
- private final Object topReaderContextIdentity;
- private final TermState[] states;
- private int docFreq;
- private long totalTermFreq;
-
- //public static boolean DEBUG = BlockTreeTermsWriter.DEBUG;
-
- /**
- * Creates an empty {@link TermContext} from a {@link IndexReaderContext}
- */
- public TermContext(IndexReaderContext context) {
- assert context != null && context.isTopLevel;
- topReaderContextIdentity = context.identity;
- docFreq = 0;
- totalTermFreq = 0;
- final int len;
- if (context.leaves() == null) {
- len = 1;
- } else {
- len = context.leaves().size();
- }
- states = new TermState[len];
- }
-
- /**
- * Expert: Return whether this {@link TermContext} was built for the given
- * {@link IndexReaderContext}. This is typically used for assertions.
- * @lucene.internal
- */
- public boolean wasBuiltFor(IndexReaderContext context) {
- return topReaderContextIdentity == context.identity;
- }
-
- /**
- * Creates a {@link TermContext} with an initial {@link TermState},
- * {@link IndexReader} pair.
- */
- public TermContext(IndexReaderContext context, TermState state, int ord, int docFreq, long totalTermFreq) {
- this(context);
- register(state, ord, docFreq, totalTermFreq);
- }
-
- /**
- * Creates a {@link TermContext} from a top-level {@link IndexReaderContext} and the
- * given {@link Term}. This method will lookup the given term in all context's leaf readers
- * and register each of the readers containing the term in the returned {@link TermContext}
- * using the leaf reader's ordinal.
- * <p>
- * Note: the given context must be a top-level context.
- */
- public static TermContext build(IndexReaderContext context, Term term)
- throws IOException {
- assert context != null && context.isTopLevel;
- final String field = term.field();
- final BytesRef bytes = term.bytes();
- final TermContext perReaderTermState = new TermContext(context);
- //if (DEBUG) System.out.println("prts.build term=" + term);
- for (final LeafReaderContext ctx : context.leaves()) {
- //if (DEBUG) System.out.println(" r=" + leaves[i].reader);
- final Terms terms = ctx.reader().terms(field);
- if (terms != null) {
- final TermsEnum termsEnum = terms.iterator();
- if (termsEnum.seekExact(bytes)) {
- final TermState termState = termsEnum.termState();
- //if (DEBUG) System.out.println(" found");
- perReaderTermState.register(termState, ctx.ord, termsEnum.docFreq(), termsEnum.totalTermFreq());
- }
- }
- }
- return perReaderTermState;
- }
-
- /**
- * Clears the {@link TermContext} internal state and removes all
- * registered {@link TermState}s
- */
- public void clear() {
- docFreq = 0;
- totalTermFreq = 0;
- Arrays.fill(states, null);
- }
-
- /**
- * Registers and associates a {@link TermState} with an leaf ordinal. The leaf ordinal
- * should be derived from a {@link IndexReaderContext}'s leaf ord.
- */
- public void register(TermState state, final int ord, final int docFreq, final long totalTermFreq) {
- register(state, ord);
- accumulateStatistics(docFreq, totalTermFreq);
- }
-
- /**
- * Expert: Registers and associates a {@link TermState} with an leaf ordinal. The
- * leaf ordinal should be derived from a {@link IndexReaderContext}'s leaf ord.
- * On the contrary to {@link #register(TermState, int, int, long)} this method
- * does NOT update term statistics.
- */
- public void register(TermState state, final int ord) {
- assert state != null : "state must not be null";
- assert ord >= 0 && ord < states.length;
- assert states[ord] == null : "state for ord: " + ord
- + " already registered";
- states[ord] = state;
- }
-
- /** Expert: Accumulate term statistics. */
- public void accumulateStatistics(final int docFreq, final long totalTermFreq) {
- assert docFreq >= 0;
- assert totalTermFreq >= 0;
- assert docFreq <= totalTermFreq;
- this.docFreq += docFreq;
- this.totalTermFreq += totalTermFreq;
- }
-
- /**
- * Returns the {@link TermState} for an leaf ordinal or <code>null</code> if no
- * {@link TermState} for the ordinal was registered.
- *
- * @param ord
- * the readers leaf ordinal to get the {@link TermState} for.
- * @return the {@link TermState} for the given readers ord or <code>null</code> if no
- * {@link TermState} for the reader was registered
- */
- public TermState get(int ord) {
- assert ord >= 0 && ord < states.length;
- return states[ord];
- }
-
- /**
- * Returns the accumulated document frequency of all {@link TermState}
- * instances passed to {@link #register(TermState, int, int, long)}.
- * @return the accumulated document frequency of all {@link TermState}
- * instances passed to {@link #register(TermState, int, int, long)}.
- */
- public int docFreq() {
- return docFreq;
- }
-
- /**
- * Returns the accumulated term frequency of all {@link TermState}
- * instances passed to {@link #register(TermState, int, int, long)}.
- * @return the accumulated term frequency of all {@link TermState}
- * instances passed to {@link #register(TermState, int, int, long)}.
- */
- public long totalTermFreq() {
- return totalTermFreq;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("TermContext\n");
- for(TermState termState : states) {
- sb.append(" state=");
- sb.append(termState.toString());
- sb.append('\n');
- }
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/index/TermStates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/TermStates.java b/lucene/core/src/java/org/apache/lucene/index/TermStates.java
new file mode 100644
index 0000000..3282ac8
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/TermStates.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Maintains a {@link IndexReader} {@link TermState} view over
+ * {@link IndexReader} instances containing a single term. The
+ * {@link TermStates} doesn't track if the given {@link TermState}
+ * objects are valid, neither if the {@link TermState} instances refer to the
+ * same terms in the associated readers.
+ *
+ * @lucene.experimental
+ */
+public final class TermStates {
+
+ private static final TermState EMPTY_TERMSTATE = new TermState() {
+ @Override
+ public void copyFrom(TermState other) {
+
+ }
+ };
+
+ // Important: do NOT keep hard references to index readers
+ private final Object topReaderContextIdentity;
+ private final TermState[] states;
+ private final Term term; // null if stats are to be used
+ private int docFreq;
+ private long totalTermFreq;
+
+ //public static boolean DEBUG = BlockTreeTermsWriter.DEBUG;
+
+ private TermStates(Term term, IndexReaderContext context) {
+ assert context != null && context.isTopLevel;
+ topReaderContextIdentity = context.identity;
+ docFreq = 0;
+ totalTermFreq = 0;
+ final int len;
+ if (context.leaves() == null) {
+ len = 1;
+ } else {
+ len = context.leaves().size();
+ }
+ states = new TermState[len];
+ this.term = term;
+ }
+
+ /**
+ * Creates an empty {@link TermStates} from a {@link IndexReaderContext}
+ */
+ public TermStates(IndexReaderContext context) {
+ this(null, context);
+ }
+
+ /**
+ * Expert: Return whether this {@link TermStates} was built for the given
+ * {@link IndexReaderContext}. This is typically used for assertions.
+ * @lucene.internal
+ */
+ public boolean wasBuiltFor(IndexReaderContext context) {
+ return topReaderContextIdentity == context.identity;
+ }
+
+ /**
+ * Creates a {@link TermStates} with an initial {@link TermState},
+ * {@link IndexReader} pair.
+ */
+ public TermStates(IndexReaderContext context, TermState state, int ord, int docFreq, long totalTermFreq) {
+ this(null, context);
+ register(state, ord, docFreq, totalTermFreq);
+ }
+
+ /**
+ * Creates a {@link TermStates} from a top-level {@link IndexReaderContext} and the
+ * given {@link Term}. This method will lookup the given term in all context's leaf readers
+ * and register each of the readers containing the term in the returned {@link TermStates}
+ * using the leaf reader's ordinal.
+ * <p>
+ * Note: the given context must be a top-level context.
+ *
+ * @param needsStats if {@code true} then all leaf contexts will be visited up-front to
+ * collect term statistics. Otherwise, the {@link TermState} objects
+ * will be built only when requested
+ */
+ public static TermStates build(IndexReaderContext context, Term term, boolean needsStats)
+ throws IOException {
+ assert context != null && context.isTopLevel;
+ final TermStates perReaderTermState = new TermStates(needsStats ? null : term, context);
+ if (needsStats) {
+ for (final LeafReaderContext ctx : context.leaves()) {
+ //if (DEBUG) System.out.println(" r=" + leaves[i].reader);
+ TermsEnum termsEnum = loadTermsEnum(ctx, term);
+ if (termsEnum != null) {
+ final TermState termState = termsEnum.termState();
+ //if (DEBUG) System.out.println(" found");
+ perReaderTermState.register(termState, ctx.ord, termsEnum.docFreq(), termsEnum.totalTermFreq());
+ }
+ }
+ }
+ return perReaderTermState;
+ }
+
+ private static TermsEnum loadTermsEnum(LeafReaderContext ctx, Term term) throws IOException {
+ final Terms terms = ctx.reader().terms(term.field());
+ if (terms != null) {
+ final TermsEnum termsEnum = terms.iterator();
+ if (termsEnum.seekExact(term.bytes())) {
+ return termsEnum;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Clears the {@link TermStates} internal state and removes all
+ * registered {@link TermState}s
+ */
+ public void clear() {
+ docFreq = 0;
+ totalTermFreq = 0;
+ Arrays.fill(states, null);
+ }
+
+ /**
+ * Registers and associates a {@link TermState} with an leaf ordinal. The leaf ordinal
+ * should be derived from a {@link IndexReaderContext}'s leaf ord.
+ */
+ public void register(TermState state, final int ord, final int docFreq, final long totalTermFreq) {
+ register(state, ord);
+ accumulateStatistics(docFreq, totalTermFreq);
+ }
+
+ /**
+ * Expert: Registers and associates a {@link TermState} with an leaf ordinal. The
+ * leaf ordinal should be derived from a {@link IndexReaderContext}'s leaf ord.
+ * On the contrary to {@link #register(TermState, int, int, long)} this method
+ * does NOT update term statistics.
+ */
+ public void register(TermState state, final int ord) {
+ assert state != null : "state must not be null";
+ assert ord >= 0 && ord < states.length;
+ assert states[ord] == null : "state for ord: " + ord
+ + " already registered";
+ states[ord] = state;
+ }
+
+ /** Expert: Accumulate term statistics. */
+ public void accumulateStatistics(final int docFreq, final long totalTermFreq) {
+ assert docFreq >= 0;
+ assert totalTermFreq >= 0;
+ assert docFreq <= totalTermFreq;
+ this.docFreq += docFreq;
+ this.totalTermFreq += totalTermFreq;
+ }
+
+ /**
+ * Returns the {@link TermState} for a leaf reader context or <code>null</code> if no
+ * {@link TermState} for the context was registered.
+ *
+ * @param ctx
+ * the {@link LeafReaderContext} to get the {@link TermState} for.
+ * @return the {@link TermState} for the given readers ord or <code>null</code> if no
+ * {@link TermState} for the reader was registered
+ */
+ public TermState get(LeafReaderContext ctx) throws IOException {
+ assert ctx.ord >= 0 && ctx.ord < states.length;
+ if (term == null)
+ return states[ctx.ord];
+ if (this.states[ctx.ord] == null) {
+ TermsEnum te = loadTermsEnum(ctx, term);
+ this.states[ctx.ord] = te == null ? EMPTY_TERMSTATE : te.termState();
+ }
+ if (this.states[ctx.ord] == EMPTY_TERMSTATE)
+ return null;
+ return this.states[ctx.ord];
+ }
+
+ /**
+ * Returns the accumulated document frequency of all {@link TermState}
+ * instances passed to {@link #register(TermState, int, int, long)}.
+ * @return the accumulated document frequency of all {@link TermState}
+ * instances passed to {@link #register(TermState, int, int, long)}.
+ */
+ public int docFreq() {
+ if (term != null) {
+ throw new IllegalStateException("Cannot call docFreq() when needsStats=false");
+ }
+ return docFreq;
+ }
+
+ /**
+ * Returns the accumulated term frequency of all {@link TermState}
+ * instances passed to {@link #register(TermState, int, int, long)}.
+ * @return the accumulated term frequency of all {@link TermState}
+ * instances passed to {@link #register(TermState, int, int, long)}.
+ */
+ public long totalTermFreq() {
+ if (term != null) {
+ throw new IllegalStateException("Cannot call totalTermFreq() when needsStats=false");
+ }
+ return totalTermFreq;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("TermStates\n");
+ for(TermState termState : states) {
+ sb.append(" state=");
+ sb.append(termState.toString());
+ sb.append('\n');
+ }
+
+ return sb.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java b/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java
index 219d453..cca6675 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java
@@ -25,7 +25,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.util.ArrayUtil;
@@ -53,7 +53,7 @@ public final class BlendedTermQuery extends Query {
private int numTerms = 0;
private Term[] terms = new Term[0];
private float[] boosts = new float[0];
- private TermContext[] contexts = new TermContext[0];
+ private TermStates[] contexts = new TermStates[0];
private RewriteMethod rewriteMethod = DISJUNCTION_MAX_REWRITE;
/** Sole constructor. */
@@ -82,10 +82,10 @@ public final class BlendedTermQuery extends Query {
/**
* Expert: Add a {@link Term} with the provided boost and context.
- * This method is useful if you already have a {@link TermContext}
+ * This method is useful if you already have a {@link TermStates}
* object constructed for the given term.
*/
- public Builder add(Term term, float boost, TermContext context) {
+ public Builder add(Term term, float boost, TermStates context) {
if (numTerms >= BooleanQuery.getMaxClauseCount()) {
throw new BooleanQuery.TooManyClauses();
}
@@ -184,10 +184,10 @@ public final class BlendedTermQuery extends Query {
private final Term[] terms;
private final float[] boosts;
- private final TermContext[] contexts;
+ private final TermStates[] contexts;
private final RewriteMethod rewriteMethod;
- private BlendedTermQuery(Term[] terms, float[] boosts, TermContext[] contexts,
+ private BlendedTermQuery(Term[] terms, float[] boosts, TermStates[] contexts,
RewriteMethod rewriteMethod) {
assert terms.length == boosts.length;
assert terms.length == contexts.length;
@@ -205,7 +205,7 @@ public final class BlendedTermQuery extends Query {
terms[i] = terms[j];
terms[j] = tmpTerm;
- TermContext tmpContext = contexts[i];
+ TermStates tmpContext = contexts[i];
contexts[i] = contexts[j];
contexts[j] = tmpContext;
@@ -263,10 +263,10 @@ public final class BlendedTermQuery extends Query {
@Override
public final Query rewrite(IndexReader reader) throws IOException {
- final TermContext[] contexts = Arrays.copyOf(this.contexts, this.contexts.length);
+ final TermStates[] contexts = Arrays.copyOf(this.contexts, this.contexts.length);
for (int i = 0; i < contexts.length; ++i) {
if (contexts[i] == null || contexts[i].wasBuiltFor(reader.getContext()) == false) {
- contexts[i] = TermContext.build(reader.getContext(), terms[i]);
+ contexts[i] = TermStates.build(reader.getContext(), terms[i], true);
}
}
@@ -275,7 +275,7 @@ public final class BlendedTermQuery extends Query {
// ttf will be the sum of all total term freqs
int df = 0;
long ttf = 0;
- for (TermContext ctx : contexts) {
+ for (TermStates ctx : contexts) {
df = Math.max(df, ctx.docFreq());
ttf += ctx.totalTermFreq();
}
@@ -294,8 +294,8 @@ public final class BlendedTermQuery extends Query {
return rewriteMethod.rewrite(termQueries);
}
- private static TermContext adjustFrequencies(IndexReaderContext readerContext,
- TermContext ctx, int artificialDf, long artificialTtf) {
+ private static TermStates adjustFrequencies(IndexReaderContext readerContext,
+ TermStates ctx, int artificialDf, long artificialTtf) throws IOException {
List<LeafReaderContext> leaves = readerContext.leaves();
final int len;
if (leaves == null) {
@@ -303,9 +303,9 @@ public final class BlendedTermQuery extends Query {
} else {
len = leaves.size();
}
- TermContext newCtx = new TermContext(readerContext);
+ TermStates newCtx = new TermStates(readerContext);
for (int i = 0; i < len; ++i) {
- TermState termState = ctx.get(i);
+ TermState termState = ctx.get(leaves.get(i));
if (termState == null) {
continue;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
index fc87563..da5ed03 100644
--- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
+++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
@@ -39,7 +39,7 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.Terms;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.Similarity;
@@ -739,7 +739,7 @@ public class IndexSearcher {
* across a distributed collection.
* @lucene.experimental
*/
- public TermStatistics termStatistics(Term term, TermContext context) throws IOException {
+ public TermStatistics termStatistics(Term term, TermStates context) throws IOException {
if (context.docFreq() == 0) {
return null;
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java b/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java
index 52b7d92..46f9dc2 100644
--- a/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java
@@ -37,7 +37,7 @@ public final class LeafSimScorer {
public LeafSimScorer(SimScorer scorer, LeafReader reader, boolean needsScores, float maxFreq) throws IOException {
this.scorer = scorer;
norms = needsScores ? reader.getNormValues(scorer.getField()) : null;
- maxScore = scorer.maxScore(maxFreq);
+ maxScore = needsScores ? scorer.maxScore(maxFreq) : Integer.MAX_VALUE;
}
private long getNormValue(int doc) throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
index 941416e..c76f7ea 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
@@ -34,7 +34,7 @@ import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -191,7 +191,7 @@ public class MultiPhraseQuery extends Query {
private class MultiPhraseWeight extends Weight {
private final Similarity similarity;
private final Similarity.SimScorer stats;
- private final Map<Term,TermContext> termContexts = new HashMap<>();
+ private final Map<Term,TermStates> termStates = new HashMap<>();
private final ScoreMode scoreMode;
public MultiPhraseWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost)
@@ -205,14 +205,16 @@ public class MultiPhraseQuery extends Query {
ArrayList<TermStatistics> allTermStats = new ArrayList<>();
for(final Term[] terms: termArrays) {
for (Term term: terms) {
- TermContext termContext = termContexts.get(term);
- if (termContext == null) {
- termContext = TermContext.build(context, term);
- termContexts.put(term, termContext);
+ TermStates ts = termStates.get(term);
+ if (ts == null) {
+ ts = TermStates.build(context, term, scoreMode.needsScores());
+ termStates.put(term, ts);
}
- TermStatistics termStatistics = searcher.termStatistics(term, termContext);
- if (termStatistics != null) {
- allTermStats.add(termStatistics);
+ if (scoreMode.needsScores()) {
+ TermStatistics termStatistics = searcher.termStatistics(term, ts);
+ if (termStatistics != null) {
+ allTermStats.add(termStatistics);
+ }
}
}
}
@@ -260,7 +262,7 @@ public class MultiPhraseQuery extends Query {
List<PostingsEnum> postings = new ArrayList<>();
for (Term term : terms) {
- TermState termState = termContexts.get(term).get(context.ord);
+ TermState termState = termStates.get(term).get(context);
if (termState != null) {
termsEnum.seekExact(term.bytes(), termState);
postings.add(termsEnum.postings(null, PostingsEnum.POSITIONS));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java
index d0869d6..636a7d6 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java
@@ -24,7 +24,7 @@ import org.apache.lucene.index.FilteredTermsEnum; // javadocs
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.SingleTermsEnum; // javadocs
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanQuery.Builder;
@@ -166,7 +166,7 @@ public abstract class MultiTermQuery extends Query {
}
@Override
- protected void addClause(BooleanQuery.Builder topLevel, Term term, int docCount, float boost, TermContext states) {
+ protected void addClause(BooleanQuery.Builder topLevel, Term term, int docCount, float boost, TermStates states) {
final TermQuery tq = new TermQuery(term, states);
topLevel.add(new BoostQuery(tq, boost), BooleanClause.Occur.SHOULD);
}
@@ -218,7 +218,7 @@ public abstract class MultiTermQuery extends Query {
@Override
protected void addClause(BlendedTermQuery.Builder topLevel, Term term, int docCount,
- float boost, TermContext states) {
+ float boost, TermStates states) {
topLevel.add(term, boost, states);
}
}
@@ -262,7 +262,7 @@ public abstract class MultiTermQuery extends Query {
}
@Override
- protected void addClause(BooleanQuery.Builder topLevel, Term term, int docFreq, float boost, TermContext states) {
+ protected void addClause(BooleanQuery.Builder topLevel, Term term, int docFreq, float boost, TermStates states) {
final Query q = new ConstantScoreQuery(new TermQuery(term, states));
topLevel.add(new BoostQuery(q, boost), BooleanClause.Occur.SHOULD);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java
index f82316d..3a46b96 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java
@@ -25,7 +25,7 @@ import java.util.Objects;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -148,9 +148,9 @@ final class MultiTermQueryConstantScoreWrapper<Q extends MultiTermQuery> extends
// build a boolean query
BooleanQuery.Builder bq = new BooleanQuery.Builder();
for (TermAndState t : collectedTerms) {
- final TermContext termContext = new TermContext(searcher.getTopReaderContext());
- termContext.register(t.state, context.ord, t.docFreq, t.totalTermFreq);
- bq.add(new TermQuery(new Term(query.field, t.term), termContext), Occur.SHOULD);
+ final TermStates termStates = new TermStates(searcher.getTopReaderContext());
+ termStates.register(t.state, context.ord, t.docFreq, t.totalTermFreq);
+ bq.add(new TermQuery(new Term(query.field, t.term), termStates), Occur.SHOULD);
}
Query q = new ConstantScoreQuery(bq.build());
final Weight weight = searcher.rewrite(q).createWeight(searcher, scoreMode, score());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
index 295cc90..011771e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
@@ -32,7 +32,7 @@ import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -353,7 +353,7 @@ public class PhraseQuery extends Query {
private final Similarity similarity;
private final Similarity.SimScorer stats;
private final ScoreMode scoreMode;
- private transient TermContext states[];
+ private transient TermStates states[];
public PhraseWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost)
throws IOException {
@@ -367,15 +367,17 @@ public class PhraseQuery extends Query {
this.scoreMode = scoreMode;
this.similarity = searcher.getSimilarity();
final IndexReaderContext context = searcher.getTopReaderContext();
- states = new TermContext[terms.length];
+ states = new TermStates[terms.length];
TermStatistics termStats[] = new TermStatistics[terms.length];
int termUpTo = 0;
for (int i = 0; i < terms.length; i++) {
final Term term = terms[i];
- states[i] = TermContext.build(context, term);
- TermStatistics termStatistics = searcher.termStatistics(term, states[i]);
- if (termStatistics != null) {
- termStats[termUpTo++] = termStatistics;
+ states[i] = TermStates.build(context, term, scoreMode.needsScores());
+ if (scoreMode.needsScores()) {
+ TermStatistics termStatistics = searcher.termStatistics(term, states[i]);
+ if (termStatistics != null) {
+ termStats[termUpTo++] = termStatistics;
+ }
}
}
if (termUpTo > 0) {
@@ -414,7 +416,7 @@ public class PhraseQuery extends Query {
for (int i = 0; i < terms.length; i++) {
final Term t = terms[i];
- final TermState state = states[i].get(context.ord);
+ final TermState state = states[i].get(context);
if (state == null) { /* term doesnt exist in this segment */
assert termNotInReader(reader, t): "no termstate found but term exists in reader";
return null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java b/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
index 6f54a86..9d02b35 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.MultiTermQuery.RewriteMethod;
@@ -64,7 +64,7 @@ public abstract class ScoringRewrite<B> extends TermCollectingRewrite<B> {
@Override
protected void addClause(BooleanQuery.Builder topLevel, Term term, int docCount,
- float boost, TermContext states) {
+ float boost, TermStates states) {
final TermQuery tq = new TermQuery(term, states);
topLevel.add(new BoostQuery(tq, boost), BooleanClause.Occur.SHOULD);
}
@@ -109,7 +109,7 @@ public abstract class ScoringRewrite<B> extends TermCollectingRewrite<B> {
if (size > 0) {
final int sort[] = col.terms.sort();
final float[] boost = col.array.boost;
- final TermContext[] termStates = col.array.termState;
+ final TermStates[] termStates = col.array.termState;
for (int i = 0; i < size; i++) {
final int pos = sort[i];
final Term term = new Term(query.getField(), col.terms.get(pos, new BytesRef()));
@@ -146,7 +146,7 @@ public abstract class ScoringRewrite<B> extends TermCollectingRewrite<B> {
} else {
// new entry: we populate the entry initially
array.boost[e] = boostAtt.getBoost();
- array.termState[e] = new TermContext(topReaderContext, state, readerContext.ord, termsEnum.docFreq(), termsEnum.totalTermFreq());
+ array.termState[e] = new TermStates(topReaderContext, state, readerContext.ord, termsEnum.docFreq(), termsEnum.totalTermFreq());
ScoringRewrite.this.checkMaxClauseCount(terms.size());
}
return true;
@@ -156,7 +156,7 @@ public abstract class ScoringRewrite<B> extends TermCollectingRewrite<B> {
/** Special implementation of BytesStartArray that keeps parallel arrays for boost and docFreq */
static final class TermFreqBoostByteStart extends DirectBytesStartArray {
float[] boost;
- TermContext[] termState;
+ TermStates[] termState;
public TermFreqBoostByteStart(int initSize) {
super(initSize);
@@ -166,7 +166,7 @@ public abstract class ScoringRewrite<B> extends TermCollectingRewrite<B> {
public int[] init() {
final int[] ord = super.init();
boost = new float[ArrayUtil.oversize(ord.length, Float.BYTES)];
- termState = new TermContext[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ termState = new TermStates[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
assert termState.length >= ord.length && boost.length >= ord.length;
return ord;
}
@@ -176,7 +176,7 @@ public abstract class ScoringRewrite<B> extends TermCollectingRewrite<B> {
final int[] ord = super.grow();
boost = ArrayUtil.grow(boost, ord.length);
if (termState.length < ord.length) {
- TermContext[] tmpTermState = new TermContext[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ TermStates[] tmpTermState = new TermStates[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
System.arraycopy(termState, 0, tmpTermState, 0, termState.length);
termState = tmpTermState;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
index 3f4c06d..10a5d42 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
@@ -31,7 +31,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.similarities.Similarity;
@@ -126,7 +126,7 @@ public final class SynonymQuery extends Query {
}
class SynonymWeight extends Weight {
- private final TermContext termContexts[];
+ private final TermStates termStates[];
private final Similarity similarity;
private final Similarity.SimScorer simWeight;
@@ -135,10 +135,10 @@ public final class SynonymQuery extends Query {
CollectionStatistics collectionStats = searcher.collectionStatistics(terms[0].field());
long docFreq = 0;
long totalTermFreq = 0;
- termContexts = new TermContext[terms.length];
- for (int i = 0; i < termContexts.length; i++) {
- termContexts[i] = TermContext.build(searcher.getTopReaderContext(), terms[i]);
- TermStatistics termStats = searcher.termStatistics(terms[i], termContexts[i]);
+ termStates = new TermStates[terms.length];
+ for (int i = 0; i < termStates.length; i++) {
+ termStates[i] = TermStates.build(searcher.getTopReaderContext(), terms[i], true);
+ TermStatistics termStats = searcher.termStatistics(terms[i], termStates[i]);
if (termStats != null) {
docFreq = Math.max(termStats.docFreq(), docFreq);
totalTermFreq += termStats.totalTermFreq();
@@ -202,7 +202,7 @@ public final class SynonymQuery extends Query {
List<Scorer> subScorers = new ArrayList<>();
long totalMaxFreq = 0;
for (int i = 0; i < terms.length; i++) {
- TermState state = termContexts[i].get(context.ord);
+ TermState state = termStates[i].get(context);
if (state != null) {
TermsEnum termsEnum = context.reader().terms(terms[i].field()).iterator();
termsEnum.seekExact(terms[i].bytes(), state);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java b/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java
index fffa5a8..86bf34f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java
@@ -23,7 +23,7 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.AttributeSource;
@@ -43,7 +43,7 @@ abstract class TermCollectingRewrite<B> extends MultiTermQuery.RewriteMethod {
addClause(topLevel, term, docCount, boost, null);
}
- protected abstract void addClause(B topLevel, Term term, int docCount, float boost, TermContext states) throws IOException;
+ protected abstract void addClause(B topLevel, Term term, int docCount, float boost, TermStates states) throws IOException;
final void collectTerms(IndexReader reader, MultiTermQuery query, TermCollector collector) throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java
index 4049e10..a8bf5b0 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java
@@ -33,7 +33,7 @@ import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.PrefixCodedTerms;
import org.apache.lucene.index.PrefixCodedTerms.TermIterator;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -268,9 +268,9 @@ public class TermInSetQuery extends Query implements Accountable {
assert builder == null;
BooleanQuery.Builder bq = new BooleanQuery.Builder();
for (TermAndState t : matchingTerms) {
- final TermContext termContext = new TermContext(searcher.getTopReaderContext());
- termContext.register(t.state, context.ord, t.docFreq, t.totalTermFreq);
- bq.add(new TermQuery(new Term(t.field, t.term), termContext), Occur.SHOULD);
+ final TermStates termStates = new TermStates(searcher.getTopReaderContext());
+ termStates.register(t.state, context.ord, t.docFreq, t.totalTermFreq);
+ bq.add(new TermQuery(new Term(t.field, t.term), termStates), Occur.SHOULD);
}
Query q = new ConstantScoreQuery(bq.build());
final Weight weight = searcher.rewrite(q).createWeight(searcher, scoreMode, score());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
index 3fa465d..d629acd 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
@@ -28,9 +28,8 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
-import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.similarities.Similarity;
@@ -41,16 +40,16 @@ import org.apache.lucene.search.similarities.Similarity;
public class TermQuery extends Query {
private final Term term;
- private final TermContext perReaderTermState;
+ private final TermStates perReaderTermState;
final class TermWeight extends Weight {
private final Similarity similarity;
private final Similarity.SimScorer simScorer;
- private final TermContext termStates;
+ private final TermStates termStates;
private final boolean needsScores;
public TermWeight(IndexSearcher searcher, boolean needsScores,
- float boost, TermContext termStates) throws IOException {
+ float boost, TermStates termStates) throws IOException {
super(TermQuery.this);
if (needsScores && termStates == null) {
throw new IllegalStateException("termStates are required when scores are needed");
@@ -125,30 +124,17 @@ public class TermQuery extends Query {
* the term does not exist in the given context
*/
private TermsEnum getTermsEnum(LeafReaderContext context) throws IOException {
- if (termStates != null) {
- // TermQuery either used as a Query or the term states have been provided at construction time
- assert termStates.wasBuiltFor(ReaderUtil.getTopLevelContext(context)) : "The top-reader used to create Weight is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
- final TermState state = termStates.get(context.ord);
- if (state == null) { // term is not present in that reader
- assert termNotInReader(context.reader(), term) : "no termstate found but term exists in reader term=" + term;
- return null;
- }
- final TermsEnum termsEnum = context.reader().terms(term.field()).iterator();
- termsEnum.seekExact(term.bytes(), state);
- return termsEnum;
- } else {
- // TermQuery used as a filter, so the term states have not been built up front
- Terms terms = context.reader().terms(term.field());
- if (terms == null) {
- return null;
- }
- final TermsEnum termsEnum = terms.iterator();
- if (termsEnum.seekExact(term.bytes())) {
- return termsEnum;
- } else {
- return null;
- }
+ assert termStates != null;
+ assert termStates.wasBuiltFor(ReaderUtil.getTopLevelContext(context)) :
+ "The top-reader used to create Weight is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
+ final TermState state = termStates.get(context);
+ if (state == null) { // term is not present in that reader
+ assert termNotInReader(context.reader(), term) : "no termstate found but term exists in reader term=" + term;
+ return null;
}
+ final TermsEnum termsEnum = context.reader().terms(term.field()).iterator();
+ termsEnum.seekExact(term.bytes(), state);
+ return termsEnum;
}
private boolean termNotInReader(LeafReader reader, Term term) throws IOException {
@@ -189,7 +175,7 @@ public class TermQuery extends Query {
* Expert: constructs a TermQuery that will use the provided docFreq instead
* of looking up the docFreq against the searcher.
*/
- public TermQuery(Term t, TermContext states) {
+ public TermQuery(Term t, TermStates states) {
assert states != null;
term = Objects.requireNonNull(t);
perReaderTermState = Objects.requireNonNull(states);
@@ -203,18 +189,10 @@ public class TermQuery extends Query {
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
final IndexReaderContext context = searcher.getTopReaderContext();
- final TermContext termState;
+ final TermStates termState;
if (perReaderTermState == null
|| perReaderTermState.wasBuiltFor(context) == false) {
- if (scoreMode.needsScores()) {
- // make TermQuery single-pass if we don't have a PRTS or if the context
- // differs!
- termState = TermContext.build(context, term);
- } else {
- // do not compute the term state, this will help save seeks in the terms
- // dict on segments that have a cache entry for this query
- termState = null;
- }
+ termState = TermStates.build(context, term, scoreMode.needsScores());
} else {
// PRTS was pre-build for this IS
termState = this.perReaderTermState;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java b/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java
index b75836e..dea4b0e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java
@@ -25,7 +25,7 @@ import java.util.PriorityQueue;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.ArrayUtil;
@@ -82,7 +82,7 @@ public abstract class TopTermsRewrite<B> extends TermCollectingRewrite<B> {
// lazy init the initial ScoreTerm because comparator is not known on ctor:
if (st == null)
- st = new ScoreTerm(new TermContext(topReaderContext));
+ st = new ScoreTerm(new TermStates(topReaderContext));
boostAtt = termsEnum.attributes().addAttribute(BoostAttribute.class);
}
@@ -139,7 +139,7 @@ public abstract class TopTermsRewrite<B> extends TermCollectingRewrite<B> {
visitedTerms.remove(st.bytes.get());
st.termState.clear(); // reset the termstate!
} else {
- st = new ScoreTerm(new TermContext(topReaderContext));
+ st = new ScoreTerm(new TermStates(topReaderContext));
}
assert stQueue.size() <= maxSize : "the PQ size must be limited to maxSize";
// set maxBoostAtt with values to help FuzzyTermsEnum to optimize
@@ -193,8 +193,8 @@ public abstract class TopTermsRewrite<B> extends TermCollectingRewrite<B> {
static final class ScoreTerm implements Comparable<ScoreTerm> {
public final BytesRefBuilder bytes = new BytesRefBuilder();
public float boost;
- public final TermContext termState;
- public ScoreTerm(TermContext termState) {
+ public final TermStates termState;
+ public ScoreTerm(TermStates termState) {
this.termState = termState;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java
index 8bb2633..23c1e2b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java
@@ -20,7 +20,7 @@ package org.apache.lucene.search.spans;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@@ -61,7 +61,7 @@ abstract class SpanContainQuery extends SpanQuery implements Cloneable {
final SpanWeight bigWeight;
final SpanWeight littleWeight;
- public SpanContainWeight(IndexSearcher searcher, Map<Term, TermContext> terms,
+ public SpanContainWeight(IndexSearcher searcher, Map<Term, TermStates> terms,
SpanWeight bigWeight, SpanWeight littleWeight, float boost) throws IOException {
super(SpanContainQuery.this, searcher, terms, boost);
this.bigWeight = bigWeight;
@@ -93,9 +93,9 @@ abstract class SpanContainQuery extends SpanQuery implements Cloneable {
}
@Override
- public void extractTermContexts(Map<Term, TermContext> contexts) {
- bigWeight.extractTermContexts(contexts);
- littleWeight.extractTermContexts(contexts);
+ public void extractTermStates(Map<Term, TermStates> contexts) {
+ bigWeight.extractTermStates(contexts);
+ littleWeight.extractTermStates(contexts);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java
index 0d62f74..6366299 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java
@@ -23,7 +23,7 @@ import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreMode;
@@ -45,15 +45,15 @@ public final class SpanContainingQuery extends SpanContainQuery {
@Override
public SpanWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- SpanWeight bigWeight = big.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);
- SpanWeight littleWeight = little.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);
- return new SpanContainingWeight(searcher, scoreMode.needsScores() ? getTermContexts(bigWeight, littleWeight) : null,
+ SpanWeight bigWeight = big.createWeight(searcher, scoreMode, boost);
+ SpanWeight littleWeight = little.createWeight(searcher, scoreMode, boost);
+ return new SpanContainingWeight(searcher, scoreMode.needsScores() ? getTermStates(bigWeight, littleWeight) : null,
bigWeight, littleWeight, boost);
}
public class SpanContainingWeight extends SpanContainWeight {
- public SpanContainingWeight(IndexSearcher searcher, Map<Term, TermContext> terms,
+ public SpanContainingWeight(IndexSearcher searcher, Map<Term, TermStates> terms,
SpanWeight bigWeight, SpanWeight littleWeight, float boost) throws IOException {
super(searcher, terms, bigWeight, littleWeight, boost);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
index ee3f5de..088e730 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
@@ -24,7 +24,7 @@ import java.util.Objects;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MultiTermQuery;
@@ -163,7 +163,7 @@ public class SpanMultiTermQueryWrapper<Q extends MultiTermQuery> extends SpanQue
}
@Override
- protected void addClause(List<SpanQuery> topLevel, Term term, int docCount, float boost, TermContext states) {
+ protected void addClause(List<SpanQuery> topLevel, Term term, int docCount, float boost, TermStates states) {
final SpanTermQuery q = new SpanTermQuery(term, states);
topLevel.add(q);
}
@@ -211,7 +211,7 @@ public class SpanMultiTermQueryWrapper<Q extends MultiTermQuery> extends SpanQue
}
@Override
- protected void addClause(List<SpanQuery> topLevel, Term term, int docFreq, float boost, TermContext states) {
+ protected void addClause(List<SpanQuery> topLevel, Term term, int docFreq, float boost, TermStates states) {
final SpanTermQuery q = new SpanTermQuery(term, states);
topLevel.add(q);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
index 24a047f..17b9e51 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
@@ -29,7 +29,7 @@ import java.util.Set;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.Terms;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@@ -181,24 +181,24 @@ public class SpanNearQuery extends SpanQuery implements Cloneable {
public SpanWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
List<SpanWeight> subWeights = new ArrayList<>();
for (SpanQuery q : clauses) {
- subWeights.add(q.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost));
+ subWeights.add(q.createWeight(searcher, scoreMode, boost));
}
- return new SpanNearWeight(subWeights, searcher, scoreMode.needsScores() ? getTermContexts(subWeights) : null, boost);
+ return new SpanNearWeight(subWeights, searcher, scoreMode.needsScores() ? getTermStates(subWeights) : null, boost);
}
public class SpanNearWeight extends SpanWeight {
final List<SpanWeight> subWeights;
- public SpanNearWeight(List<SpanWeight> subWeights, IndexSearcher searcher, Map<Term, TermContext> terms, float boost) throws IOException {
+ public SpanNearWeight(List<SpanWeight> subWeights, IndexSearcher searcher, Map<Term, TermStates> terms, float boost) throws IOException {
super(SpanNearQuery.this, searcher, terms, boost);
this.subWeights = subWeights;
}
@Override
- public void extractTermContexts(Map<Term, TermContext> contexts) {
+ public void extractTermStates(Map<Term, TermStates> contexts) {
for (SpanWeight w : subWeights) {
- w.extractTermContexts(contexts);
+ w.extractTermStates(contexts);
}
}
@@ -318,7 +318,7 @@ public class SpanNearQuery extends SpanQuery implements Cloneable {
}
@Override
- public void extractTermContexts(Map<Term, TermContext> contexts) {
+ public void extractTermStates(Map<Term, TermStates> contexts) {
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
index 5b97f8d..6c56df3 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
@@ -25,7 +25,7 @@ import java.util.Set;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@@ -99,9 +99,9 @@ public final class SpanNotQuery extends SpanQuery {
@Override
public SpanWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- SpanWeight includeWeight = include.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);
+ SpanWeight includeWeight = include.createWeight(searcher, scoreMode, boost);
SpanWeight excludeWeight = exclude.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);
- return new SpanNotWeight(searcher, scoreMode.needsScores() ? getTermContexts(includeWeight, excludeWeight) : null,
+ return new SpanNotWeight(searcher, scoreMode.needsScores() ? getTermStates(includeWeight) : null,
includeWeight, excludeWeight, boost);
}
@@ -110,7 +110,7 @@ public final class SpanNotQuery extends SpanQuery {
final SpanWeight includeWeight;
final SpanWeight excludeWeight;
- public SpanNotWeight(IndexSearcher searcher, Map<Term, TermContext> terms,
+ public SpanNotWeight(IndexSearcher searcher, Map<Term, TermStates> terms,
SpanWeight includeWeight, SpanWeight excludeWeight, float boost) throws IOException {
super(SpanNotQuery.this, searcher, terms, boost);
this.includeWeight = includeWeight;
@@ -118,8 +118,8 @@ public final class SpanNotQuery extends SpanQuery {
}
@Override
- public void extractTermContexts(Map<Term, TermContext> contexts) {
- includeWeight.extractTermContexts(contexts);
+ public void extractTermStates(Map<Term, TermStates> contexts) {
+ includeWeight.extractTermStates(contexts);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
index 2e15c92..849edaa 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
@@ -27,7 +27,7 @@ import java.util.Set;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.DisiPriorityQueue;
import org.apache.lucene.search.DisiWrapper;
import org.apache.lucene.search.DisjunctionDISIApproximation;
@@ -119,16 +119,16 @@ public final class SpanOrQuery extends SpanQuery {
public SpanWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
List<SpanWeight> subWeights = new ArrayList<>(clauses.size());
for (SpanQuery q : clauses) {
- subWeights.add(q.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost));
+ subWeights.add(q.createWeight(searcher, scoreMode, boost));
}
- return new SpanOrWeight(searcher, scoreMode.needsScores() ? getTermContexts(subWeights) : null, subWeights, boost);
+ return new SpanOrWeight(searcher, scoreMode.needsScores() ? getTermStates(subWeights) : null, subWeights, boost);
}
public class SpanOrWeight extends SpanWeight {
final List<SpanWeight> subWeights;
- public SpanOrWeight(IndexSearcher searcher, Map<Term, TermContext> terms, List<SpanWeight> subWeights, float boost) throws IOException {
+ public SpanOrWeight(IndexSearcher searcher, Map<Term, TermStates> terms, List<SpanWeight> subWeights, float boost) throws IOException {
super(SpanOrQuery.this, searcher, terms, boost);
this.subWeights = subWeights;
}
@@ -150,9 +150,9 @@ public final class SpanOrQuery extends SpanQuery {
}
@Override
- public void extractTermContexts(Map<Term, TermContext> contexts) {
+ public void extractTermStates(Map<Term, TermStates> contexts) {
for (SpanWeight w : subWeights) {
- w.extractTermContexts(contexts);
+ w.extractTermStates(contexts);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
index f9b7697..099b627 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
@@ -25,7 +25,7 @@ import java.util.Set;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
@@ -69,15 +69,15 @@ public abstract class SpanPositionCheckQuery extends SpanQuery implements Clonea
@Override
public SpanWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- SpanWeight matchWeight = match.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);
- return new SpanPositionCheckWeight(matchWeight, searcher, scoreMode.needsScores() ? getTermContexts(matchWeight) : null, boost);
+ SpanWeight matchWeight = match.createWeight(searcher, scoreMode, boost);
+ return new SpanPositionCheckWeight(matchWeight, searcher, scoreMode.needsScores() ? getTermStates(matchWeight) : null, boost);
}
public class SpanPositionCheckWeight extends SpanWeight {
final SpanWeight matchWeight;
- public SpanPositionCheckWeight(SpanWeight matchWeight, IndexSearcher searcher, Map<Term, TermContext> terms, float boost) throws IOException {
+ public SpanPositionCheckWeight(SpanWeight matchWeight, IndexSearcher searcher, Map<Term, TermStates> terms, float boost) throws IOException {
super(SpanPositionCheckQuery.this, searcher, terms, boost);
this.matchWeight = matchWeight;
}
@@ -93,8 +93,8 @@ public abstract class SpanPositionCheckQuery extends SpanQuery implements Clonea
}
@Override
- public void extractTermContexts(Map<Term, TermContext> contexts) {
- matchWeight.extractTermContexts(contexts);
+ public void extractTermStates(Map<Term, TermStates> contexts) {
+ matchWeight.extractTermStates(contexts);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java
index 607a375..ca657b6 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java
@@ -23,7 +23,7 @@ import java.util.Map;
import java.util.TreeMap;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
@@ -40,25 +40,25 @@ public abstract class SpanQuery extends Query {
public abstract SpanWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException;
/**
- * Build a map of terms to termcontexts, for use in constructing SpanWeights
+ * Build a map of terms to {@link TermStates}, for use in constructing SpanWeights
* @lucene.internal
*/
- public static Map<Term, TermContext> getTermContexts(SpanWeight... weights) {
- Map<Term, TermContext> terms = new TreeMap<>();
+ public static Map<Term, TermStates> getTermStates(SpanWeight... weights) {
+ Map<Term, TermStates> terms = new TreeMap<>();
for (SpanWeight w : weights) {
- w.extractTermContexts(terms);
+ w.extractTermStates(terms);
}
return terms;
}
/**
- * Build a map of terms to termcontexts, for use in constructing SpanWeights
+ * Build a map of terms to {@link TermStates}, for use in constructing SpanWeights
* @lucene.internal
*/
- public static Map<Term, TermContext> getTermContexts(Collection<SpanWeight> weights) {
- Map<Term, TermContext> terms = new TreeMap<>();
+ public static Map<Term, TermStates> getTermStates(Collection<SpanWeight> weights) {
+ Map<Term, TermStates> terms = new TreeMap<>();
for (SpanWeight w : weights) {
- w.extractTermContexts(terms);
+ w.extractTermStates(terms);
}
return terms;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
index 9eea3aa..9ac7afb 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
@@ -28,7 +28,7 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -41,21 +41,21 @@ import org.apache.lucene.search.ScoreMode;
public class SpanTermQuery extends SpanQuery {
protected final Term term;
- protected final TermContext termContext;
+ protected final TermStates termStates;
/** Construct a SpanTermQuery matching the named term's spans. */
public SpanTermQuery(Term term) {
this.term = Objects.requireNonNull(term);
- this.termContext = null;
+ this.termStates = null;
}
/**
* Expert: Construct a SpanTermQuery matching the named term's spans, using
- * the provided TermContext
+ * the provided TermStates
*/
- public SpanTermQuery(Term term, TermContext context) {
+ public SpanTermQuery(Term term, TermStates termStates) {
this.term = Objects.requireNonNull(term);
- this.termContext = context;
+ this.termStates = termStates;
}
/** Return the term whose spans are matched. */
@@ -66,25 +66,25 @@ public class SpanTermQuery extends SpanQuery {
@Override
public SpanWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- final TermContext context;
+ final TermStates context;
final IndexReaderContext topContext = searcher.getTopReaderContext();
- if (termContext == null || termContext.wasBuiltFor(topContext) == false) {
- context = TermContext.build(topContext, term);
+ if (termStates == null || termStates.wasBuiltFor(topContext) == false) {
+ context = TermStates.build(topContext, term, scoreMode.needsScores());
}
else {
- context = termContext;
+ context = termStates;
}
return new SpanTermWeight(context, searcher, scoreMode.needsScores() ? Collections.singletonMap(term, context) : null, boost);
}
public class SpanTermWeight extends SpanWeight {
- final TermContext termContext;
+ final TermStates termStates;
- public SpanTermWeight(TermContext termContext, IndexSearcher searcher, Map<Term, TermContext> terms, float boost) throws IOException {
+ public SpanTermWeight(TermStates termStates, IndexSearcher searcher, Map<Term, TermStates> terms, float boost) throws IOException {
super(SpanTermQuery.this, searcher, terms, boost);
- this.termContext = termContext;
- assert termContext != null : "TermContext must not be null";
+ this.termStates = termStates;
+ assert termStates != null : "TermStates must not be null";
}
@Override
@@ -98,16 +98,16 @@ public class SpanTermQuery extends SpanQuery {
}
@Override
- public void extractTermContexts(Map<Term, TermContext> contexts) {
- contexts.put(term, termContext);
+ public void extractTermStates(Map<Term, TermStates> contexts) {
+ contexts.put(term, termStates);
}
@Override
public Spans getSpans(final LeafReaderContext context, Postings requiredPostings) throws IOException {
- assert termContext.wasBuiltFor(ReaderUtil.getTopLevelContext(context)) : "The top-reader used to create Weight is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
+ assert termStates.wasBuiltFor(ReaderUtil.getTopLevelContext(context)) : "The top-reader used to create Weight is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
- final TermState state = termContext.get(context.ord);
+ final TermState state = termStates.get(context);
if (state == null) { // term is not present in that reader
assert context.reader().docFreq(term) == 0 : "no termstate found but term exists in reader term=" + term;
return null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
index f1d8894..e971610 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
@@ -24,7 +24,7 @@ import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
@@ -79,24 +79,24 @@ public abstract class SpanWeight extends Weight {
* Create a new SpanWeight
* @param query the parent query
* @param searcher the IndexSearcher to query against
- * @param termContexts a map of terms to termcontexts for use in building the similarity. May
+ * @param termStates a map of terms to {@link TermStates} for use in building the similarity. May
* be null if scores are not required
* @throws IOException on error
*/
- public SpanWeight(SpanQuery query, IndexSearcher searcher, Map<Term, TermContext> termContexts, float boost) throws IOException {
+ public SpanWeight(SpanQuery query, IndexSearcher searcher, Map<Term, TermStates> termStates, float boost) throws IOException {
super(query);
this.field = query.getField();
this.similarity = searcher.getSimilarity();
- this.simScorer = buildSimWeight(query, searcher, termContexts, boost);
+ this.simScorer = buildSimWeight(query, searcher, termStates, boost);
}
- private Similarity.SimScorer buildSimWeight(SpanQuery query, IndexSearcher searcher, Map<Term, TermContext> termContexts, float boost) throws IOException {
- if (termContexts == null || termContexts.size() == 0 || query.getField() == null)
+ private Similarity.SimScorer buildSimWeight(SpanQuery query, IndexSearcher searcher, Map<Term, TermStates> termStates, float boost) throws IOException {
+ if (termStates == null || termStates.size() == 0 || query.getField() == null)
return null;
- TermStatistics[] termStats = new TermStatistics[termContexts.size()];
+ TermStatistics[] termStats = new TermStatistics[termStates.size()];
int termUpTo = 0;
- for (Term term : termContexts.keySet()) {
- TermStatistics termStatistics = searcher.termStatistics(term, termContexts.get(term));
+ for (Term term : termStates.keySet()) {
+ TermStatistics termStatistics = searcher.termStatistics(term, termStates.get(term));
if (termStatistics != null) {
termStats[termUpTo++] = termStatistics;
}
@@ -110,10 +110,10 @@ public abstract class SpanWeight extends Weight {
}
/**
- * Collect all TermContexts used by this Weight
- * @param contexts a map to add the TermContexts to
+ * Collect all TermStates used by this Weight
+ * @param contexts a map to add the TermStates to
*/
- public abstract void extractTermContexts(Map<Term, TermContext> contexts);
+ public abstract void extractTermStates(Map<Term, TermStates> contexts);
/**
* Expert: Return a Spans object iterating over matches from this Weight
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java
index 9c618dd..fba85fe 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java
@@ -23,7 +23,7 @@ import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreMode;
@@ -46,15 +46,15 @@ public final class SpanWithinQuery extends SpanContainQuery {
@Override
public SpanWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- SpanWeight bigWeight = big.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);
- SpanWeight littleWeight = little.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);
- return new SpanWithinWeight(searcher, scoreMode.needsScores() ? getTermContexts(bigWeight, littleWeight) : null,
+ SpanWeight bigWeight = big.createWeight(searcher, scoreMode, boost);
+ SpanWeight littleWeight = little.createWeight(searcher, scoreMode, boost);
+ return new SpanWithinWeight(searcher, scoreMode.needsScores() ? getTermStates(bigWeight, littleWeight) : null,
bigWeight, littleWeight, boost);
}
public class SpanWithinWeight extends SpanContainWeight {
- public SpanWithinWeight(IndexSearcher searcher, Map<Term, TermContext> terms,
+ public SpanWithinWeight(IndexSearcher searcher, Map<Term, TermStates> terms,
SpanWeight bigWeight, SpanWeight littleWeight, float boost) throws IOException {
super(searcher, terms, bigWeight, littleWeight, boost);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d250a146/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java b/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
index 268e8fc..30b03ac 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
@@ -34,7 +34,7 @@ import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.Directory;
@@ -329,7 +329,7 @@ public class TestMinShouldMatch2 extends LuceneTestCase {
if (ord >= 0) {
boolean success = ords.add(ord);
assert success; // no dups
- TermContext context = TermContext.build(reader.getContext(), term);
+ TermStates context = TermStates.build(reader.getContext(), term, true);
SimScorer w = weight.similarity.scorer(1f,
searcher.collectionStatistics("field"),
searcher.termStatistics(term, context));
[46/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11825: stop testing
randomized dates with function query sorting
Posted by da...@apache.org.
SOLR-11825: stop testing randomized dates with function query sorting
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/077e8b1d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/077e8b1d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/077e8b1d
Branch: refs/heads/jira/solr-11702
Commit: 077e8b1da4f5a2ab99ac5fefcb2891869c2a43c6
Parents: ae1e192
Author: Steve Rowe <sa...@apache.org>
Authored: Tue Jan 9 21:02:31 2018 -0500
Committer: Steve Rowe <sa...@apache.org>
Committed: Tue Jan 9 21:04:21 2018 -0500
----------------------------------------------------------------------
.../org/apache/solr/schema/TestPointFields.java | 66 ++++++++------------
1 file changed, 27 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/077e8b1d/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
index 34c48e6..af5b022 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
@@ -3562,51 +3562,39 @@ public class TestPointFields extends SolrTestCaseJ4 {
}
}
- private void doTestDatePointFunctionQuery(String field) throws Exception {
- assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof DatePointField);
- int numVals = 10 * RANDOM_MULTIPLIER;
- List<Long> values = getRandomLongs(numVals, false, MAX_DATE_EPOCH_MILLIS);
+ private void doTestDatePointFunctionQuery(String field) {
+ // This method is intentionally not randomized, because sorting by function happens
+ // at float precision, which causes ms(date) to give the same value for different dates.
+ // See https://issues.apache.org/jira/browse/SOLR-11825
- String assertNumFound = "//*[@numFound='" + numVals + "']";
- String[] idAscXpathChecks = new String[numVals + 1];
- String[] idAscNegXpathChecks = new String[numVals + 1];
+ final String baseDate = "1995-01-10T10:59:10Z";
- idAscXpathChecks[0] = assertNumFound;
- idAscNegXpathChecks[0] = assertNumFound;
- for (int i = 0 ; i < values.size() ; ++i) {
- assertU(adoc("id", Character.valueOf((char)('A' + i)).toString(), field, Instant.ofEpochMilli(values.get(i)).toString()));
- // reminder: xpath array indexes start at 1
- idAscXpathChecks[i + 1] = "//result/doc[" + (1 + i) + "]/date[@name='field(" + field
- + ")'][.='" + Instant.ofEpochMilli(values.get(i)) + "']";
- idAscNegXpathChecks[i + 1] = "//result/doc[" + (1 + i) + "]/float[@name='product(-1,ms(" + field + "))'][.='"
- + (-1.0f * (float)values.get(i)) + "']";
+ for (int i = 9; i >= 0; i--) {
+ String date = String.format(Locale.ROOT, "%s+%dSECONDS", baseDate, i+1);
+ assertU(adoc("id", String.valueOf(i), field, date));
}
assertU(commit());
- assertQ(req("q", "*:*", "fl", "id, " + field + ", field(" + field + ")", "rows", String.valueOf(numVals), "sort", "id asc"),
- idAscXpathChecks);
- assertQ(req("q", "*:*", "fl", "id, " + field + ", product(-1,ms(" + field + "))", "rows", String.valueOf(numVals), "sort", "id asc"),
- idAscNegXpathChecks);
+ assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof DatePointField);
+ assertQ(req("q", "*:*", "fl", "id, " + field, "sort", "product(-1,ms(" + field + "," + baseDate +")) asc"),
+ "//*[@numFound='10']",
+ "//result/doc[1]/date[@name='" + field + "'][.='1995-01-10T10:59:20Z']",
+ "//result/doc[2]/date[@name='" + field + "'][.='1995-01-10T10:59:19Z']",
+ "//result/doc[3]/date[@name='" + field + "'][.='1995-01-10T10:59:18Z']",
+ "//result/doc[10]/date[@name='" + field + "'][.='1995-01-10T10:59:11Z']");
- List<PosVal<Long>> ascNegPosVals
- = toAscendingPosVals(values.stream().map(v -> -v).collect(Collectors.toList()), true);
- String[] ascNegXpathChecks = new String[numVals + 1];
- ascNegXpathChecks[0] = assertNumFound;
- for (int i = 0 ; i < ascNegPosVals.size() ; ++i) {
- PosVal<Long> posVal = ascNegPosVals.get(i);
- ascNegXpathChecks[i + 1] = "//result/doc[" + (1 + i) + "]/date[@name='" + field
- + "'][.='" + Instant.ofEpochMilli(values.get(posVal.pos)) + "']";
- }
- assertQ(req("q", "*:*", "fl", "id, " + field, "rows", String.valueOf(numVals), "sort", "product(-1,ms(" + field + ")) asc"),
- ascNegXpathChecks);
+ assertQ(req("q", "*:*", "fl", "id, " + field + ", ms(" + field + ","+baseDate+")", "sort", "id asc"),
+ "//*[@numFound='10']",
+ "//result/doc[1]/float[@name='ms(" + field + "," + baseDate + ")'][.='1000.0']",
+ "//result/doc[2]/float[@name='ms(" + field + "," + baseDate + ")'][.='2000.0']",
+ "//result/doc[3]/float[@name='ms(" + field + "," + baseDate + ")'][.='3000.0']",
+ "//result/doc[10]/float[@name='ms(" + field + "," + baseDate + ")'][.='10000.0']");
- clearIndex();
- assertU(adoc("id", "1", field, "1995-12-31T10:59:59Z"));
- assertU(adoc("id", "2", field, "1996-12-31T10:59:59Z"));
- assertU(commit());
- assertQ(req("q", field + ":[* TO *]", "fl", "id, " + field),
- "//*[@numFound='2']");
- assertQ(req("q", field + ":{* TO *}", "fl", "id, " + field),
- "//*[@numFound='2']");
+ assertQ(req("q", "*:*", "fl", "id, " + field + ", field(" + field + ")", "sort", "id asc"),
+ "//*[@numFound='10']",
+ "//result/doc[1]/date[@name='field(" + field + ")'][.='1995-01-10T10:59:11Z']",
+ "//result/doc[2]/date[@name='field(" + field + ")'][.='1995-01-10T10:59:12Z']",
+ "//result/doc[3]/date[@name='field(" + field + ")'][.='1995-01-10T10:59:13Z']",
+ "//result/doc[10]/date[@name='field(" + field + ")'][.='1995-01-10T10:59:20Z']");
}
private void doTestDatePointStats(String field, String dvField, String[] dates) {
[27/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8115: fail
precommit on unnecessary-on-its-own {@inheritDoc} annotations.
Posted by da...@apache.org.
LUCENE-8115: fail precommit on unnecessary-on-its-own {@inheritDoc} annotations.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ad6e8b82
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ad6e8b82
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ad6e8b82
Branch: refs/heads/jira/solr-11702
Commit: ad6e8b82ec9c2f886c3fd14efc0b9a8634776434
Parents: 07afc23
Author: Christine Poerschke <cp...@apache.org>
Authored: Mon Jan 8 19:46:22 2018 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Mon Jan 8 19:46:22 2018 +0000
----------------------------------------------------------------------
build.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ad6e8b82/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 9db8312..78e0295 100755
--- a/build.xml
+++ b/build.xml
@@ -145,6 +145,7 @@
(~$/(?i)\bno(n|)commit\b/$) : 'nocommit',
(~$/\bTOOD:/$) : 'TOOD instead TODO',
(~$/\t/$) : 'tabs instead spaces',
+ (~$/\Q/**\E((?:\s)|(?:\*))*\Q{@inheritDoc}\E((?:\s)|(?:\*))*\Q*/\E/$) : '{@inheritDoc} on its own is unnecessary',
(~$/\$$(?:LastChanged)?Date\b/$) : 'svn keyword',
(~$/\$$(?:(?:LastChanged)?Revision|Rev)\b/$) : 'svn keyword',
(~$/\$$(?:LastChangedBy|Author)\b/$) : 'svn keyword',
[43/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11631: fix precommit
Posted by da...@apache.org.
SOLR-11631: fix precommit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e538792d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e538792d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e538792d
Branch: refs/heads/jira/solr-11702
Commit: e538792d298d85637ebaa88b5e0ac090eeb1bb20
Parents: f6eae40
Author: Steve Rowe <sa...@apache.org>
Authored: Tue Jan 9 18:00:16 2018 -0500
Committer: Steve Rowe <sa...@apache.org>
Committed: Tue Jan 9 18:00:49 2018 -0500
----------------------------------------------------------------------
.../client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java | 4 ----
.../test/org/apache/solr/client/solrj/request/SchemaTest.java | 1 -
2 files changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e538792d/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java
index 0539c70..9033a86 100644
--- a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java
+++ b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java
@@ -20,19 +20,15 @@ import java.io.IOException;
import java.nio.file.Path;
import java.util.Collections;
import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.api.ApiBag;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.schema.SchemaRequest;
import org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.apache.solr.client.solrj.response.schema.SchemaResponse.FieldResponse;
import org.apache.solr.common.SolrException;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SimpleOrderedMap;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e538792d/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java
index bab4a41..75301df 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java
@@ -28,7 +28,6 @@ import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.commons.io.FileUtils;
-import org.apache.solr.api.ApiBag;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.schema.AnalyzerDefinition;
[04/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8116: SimScorer
now only takes a frequency and a norm as per-document scoring factors.
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
index 0452371..de317d0 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
@@ -22,8 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
@@ -511,7 +509,7 @@ public abstract class TFIDFSimilarity extends Similarity {
}
@Override
- public final SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public final SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
final Explanation idf = termStats.length == 1
? idfExplain(collectionStats, termStats[0])
: idfExplain(collectionStats, termStats);
@@ -522,110 +520,69 @@ public abstract class TFIDFSimilarity extends Similarity {
normTable[i] = norm;
}
normTable[0] = 1f / normTable[255];
- return new IDFStats(collectionStats.field(), boost, idf, normTable);
+ return new TFIDFScorer(collectionStats.field(), boost, idf, normTable);
}
- @Override
- public final SimScorer simScorer(SimWeight stats, LeafReaderContext context) throws IOException {
- IDFStats idfstats = (IDFStats) stats;
- // the norms only encode the length, we need a translation table that depends on how lengthNorm is implemented
- final float[] normTable = idfstats.normTable;
- return new TFIDFSimScorer(idfstats, context.reader().getNormValues(idfstats.field), normTable);
- }
-
- private final class TFIDFSimScorer extends SimScorer {
- private final IDFStats stats;
- private final float weightValue;
- private final NumericDocValues norms;
- private final float[] normTable;
-
- TFIDFSimScorer(IDFStats stats, NumericDocValues norms, float[] normTable) throws IOException {
- this.stats = stats;
- this.weightValue = stats.queryWeight;
- this.norms = norms;
- this.normTable = normTable;
- }
-
- @Override
- public float score(int doc, float freq) throws IOException {
- final float raw = tf(freq) * weightValue; // compute tf(f)*weight
-
- if (norms == null) {
- return raw;
- } else {
- boolean found = norms.advanceExact(doc);
- assert found;
- float normValue = normTable[(int) (norms.longValue() & 0xFF)];
- return raw * normValue; // normalize for field
- }
- }
-
- @Override
- public float maxScore(float maxFreq) {
- final float raw = tf(maxFreq) * weightValue;
- if (norms == null) {
- return raw;
- } else {
- float maxNormValue = Float.NEGATIVE_INFINITY;
- for (float norm : normTable) {
- maxNormValue = Math.max(maxNormValue, norm);
- }
- return raw * maxNormValue;
- }
- }
-
- @Override
- public Explanation explain(int doc, Explanation freq) throws IOException {
- return explainScore(doc, freq, stats, norms, normTable);
- }
- }
/** Collection statistics for the TF-IDF model. The only statistic of interest
* to this model is idf. */
- static class IDFStats extends SimWeight {
- private final String field;
+ class TFIDFScorer extends SimScorer {
/** The idf and its explanation */
private final Explanation idf;
private final float boost;
private final float queryWeight;
final float[] normTable;
- public IDFStats(String field, float boost, Explanation idf, float[] normTable) {
+ public TFIDFScorer(String field, float boost, Explanation idf, float[] normTable) {
+ super(field);
// TODO: Validate?
- this.field = field;
this.idf = idf;
this.boost = boost;
this.queryWeight = boost * idf.getValue().floatValue();
this.normTable = normTable;
}
- }
- private Explanation explainScore(int doc, Explanation freq, IDFStats stats, NumericDocValues norms, float[] normTable) throws IOException {
- List<Explanation> subs = new ArrayList<Explanation>();
- if (stats.boost != 1F) {
- subs.add(Explanation.match(stats.boost, "boost"));
+ @Override
+ public float score(float freq, long norm) throws IOException {
+ final float raw = tf(freq) * queryWeight; // compute tf(f)*weight
+ float normValue = normTable[(int) (norm & 0xFF)];
+ return raw * normValue; // normalize for field
}
- subs.add(stats.idf);
- Explanation tf = Explanation.match(tf(freq.getValue().floatValue()), "tf(freq="+freq.getValue()+"), with freq of:", freq);
- subs.add(tf);
- float norm;
- if (norms == null) {
- norm = 1f;
- } else {
- boolean found = norms.advanceExact(doc);
- assert found;
- norm = normTable[(int) (norms.longValue() & 0xFF)];
+ @Override
+ public float maxScore(float maxFreq) {
+ final float raw = tf(maxFreq) * queryWeight;
+ float maxNormValue = Float.NEGATIVE_INFINITY;
+ for (float norm : normTable) {
+ maxNormValue = Math.max(maxNormValue, norm);
+ }
+ return raw * maxNormValue;
}
-
- Explanation fieldNorm = Explanation.match(
- norm,
- "fieldNorm(doc=" + doc + ")");
- subs.add(fieldNorm);
-
- return Explanation.match(
- stats.queryWeight * tf.getValue().floatValue() * norm,
- "score(doc="+doc+",freq="+freq.getValue()+"), product of:",
- subs);
- }
+
+ @Override
+ public Explanation explain(Explanation freq, long norm) throws IOException {
+ return explainScore(freq, norm, normTable);
+ }
+
+ private Explanation explainScore(Explanation freq, long encodedNorm, float[] normTable) throws IOException {
+ List<Explanation> subs = new ArrayList<Explanation>();
+ if (boost != 1F) {
+ subs.add(Explanation.match(boost, "boost"));
+ }
+ subs.add(idf);
+ Explanation tf = Explanation.match(tf(freq.getValue().floatValue()), "tf(freq="+freq.getValue()+"), with freq of:", freq);
+ subs.add(tf);
+
+ float norm = normTable[(int) (encodedNorm & 0xFF)];
+
+ Explanation fieldNorm = Explanation.match(norm, "fieldNorm");
+ subs.add(fieldNorm);
+
+ return Explanation.match(
+ queryWeight * tf.getValue().floatValue() * norm,
+ "score(freq="+freq.getValue()+"), product of:",
+ subs);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java
index 57a68e4..044ac7a 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java
@@ -21,9 +21,9 @@ import java.io.IOException;
import java.util.Objects;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.LeafSimScorer;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TwoPhaseIterator;
-import org.apache.lucene.search.similarities.Similarity;
/**
* A basic {@link Scorer} over {@link Spans}.
@@ -32,7 +32,7 @@ import org.apache.lucene.search.similarities.Similarity;
public class SpanScorer extends Scorer {
protected final Spans spans;
- protected final Similarity.SimScorer docScorer;
+ protected final LeafSimScorer docScorer;
/** accumulated sloppy freq (computed in setFreqCurrentDoc) */
private float freq;
@@ -41,7 +41,7 @@ public class SpanScorer extends Scorer {
private int lastScoredDoc = -1; // last doc we called setFreqCurrentDoc() for
/** Sole constructor. */
- public SpanScorer(SpanWeight weight, Spans spans, Similarity.SimScorer docScorer) {
+ public SpanScorer(SpanWeight weight, Spans spans, LeafSimScorer docScorer) {
super(weight);
this.spans = Objects.requireNonNull(spans);
this.docScorer = docScorer;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
index 0dad614..f1d8894 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
@@ -28,10 +28,10 @@ import org.apache.lucene.index.TermContext;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LeafSimScorer;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
/**
* Expert-only. Public for use by other weight implementations
@@ -72,7 +72,7 @@ public abstract class SpanWeight extends Weight {
}
protected final Similarity similarity;
- protected final Similarity.SimWeight simWeight;
+ protected final Similarity.SimScorer simScorer;
protected final String field;
/**
@@ -86,11 +86,11 @@ public abstract class SpanWeight extends Weight {
public SpanWeight(SpanQuery query, IndexSearcher searcher, Map<Term, TermContext> termContexts, float boost) throws IOException {
super(query);
this.field = query.getField();
- this.similarity = searcher.getSimilarity(termContexts != null);
- this.simWeight = buildSimWeight(query, searcher, termContexts, boost);
+ this.similarity = searcher.getSimilarity();
+ this.simScorer = buildSimWeight(query, searcher, termContexts, boost);
}
- private Similarity.SimWeight buildSimWeight(SpanQuery query, IndexSearcher searcher, Map<Term, TermContext> termContexts, float boost) throws IOException {
+ private Similarity.SimScorer buildSimWeight(SpanQuery query, IndexSearcher searcher, Map<Term, TermContext> termContexts, float boost) throws IOException {
if (termContexts == null || termContexts.size() == 0 || query.getField() == null)
return null;
TermStatistics[] termStats = new TermStatistics[termContexts.size()];
@@ -103,7 +103,7 @@ public abstract class SpanWeight extends Weight {
}
CollectionStatistics collectionStats = searcher.collectionStatistics(query.getField());
if (termUpTo > 0) {
- return similarity.computeWeight(boost, collectionStats, Arrays.copyOf(termStats, termUpTo));
+ return similarity.scorer(boost, collectionStats, Arrays.copyOf(termStats, termUpTo));
} else {
return null; // no terms at all exist, we won't use similarity
}
@@ -129,18 +129,18 @@ public abstract class SpanWeight extends Weight {
if (spans == null) {
return null;
}
- final Similarity.SimScorer docScorer = getSimScorer(context);
+ final LeafSimScorer docScorer = getSimScorer(context);
return new SpanScorer(this, spans, docScorer);
}
/**
- * Return a SimScorer for this context
+ * Return a LeafSimScorer for this context
* @param context the LeafReaderContext
* @return a SimWeight
* @throws IOException on error
*/
- public Similarity.SimScorer getSimScorer(LeafReaderContext context) throws IOException {
- return simWeight == null ? null : similarity.simScorer(simWeight, context);
+ public LeafSimScorer getSimScorer(LeafReaderContext context) throws IOException {
+ return simScorer == null ? null : new LeafSimScorer(simScorer, context.reader(), true, Float.POSITIVE_INFINITY);
}
@Override
@@ -150,7 +150,7 @@ public abstract class SpanWeight extends Weight {
int newDoc = scorer.iterator().advance(doc);
if (newDoc == doc) {
float freq = scorer.sloppyFreq();
- SimScorer docScorer = similarity.simScorer(simWeight, context);
+ LeafSimScorer docScorer = new LeafSimScorer(simScorer, context.reader(), true, Float.POSITIVE_INFINITY);
Explanation freqExplanation = Explanation.match(freq, "phraseFreq=" + freq);
Explanation scoreExplanation = docScorer.explain(doc, freqExplanation);
return Explanation.match(scoreExplanation.getValue(),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java b/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
index f1e1aed..625bb0e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
@@ -22,7 +22,7 @@ import java.util.Objects;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.LeafSimScorer;
/**
* Expert:
@@ -39,7 +39,7 @@ public class TermSpans extends Spans {
protected boolean readPayload;
private final float positionsCost;
- public TermSpans(Similarity.SimScorer scorer,
+ public TermSpans(LeafSimScorer scorer,
PostingsEnum postings, Term term, float positionsCost) {
this.postings = Objects.requireNonNull(postings);
this.term = Objects.requireNonNull(term);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
index a811192..7737de1 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
@@ -103,12 +103,7 @@ public class TestCustomNorms extends LuceneTestCase {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
throw new UnsupportedOperationException();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java b/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java
index d2eff25..8bb81d2 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java
@@ -17,8 +17,6 @@
package org.apache.lucene.index;
-import java.io.IOException;
-
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -432,12 +430,7 @@ public class TestCustomTermFreq extends LuceneTestCase {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
throw new UnsupportedOperationException();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java
index f78b7fa..08635fc 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java
@@ -17,7 +17,6 @@
package org.apache.lucene.index;
-import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@@ -54,12 +53,7 @@ public class TestFieldInvertState extends LuceneTestCase {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
throw new UnsupportedOperationException();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index 0a8799d..6b43c16 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -1947,13 +1947,8 @@ public class TestIndexSorting extends LuceneTestCase {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return in.computeWeight(boost, collectionStats, termStats);
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
- return in.simScorer(weight, context);
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ return in.scorer(boost, collectionStats, termStats);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java b/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
index f391c5a..cf29392 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
@@ -109,17 +109,11 @@ public class TestMaxTermFrequency extends LuceneTestCase {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return new SimWeight() {};
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
-
- return new SimScorer() {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ return new SimScorer(collectionStats.field()) {
@Override
- public float score(int doc, float freq) throws IOException {
+ public float score(float freq, long norm) throws IOException {
return 0;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
index 70c7a32..805c7e5 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
@@ -116,12 +116,7 @@ public class TestNorms extends LuceneTestCase {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
throw new UnsupportedOperationException();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java b/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java
index a0fca4c..2de0234 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java
@@ -17,7 +17,6 @@
package org.apache.lucene.index;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
@@ -106,12 +105,7 @@ public class TestUniqueTermCount extends LuceneTestCase {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
throw new UnsupportedOperationException();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java
index 0523e2c..c85732e 100644
--- a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java
+++ b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java
@@ -194,12 +194,7 @@ final class JustCompileSearch {
static final class JustCompileSimilarity extends Similarity {
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- throw new UnsupportedOperationException(UNSUPPORTED_MSG);
- }
-
- @Override
- public SimScorer simScorer(SimWeight stats, LeafReaderContext context) {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java
index 9556281..a9e2891 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java
@@ -148,7 +148,7 @@ public class TestBoolean2 extends LuceneTestCase {
}
singleSegmentReader = DirectoryReader.open(singleSegmentDirectory);
singleSegmentSearcher = newSearcher(singleSegmentReader);
- singleSegmentSearcher.setSimilarity(searcher.getSimilarity(true));
+ singleSegmentSearcher.setSimilarity(searcher.getSimilarity());
// Make big index
dir2 = copyOf(directory);
@@ -379,7 +379,7 @@ public class TestBoolean2 extends LuceneTestCase {
QueryUtils.check(random(), q1,searcher); // baseline sim
try {
// a little hackish, QueryUtils.check is too costly to do on bigSearcher in this loop.
- searcher.setSimilarity(bigSearcher.getSimilarity(true)); // random sim
+ searcher.setSimilarity(bigSearcher.getSimilarity()); // random sim
QueryUtils.check(random(), q1, searcher);
} finally {
searcher.setSimilarity(new ClassicSimilarity()); // restore
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java
index 19f45f8..37a1b57 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java
@@ -38,7 +38,6 @@ import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.Scorer.ChildScorer;
-import org.apache.lucene.search.similarities.BasicStats;
import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
@@ -329,15 +328,10 @@ public class TestBooleanQueryVisitSubscorers extends LuceneTestCase {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return new BasicStats("", boost);
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
- return new SimScorer() {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ return new SimScorer(collectionStats.field()) {
@Override
- public float score(int doc, float freq) throws IOException {
+ public float score(float freq, long norm) throws IOException {
return freq;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/search/TestBooleanRewrites.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanRewrites.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanRewrites.java
index d21f373..292dfa9 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanRewrites.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanRewrites.java
@@ -357,7 +357,7 @@ public class TestBooleanRewrites extends LuceneTestCase {
return original;
}
};
- searcher2.setSimilarity(searcher1.getSimilarity(true));
+ searcher2.setSimilarity(searcher1.getSimilarity());
final int iters = atLeast(1000);
for (int i = 0; i < iters; ++i) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java b/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java
index a4e9596..9b79675 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java
@@ -34,7 +34,6 @@ import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.Similarity;
@@ -100,16 +99,11 @@ public class TestConjunctions extends LuceneTestCase {
}
@Override
- public SimWeight computeWeight(float boost,
+ public SimScorer scorer(float boost,
CollectionStatistics collectionStats, TermStatistics... termStats) {
- return new SimWeight() {};
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
- return new SimScorer() {
+ return new SimScorer(collectionStats.field()) {
@Override
- public float score(int doc, float freq) {
+ public float score(float freq, long norm) {
return freq;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java
deleted file mode 100644
index 8856431..0000000
--- a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search;
-
-
-import java.io.IOException;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
-import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase;
-
-/**
- * Tests the use of indexdocvalues in scoring.
- *
- * In the example, a docvalues field is used as a per-document boost (separate from the norm)
- * @lucene.experimental
- */
-public class TestDocValuesScoring extends LuceneTestCase {
- private static final float SCORE_EPSILON = 0.001f; /* for comparing floats */
-
- public void testSimple() throws Exception {
- Directory dir = newDirectory();
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
- Document doc = new Document();
- Field field = newTextField("foo", "", Field.Store.NO);
- doc.add(field);
- Field dvField = new FloatDocValuesField("foo_boost", 0.0F);
- doc.add(dvField);
- Field field2 = newTextField("bar", "", Field.Store.NO);
- doc.add(field2);
-
- field.setStringValue("quick brown fox");
- field2.setStringValue("quick brown fox");
- dvField.setFloatValue(2f); // boost x2
- iw.addDocument(doc);
- field.setStringValue("jumps over lazy brown dog");
- field2.setStringValue("jumps over lazy brown dog");
- dvField.setFloatValue(4f); // boost x4
- iw.addDocument(doc);
- IndexReader ir = iw.getReader();
- iw.close();
-
- // no boosting
- IndexSearcher searcher1 = newSearcher(ir, false);
- final Similarity base = searcher1.getSimilarity(true);
- // boosting
- IndexSearcher searcher2 = newSearcher(ir, false);
- searcher2.setSimilarity(new PerFieldSimilarityWrapper() {
- final Similarity fooSim = new BoostingSimilarity(base, "foo_boost");
-
- @Override
- public Similarity get(String field) {
- return "foo".equals(field) ? fooSim : base;
- }
- });
-
- // in this case, we searched on field "foo". first document should have 2x the score.
- TermQuery tq = new TermQuery(new Term("foo", "quick"));
- QueryUtils.check(random(), tq, searcher1);
- QueryUtils.check(random(), tq, searcher2);
-
- TopDocs noboost = searcher1.search(tq, 10);
- TopDocs boost = searcher2.search(tq, 10);
- assertEquals(1, noboost.totalHits);
- assertEquals(1, boost.totalHits);
-
- //System.out.println(searcher2.explain(tq, boost.scoreDocs[0].doc));
- assertEquals(boost.scoreDocs[0].score, noboost.scoreDocs[0].score*2f, SCORE_EPSILON);
-
- // this query matches only the second document, which should have 4x the score.
- tq = new TermQuery(new Term("foo", "jumps"));
- QueryUtils.check(random(), tq, searcher1);
- QueryUtils.check(random(), tq, searcher2);
-
- noboost = searcher1.search(tq, 10);
- boost = searcher2.search(tq, 10);
- assertEquals(1, noboost.totalHits);
- assertEquals(1, boost.totalHits);
-
- assertEquals(boost.scoreDocs[0].score, noboost.scoreDocs[0].score*4f, SCORE_EPSILON);
-
- // search on on field bar just for kicks, nothing should happen, since we setup
- // our sim provider to only use foo_boost for field foo.
- tq = new TermQuery(new Term("bar", "quick"));
- QueryUtils.check(random(), tq, searcher1);
- QueryUtils.check(random(), tq, searcher2);
-
- noboost = searcher1.search(tq, 10);
- boost = searcher2.search(tq, 10);
- assertEquals(1, noboost.totalHits);
- assertEquals(1, boost.totalHits);
-
- assertEquals(boost.scoreDocs[0].score, noboost.scoreDocs[0].score, SCORE_EPSILON);
-
- ir.close();
- dir.close();
- }
-
- /**
- * Similarity that wraps another similarity and boosts the final score
- * according to whats in a docvalues field.
- *
- * @lucene.experimental
- */
- static class BoostingSimilarity extends Similarity {
- private final Similarity sim;
- private final String boostField;
-
- public BoostingSimilarity(Similarity sim, String boostField) {
- this.sim = sim;
- this.boostField = boostField;
- }
-
- @Override
- public long computeNorm(FieldInvertState state) {
- return sim.computeNorm(state);
- }
-
- @Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return sim.computeWeight(boost, collectionStats, termStats);
- }
-
- @Override
- public SimScorer simScorer(SimWeight stats, LeafReaderContext context) throws IOException {
- final SimScorer sub = sim.simScorer(stats, context);
- final NumericDocValues values = DocValues.getNumeric(context.reader(), boostField);
-
- return new SimScorer() {
-
- private float getValueForDoc(int doc) throws IOException {
- int curDocID = values.docID();
- if (doc < curDocID) {
- throw new IllegalArgumentException("doc=" + doc + " is before curDocID=" + curDocID);
- }
- if (doc > curDocID) {
- curDocID = values.advance(doc);
- }
- if (curDocID == doc) {
- return Float.intBitsToFloat((int)values.longValue());
- } else {
- return 0f;
- }
- }
-
- @Override
- public float score(int doc, float freq) throws IOException {
- return getValueForDoc(doc) * sub.score(doc, freq);
- }
-
- @Override
- public float maxScore(float maxFreq) {
- return Float.POSITIVE_INFINITY;
- }
-
- @Override
- public Explanation explain(int doc, Explanation freq) throws IOException {
- Explanation boostExplanation = Explanation.match(getValueForDoc(doc), "indexDocValue(" + boostField + ")");
- Explanation simExplanation = sub.explain(doc, freq);
- return Explanation.match(
- boostExplanation.getValue().doubleValue() * simExplanation.getValue().doubleValue(),
- "product of:", boostExplanation, simExplanation);
- }
- };
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java b/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
index 924a1af..268e8fc 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
@@ -37,7 +37,6 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.Similarity.SimScorer;
-import org.apache.lucene.search.similarities.Similarity.SimWeight;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -310,7 +309,7 @@ public class TestMinShouldMatch2 extends LuceneTestCase {
final int maxDoc;
final Set<Long> ords = new HashSet<>();
- final SimScorer[] sims;
+ final LeafSimScorer[] sims;
final int minNrShouldMatch;
double score = Float.NaN;
@@ -321,7 +320,7 @@ public class TestMinShouldMatch2 extends LuceneTestCase {
this.maxDoc = reader.maxDoc();
BooleanQuery bq = (BooleanQuery) weight.getQuery();
this.minNrShouldMatch = bq.getMinimumNumberShouldMatch();
- this.sims = new SimScorer[(int)dv.getValueCount()];
+ this.sims = new LeafSimScorer[(int)dv.getValueCount()];
for (BooleanClause clause : bq.clauses()) {
assert !clause.isProhibited();
assert !clause.isRequired();
@@ -331,10 +330,10 @@ public class TestMinShouldMatch2 extends LuceneTestCase {
boolean success = ords.add(ord);
assert success; // no dups
TermContext context = TermContext.build(reader.getContext(), term);
- SimWeight w = weight.similarity.computeWeight(1f,
+ SimScorer w = weight.similarity.scorer(1f,
searcher.collectionStatistics("field"),
searcher.termStatistics(term, context));
- sims[(int)ord] = weight.similarity.simScorer(w, reader.getContext());
+ sims[(int)ord] = new LeafSimScorer(w, reader, true, 1);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
index a6970f9..f0e3b22 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
@@ -25,7 +25,6 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomIndexWriter;
@@ -113,16 +112,11 @@ public class TestSimilarityProvider extends LuceneTestCase {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return new SimWeight() {};
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
- return new SimScorer() {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ return new SimScorer(collectionStats.field()) {
@Override
- public float score(int doc, float freq) throws IOException {
+ public float score(float freq, long norm) throws IOException {
return 1;
}
@@ -143,16 +137,11 @@ public class TestSimilarityProvider extends LuceneTestCase {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return new SimWeight() {};
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
- return new SimScorer() {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ return new SimScorer(collectionStats.field()) {
@Override
- public float score(int doc, float freq) throws IOException {
+ public float score(float freq, long norm) throws IOException {
return 10;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java b/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
index 7278a3b..1cc4d59 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
@@ -34,7 +34,6 @@ import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.Scorer.ChildScorer;
-import org.apache.lucene.search.similarities.BasicStats;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
@@ -231,15 +230,10 @@ public class TestSubScorerFreqs extends LuceneTestCase {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return new BasicStats("", boost);
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
- return new SimScorer() {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ return new SimScorer(collectionStats.field()) {
@Override
- public float score(int doc, float freq) throws IOException {
+ public float score(float freq, long norm) throws IOException {
return freq;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java
index a0fa0f3..eb7a590 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java
@@ -36,7 +36,6 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.similarities.TFIDFSimilarity.IDFStats;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
@@ -157,7 +156,7 @@ public class TestClassicSimilarity extends BaseSimilarityTestCase {
public void testSaneNormValues() throws IOException {
ClassicSimilarity sim = new ClassicSimilarity();
- TFIDFSimilarity.IDFStats stats = (IDFStats) sim.computeWeight(1f, indexSearcher.collectionStatistics("test"));
+ TFIDFSimilarity.TFIDFScorer stats = (TFIDFSimilarity.TFIDFScorer) sim.scorer(1f, indexSearcher.collectionStatistics("test"));
for (int i = 0; i < 256; i++) {
float boost = stats.normTable[i];
assertFalse("negative boost: " + boost + ", byte=" + i, boost < 0.0f);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
index 279e30c..b263582 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
@@ -37,6 +37,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.SimilarityBase.BasicSimScorer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -207,13 +208,13 @@ public class TestSimilarityBase extends LuceneTestCase {
*/
private void unitTestCore(BasicStats stats, float freq, int docLen) {
for (SimilarityBase sim : sims) {
- BasicStats realStats = (BasicStats) sim.computeWeight(
+ BasicStats realStats = ((BasicSimScorer) sim.scorer(
(float)stats.getBoost(),
toCollectionStats(stats),
- toTermStats(stats));
+ toTermStats(stats))).stats;
float score = (float)sim.score(realStats, freq, docLen);
float explScore = sim.explain(
- realStats, 1, Explanation.match(freq, "freq"), docLen).getValue().floatValue();
+ realStats, Explanation.match(freq, "freq"), docLen).getValue().floatValue();
assertFalse("Score infinite: " + sim.toString(), Float.isInfinite(score));
assertFalse("Score NaN: " + sim.toString(), Float.isNaN(score));
assertTrue("Score negative: " + sim.toString(), score >= 0);
@@ -489,10 +490,10 @@ public class TestSimilarityBase extends LuceneTestCase {
*/
private void correctnessTestCore(SimilarityBase sim, float gold) {
BasicStats stats = createStats();
- BasicStats realStats = (BasicStats) sim.computeWeight(
+ BasicStats realStats = ((BasicSimScorer) sim.scorer(
(float)stats.getBoost(),
toCollectionStats(stats),
- toTermStats(stats));
+ toTermStats(stats))).stats;
float score = (float) sim.score(realStats, FREQ, DOC_LEN);
assertEquals(
sim.toString() + " score not correct.", gold, score, FLOAT_EPSILON);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java
index 8ed0462..f72ea66 100644
--- a/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java
@@ -235,7 +235,7 @@ public class TestFieldMaskingSpanQuery extends LuceneTestCase {
public void testSimple2() throws Exception {
assumeTrue("Broken scoring: LUCENE-3723",
- searcher.getSimilarity(true) instanceof TFIDFSimilarity);
+ searcher.getSimilarity() instanceof TFIDFSimilarity);
SpanQuery q1 = new SpanTermQuery(new Term("gender", "female"));
SpanQuery q2 = new SpanTermQuery(new Term("last", "smith"));
SpanQuery q = new SpanNearQuery(new SpanQuery[]
@@ -291,7 +291,7 @@ public class TestFieldMaskingSpanQuery extends LuceneTestCase {
public void testSpans2() throws Exception {
assumeTrue("Broken scoring: LUCENE-3723",
- searcher.getSimilarity(true) instanceof TFIDFSimilarity);
+ searcher.getSimilarity() instanceof TFIDFSimilarity);
SpanQuery qA1 = new SpanTermQuery(new Term("gender", "female"));
SpanQuery qA2 = new SpanTermQuery(new Term("first", "james"));
SpanQuery qA = new SpanOrQuery(qA1, new FieldMaskingSpanQuery(qA2, "gender"));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
index a4b9d71..94cf974 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
@@ -50,7 +50,6 @@ import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SortedDocValues;
@@ -200,12 +199,7 @@ public class TestMemoryIndex extends LuceneTestCase {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java
index 2e8491d..5e98349f 100644
--- a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java
+++ b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -27,7 +28,6 @@ import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
@@ -36,7 +36,6 @@ import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -358,7 +357,7 @@ public class TestDiversifiedTopDocsCollector extends LuceneTestCase {
Occur.SHOULD));
testQuery.add(new BooleanClause(new TermQuery(new Term("year", "1969")),
Occur.SHOULD));
- return testQuery.build();
+ return new DocValueScoreQuery(testQuery.build(), "weeksAtNumberOne");
}
@Override
@@ -411,10 +410,6 @@ public class TestDiversifiedTopDocsCollector extends LuceneTestCase {
writer.close();
searcher = newSearcher(reader);
artistDocValues = MultiDocValues.getSortedValues(reader, "artist");
-
- // All searches sort by song popularity
- final Similarity base = searcher.getSimilarity(true);
- searcher.setSimilarity(new DocValueSimilarity(base, "weeksAtNumberOne"));
}
@Override
@@ -442,61 +437,108 @@ public class TestDiversifiedTopDocsCollector extends LuceneTestCase {
return result;
}
- /**
- * Similarity that wraps another similarity and replaces the final score
- * according to whats in a docvalues field.
- *
- * @lucene.experimental
- */
- static class DocValueSimilarity extends Similarity {
- private final Similarity sim;
- private final String scoreValueField;
-
- public DocValueSimilarity(Similarity sim, String scoreValueField) {
- this.sim = sim;
- this.scoreValueField = scoreValueField;
- }
+ private static final class DocValueScoreQuery extends Query {
+ private final Query query;
+ private final String scoreField;
+
+ DocValueScoreQuery(Query query, String scoreField) {
+ this.query = query;
+ this.scoreField = scoreField;
+ }
+
@Override
- public long computeNorm(FieldInvertState state) {
- return sim.computeNorm(state);
+ public String toString(String field) {
+ return "DocValueScore(" + query.toString(field) + ")";
}
@Override
- public SimWeight computeWeight(float boost,
- CollectionStatistics collectionStats, TermStatistics... termStats) {
- return sim.computeWeight(boost, collectionStats, termStats);
+ public boolean equals(Object obj) {
+ if (obj instanceof DocValueScoreQuery == false) {
+ return false;
+ }
+ return query.equals(((DocValueScoreQuery) obj).query);
}
@Override
- public SimScorer simScorer(SimWeight stats, LeafReaderContext context)
- throws IOException {
+ public int hashCode() {
+ int h = getClass().hashCode();
+ h = 31 * h + query.hashCode();
+ h = 31 * h + scoreField.hashCode();
+ return h;
+ }
- final NumericDocValues values = DocValues.getNumeric(context.reader(), scoreValueField);
+ @Override
+ public Query rewrite(IndexReader reader) throws IOException {
+ Query rewritten = query.rewrite(reader);
+ if (rewritten != query) {
+ return new DocValueScoreQuery(rewritten, scoreField);
+ }
+ return super.rewrite(reader);
+ }
- return new SimScorer() {
+ @Override
+ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+ if (scoreMode.needsScores() == false) {
+ return query.createWeight(searcher, scoreMode, boost);
+ }
+ Weight inner = query.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);
+ return new Weight(this) {
+
@Override
- public float score(int doc, float freq) throws IOException {
- if (doc != values.docID()) {
- values.advance(doc);
- }
- if (doc == values.docID()) {
- return Float.intBitsToFloat((int) values.longValue());
- } else {
- return 0f;
- }
+ public boolean isCacheable(LeafReaderContext ctx) {
+ return true;
}
-
+
@Override
- public float maxScore(float maxFreq) {
- return Float.MAX_VALUE;
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ Scorer innerScorer = inner.scorer(context);
+ NumericDocValues scoreFactors = DocValues.getNumeric(context.reader(), scoreField);
+ return new Scorer(this) {
+
+ @Override
+ public float score() throws IOException {
+ if (scoreFactors.advanceExact(docID())) {
+ return Float.intBitsToFloat((int) scoreFactors.longValue());
+ }
+ return 0;
+ }
+
+ @Override
+ public float maxScore() {
+ return Float.POSITIVE_INFINITY;
+ }
+
+ @Override
+ public DocIdSetIterator iterator() {
+ return innerScorer.iterator();
+ }
+
+ @Override
+ public int docID() {
+ return innerScorer.docID();
+ }
+ };
}
-
+
@Override
- public Explanation explain(int doc, Explanation freq) throws IOException {
- return Explanation.match(score(doc, 0f), "indexDocValue(" + scoreValueField + ")");
+ public void extractTerms(Set<Term> terms) {
+ inner.extractTerms(terms);
+ }
+
+ @Override
+ public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+ Scorer s = scorer(context);
+ if (s != null) {
+ int advanced = s.iterator().advance(doc);
+ if (doc != advanced) {
+ return Explanation.match(s.score(), "match");
+ }
+ }
+ return Explanation.noMatch("no match");
}
};
}
}
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java
index 34e5697..4192f2d 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java
@@ -47,7 +47,7 @@ public class IDFValueSource extends DocFreqValueSource {
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
IndexSearcher searcher = (IndexSearcher)context.get("searcher");
- TFIDFSimilarity sim = asTFIDF(searcher.getSimilarity(true), field);
+ TFIDFSimilarity sim = asTFIDF(searcher.getSimilarity(), field);
if (sim == null) {
throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as ClassicSimilarity)");
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
index ea63de9..682c0c0 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
@@ -25,11 +25,11 @@ import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.FloatDocValues;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LeafSimScorer;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.search.similarities.Similarity.SimScorer;
-import org.apache.lucene.search.similarities.Similarity.SimWeight;
/**
* Function that returns the decoded norm for every document.
@@ -62,16 +62,16 @@ public class NormValueSource extends ValueSource {
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
IndexSearcher searcher = (IndexSearcher)context.get("searcher");
- final TFIDFSimilarity similarity = IDFValueSource.asTFIDF(searcher.getSimilarity(true), field);
+ final TFIDFSimilarity similarity = IDFValueSource.asTFIDF(searcher.getSimilarity(), field);
if (similarity == null) {
throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as ClassicSimilarity)");
}
// Only works if the contribution of the tf is 1 when the freq is 1 and contribution of the idf
// is 1 when docCount == docFreq == 1
- final SimWeight simWeight = similarity.computeWeight(1f,
+ final SimScorer simScorer = similarity.scorer(1f,
new CollectionStatistics(field, 1, 1, 1, 1),
new TermStatistics(new BytesRef("bogus"), 1, 1));
- final SimScorer simScorer = similarity.simScorer(simWeight, readerContext);
+ final LeafSimScorer leafSimScorer = new LeafSimScorer(simScorer, readerContext.reader(), true, Float.POSITIVE_INFINITY);
return new FloatDocValues(this) {
int lastDocID = -1;
@@ -81,7 +81,7 @@ public class NormValueSource extends ValueSource {
throw new AssertionError("docs out of order: lastDocID=" + lastDocID + " docID=" + docID);
}
lastDocID = docID;
- return simScorer.score(docID, 1f);
+ return leafSimScorer.score(docID, 1f);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java
index baed0ff..731ab1f 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java
@@ -51,7 +51,7 @@ public class TFValueSource extends TermFreqValueSource {
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
final Terms terms = readerContext.reader().terms(indexedField);
IndexSearcher searcher = (IndexSearcher)context.get("searcher");
- final TFIDFSimilarity similarity = IDFValueSource.asTFIDF(searcher.getSimilarity(true), indexedField);
+ final TFIDFSimilarity similarity = IDFValueSource.asTFIDF(searcher.getSimilarity(), indexedField);
if (similarity == null) {
throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as ClassicSimilarity)");
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadScoreQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadScoreQuery.java
index 6704ba1..c618886 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadScoreQuery.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadScoreQuery.java
@@ -28,10 +28,9 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LeafSimScorer;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.search.spans.FilterSpans;
import org.apache.lucene.search.spans.SpanCollector;
import org.apache.lucene.search.spans.SpanQuery;
@@ -150,7 +149,7 @@ public class PayloadScoreQuery extends SpanQuery {
Spans spans = getSpans(context, Postings.PAYLOADS);
if (spans == null)
return null;
- SimScorer docScorer = innerWeight.getSimScorer(context);
+ LeafSimScorer docScorer = innerWeight.getSimScorer(context);
PayloadSpans payloadSpans = new PayloadSpans(spans, decoder);
return new PayloadSpanScorer(this, payloadSpans, docScorer);
}
@@ -228,7 +227,7 @@ public class PayloadScoreQuery extends SpanQuery {
private final PayloadSpans spans;
- private PayloadSpanScorer(SpanWeight weight, PayloadSpans spans, Similarity.SimScorer docScorer) throws IOException {
+ private PayloadSpanScorer(SpanWeight weight, PayloadSpans spans, LeafSimScorer docScorer) throws IOException {
super(weight, spans, docScorer);
this.spans = spans;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/queries/src/java/org/apache/lucene/queries/payloads/SpanPayloadCheckQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/payloads/SpanPayloadCheckQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/payloads/SpanPayloadCheckQuery.java
index 8b23122..05d6682 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/payloads/SpanPayloadCheckQuery.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/payloads/SpanPayloadCheckQuery.java
@@ -28,9 +28,9 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
import org.apache.lucene.index.Terms;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LeafSimScorer;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.spans.FilterSpans;
import org.apache.lucene.search.spans.FilterSpans.AcceptStatus;
import org.apache.lucene.search.spans.SpanCollector;
@@ -127,7 +127,7 @@ public class SpanPayloadCheckQuery extends SpanQuery {
if (spans == null) {
return null;
}
- final Similarity.SimScorer docScorer = getSimScorer(context);
+ final LeafSimScorer docScorer = getSimScorer(context);
return new SpanScorer(this, spans, docScorer);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java
index 842c117..ef0f476 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java
@@ -82,7 +82,7 @@ public class TestLongNormValueSource extends LuceneTestCase {
}
public void testNorm() throws Exception {
- Similarity saved = searcher.getSimilarity(true);
+ Similarity saved = searcher.getSimilarity();
try {
// no norm field (so agnostic to indexed similarity)
searcher.setSimilarity(sim);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
index 876fec8..b12f026 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
@@ -225,7 +225,7 @@ public class TestValueSources extends LuceneTestCase {
}
public void testIDF() throws Exception {
- Similarity saved = searcher.getSimilarity(true);
+ Similarity saved = searcher.getSimilarity();
try {
searcher.setSimilarity(new ClassicSimilarity());
ValueSource vs = new IDFValueSource("bogus", "bogus", "text", new BytesRef("test"));
@@ -362,7 +362,7 @@ public class TestValueSources extends LuceneTestCase {
}
public void testNorm() throws Exception {
- Similarity saved = searcher.getSimilarity(true);
+ Similarity saved = searcher.getSimilarity();
try {
// no norm field (so agnostic to indexed similarity)
searcher.setSimilarity(new ClassicSimilarity());
@@ -414,7 +414,7 @@ public class TestValueSources extends LuceneTestCase {
}
public void testQuery() throws Exception {
- Similarity saved = searcher.getSimilarity(true);
+ Similarity saved = searcher.getSimilarity();
try {
searcher.setSimilarity(new ClassicSimilarity());
@@ -521,7 +521,7 @@ public class TestValueSources extends LuceneTestCase {
}
public void testTF() throws Exception {
- Similarity saved = searcher.getSimilarity(true);
+ Similarity saved = searcher.getSimilarity();
try {
// no norm field (so agnostic to indexed similarity)
searcher.setSimilarity(new ClassicSimilarity());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
index d5607da..92c8d59 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
@@ -335,14 +335,14 @@ public class TermAutomatonQuery extends Query {
final class TermAutomatonWeight extends Weight {
final Automaton automaton;
private final Map<Integer,TermContext> termStates;
- private final Similarity.SimWeight stats;
+ private final Similarity.SimScorer stats;
private final Similarity similarity;
public TermAutomatonWeight(Automaton automaton, IndexSearcher searcher, Map<Integer,TermContext> termStates, float boost) throws IOException {
super(TermAutomatonQuery.this);
this.automaton = automaton;
this.termStates = termStates;
- this.similarity = searcher.getSimilarity(true);
+ this.similarity = searcher.getSimilarity();
List<TermStatistics> allTermStats = new ArrayList<>();
for(Map.Entry<Integer,BytesRef> ent : idToTerm.entrySet()) {
Integer termID = ent.getKey();
@@ -357,7 +357,7 @@ public class TermAutomatonQuery extends Query {
if (allTermStats.isEmpty()) {
stats = null; // no terms matched at all, will not use sim
} else {
- stats = similarity.computeWeight(boost, searcher.collectionStatistics(field),
+ stats = similarity.scorer(boost, searcher.collectionStatistics(field),
allTermStats.toArray(new TermStatistics[allTermStats.size()]));
}
}
@@ -397,7 +397,7 @@ public class TermAutomatonQuery extends Query {
}
if (any) {
- return new TermAutomatonScorer(this, enums, anyTermID, idToTerm, similarity.simScorer(stats, context));
+ return new TermAutomatonScorer(this, enums, anyTermID, idToTerm, new LeafSimScorer(stats, context.reader(), true, Float.POSITIVE_INFINITY));
} else {
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java
index 27270e7..6094c01 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java
@@ -21,7 +21,6 @@ import java.util.Map;
import org.apache.lucene.search.TermAutomatonQuery.EnumAndScorer;
import org.apache.lucene.search.TermAutomatonQuery.TermAutomatonWeight;
-import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.PriorityQueue;
@@ -47,7 +46,7 @@ class TermAutomatonScorer extends Scorer {
// This is -1 if wildcard (null) terms were not used, else it's the id
// of the wildcard term:
private final int anyTermID;
- private final Similarity.SimScorer docScorer;
+ private final LeafSimScorer docScorer;
private int numSubsOnDoc;
@@ -56,7 +55,7 @@ class TermAutomatonScorer extends Scorer {
private int docID = -1;
private int freq;
- public TermAutomatonScorer(TermAutomatonWeight weight, EnumAndScorer[] subs, int anyTermID, Map<Integer,BytesRef> idToTerm, Similarity.SimScorer docScorer) throws IOException {
+ public TermAutomatonScorer(TermAutomatonWeight weight, EnumAndScorer[] subs, int anyTermID, Map<Integer,BytesRef> idToTerm, LeafSimScorer docScorer) throws IOException {
super(weight);
//System.out.println(" automaton:\n" + weight.automaton.toDot());
this.runAutomaton = new TermRunAutomaton(weight.automaton, subs.length);
@@ -362,7 +361,7 @@ class TermAutomatonScorer extends Scorer {
@Override
public float maxScore() {
- return docScorer.maxScore(Float.POSITIVE_INFINITY);
+ return docScorer.maxScore();
}
static class TermRunAutomaton extends RunAutomaton {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
index 370d009..617a721 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
@@ -534,12 +534,7 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
throw new UnsupportedOperationException();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java b/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
index 1773977..fa11311 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
@@ -161,7 +161,7 @@ public class QueryUtils {
};
IndexSearcher out = LuceneTestCase.newSearcher(new MultiReader(readers));
- out.setSimilarity(s.getSimilarity(true));
+ out.setSimilarity(s.getSimilarity());
return out;
}
@@ -373,7 +373,7 @@ public class QueryUtils {
if (lastReader[0] != null) {
final LeafReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader, false);
- indexSearcher.setSimilarity(s.getSimilarity(true));
+ indexSearcher.setSimilarity(s.getSimilarity());
Weight w = indexSearcher.createNormalizedWeight(q, ScoreMode.COMPLETE);
LeafReaderContext ctx = (LeafReaderContext)indexSearcher.getTopReaderContext();
Scorer scorer = w.scorer(ctx);
@@ -403,7 +403,7 @@ public class QueryUtils {
// previous reader, hits NO_MORE_DOCS
final LeafReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader, false);
- indexSearcher.setSimilarity(s.getSimilarity(true));
+ indexSearcher.setSimilarity(s.getSimilarity());
Weight w = indexSearcher.createNormalizedWeight(q, ScoreMode.COMPLETE);
LeafReaderContext ctx = previousReader.getContext();
Scorer scorer = w.scorer(ctx);
@@ -475,7 +475,7 @@ public class QueryUtils {
if (lastReader[0] != null) {
final LeafReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader, false);
- indexSearcher.setSimilarity(s.getSimilarity(true));
+ indexSearcher.setSimilarity(s.getSimilarity());
Weight w = indexSearcher.createNormalizedWeight(q, ScoreMode.COMPLETE);
Scorer scorer = w.scorer((LeafReaderContext)indexSearcher.getTopReaderContext());
if (scorer != null) {
@@ -503,7 +503,7 @@ public class QueryUtils {
// previous reader, hits NO_MORE_DOCS
final LeafReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader, false);
- indexSearcher.setSimilarity(s.getSimilarity(true));
+ indexSearcher.setSimilarity(s.getSimilarity());
Weight w = indexSearcher.createNormalizedWeight(q, ScoreMode.COMPLETE);
Scorer scorer = w.scorer((LeafReaderContext)indexSearcher.getTopReaderContext());
if (scorer != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/test-framework/src/java/org/apache/lucene/search/similarities/AssertingSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/similarities/AssertingSimilarity.java b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/AssertingSimilarity.java
index ac41ea91..afc33ba 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/similarities/AssertingSimilarity.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/AssertingSimilarity.java
@@ -19,7 +19,6 @@ package org.apache.lucene.search.similarities;
import java.io.IOException;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -48,7 +47,7 @@ public class AssertingSimilarity extends Similarity {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
assert boost >= 0;
assert collectionStats != null;
assert termStats.length > 0;
@@ -56,70 +55,54 @@ public class AssertingSimilarity extends Similarity {
assert term != null;
}
// TODO: check that TermStats is in bounds with respect to collection? e.g. docFreq <= maxDoc
- SimWeight weight = delegate.computeWeight(boost, collectionStats, termStats);
- assert weight != null;
- return new AssertingWeight(weight, boost);
+ SimScorer scorer = delegate.scorer(boost, collectionStats, termStats);
+ assert scorer != null;
+ return new AssertingSimScorer(scorer, boost);
}
- static class AssertingWeight extends SimWeight {
- final SimWeight delegate;
+ static class AssertingSimScorer extends SimScorer {
+ final SimScorer delegate;
final float boost;
- AssertingWeight(SimWeight delegate, float boost) {
+ AssertingSimScorer(SimScorer delegate, float boost) {
+ super(delegate.getField());
this.delegate = delegate;
this.boost = boost;
}
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
- assert weight != null;
- assert context != null;
- AssertingWeight assertingWeight = (AssertingWeight)weight;
- SimScorer delegateScorer = delegate.simScorer(assertingWeight.delegate, context);
- assert delegateScorer != null;
- return new SimScorer() {
- @Override
- public float score(int doc, float freq) throws IOException {
- // doc in bounds
- assert doc >= 0;
- assert doc < context.reader().maxDoc();
- // freq in bounds
- assert Float.isFinite(freq);
- assert freq > 0;
- // result in bounds
- float score = delegateScorer.score(doc, freq);
- assert Float.isFinite(score);
- assert score <= maxScore(freq);
- assert score >= 0;
- return score;
- }
+ @Override
+ public float score(float freq, long norm) throws IOException {
+ // freq in bounds
+ assert Float.isFinite(freq);
+ assert freq > 0;
+ // result in bounds
+ float score = delegate.score(freq, norm);
+ assert Float.isFinite(score);
+ assert score <= maxScore(freq);
+ assert score >= 0;
+ return score;
+ }
- @Override
- public float maxScore(float maxFreq) {
- float maxScore = delegateScorer.maxScore(maxFreq);
- assert Float.isNaN(maxScore) == false;
- return maxScore;
- }
+ @Override
+ public float maxScore(float maxFreq) {
+ float maxScore = delegate.maxScore(maxFreq);
+ assert Float.isNaN(maxScore) == false;
+ return maxScore;
+ }
- @Override
- public Explanation explain(int doc, Explanation freq) throws IOException {
- // doc in bounds
- assert doc >= 0;
- assert doc < context.reader().maxDoc();
- // freq in bounds
- assert freq != null;
- assert Float.isFinite(freq.getValue().floatValue());
- // result in bounds
- Explanation explanation = delegateScorer.explain(doc, freq);
- assert explanation != null;
- assert Float.isFinite(explanation.getValue().floatValue());
- // result matches score exactly
- assert explanation.getValue().floatValue() == delegateScorer.score(doc, freq.getValue().floatValue());
- return explanation;
- }
- };
+ @Override
+ public Explanation explain(Explanation freq, long norm) throws IOException {
+ // freq in bounds
+ assert freq != null;
+ assert Float.isFinite(freq.getValue().floatValue());
+ // result in bounds
+ Explanation explanation = delegate.explain(freq, norm);
+ assert explanation != null;
+ assert Float.isFinite(explanation.getValue().floatValue());
+ // result matches score exactly
+ assert explanation.getValue().floatValue() == delegate.score(freq.getValue().floatValue(), norm);
+ return explanation;
+ }
}
@Override
[21/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11821:
ConcurrentModificationException in SimSolrCloudTestCase.tearDown
Posted by da...@apache.org.
SOLR-11821: ConcurrentModificationException in SimSolrCloudTestCase.tearDown
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3d8ef981
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3d8ef981
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3d8ef981
Branch: refs/heads/jira/solr-11702
Commit: 3d8ef981cbe150d5f443ee15081b3fff2a6ab50e
Parents: 7695544
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Mon Jan 8 14:04:11 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Mon Jan 8 14:04:11 2018 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 1 +
.../cloud/autoscaling/sim/SimSolrCloudTestCase.java | 16 ++++++++++------
2 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3d8ef981/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 4b4820c..011766f 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -88,6 +88,7 @@ Bug Fixes
* SOLR-11824: Fixed bucket ordering in distributed json.facet type:range when mincount>0 (hossman)
+* SOLR-11821: ConcurrentModificationException in SimSolrCloudTestCase.tearDown (shalin)
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3d8ef981/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
index 77ddbc0..48d46b6 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
@@ -138,13 +138,17 @@ public class SimSolrCloudTestCase extends SolrTestCaseJ4 {
cluster.simGetOpCounts().forEach((k, cnt) -> log.info("##\t\t- " + String.format(Locale.ROOT, "%-14s %4d", k, cnt.get())));
log.info("######### Autoscaling event counts ###########");
TreeMap<String, Map<String, AtomicInteger>> counts = new TreeMap<>();
- for (SolrInputDocument d : cluster.simGetSystemCollection()) {
- if (!"autoscaling_event".equals(d.getFieldValue("type"))) {
- continue;
+
+ List<SolrInputDocument> solrInputDocuments = cluster.simGetSystemCollection();
+ synchronized (solrInputDocuments) {
+ for (SolrInputDocument d : solrInputDocuments) {
+ if (!"autoscaling_event".equals(d.getFieldValue("type"))) {
+ continue;
+ }
+ counts.computeIfAbsent((String)d.getFieldValue("event.source_s"), s -> new TreeMap<>())
+ .computeIfAbsent((String)d.getFieldValue("stage_s"), s -> new AtomicInteger())
+ .incrementAndGet();
}
- counts.computeIfAbsent((String)d.getFieldValue("event.source_s"), s -> new TreeMap<>())
- .computeIfAbsent((String)d.getFieldValue("stage_s"), s -> new AtomicInteger())
- .incrementAndGet();
}
counts.forEach((trigger, map) -> {
log.info("## * Trigger: " + trigger);
[30/50] [abbrv] lucene-solr:jira/solr-11702: Revert "LUCENE-8115:
remove one TODO-on-its-own javadoc."
Posted by da...@apache.org.
Revert "LUCENE-8115: remove one TODO-on-its-own javadoc."
This reverts commit bd69d64ad04fb0fe6f17f68dcc1fa685e15a9317.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a3a0e0b1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a3a0e0b1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a3a0e0b1
Branch: refs/heads/jira/solr-11702
Commit: a3a0e0b11e4538ccdff998c09b1145ce9036ac33
Parents: 144616b
Author: Christine Poerschke <cp...@apache.org>
Authored: Mon Jan 8 20:18:45 2018 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Mon Jan 8 20:18:45 2018 +0000
----------------------------------------------------------------------
.../apache/lucene/classification/KNearestNeighborClassifier.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a3a0e0b1/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
index b12bb5c..1bc53b0 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
@@ -124,6 +124,9 @@ public class KNearestNeighborClassifier implements Classifier<BytesRef> {
return classifyFromTopDocs(knnSearch(text));
}
+ /**
+ * TODO
+ */
protected ClassificationResult<BytesRef> classifyFromTopDocs(TopDocs knnResults) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = buildListFromTopDocs(knnResults);
ClassificationResult<BytesRef> assignedClass = null;
[48/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8121: UH switch
to SpanCollector API. Better accuracy. * Use the filtered freq in position
sensitive terms (better scores) * Refactored UH's OffsetsEnum * Improved test
randomization in TestUnifiedHighl
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/352ec01a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterStrictPhrases.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterStrictPhrases.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterStrictPhrases.java
index acc4bd7..08820aa 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterStrictPhrases.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterStrictPhrases.java
@@ -17,6 +17,7 @@
package org.apache.lucene.search.uhighlight;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -38,6 +39,7 @@ import org.apache.lucene.search.MultiPhraseQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TermQuery;
@@ -46,6 +48,7 @@ import org.apache.lucene.search.Weight;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
import org.apache.lucene.search.spans.SpanNearQuery;
+import org.apache.lucene.search.spans.SpanOrQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.store.Directory;
@@ -55,6 +58,7 @@ import org.apache.lucene.util.QueryBuilder;
import org.junit.After;
import org.junit.Before;
+//TODO rename to reflect position sensitivity
public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {
final FieldType fieldType;
@@ -151,6 +155,16 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {
String[] snippets = highlighter.highlight("body", query, topDocs);
assertArrayEquals(new String[]{"<b>Yin</b> <b>yang</b>, <b>yin</b> gap yang"}, snippets);
+
+ // test the Passage only has 3 matches. We don't want duplicates from "Yin" being in TermQuery & PhraseQuery.
+ highlighter.setFormatter(new PassageFormatter() {
+ @Override
+ public Object format(Passage[] passages, String content) {
+ return Arrays.toString(passages);
+ }
+ });
+ assertArrayEquals(new String[]{"[Passage[0-22]{yin[0-3],yang[4-8],yin[10-13]}score=2.4964213]"},
+ highlighter.highlight("body", query, topDocs));
}
public void testPhraseNotInDoc() throws IOException {
@@ -185,6 +199,16 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {
String[] snippets = highlighter.highlight("body", query, topDocs);
assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo alpha"}, snippets);
+
+ // test the Passage only has 3 matches. We don't want duplicates from both PhraseQuery
+ highlighter.setFormatter(new PassageFormatter() {
+ @Override
+ public Object format(Passage[] passages, String content) {
+ return Arrays.toString(passages);
+ }
+ });
+ assertArrayEquals(new String[]{"[Passage[0-41]{alpha[0-5],bravo[6-11],charlie[12-19]}score=3.931102]"},
+ highlighter.highlight("body", query, topDocs));
}
public void testSynonyms() throws IOException {
@@ -477,4 +501,68 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {
return wrapped.hashCode();
}
}
+
+ // Ported from LUCENE-5455 (fixed in LUCENE-8121). Also see LUCENE-2287.
+ public void testNestedSpanQueryHighlight() throws Exception {
+ // For a long time, the highlighters used to assume all query terms within the SpanQuery were valid at the Spans'
+ // position range. This would highlight occurrences of terms that were actually not matched by the query.
+ // But now using the SpanCollector API we don't make this kind of mistake.
+ final String FIELD_NAME = "body";
+ final String indexedText = "x y z x z x a";
+ indexWriter.addDocument(newDoc(indexedText));
+ initReaderSearcherHighlighter();
+ TopDocs topDocs = new TopDocs(1, new ScoreDoc[]{new ScoreDoc(0, 1f)}, 1f);
+
+ String expected = "<b>x</b> <b>y</b> <b>z</b> x z x <b>a</b>";
+ Query q = new SpanNearQuery(new SpanQuery[] {
+ new SpanNearQuery(new SpanQuery[] {
+ new SpanTermQuery(new Term(FIELD_NAME, "x")),
+ new SpanTermQuery(new Term(FIELD_NAME, "y")),
+ new SpanTermQuery(new Term(FIELD_NAME, "z"))}, 0, true),
+ new SpanTermQuery(new Term(FIELD_NAME, "a"))}, 10, false);
+ String observed = highlighter.highlight(FIELD_NAME, q, topDocs)[0];
+ if (VERBOSE) System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);
+ assertEquals("Nested SpanNear query not properly highlighted.", expected, observed);
+
+ expected = "x <b>y</b> <b>z</b> <b>x</b> <b>z</b> x <b>a</b>";
+ q = new SpanNearQuery(new SpanQuery[] {
+ new SpanOrQuery(
+ new SpanNearQuery(new SpanQuery[] {
+ new SpanTermQuery(new Term(FIELD_NAME, "x")),
+ new SpanTermQuery(new Term(FIELD_NAME, "z"))}, 0, true),
+ new SpanNearQuery(new SpanQuery[] {
+ new SpanTermQuery(new Term(FIELD_NAME, "y")),
+ new SpanTermQuery(new Term(FIELD_NAME, "z"))}, 0, true)),
+ new SpanOrQuery(
+ new SpanTermQuery(new Term(FIELD_NAME, "a")),
+ new SpanTermQuery(new Term(FIELD_NAME, "b")))}, 10, false);
+ observed = highlighter.highlight(FIELD_NAME, q, topDocs)[0];
+ if (VERBOSE) System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);
+ assertEquals("Nested SpanNear query within SpanOr not properly highlighted.", expected, observed);
+
+ expected = "x <b>y</b> <b>z</b> <b>x</b> <b>z</b> x <b>a</b>";
+ q = new SpanNearQuery(new SpanQuery[] {
+ new SpanNearQuery(new SpanQuery[] {
+ new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term(FIELD_NAME, "*"))),
+ new SpanTermQuery(new Term(FIELD_NAME, "z"))}, 0, true),
+ new SpanTermQuery(new Term(FIELD_NAME, "a"))}, 10, false);
+ observed = highlighter.highlight(FIELD_NAME, q, topDocs)[0];
+ if (VERBOSE) System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);
+ assertEquals("Nested SpanNear query with wildcard not properly highlighted.", expected, observed);
+
+ expected = "<b>x</b> <b>y</b> z x z x <b>a</b>";
+ q = new SpanNearQuery(new SpanQuery[] {
+ new SpanOrQuery(
+ new SpanNearQuery(new SpanQuery[] {
+ new SpanTermQuery(new Term(FIELD_NAME, "x")),
+ new SpanTermQuery(new Term(FIELD_NAME, "y"))}, 0, true),
+ new SpanNearQuery(new SpanQuery[] { //No hit span query
+ new SpanTermQuery(new Term(FIELD_NAME, "z")),
+ new SpanTermQuery(new Term(FIELD_NAME, "a"))}, 0, true)),
+ new SpanTermQuery(new Term(FIELD_NAME, "a"))}, 10, false);
+ observed = highlighter.highlight(FIELD_NAME, q, topDocs)[0];
+ if (VERBOSE) System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);
+ assertEquals("Nested SpanNear query within SpanOr not properly highlighted.", expected, observed);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/352ec01a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/visibility/TestUnifiedHighlighterExtensibility.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/visibility/TestUnifiedHighlighterExtensibility.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/visibility/TestUnifiedHighlighterExtensibility.java
index 738a0b9..e60b17b 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/visibility/TestUnifiedHighlighterExtensibility.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/visibility/TestUnifiedHighlighterExtensibility.java
@@ -218,11 +218,9 @@ public class TestUnifiedHighlighterExtensibility extends LuceneTestCase {
// this code never runs; just for compilation
Passage p;
- try (OffsetsEnum oe = new OffsetsEnum(null, EMPTY)) {
+ try (OffsetsEnum oe = new OffsetsEnum.OfPostings(null, EMPTY)) {
oe.getTerm();
- oe.getPostingsEnum();
oe.freq();
- oe.hasMorePositions();
oe.nextPosition();
oe.startOffset();
oe.endOffset();
[05/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8116: SimScorer
now only takes a frequency and a norm as per-document scoring factors.
Posted by da...@apache.org.
LUCENE-8116: SimScorer now only takes a frequency and a norm as per-document scoring factors.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8fd7ead9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8fd7ead9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8fd7ead9
Branch: refs/heads/jira/solr-11702
Commit: 8fd7ead940f69a892dfc951a1aa042e8cae806c1
Parents: 8836fda
Author: Adrien Grand <jp...@gmail.com>
Authored: Thu Jan 4 15:13:36 2018 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Thu Jan 4 15:13:36 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 6 +
.../classification/KNearestFuzzyClassifier.java | 2 +-
.../KNearestNeighborClassifier.java | 2 +-
.../org/apache/lucene/search/BooleanWeight.java | 2 +-
.../apache/lucene/search/ExactPhraseScorer.java | 7 +-
.../org/apache/lucene/search/IndexSearcher.java | 45 +----
.../org/apache/lucene/search/LeafSimScorer.java | 74 +++++++
.../apache/lucene/search/MultiPhraseQuery.java | 29 +--
.../org/apache/lucene/search/PhraseQuery.java | 13 +-
.../lucene/search/SloppyPhraseScorer.java | 7 +-
.../org/apache/lucene/search/SynonymQuery.java | 35 ++--
.../org/apache/lucene/search/TermQuery.java | 15 +-
.../org/apache/lucene/search/TermScorer.java | 11 +-
.../org/apache/lucene/search/package-info.java | 4 +-
.../lucene/search/similarities/Axiomatic.java | 10 +-
.../search/similarities/BM25Similarity.java | 142 +++++---------
.../lucene/search/similarities/BasicStats.java | 2 +-
.../search/similarities/BooleanSimilarity.java | 51 ++---
.../search/similarities/DFISimilarity.java | 6 +-
.../search/similarities/DFRSimilarity.java | 8 +-
.../search/similarities/IBSimilarity.java | 8 +-
.../similarities/LMDirichletSimilarity.java | 10 +-
.../similarities/LMJelinekMercerSimilarity.java | 10 +-
.../search/similarities/LMSimilarity.java | 2 +-
.../search/similarities/MultiSimilarity.java | 37 +---
.../similarities/PerFieldSimilarityWrapper.java | 22 +--
.../lucene/search/similarities/Similarity.java | 99 ++++------
.../search/similarities/SimilarityBase.java | 79 +++-----
.../search/similarities/TFIDFSimilarity.java | 133 +++++--------
.../apache/lucene/search/spans/SpanScorer.java | 6 +-
.../apache/lucene/search/spans/SpanWeight.java | 22 +--
.../apache/lucene/search/spans/TermSpans.java | 4 +-
.../apache/lucene/index/TestCustomNorms.java | 7 +-
.../apache/lucene/index/TestCustomTermFreq.java | 9 +-
.../lucene/index/TestFieldInvertState.java | 8 +-
.../apache/lucene/index/TestIndexSorting.java | 9 +-
.../lucene/index/TestMaxTermFrequency.java | 12 +-
.../test/org/apache/lucene/index/TestNorms.java | 7 +-
.../lucene/index/TestUniqueTermCount.java | 8 +-
.../apache/lucene/search/JustCompileSearch.java | 7 +-
.../org/apache/lucene/search/TestBoolean2.java | 4 +-
.../search/TestBooleanQueryVisitSubscorers.java | 12 +-
.../lucene/search/TestBooleanRewrites.java | 2 +-
.../apache/lucene/search/TestConjunctions.java | 12 +-
.../lucene/search/TestDocValuesScoring.java | 192 -------------------
.../lucene/search/TestMinShouldMatch2.java | 9 +-
.../lucene/search/TestSimilarityProvider.java | 23 +--
.../lucene/search/TestSubScorerFreqs.java | 12 +-
.../similarities/TestClassicSimilarity.java | 3 +-
.../search/similarities/TestSimilarityBase.java | 11 +-
.../search/spans/TestFieldMaskingSpanQuery.java | 4 +-
.../lucene/index/memory/TestMemoryIndex.java | 8 +-
.../search/TestDiversifiedTopDocsCollector.java | 132 ++++++++-----
.../function/valuesource/IDFValueSource.java | 2 +-
.../function/valuesource/NormValueSource.java | 10 +-
.../function/valuesource/TFValueSource.java | 2 +-
.../queries/payloads/PayloadScoreQuery.java | 7 +-
.../queries/payloads/SpanPayloadCheckQuery.java | 4 +-
.../function/TestLongNormValueSource.java | 2 +-
.../queries/function/TestValueSources.java | 8 +-
.../lucene/search/TermAutomatonQuery.java | 8 +-
.../lucene/search/TermAutomatonScorer.java | 7 +-
.../lucene/index/BaseNormsFormatTestCase.java | 7 +-
.../org/apache/lucene/search/QueryUtils.java | 10 +-
.../similarities/AssertingSimilarity.java | 95 ++++-----
.../similarities/BaseSimilarityTestCase.java | 140 ++------------
.../search/spans/AssertingSpanWeight.java | 4 +-
.../similarities/BaseSimilarityTestCase.java | 2 +-
68 files changed, 606 insertions(+), 1096 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ff94809..16050d1 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -32,6 +32,9 @@ API Changes
* LUCENE-8012: Explanation now takes Number rather than float (Alan Woodward,
Robert Muir)
+* LUCENE-8116: SimScorer now only takes a frequency and a norm as per-document
+ scoring factors. (Adrien Grand)
+
Changes in Runtime Behavior
* LUCENE-7837: Indices that were created before the previous major version
@@ -46,6 +49,9 @@ Changes in Runtime Behavior
* LUCENE-7996: FunctionQuery and FunctionScoreQuery now return a score of 0
when the function produces a negative value. (Adrien Grand)
+* LUCENE-8116: Similarities now score fields that omit norms as if the norm was
+ 1. This might change score values on fields that omit norms. (Adrien Grand)
+
Improvements
* LUCENE-7997: Add BaseSimilarityTestCase to sanity check similarities.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
index cbd241b..14f9a27 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
@@ -213,7 +213,7 @@ public class KNearestFuzzyClassifier implements Classifier<BytesRef> {
", classFieldName='" + classFieldName + '\'' +
", k=" + k +
", query=" + query +
- ", similarity=" + indexSearcher.getSimilarity(true) +
+ ", similarity=" + indexSearcher.getSimilarity() +
'}';
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
index f0391f4..e6ad4a3 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
@@ -251,7 +251,7 @@ public class KNearestNeighborClassifier implements Classifier<BytesRef> {
", classFieldName='" + classFieldName + '\'' +
", k=" + k +
", query=" + query +
- ", similarity=" + indexSearcher.getSimilarity(true) +
+ ", similarity=" + indexSearcher.getSimilarity() +
'}';
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
index 900a77f..fffdd09 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
@@ -48,7 +48,7 @@ final class BooleanWeight extends Weight {
super(query);
this.query = query;
this.scoreMode = scoreMode;
- this.similarity = searcher.getSimilarity(scoreMode.needsScores());
+ this.similarity = searcher.getSimilarity();
weights = new ArrayList<>();
for (BooleanClause c : query) {
Weight w = searcher.createWeight(c.getQuery(), c.isScoring() ? scoreMode : ScoreMode.COMPLETE_NO_SCORES, boost);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java b/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java
index f4a7ca7..e2d6d80 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.search.similarities.Similarity;
final class ExactPhraseScorer extends Scorer {
@@ -42,13 +41,13 @@ final class ExactPhraseScorer extends Scorer {
private int freq;
- private final Similarity.SimScorer docScorer;
+ private final LeafSimScorer docScorer;
private final boolean needsScores, needsTotalHitCount;
private float matchCost;
private float minCompetitiveScore;
ExactPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings,
- Similarity.SimScorer docScorer, ScoreMode scoreMode,
+ LeafSimScorer docScorer, ScoreMode scoreMode,
float matchCost) throws IOException {
super(weight);
this.docScorer = docScorer;
@@ -123,7 +122,7 @@ final class ExactPhraseScorer extends Scorer {
@Override
public float maxScore() {
- return docScorer.maxScore(Integer.MAX_VALUE);
+ return docScorer.maxScore();
}
/** Advance the given pos enum to the first doc on or after {@code target}.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
index 5ee815c..fc87563 100644
--- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
+++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
@@ -32,7 +32,6 @@ import java.util.concurrent.Future;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.IndexWriter;
@@ -75,36 +74,6 @@ import org.apache.lucene.util.ThreadInterruptedException;
*/
public class IndexSearcher {
- /** A search-time {@link Similarity} that does not make use of scoring factors
- * and may be used when scores are not needed. */
- private static final Similarity NON_SCORING_SIMILARITY = new Similarity() {
-
- @Override
- public long computeNorm(FieldInvertState state) {
- throw new UnsupportedOperationException("This Similarity may only be used for searching, not indexing");
- }
-
- @Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return new SimWeight() {};
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
- return new SimScorer() {
- @Override
- public float score(int doc, float freq) {
- return 0f;
- }
- @Override
- public float maxScore(float maxFreq) {
- return 0f;
- }
- };
- }
-
- };
-
private static QueryCache DEFAULT_QUERY_CACHE;
private static QueryCachingPolicy DEFAULT_CACHING_POLICY = new UsageTrackingQueryCachingPolicy();
static {
@@ -136,7 +105,7 @@ public class IndexSearcher {
* Expert: returns a default Similarity instance.
* In general, this method is only called to initialize searchers and writers.
* User code and query implementations should respect
- * {@link IndexSearcher#getSimilarity(boolean)}.
+ * {@link IndexSearcher#getSimilarity()}.
* @lucene.internal
*/
public static Similarity getDefaultSimilarity() {
@@ -329,15 +298,11 @@ public class IndexSearcher {
this.similarity = similarity;
}
- /** Expert: Get the {@link Similarity} to use to compute scores. When
- * {@code needsScores} is {@code false}, this method will return a simple
- * {@link Similarity} that does not leverage scoring factors such as norms.
- * When {@code needsScores} is {@code true}, this returns the
+ /** Expert: Get the {@link Similarity} to use to compute scores. This returns the
* {@link Similarity} that has been set through {@link #setSimilarity(Similarity)}
- * or the {@link #getDefaultSimilarity()} default {@link Similarity} if none
- * has been set explicitly. */
- public Similarity getSimilarity(boolean needsScores) {
- return needsScores ? similarity : NON_SCORING_SIMILARITY;
+ * or the default {@link Similarity} if none has been set explicitly. */
+ public Similarity getSimilarity() {
+ return similarity;
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java b/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java
new file mode 100644
index 0000000..52b7d92
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.search.similarities.Similarity.SimScorer;
+
+/**
+ * {@link SimScorer} on a specific {@link LeafReader}.
+ */
+public final class LeafSimScorer {
+
+ private final SimScorer scorer;
+ private final NumericDocValues norms;
+ private final float maxScore;
+
+ /**
+ * Sole constructor: Score documents of {@code reader} with {@code scorer}.
+ */
+ public LeafSimScorer(SimScorer scorer, LeafReader reader, boolean needsScores, float maxFreq) throws IOException {
+ this.scorer = scorer;
+ norms = needsScores ? reader.getNormValues(scorer.getField()) : null;
+ maxScore = scorer.maxScore(maxFreq);
+ }
+
+ private long getNormValue(int doc) throws IOException {
+ if (norms != null) {
+ boolean found = norms.advanceExact(doc);
+ assert found;
+ return norms.longValue();
+ } else {
+ return 1L; // default norm
+ }
+ }
+
+ /** Score the provided document assuming the given term document frequency.
+ * This method must be called on non-decreasing sequences of doc ids.
+ * @see SimScorer#score(float, long) */
+ public float score(int doc, float freq) throws IOException {
+ return scorer.score(freq, getNormValue(doc));
+ }
+
+ /** Explain the score for the provided document assuming the given term document frequency.
+ * This method must be called on non-decreasing sequences of doc ids.
+ * @see SimScorer#explain(Explanation, long) */
+ public Explanation explain(int doc, Explanation freqExpl) throws IOException {
+ return scorer.explain(freqExpl, getNormValue(doc));
+ }
+
+ /**
+ * Return an upper bound of the score.
+ * @see SimScorer#maxScore(float)
+ */
+ public float maxScore() {
+ return maxScore;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
index 34361a7..941416e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
@@ -18,19 +18,26 @@ package org.apache.lucene.search;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
@@ -183,7 +190,7 @@ public class MultiPhraseQuery extends Query {
private class MultiPhraseWeight extends Weight {
private final Similarity similarity;
- private final Similarity.SimWeight stats;
+ private final Similarity.SimScorer stats;
private final Map<Term,TermContext> termContexts = new HashMap<>();
private final ScoreMode scoreMode;
@@ -191,7 +198,7 @@ public class MultiPhraseQuery extends Query {
throws IOException {
super(MultiPhraseQuery.this);
this.scoreMode = scoreMode;
- this.similarity = searcher.getSimilarity(scoreMode.needsScores());
+ this.similarity = searcher.getSimilarity();
final IndexReaderContext context = searcher.getTopReaderContext();
// compute idf
@@ -212,7 +219,7 @@ public class MultiPhraseQuery extends Query {
if (allTermStats.isEmpty()) {
stats = null; // none of the terms were found, we won't use sim at all
} else {
- stats = similarity.computeWeight(
+ stats = similarity.scorer(
boost,
searcher.collectionStatistics(field),
allTermStats.toArray(new TermStatistics[allTermStats.size()]));
@@ -282,11 +289,11 @@ public class MultiPhraseQuery extends Query {
if (slop == 0) {
return new ExactPhraseScorer(this, postingsFreqs,
- similarity.simScorer(stats, context),
+ new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Integer.MAX_VALUE),
scoreMode, totalMatchCost);
} else {
return new SloppyPhraseScorer(this, postingsFreqs, slop,
- similarity.simScorer(stats, context),
+ new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Float.POSITIVE_INFINITY),
scoreMode.needsScores(), totalMatchCost);
}
}
@@ -303,7 +310,7 @@ public class MultiPhraseQuery extends Query {
int newDoc = scorer.iterator().advance(doc);
if (newDoc == doc) {
float freq = slop == 0 ? ((ExactPhraseScorer)scorer).freq() : ((SloppyPhraseScorer)scorer).sloppyFreq();
- SimScorer docScorer = similarity.simScorer(stats, context);
+ LeafSimScorer docScorer = new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Float.POSITIVE_INFINITY);
Explanation freqExplanation = Explanation.match(freq, "phraseFreq=" + freq);
Explanation scoreExplanation = docScorer.explain(doc, freqExplanation);
return Explanation.match(
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
index 3d359b4..295cc90 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
@@ -37,7 +37,6 @@ import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
@@ -352,7 +351,7 @@ public class PhraseQuery extends Query {
private class PhraseWeight extends Weight {
private final Similarity similarity;
- private final Similarity.SimWeight stats;
+ private final Similarity.SimScorer stats;
private final ScoreMode scoreMode;
private transient TermContext states[];
@@ -366,7 +365,7 @@ public class PhraseQuery extends Query {
throw new IllegalStateException("PhraseWeight requires that the first position is 0, call rewrite first");
}
this.scoreMode = scoreMode;
- this.similarity = searcher.getSimilarity(scoreMode.needsScores());
+ this.similarity = searcher.getSimilarity();
final IndexReaderContext context = searcher.getTopReaderContext();
states = new TermContext[terms.length];
TermStatistics termStats[] = new TermStatistics[terms.length];
@@ -380,7 +379,7 @@ public class PhraseQuery extends Query {
}
}
if (termUpTo > 0) {
- stats = similarity.computeWeight(boost, searcher.collectionStatistics(field), Arrays.copyOf(termStats, termUpTo));
+ stats = similarity.scorer(boost, searcher.collectionStatistics(field), Arrays.copyOf(termStats, termUpTo));
} else {
stats = null; // no terms at all, we won't use similarity
}
@@ -433,11 +432,11 @@ public class PhraseQuery extends Query {
if (slop == 0) { // optimize exact case
return new ExactPhraseScorer(this, postingsFreqs,
- similarity.simScorer(stats, context),
+ new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Integer.MAX_VALUE),
scoreMode, totalMatchCost);
} else {
return new SloppyPhraseScorer(this, postingsFreqs, slop,
- similarity.simScorer(stats, context),
+ new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Float.POSITIVE_INFINITY),
scoreMode.needsScores(), totalMatchCost);
}
}
@@ -459,7 +458,7 @@ public class PhraseQuery extends Query {
int newDoc = scorer.iterator().advance(doc);
if (newDoc == doc) {
float freq = slop == 0 ? ((ExactPhraseScorer)scorer).freq() : ((SloppyPhraseScorer)scorer).sloppyFreq();
- SimScorer docScorer = similarity.simScorer(stats, context);
+ LeafSimScorer docScorer = new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Float.POSITIVE_INFINITY);
Explanation freqExplanation = Explanation.match(freq, "phraseFreq=" + freq);
Explanation scoreExplanation = docScorer.explain(doc, freqExplanation);
return Explanation.match(
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java b/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
index dc5490a..60b77c5 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
@@ -26,7 +26,6 @@ import java.util.HashSet;
import java.util.LinkedHashMap;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.FixedBitSet;
final class SloppyPhraseScorer extends Scorer {
@@ -36,7 +35,7 @@ final class SloppyPhraseScorer extends Scorer {
private float sloppyFreq; //phrase frequency in current doc as computed by phraseFreq().
- private final Similarity.SimScorer docScorer;
+ private final LeafSimScorer docScorer;
private final int slop;
private final int numPostings;
@@ -55,7 +54,7 @@ final class SloppyPhraseScorer extends Scorer {
private final float matchCost;
SloppyPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings,
- int slop, Similarity.SimScorer docScorer, boolean needsScores,
+ int slop, LeafSimScorer docScorer, boolean needsScores,
float matchCost) {
super(weight);
this.docScorer = docScorer;
@@ -558,7 +557,7 @@ final class SloppyPhraseScorer extends Scorer {
@Override
public float maxScore() {
- return docScorer.maxScore(Float.POSITIVE_INFINITY);
+ return docScorer.maxScore();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
index ce9d6e0..3f4c06d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
@@ -35,7 +35,6 @@ import org.apache.lucene.index.TermContext;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.BytesRef;
/**
@@ -129,7 +128,7 @@ public final class SynonymQuery extends Query {
class SynonymWeight extends Weight {
private final TermContext termContexts[];
private final Similarity similarity;
- private final Similarity.SimWeight simWeight;
+ private final Similarity.SimScorer simWeight;
SynonymWeight(Query query, IndexSearcher searcher, float boost) throws IOException {
super(query);
@@ -145,10 +144,10 @@ public final class SynonymQuery extends Query {
totalTermFreq += termStats.totalTermFreq();
}
}
- this.similarity = searcher.getSimilarity(true);
+ this.similarity = searcher.getSimilarity();
if (docFreq > 0) {
TermStatistics pseudoStats = new TermStatistics(new BytesRef("synonym pseudo-term"), docFreq, totalTermFreq);
- this.simWeight = similarity.computeWeight(boost, collectionStats, pseudoStats);
+ this.simWeight = similarity.scorer(boost, collectionStats, pseudoStats);
} else {
this.simWeight = null; // no terms exist at all, we won't use similarity
}
@@ -175,7 +174,7 @@ public final class SynonymQuery extends Query {
assert scorer instanceof TermScorer;
freq = ((TermScorer)scorer).freq();
}
- SimScorer docScorer = similarity.simScorer(simWeight, context);
+ LeafSimScorer docScorer = new LeafSimScorer(simWeight, context.reader(), true, Float.POSITIVE_INFINITY);
Explanation freqExplanation = Explanation.match(freq, "termFreq=" + freq);
Explanation scoreExplanation = docScorer.explain(doc, freqExplanation);
return Explanation.match(
@@ -190,7 +189,6 @@ public final class SynonymQuery extends Query {
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
- Similarity.SimScorer simScorer = null;
IndexOptions indexOptions = IndexOptions.NONE;
if (terms.length > 0) {
FieldInfo info = context.reader()
@@ -202,21 +200,17 @@ public final class SynonymQuery extends Query {
}
// we use termscorers + disjunction as an impl detail
List<Scorer> subScorers = new ArrayList<>();
- long maxFreq = 0;
+ long totalMaxFreq = 0;
for (int i = 0; i < terms.length; i++) {
TermState state = termContexts[i].get(context.ord);
if (state != null) {
TermsEnum termsEnum = context.reader().terms(terms[i].field()).iterator();
termsEnum.seekExact(terms[i].bytes(), state);
-
- maxFreq += getMaxFreq(indexOptions, termsEnum.totalTermFreq(), termsEnum.docFreq());
-
+ long termMaxFreq = getMaxFreq(indexOptions, termsEnum.totalTermFreq(), termsEnum.docFreq());
+ totalMaxFreq += termMaxFreq;
PostingsEnum postings = termsEnum.postings(null, PostingsEnum.FREQS);
- // lazy init sim, in case no terms exist
- if (simScorer == null) {
- simScorer = similarity.simScorer(simWeight, context);
- }
- subScorers.add(new TermScorer(this, postings, simScorer, Float.POSITIVE_INFINITY));
+ LeafSimScorer simScorer = new LeafSimScorer(simWeight, context.reader(), true, termMaxFreq);
+ subScorers.add(new TermScorer(this, postings, simScorer));
}
}
if (subScorers.isEmpty()) {
@@ -225,7 +219,8 @@ public final class SynonymQuery extends Query {
// we must optimize this case (term not in segment), disjunctionscorer requires >= 2 subs
return subScorers.get(0);
} else {
- return new SynonymScorer(simScorer, this, subScorers, maxFreq);
+ LeafSimScorer simScorer = new LeafSimScorer(simWeight, context.reader(), true, totalMaxFreq);
+ return new SynonymScorer(simScorer, this, subScorers);
}
}
@@ -248,13 +243,11 @@ public final class SynonymQuery extends Query {
}
static class SynonymScorer extends DisjunctionScorer {
- private final Similarity.SimScorer similarity;
- private final float maxFreq;
+ private final LeafSimScorer similarity;
- SynonymScorer(Similarity.SimScorer similarity, Weight weight, List<Scorer> subScorers, float maxFreq) {
+ SynonymScorer(LeafSimScorer similarity, Weight weight, List<Scorer> subScorers) {
super(weight, subScorers, true);
this.similarity = similarity;
- this.maxFreq = maxFreq;
}
@Override
@@ -264,7 +257,7 @@ public final class SynonymQuery extends Query {
@Override
public float maxScore() {
- return similarity.maxScore(maxFreq);
+ return similarity.maxScore();
}
/** combines TF of all subs. */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
index 925fe93..3fa465d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
@@ -33,7 +33,6 @@ import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
/**
* A Query that matches documents containing a term. This may be combined with
@@ -46,7 +45,7 @@ public class TermQuery extends Query {
final class TermWeight extends Weight {
private final Similarity similarity;
- private final Similarity.SimWeight stats;
+ private final Similarity.SimScorer simScorer;
private final TermContext termStates;
private final boolean needsScores;
@@ -58,7 +57,7 @@ public class TermQuery extends Query {
}
this.needsScores = needsScores;
this.termStates = termStates;
- this.similarity = searcher.getSimilarity(needsScores);
+ this.similarity = searcher.getSimilarity();
final CollectionStatistics collectionStats;
final TermStatistics termStats;
@@ -72,9 +71,9 @@ public class TermQuery extends Query {
}
if (termStats == null) {
- this.stats = null; // term doesn't exist in any segment, we won't use similarity at all
+ this.simScorer = null; // term doesn't exist in any segment, we won't use similarity at all
} else {
- this.stats = similarity.computeWeight(boost, collectionStats, termStats);
+ this.simScorer = similarity.scorer(boost, collectionStats, termStats);
}
}
@@ -101,8 +100,8 @@ public class TermQuery extends Query {
.getIndexOptions();
PostingsEnum docs = termsEnum.postings(null, needsScores ? PostingsEnum.FREQS : PostingsEnum.NONE);
assert docs != null;
- return new TermScorer(this, docs, similarity.simScorer(stats, context),
- getMaxFreq(indexOptions, termsEnum.totalTermFreq(), termsEnum.docFreq()));
+ float maxFreq = getMaxFreq(indexOptions, termsEnum.totalTermFreq(), termsEnum.docFreq());
+ return new TermScorer(this, docs, new LeafSimScorer(simScorer, context.reader(), needsScores, maxFreq));
}
private long getMaxFreq(IndexOptions indexOptions, long ttf, long df) {
@@ -166,7 +165,7 @@ public class TermQuery extends Query {
int newDoc = scorer.iterator().advance(doc);
if (newDoc == doc) {
float freq = scorer.freq();
- SimScorer docScorer = similarity.simScorer(stats, context);
+ LeafSimScorer docScorer = new LeafSimScorer(simScorer, context.reader(), true, Integer.MAX_VALUE);
Explanation freqExplanation = Explanation.match(freq, "freq, occurrences of term within document");
Explanation scoreExplanation = docScorer.explain(doc, freqExplanation);
return Explanation.match(
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
index a4aeb04..653a60e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
@@ -20,14 +20,12 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.search.similarities.Similarity;
/** Expert: A <code>Scorer</code> for documents matching a <code>Term</code>.
*/
final class TermScorer extends Scorer {
private final PostingsEnum postingsEnum;
- private final Similarity.SimScorer docScorer;
- private final float maxFreq;
+ private final LeafSimScorer docScorer;
/**
* Construct a <code>TermScorer</code>.
@@ -39,14 +37,11 @@ final class TermScorer extends Scorer {
* @param docScorer
* The <code>Similarity.SimScorer</code> implementation
* to be used for score computations.
- * @param maxFreq
- * An upper bound of the term frequency of the searched term in any document.
*/
- TermScorer(Weight weight, PostingsEnum td, Similarity.SimScorer docScorer, float maxFreq) {
+ TermScorer(Weight weight, PostingsEnum td, LeafSimScorer docScorer) {
super(weight);
this.docScorer = docScorer;
this.postingsEnum = td;
- this.maxFreq = maxFreq;
}
@Override
@@ -71,7 +66,7 @@ final class TermScorer extends Scorer {
@Override
public float maxScore() {
- return docScorer.maxScore(maxFreq);
+ return docScorer.maxScore();
}
/** Returns a string representation of this <code>TermScorer</code>. */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/package-info.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/package-info.java b/lucene/core/src/java/org/apache/lucene/search/package-info.java
index 69c5c2a..7e53da4 100644
--- a/lucene/core/src/java/org/apache/lucene/search/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/search/package-info.java
@@ -378,7 +378,7 @@
* scored the way it was.
* Typically a weight such as TermWeight
* that scores via a {@link org.apache.lucene.search.similarities.Similarity Similarity} will make use of the Similarity's implementation:
- * {@link org.apache.lucene.search.similarities.Similarity.SimScorer#explain(int, Explanation) SimScorer#explain(int doc, Explanation freq)}.
+ * {@link org.apache.lucene.search.similarities.Similarity.SimScorer#explain(Explanation, long) SimScorer#explain(Explanation freq, long norm)}.
* </li>
* </ol>
* <a name="scorerClass"></a>
@@ -402,7 +402,7 @@
* {@link org.apache.lucene.search.Scorer#score score()} — Return the score of the
* current document. This value can be determined in any appropriate way for an application. For instance, the
* {@link org.apache.lucene.search.TermScorer TermScorer} simply defers to the configured Similarity:
- * {@link org.apache.lucene.search.similarities.Similarity.SimScorer#score(int, float) SimScorer.score(int doc, float freq)}.
+ * {@link org.apache.lucene.search.similarities.Similarity.SimScorer#score(float, long) SimScorer.score(float freq, long norm)}.
* </li>
* <li>
* {@link org.apache.lucene.search.Scorer#getChildren getChildren()} — Returns any child subscorers
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
index 1522e5d..3865933 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
@@ -120,10 +120,10 @@ public abstract class Axiomatic extends SimilarityBase {
@Override
protected Explanation explain(
- BasicStats stats, int doc, Explanation freq, double docLen) {
+ BasicStats stats, Explanation freq, double docLen) {
List<Explanation> subs = new ArrayList<>();
double f = freq.getValue().doubleValue();
- explain(subs, stats, doc, f, docLen);
+ explain(subs, stats, f, docLen);
double score = tf(stats, f, docLen)
* ln(stats, f, docLen)
@@ -132,7 +132,7 @@ public abstract class Axiomatic extends SimilarityBase {
- gamma(stats, f, docLen);
Explanation explanation = Explanation.match((float) score,
- "score(" + getClass().getSimpleName() + ", doc=" + doc + ", freq=" + freq.getValue() +"), computed from:",
+ "score(" + getClass().getSimpleName() + ", freq=" + freq.getValue() +"), computed from:",
subs);
if (stats.boost != 1f) {
explanation = Explanation.match((float) (score * stats.boost), "Boosted score, computed as (score * boost) from:",
@@ -148,7 +148,7 @@ public abstract class Axiomatic extends SimilarityBase {
}
@Override
- protected void explain(List<Explanation> subs, BasicStats stats, int doc,
+ protected void explain(List<Explanation> subs, BasicStats stats,
double freq, double docLen) {
if (stats.getBoost() != 1.0d) {
subs.add(Explanation.match((float) stats.getBoost(),
@@ -165,7 +165,7 @@ public abstract class Axiomatic extends SimilarityBase {
subs.add(tflnExplain(stats, freq, docLen));
subs.add(idfExplain(stats, freq, docLen));
subs.add(Explanation.match((float) gamma(stats, freq, docLen), "gamma"));
- super.explain(subs, stats, doc, freq, docLen);
+ super.explain(subs, stats, freq, docLen);
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
index dce156b..09bef40 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
@@ -22,8 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -176,7 +174,7 @@ public class BM25Similarity extends Similarity {
}
@Override
- public final SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public final SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
Explanation idf = termStats.length == 1 ? idfExplain(collectionStats, termStats[0]) : idfExplain(collectionStats, termStats);
float avgdl = avgFieldLength(collectionStats);
@@ -184,100 +182,17 @@ public class BM25Similarity extends Similarity {
for (int i = 0; i < cache.length; i++) {
cache[i] = k1 * ((1 - b) + b * LENGTH_TABLE[i] / avgdl);
}
- return new BM25Stats(collectionStats.field(), boost, k1, idf, avgdl, cache);
- }
-
- @Override
- public final SimScorer simScorer(SimWeight stats, LeafReaderContext context) throws IOException {
- BM25Stats bm25stats = (BM25Stats) stats;
- return new BM25DocScorer(bm25stats, context.reader().getNormValues(bm25stats.field));
- }
-
- private class BM25DocScorer extends SimScorer {
- private final BM25Stats stats;
- private final float weightValue; // boost * idf * (k1 + 1)
- private final NumericDocValues norms;
- /** precomputed cache for all length values */
- private final float[] lengthCache;
- /** precomputed norm[256] with k1 * ((1 - b) + b * dl / avgdl) */
- private final float[] cache;
-
- BM25DocScorer(BM25Stats stats, NumericDocValues norms) throws IOException {
- this.stats = stats;
- this.weightValue = stats.weight;
- this.norms = norms;
- lengthCache = LENGTH_TABLE;
- cache = stats.cache;
- }
-
- @Override
- public float score(int doc, float freq) throws IOException {
- // if there are no norms, we act as if b=0
- double norm;
- if (norms == null) {
- norm = k1;
- } else {
- boolean found = norms.advanceExact(doc);
- assert found;
- norm = cache[((byte) norms.longValue()) & 0xFF];
- }
- return weightValue * (float) (freq / (freq + norm));
- }
-
- @Override
- public float maxScore(float maxFreq) {
- // TODO: leverage maxFreq and the min norm from the cache
- return weightValue;
- }
-
- @Override
- public Explanation explain(int doc, Explanation freq) throws IOException {
- List<Explanation> subs = new ArrayList<>();
- subs.addAll(stats.explain());
- Explanation tfExpl = explainTF(doc, freq);
- subs.add(tfExpl);
- return Explanation.match(stats.weight * tfExpl.getValue().floatValue(),
- "score(doc="+doc+",freq="+freq.getValue()+"), product of:", subs);
- }
-
- private Explanation explainTF(int doc, Explanation freq) throws IOException {
- List<Explanation> subs = new ArrayList<>();
- subs.add(freq);
- subs.add(Explanation.match(k1, "k1, term saturation parameter"));
- if (norms == null) {
- subs.add(Explanation.match(0, "b, field omits length norms"));
- return Explanation.match(
- (float) (freq.getValue().floatValue() / (freq.getValue().floatValue() + (double) k1)),
- "tf, computed as freq / (freq + k1) from:", subs);
- } else {
- boolean found = norms.advanceExact(doc);
- assert found;
- byte norm = (byte) norms.longValue();
- float doclen = lengthCache[norm & 0xff];
- subs.add(Explanation.match(b, "b, length normalization parameter"));
- if ((norm & 0xFF) > 39) {
- subs.add(Explanation.match(doclen, "dl, length of field (approximate)"));
- } else {
- subs.add(Explanation.match(doclen, "dl, length of field"));
- }
- subs.add(Explanation.match(stats.avgdl, "avgdl, average length of field"));
- float normValue = k1 * ((1 - b) + b * doclen / stats.avgdl);
- return Explanation.match(
- (float) (freq.getValue().floatValue() / (freq.getValue().floatValue() + (double) normValue)),
- "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:", subs);
- }
- }
-
+ return new BM25Scorer(collectionStats.field(), boost, k1, b, idf, avgdl, cache);
}
/** Collection statistics for the BM25 model. */
- private static class BM25Stats extends SimWeight {
- /** field name, for pulling norms */
- private final String field;
+ private static class BM25Scorer extends SimScorer {
/** query boost */
private final float boost;
/** k1 value for scale factor */
private final float k1;
+ /** b value for length normalization impact */
+ private final float b;
/** BM25's idf */
private final Explanation idf;
/** The average document length. */
@@ -287,17 +202,57 @@ public class BM25Similarity extends Similarity {
/** weight (idf * boost) */
private final float weight;
- BM25Stats(String field, float boost, float k1, Explanation idf, float avgdl, float[] cache) {
- this.field = field;
+ BM25Scorer(String field, float boost, float k1, float b, Explanation idf, float avgdl, float[] cache) {
+ super(field);
this.boost = boost;
this.idf = idf;
this.avgdl = avgdl;
this.k1 = k1;
+ this.b = b;
this.cache = cache;
this.weight = (k1 + 1) * boost * idf.getValue().floatValue();
}
- private List<Explanation> explain() {
+ @Override
+ public float score(float freq, long encodedNorm) throws IOException {
+ double norm = cache[((byte) encodedNorm) & 0xFF];
+ return weight * (float) (freq / (freq + norm));
+ }
+
+ @Override
+ public float maxScore(float maxFreq) {
+ // TODO: leverage maxFreq and the min norm from the cache
+ return weight;
+ }
+
+ @Override
+ public Explanation explain(Explanation freq, long encodedNorm) throws IOException {
+ List<Explanation> subs = new ArrayList<>(explainConstantFactors());
+ Explanation tfExpl = explainTF(freq, encodedNorm);
+ subs.add(tfExpl);
+ return Explanation.match(weight * tfExpl.getValue().floatValue(),
+ "score(freq="+freq.getValue()+"), product of:", subs);
+ }
+
+ private Explanation explainTF(Explanation freq, long norm) throws IOException {
+ List<Explanation> subs = new ArrayList<>();
+ subs.add(freq);
+ subs.add(Explanation.match(k1, "k1, term saturation parameter"));
+ float doclen = LENGTH_TABLE[((byte) norm) & 0xff];
+ subs.add(Explanation.match(b, "b, length normalization parameter"));
+ if ((norm & 0xFF) > 39) {
+ subs.add(Explanation.match(doclen, "dl, length of field (approximate)"));
+ } else {
+ subs.add(Explanation.match(doclen, "dl, length of field"));
+ }
+ subs.add(Explanation.match(avgdl, "avgdl, average length of field"));
+ float normValue = k1 * ((1 - b) + b * doclen / avgdl);
+ return Explanation.match(
+ (float) (freq.getValue().floatValue() / (freq.getValue().floatValue() + (double) normValue)),
+ "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:", subs);
+ }
+
+ private List<Explanation> explainConstantFactors() {
List<Explanation> subs = new ArrayList<>();
// scale factor
subs.add(Explanation.match(k1 + 1, "scaling factor, k1 + 1"));
@@ -311,7 +266,6 @@ public class BM25Similarity extends Similarity {
}
}
-
@Override
public String toString() {
return "BM25(k1=" + k1 + ",b=" + b + ")";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/BasicStats.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicStats.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicStats.java
index cc3cab4..dc9356f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicStats.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicStats.java
@@ -23,7 +23,7 @@ import org.apache.lucene.index.Terms;
* Stores all statistics commonly used ranking methods.
* @lucene.experimental
*/
-public class BasicStats extends Similarity.SimWeight {
+public class BasicStats {
final String field;
/** The number of documents. */
protected long numberOfDocuments;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java
index 7134172..2690365 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java
@@ -19,7 +19,6 @@ package org.apache.lucene.search.similarities;
import java.io.IOException;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -47,44 +46,36 @@ public class BooleanSimilarity extends Similarity {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return new BooleanWeight(boost);
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ return new BooleanWeight(collectionStats.field(), boost);
}
- private static class BooleanWeight extends SimWeight {
+ private static class BooleanWeight extends SimScorer {
final float boost;
- BooleanWeight(float boost) {
+ BooleanWeight(String field, float boost) {
+ super(field);
this.boost = boost;
}
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
- final float boost = ((BooleanWeight) weight).boost;
-
- return new SimScorer() {
- @Override
- public float score(int doc, float freq) throws IOException {
- return boost;
- }
-
- @Override
- public float maxScore(float maxFreq) {
- return boost;
- }
+ @Override
+ public float score(float freq, long norm) throws IOException {
+ return boost;
+ }
- @Override
- public Explanation explain(int doc, Explanation freq) throws IOException {
- Explanation queryBoostExpl = Explanation.match(boost, "boost, query boost");
- return Explanation.match(
- queryBoostExpl.getValue(),
- "score(" + getClass().getSimpleName() + ", doc=" + doc + "), computed from:",
- queryBoostExpl);
- }
+ @Override
+ public float maxScore(float maxFreq) {
+ return boost;
+ }
- };
+ @Override
+ public Explanation explain(Explanation freq, long norm) throws IOException {
+ Explanation queryBoostExpl = Explanation.match(boost, "boost, query boost");
+ return Explanation.match(
+ queryBoostExpl.getValue(),
+ "score(" + getClass().getSimpleName() + "), computed from:",
+ queryBoostExpl);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/DFISimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/DFISimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/DFISimilarity.java
index 66f22be..44da93c 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/DFISimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/DFISimilarity.java
@@ -79,12 +79,12 @@ public class DFISimilarity extends SimilarityBase {
@Override
protected Explanation explain(
- BasicStats stats, int doc, Explanation freq, double docLen) {
+ BasicStats stats, Explanation freq, double docLen) {
final double expected = (stats.getTotalTermFreq() + 1) * docLen /
(stats.getNumberOfFieldTokens() + 1);
if (freq.getValue().doubleValue() <= expected){
return Explanation.match((float) 0, "score(" +
- getClass().getSimpleName() + ", doc=" + doc + ", freq=" +
+ getClass().getSimpleName() + ", freq=" +
freq.getValue() +"), equals to 0");
}
Explanation explExpected = Explanation.match((float) expected,
@@ -103,7 +103,7 @@ public class DFISimilarity extends SimilarityBase {
return Explanation.match(
(float) score(stats, freq.getValue().doubleValue(), docLen),
- "score(" + getClass().getSimpleName() + ", doc=" + doc + ", freq=" +
+ "score(" + getClass().getSimpleName() + ", freq=" +
freq.getValue() +"), computed as boost * log2(measure + 1) from:",
Explanation.match( (float)stats.getBoost(), "boost, query boost"),
explMeasure);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java
index a41e35c..1677168 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java
@@ -121,7 +121,7 @@ public class DFRSimilarity extends SimilarityBase {
@Override
protected void explain(List<Explanation> subs,
- BasicStats stats, int doc, double freq, double docLen) {
+ BasicStats stats, double freq, double docLen) {
if (stats.getBoost() != 1.0d) {
subs.add(Explanation.match( (float)stats.getBoost(), "boost, query boost"));
}
@@ -136,13 +136,13 @@ public class DFRSimilarity extends SimilarityBase {
@Override
protected Explanation explain(
- BasicStats stats, int doc, Explanation freq, double docLen) {
+ BasicStats stats, Explanation freq, double docLen) {
List<Explanation> subs = new ArrayList<>();
- explain(subs, stats, doc, freq.getValue().doubleValue(), docLen);
+ explain(subs, stats, freq.getValue().doubleValue(), docLen);
return Explanation.match(
(float) score(stats, freq.getValue().doubleValue(), docLen),
- "score(" + getClass().getSimpleName() + ", doc=" + doc + ", freq=" +
+ "score(" + getClass().getSimpleName() + ", freq=" +
freq.getValue() +"), computed as boost * " +
"basicModel.score(stats, tfn) * afterEffect.score(stats, tfn) from:",
subs);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java
index 9a57608..231d554 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java
@@ -112,7 +112,7 @@ public class IBSimilarity extends SimilarityBase {
@Override
protected void explain(
- List<Explanation> subs, BasicStats stats, int doc, double freq, double docLen) {
+ List<Explanation> subs, BasicStats stats, double freq, double docLen) {
if (stats.getBoost() != 1.0d) {
subs.add(Explanation.match((float)stats.getBoost(), "boost, query boost"));
}
@@ -125,13 +125,13 @@ public class IBSimilarity extends SimilarityBase {
@Override
protected Explanation explain(
- BasicStats stats, int doc, Explanation freq, double docLen) {
+ BasicStats stats, Explanation freq, double docLen) {
List<Explanation> subs = new ArrayList<>();
- explain(subs, stats, doc, freq.getValue().doubleValue(), docLen);
+ explain(subs, stats, freq.getValue().doubleValue(), docLen);
return Explanation.match(
(float) score(stats, freq.getValue().doubleValue(), docLen),
- "score(" + getClass().getSimpleName() + ", doc=" + doc + ", freq=" +
+ "score(" + getClass().getSimpleName() + ", freq=" +
freq.getValue() +"), computed as boost * " +
"distribution.score(stats, normalization.tfn(stats, freq," +
" docLen), lambda.lambda(stats)) from:",
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java
index c12cba4..7522c17 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java
@@ -84,7 +84,7 @@ public class LMDirichletSimilarity extends LMSimilarity {
}
@Override
- protected void explain(List<Explanation> subs, BasicStats stats, int doc,
+ protected void explain(List<Explanation> subs, BasicStats stats,
double freq, double docLen) {
if (stats.getBoost() != 1.0d) {
subs.add(Explanation.match((float) stats.getBoost(), "query boost"));
@@ -107,18 +107,18 @@ public class LMDirichletSimilarity extends LMSimilarity {
(float)Math.log(mu / (docLen + mu)),
"document norm, computed as log(mu / (dl + mu))"));
subs.add(Explanation.match((float) docLen,"dl, length of field"));
- super.explain(subs, stats, doc, freq, docLen);
+ super.explain(subs, stats, freq, docLen);
}
@Override
protected Explanation explain(
- BasicStats stats, int doc, Explanation freq, double docLen) {
+ BasicStats stats, Explanation freq, double docLen) {
List<Explanation> subs = new ArrayList<>();
- explain(subs, stats, doc, freq.getValue().doubleValue(), docLen);
+ explain(subs, stats, freq.getValue().doubleValue(), docLen);
return Explanation.match(
(float) score(stats, freq.getValue().doubleValue(), docLen),
- "score(" + getClass().getSimpleName() + ", doc=" + doc + ", freq=" +
+ "score(" + getClass().getSimpleName() + ", freq=" +
freq.getValue() +"), computed as boost * " +
"(term weight + document norm) from:",
subs);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java
index 42e5a7b..dde0650 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java
@@ -74,7 +74,7 @@ public class LMJelinekMercerSimilarity extends LMSimilarity {
}
@Override
- protected void explain(List<Explanation> subs, BasicStats stats, int doc,
+ protected void explain(List<Explanation> subs, BasicStats stats,
double freq, double docLen) {
if (stats.getBoost() != 1.0d) {
subs.add(Explanation.match((float) stats.getBoost(), "boost"));
@@ -88,18 +88,18 @@ public class LMJelinekMercerSimilarity extends LMSimilarity {
"freq, number of occurrences of term in the document");
subs.add(explFreq);
subs.add(Explanation.match((float) docLen,"dl, length of field"));
- super.explain(subs, stats, doc, freq, docLen);
+ super.explain(subs, stats, freq, docLen);
}
@Override
protected Explanation explain(
- BasicStats stats, int doc, Explanation freq, double docLen) {
+ BasicStats stats, Explanation freq, double docLen) {
List<Explanation> subs = new ArrayList<>();
- explain(subs, stats, doc, freq.getValue().doubleValue(), docLen);
+ explain(subs, stats, freq.getValue().doubleValue(), docLen);
return Explanation.match(
(float) score(stats, freq.getValue().doubleValue(), docLen),
- "score(" + getClass().getSimpleName() + ", doc=" + doc + ", freq=" +
+ "score(" + getClass().getSimpleName() + ", freq=" +
freq.getValue() +"), computed as boost * " +
"log(1 + ((1 - lambda) * freq / dl) /(lambda * P)) from:",
subs);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java
index 8154806..73a1276 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java
@@ -70,7 +70,7 @@ public abstract class LMSimilarity extends SimilarityBase {
}
@Override
- protected void explain(List<Explanation> subExpls, BasicStats stats, int doc,
+ protected void explain(List<Explanation> subExpls, BasicStats stats,
double freq, double docLen) {
subExpls.add(Explanation.match((float) collectionModel.computeProbability(stats),
"collection probability"));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
index 2f48cc6..3526db4 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -49,35 +48,27 @@ public class MultiSimilarity extends Similarity {
}
@Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- SimWeight subStats[] = new SimWeight[sims.length];
- for (int i = 0; i < subStats.length; i++) {
- subStats[i] = sims[i].computeWeight(boost, collectionStats, termStats);
- }
- return new MultiStats(subStats);
- }
-
- @Override
- public SimScorer simScorer(SimWeight stats, LeafReaderContext context) throws IOException {
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
SimScorer subScorers[] = new SimScorer[sims.length];
for (int i = 0; i < subScorers.length; i++) {
- subScorers[i] = sims[i].simScorer(((MultiStats)stats).subStats[i], context);
+ subScorers[i] = sims[i].scorer(boost, collectionStats, termStats);
}
- return new MultiSimScorer(subScorers);
+ return new MultiSimScorer(collectionStats.field(), subScorers);
}
static class MultiSimScorer extends SimScorer {
private final SimScorer subScorers[];
- MultiSimScorer(SimScorer subScorers[]) {
+ MultiSimScorer(String field, SimScorer subScorers[]) {
+ super(field);
this.subScorers = subScorers;
}
@Override
- public float score(int doc, float freq) throws IOException {
+ public float score(float freq, long norm) throws IOException {
float sum = 0.0f;
for (SimScorer subScorer : subScorers) {
- sum += subScorer.score(doc, freq);
+ sum += subScorer.score(freq, norm);
}
return sum;
}
@@ -92,21 +83,13 @@ public class MultiSimilarity extends Similarity {
}
@Override
- public Explanation explain(int doc, Explanation freq) throws IOException {
+ public Explanation explain(Explanation freq, long norm) throws IOException {
List<Explanation> subs = new ArrayList<>();
for (SimScorer subScorer : subScorers) {
- subs.add(subScorer.explain(doc, freq));
+ subs.add(subScorer.explain(freq, norm));
}
- return Explanation.match(score(doc, freq.getValue().floatValue()), "sum of:", subs);
+ return Explanation.match(score(freq.getValue().floatValue(), norm), "sum of:", subs);
}
}
-
- static class MultiStats extends SimWeight {
- final SimWeight subStats[];
-
- MultiStats(SimWeight subStats[]) {
- this.subStats = subStats;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java b/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java
index 6c05616..ee2381f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java
@@ -17,9 +17,6 @@
package org.apache.lucene.search.similarities;
-import java.io.IOException;
-
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.TermStatistics;
@@ -46,26 +43,13 @@ public abstract class PerFieldSimilarityWrapper extends Similarity {
}
@Override
- public final SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- PerFieldSimWeight weight = new PerFieldSimWeight();
- weight.delegate = get(collectionStats.field());
- weight.delegateWeight = weight.delegate.computeWeight(boost, collectionStats, termStats);
- return weight;
- }
-
- @Override
- public final SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
- PerFieldSimWeight perFieldWeight = (PerFieldSimWeight) weight;
- return perFieldWeight.delegate.simScorer(perFieldWeight.delegateWeight, context);
+ public final SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ return get(collectionStats.field()).scorer(boost, collectionStats, termStats);
}
/**
* Returns a {@link Similarity} for scoring a field.
*/
public abstract Similarity get(String name);
-
- static class PerFieldSimWeight extends SimWeight {
- Similarity delegate;
- SimWeight delegateWeight;
- }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
index 5f0bcd0..a2ebe4a 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
@@ -19,16 +19,14 @@ package org.apache.lucene.search.similarities;
import java.io.IOException;
import java.util.Collections;
+import java.util.Objects;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermStatistics;
-import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.util.SmallFloat;
/**
@@ -38,9 +36,9 @@ import org.apache.lucene.util.SmallFloat;
* <p>
* This is a low-level API, you should only extend this API if you want to implement
* an information retrieval <i>model</i>. If you are instead looking for a convenient way
- * to alter Lucene's scoring, consider extending a higher-level implementation
- * such as {@link TFIDFSimilarity}, which implements the vector space model with this API, or
- * just tweaking the default implementation: {@link BM25Similarity}.
+ * to alter Lucene's scoring, consider just tweaking the default implementation:
+ * {@link BM25Similarity} or extend {@link SimilarityBase}, which makes it easy to compute
+ * a score from index statistics.
* <p>
* Similarity determines how Lucene weights terms, and Lucene interacts with
* this class at both <a href="#indextime">index-time</a> and
@@ -49,23 +47,22 @@ import org.apache.lucene.util.SmallFloat;
* <a name="indextime">Indexing Time</a>
* At indexing time, the indexer calls {@link #computeNorm(FieldInvertState)}, allowing
* the Similarity implementation to set a per-document value for the field that will
- * be later accessible via {@link org.apache.lucene.index.LeafReader#getNormValues(String)}. Lucene makes no assumption
- * about what is in this norm, but it is most useful for encoding length normalization
- * information.
+ * be later accessible via {@link org.apache.lucene.index.LeafReader#getNormValues(String)}.
+ * Lucene makes no assumption about what is in this norm, but it is most useful for
+ * encoding length normalization information.
* <p>
* Implementations should carefully consider how the normalization is encoded: while
- * Lucene's {@link BM25Similarity} encodes a combination of index-time boost
- * and length normalization information with {@link SmallFloat} into a single byte, this
- * might not be suitable for all purposes.
+ * Lucene's {@link BM25Similarity} encodes length normalization information with
+ * {@link SmallFloat} into a single byte, this might not be suitable for all purposes.
* <p>
* Many formulas require the use of average document length, which can be computed via a
* combination of {@link CollectionStatistics#sumTotalTermFreq()} and
- * {@link CollectionStatistics#maxDoc()} or {@link CollectionStatistics#docCount()},
- * depending upon whether the average should reflect field sparsity.
+ * {@link CollectionStatistics#docCount()}.
* <p>
- * Additional scoring factors can be stored in named
- * <code>NumericDocValuesField</code>s and accessed
- * at query-time with {@link org.apache.lucene.index.LeafReader#getNumericDocValues(String)}.
+ * Additional scoring factors can be stored in named {@link NumericDocValuesField}s and
+ * accessed at query-time with {@link org.apache.lucene.index.LeafReader#getNumericDocValues(String)}.
+ * However this should not be done in the {@link Similarity} but externally, for instance
+ * by using <tt>FunctionScoreQuery</tt>.
* <p>
* Finally, using index-time boosts (either via folding into the normalization byte or
* via DocValues), is an inefficient way to boost the scores of different fields if the
@@ -76,14 +73,13 @@ import org.apache.lucene.util.SmallFloat;
* <a name="querytime">Query time</a>
* At query-time, Queries interact with the Similarity via these steps:
* <ol>
- * <li>The {@link #computeWeight(float, CollectionStatistics, TermStatistics...)} method is called a single time,
+ * <li>The {@link #scorer(float, CollectionStatistics, TermStatistics...)} method is called a single time,
* allowing the implementation to compute any statistics (such as IDF, average document length, etc)
* across <i>the entire collection</i>. The {@link TermStatistics} and {@link CollectionStatistics} passed in
* already contain all of the raw statistics involved, so a Similarity can freely use any combination
* of statistics without causing any additional I/O. Lucene makes no assumption about what is
- * stored in the returned {@link Similarity.SimWeight} object.
- * <li>For each segment in the index, the Query creates a {@link #simScorer(SimWeight, org.apache.lucene.index.LeafReaderContext)}
- * The score() method is called for each matching document.
+ * stored in the returned {@link Similarity.SimScorer} object.
+ * <li>Then {@link SimScorer#score(float, long)} is called for every matching document to compute its score.
* </ol>
* <p>
* <a name="explaintime">Explanations</a>
@@ -126,37 +122,38 @@ public abstract class Similarity {
* @param termStats term-level statistics, such as the document frequency of a term across the collection.
* @return SimWeight object with the information this Similarity needs to score a query.
*/
- public abstract SimWeight computeWeight(float boost,
+ public abstract SimScorer scorer(float boost,
CollectionStatistics collectionStats, TermStatistics... termStats);
-
- /**
- * Creates a new {@link Similarity.SimScorer} to score matching documents from a segment of the inverted index.
- * @param weight collection information from {@link #computeWeight(float, CollectionStatistics, TermStatistics...)}
- * @param context segment of the inverted index to be scored.
- * @return SloppySimScorer for scoring documents across <code>context</code>
- * @throws IOException if there is a low-level I/O error
- */
- public abstract SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException;
- /**
- * API for scoring "sloppy" queries such as {@link TermQuery},
- * {@link SpanQuery}, and {@link PhraseQuery}.
+ /** Stores the weight for a query across the indexed collection. This abstract
+ * implementation is empty; descendants of {@code Similarity} should
+ * subclass {@code SimWeight} and define the statistics they require in the
+ * subclass. Examples include idf, average field length, etc.
*/
public static abstract class SimScorer {
-
+
+ private final String field;
+
/**
* Sole constructor. (For invocation by subclass
- * constructors, typically implicit.)
+ * constructors.)
*/
- public SimScorer() {}
+ public SimScorer(String field) {
+ this.field = Objects.requireNonNull(field);
+ }
+
+ /** Return the field that this {@link SimScorer} operates on. */
+ public final String getField() {
+ return field;
+ }
/**
* Score a single document
- * @param doc document id within the inverted index segment
* @param freq sloppy term frequency
+ * @param norm encoded normalization factor, as returned by {@link Similarity#computeNorm}, or {@code 1} if norms are disabled
* @return document's score
*/
- public abstract float score(int doc, float freq) throws IOException;
+ public abstract float score(float freq, long norm) throws IOException;
/**
* Return the maximum score that this scorer may produce for freqs in {@code ]0, maxFreq]}.
@@ -167,30 +164,16 @@ public abstract class Similarity {
/**
* Explain the score for a single document
- * @param doc document id within the inverted index segment
* @param freq Explanation of how the sloppy term frequency was computed
+ * @param norm encoded normalization factor, as returned by {@link Similarity#computeNorm}, or {@code 1} if norms are disabled
* @return document's score
*/
- public Explanation explain(int doc, Explanation freq) throws IOException {
+ public Explanation explain(Explanation freq, long norm) throws IOException {
return Explanation.match(
- score(doc, freq.getValue().floatValue()),
- "score(doc=" + doc + ",freq=" + freq.getValue() +"), with freq of:",
+ score(freq.getValue().floatValue(), norm),
+ "score(freq=" + freq.getValue() +"), with freq of:",
Collections.singleton(freq));
}
- }
-
- /** Stores the weight for a query across the indexed collection. This abstract
- * implementation is empty; descendants of {@code Similarity} should
- * subclass {@code SimWeight} and define the statistics they require in the
- * subclass. Examples include idf, average field length, etc.
- */
- public static abstract class SimWeight {
-
- /**
- * Sole constructor. (For invocation by subclass
- * constructors, typically implicit.)
- */
- public SimWeight() {}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
index f227f38..f750b12 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
@@ -22,8 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -33,7 +31,7 @@ import org.apache.lucene.util.SmallFloat;
* A subclass of {@code Similarity} that provides a simplified API for its
* descendants. Subclasses are only required to implement the {@link #score}
* and {@link #toString()} methods. Implementing
- * {@link #explain(List, BasicStats, int, double, double)} is optional,
+ * {@link #explain(List, BasicStats, double, double)} is optional,
* inasmuch as SimilarityBase already provides a basic explanation of the score
* and the term frequency. However, implementers of a subclass are encouraged to
* include as much detail about the scoring method as possible.
@@ -82,13 +80,18 @@ public abstract class SimilarityBase extends Similarity {
}
@Override
- public final SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- BasicStats stats[] = new BasicStats[termStats.length];
+ public final SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ SimScorer weights[] = new SimScorer[termStats.length];
for (int i = 0; i < termStats.length; i++) {
- stats[i] = newStats(collectionStats.field(), boost);
- fillBasicStats(stats[i], collectionStats, termStats[i]);
+ BasicStats stats = newStats(collectionStats.field(), boost);
+ fillBasicStats(stats, collectionStats, termStats[i]);
+ weights[i] = new BasicSimScorer(stats);
+ }
+ if (weights.length == 1) {
+ return weights[0];
+ } else {
+ return new MultiSimilarity.MultiSimScorer(collectionStats.field(), weights);
}
- return stats.length == 1 ? stats[0] : new MultiSimilarity.MultiStats(stats);
}
/** Factory method to return a custom stats object */
@@ -137,12 +140,11 @@ public abstract class SimilarityBase extends Similarity {
*
* @param subExpls the list of details of the explanation to extend
* @param stats the corpus level statistics.
- * @param doc the document id.
* @param freq the term frequency.
* @param docLen the document length.
*/
protected void explain(
- List<Explanation> subExpls, BasicStats stats, int doc, double freq, double docLen) {}
+ List<Explanation> subExpls, BasicStats stats, double freq, double docLen) {}
/**
* Explains the score. The implementation here provides a basic explanation
@@ -151,43 +153,24 @@ public abstract class SimilarityBase extends Similarity {
* attaches the score (computed via the {@link #score(BasicStats, double, double)}
* method) and the explanation for the term frequency. Subclasses content with
* this format may add additional details in
- * {@link #explain(List, BasicStats, int, double, double)}.
+ * {@link #explain(List, BasicStats, double, double)}.
*
* @param stats the corpus level statistics.
- * @param doc the document id.
* @param freq the term frequency and its explanation.
* @param docLen the document length.
* @return the explanation.
*/
protected Explanation explain(
- BasicStats stats, int doc, Explanation freq, double docLen) {
+ BasicStats stats, Explanation freq, double docLen) {
List<Explanation> subs = new ArrayList<>();
- explain(subs, stats, doc, freq.getValue().floatValue(), docLen);
+ explain(subs, stats, freq.getValue().floatValue(), docLen);
return Explanation.match(
(float) score(stats, freq.getValue().floatValue(), docLen),
- "score(" + getClass().getSimpleName() + ", doc=" + doc + ", freq=" + freq.getValue() +"), computed from:",
+ "score(" + getClass().getSimpleName() + ", freq=" + freq.getValue() +"), computed from:",
subs);
}
- @Override
- public final SimScorer simScorer(SimWeight stats, LeafReaderContext context) throws IOException {
- if (stats instanceof MultiSimilarity.MultiStats) {
- // a multi term query (e.g. phrase). return the summation,
- // scoring almost as if it were boolean query
- SimWeight subStats[] = ((MultiSimilarity.MultiStats) stats).subStats;
- SimScorer subScorers[] = new SimScorer[subStats.length];
- for (int i = 0; i < subScorers.length; i++) {
- BasicStats basicstats = (BasicStats) subStats[i];
- subScorers[i] = new BasicSimScorer(basicstats, context.reader().getNormValues(basicstats.field));
- }
- return new MultiSimilarity.MultiSimScorer(subScorers);
- } else {
- BasicStats basicstats = (BasicStats) stats;
- return new BasicSimScorer(basicstats, context.reader().getNormValues(basicstats.field));
- }
- }
-
/**
* Subclasses must override this method to return the name of the Similarity
* and preferably the values of parameters (if any) as well.
@@ -227,33 +210,27 @@ public abstract class SimilarityBase extends Similarity {
// --------------------------------- Classes ---------------------------------
- /** Delegates the {@link #score(int, float)} and
- * {@link #explain(int, Explanation)} methods to
+ /** Delegates the {@link #score(float, long)} and
+ * {@link #explain(Explanation, long)} methods to
* {@link SimilarityBase#score(BasicStats, double, double)} and
- * {@link SimilarityBase#explain(BasicStats, int, Explanation, double)},
+ * {@link SimilarityBase#explain(BasicStats, Explanation, double)},
* respectively.
*/
final class BasicSimScorer extends SimScorer {
- private final BasicStats stats;
- private final NumericDocValues norms;
+ final BasicStats stats;
- BasicSimScorer(BasicStats stats, NumericDocValues norms) throws IOException {
+ BasicSimScorer(BasicStats stats) {
+ super(stats.field);
this.stats = stats;
- this.norms = norms;
}
- double getLengthValue(int doc) throws IOException {
- if (norms == null) {
- return 1D;
- }
- boolean found = norms.advanceExact(doc);
- assert found;
- return LENGTH_TABLE[Byte.toUnsignedInt((byte) norms.longValue())];
+ double getLengthValue(long norm) throws IOException {
+ return LENGTH_TABLE[Byte.toUnsignedInt((byte) norm)];
}
@Override
- public float score(int doc, float freq) throws IOException {
- return (float) SimilarityBase.this.score(stats, freq, getLengthValue(doc));
+ public float score(float freq, long norm) throws IOException {
+ return (float) SimilarityBase.this.score(stats, freq, getLengthValue(norm));
}
@Override
@@ -262,8 +239,8 @@ public abstract class SimilarityBase extends Similarity {
}
@Override
- public Explanation explain(int doc, Explanation freq) throws IOException {
- return SimilarityBase.this.explain(stats, doc, freq, getLengthValue(doc));
+ public Explanation explain(Explanation freq, long norm) throws IOException {
+ return SimilarityBase.this.explain(stats, freq, getLengthValue(norm));
}
}
[22/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8099: Replace
BoostQParserPlugin.boostQuery() with FunctionScoreQuery.boostByValue()
Posted by da...@apache.org.
LUCENE-8099: Replace BoostQParserPlugin.boostQuery() with FunctionScoreQuery.boostByValue()
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0744fea8
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0744fea8
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0744fea8
Branch: refs/heads/jira/solr-11702
Commit: 0744fea821366a853b8e239e766b9786ef96cb27
Parents: 3d8ef98
Author: Alan Woodward <ro...@apache.org>
Authored: Mon Jan 8 09:51:53 2018 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Mon Jan 8 10:01:06 2018 +0000
----------------------------------------------------------------------
.../apache/solr/search/BoostQParserPlugin.java | 21 +-------------------
.../solr/search/ExtendedDismaxQParser.java | 5 +++--
.../apache/solr/search/ValueSourceParser.java | 3 ++-
3 files changed, 6 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0744fea8/solr/core/src/java/org/apache/solr/search/BoostQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/BoostQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/BoostQParserPlugin.java
index 7391f66..70e08a6 100644
--- a/solr/core/src/java/org/apache/solr/search/BoostQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/BoostQParserPlugin.java
@@ -16,18 +16,11 @@
*/
package org.apache.solr.search;
-import java.text.ParseException;
-
-import org.apache.lucene.expressions.Expression;
-import org.apache.lucene.expressions.SimpleBindings;
-import org.apache.lucene.expressions.js.JavascriptCompiler;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.FunctionScoreQuery;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.QueryValueSource;
-import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Query;
-import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
@@ -67,7 +60,7 @@ public class BoostQParserPlugin extends QParserPlugin {
} else {
vs = new QueryValueSource(bq, 0.0f);
}
- return boostQuery(q, vs);
+ return FunctionScoreQuery.boostByValue(q, vs.asDoubleValuesSource());
}
@@ -91,16 +84,4 @@ public class BoostQParserPlugin extends QParserPlugin {
};
}
- public static Query boostQuery(Query input, ValueSource vs) {
- try {
- SimpleBindings bindings = new SimpleBindings();
- bindings.add("score", DoubleValuesSource.SCORES);
- bindings.add("vs", vs.asDoubleValuesSource());
- Expression expr = JavascriptCompiler.compile("score * vs");
- return new FunctionScoreQuery(input, expr.getDoubleValuesSource(bindings));
- } catch (ParseException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); // should never happen!
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0744fea8/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java b/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
index 5e74f4a..004d1c0 100644
--- a/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
+++ b/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
@@ -34,6 +34,7 @@ import org.apache.lucene.analysis.core.StopFilterFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.FunctionQuery;
+import org.apache.lucene.queries.function.FunctionScoreQuery;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.ProductFloatFunction;
import org.apache.lucene.queries.function.valuesource.QueryValueSource;
@@ -196,9 +197,9 @@ public class ExtendedDismaxQParser extends QParser {
List<ValueSource> boosts = getMultiplicativeBoosts();
if (boosts.size()>1) {
ValueSource prod = new ProductFloatFunction(boosts.toArray(new ValueSource[boosts.size()]));
- topQuery = BoostQParserPlugin.boostQuery(topQuery, prod);
+ topQuery = FunctionScoreQuery.boostByValue(topQuery, prod.asDoubleValuesSource());
} else if (boosts.size() == 1) {
- topQuery = BoostQParserPlugin.boostQuery(topQuery, boosts.get(0));
+ topQuery = FunctionScoreQuery.boostByValue(topQuery, boosts.get(0).asDoubleValuesSource());
}
return topQuery;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0744fea8/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
index 367eb64..450d95a 100644
--- a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
+++ b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
@@ -27,6 +27,7 @@ import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.function.FunctionScoreQuery;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.BoolDocValues;
@@ -325,7 +326,7 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
public ValueSource parse(FunctionQParser fp) throws SyntaxError {
Query q = fp.parseNestedQuery();
ValueSource vs = fp.parseValueSource();
- return new QueryValueSource(BoostQParserPlugin.boostQuery(q, vs), 0.0f);
+ return new QueryValueSource(FunctionScoreQuery.boostByValue(q, vs.asDoubleValuesSource()), 0.0f);
}
});
addParser("joindf", new ValueSourceParser() {
[37/50] [abbrv] lucene-solr:jira/solr-11702: implement MapWriter
Posted by da...@apache.org.
implement MapWriter
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/74128cf6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/74128cf6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/74128cf6
Branch: refs/heads/jira/solr-11702
Commit: 74128cf695e6c6ed4d6bc8e408c5cb7e23b9ac4e
Parents: 7a375fd
Author: Noble Paul <no...@apache.org>
Authored: Tue Jan 9 19:15:58 2018 +1100
Committer: Noble Paul <no...@apache.org>
Committed: Tue Jan 9 19:15:58 2018 +1100
----------------------------------------------------------------------
.../client/solrj/cloud/autoscaling/AddReplicaSuggester.java | 9 +++++++++
.../solrj/cloud/autoscaling/MoveReplicaSuggester.java | 8 ++++++++
2 files changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74128cf6/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
index a1cd8c9..3b997bf 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
@@ -17,13 +17,16 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
+import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.util.Pair;
class AddReplicaSuggester extends Suggester {
@@ -72,4 +75,10 @@ class AddReplicaSuggester extends Suggester {
}
+ @Override
+ public void writeMap(MapWriter.EntryWriter ew) throws IOException {
+ ew.put("action", CollectionParams.CollectionAction.ADDREPLICA.toString());
+ super.writeMap(ew);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74128cf6/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
index 53b3bb7..b68b0b5 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
@@ -17,10 +17,12 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
+import java.io.IOException;
import java.util.List;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.util.Pair;
public class MoveReplicaSuggester extends Suggester {
@@ -79,4 +81,10 @@ public class MoveReplicaSuggester extends Suggester {
return null;
}
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put("action", CollectionParams.CollectionAction.MOVEREPLICA.toString());
+ super.writeMap(ew);
+ }
}
[08/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11798: Remove
support for deprecated top-level syntax in solrconfig.xml.
(master branch for 8x only)
Posted by da...@apache.org.
SOLR-11798: Remove support for deprecated top-level <highlighting> syntax in solrconfig.xml.
(master branch for 8x only)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5a08fa8b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5a08fa8b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5a08fa8b
Branch: refs/heads/jira/solr-11702
Commit: 5a08fa8bbb1cf26b4af5b71549671c31e1427f44
Parents: 65c842f
Author: Christine Poerschke <cp...@apache.org>
Authored: Thu Jan 4 15:07:34 2018 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Thu Jan 4 15:22:07 2018 +0000
----------------------------------------------------------------------
.../handler/component/HighlightComponent.java | 18 +++---------------
1 file changed, 3 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5a08fa8b/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java b/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
index 1602502..0ee6855 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
@@ -25,7 +25,6 @@ import java.util.stream.Stream;
import com.google.common.base.Objects;
import org.apache.lucene.search.Query;
-import org.apache.lucene.util.Version;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.HighlightParams;
import org.apache.solr.common.params.SolrParams;
@@ -47,7 +46,6 @@ import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
import static java.util.stream.Collectors.toMap;
-import static org.apache.solr.core.Config.assertWarnOrFail;
/**
* TODO!
@@ -132,19 +130,9 @@ public class HighlightComponent extends SearchComponent implements PluginInfoIni
public void inform(SolrCore core) {
List<PluginInfo> children = info.getChildren("highlighting");
if(children.isEmpty()) {
- PluginInfo pluginInfo = core.getSolrConfig().getPluginInfo(SolrHighlighter.class.getName()); //TODO deprecated configuration remove later
- assertWarnOrFail("solrconfig.xml <highlighting> configuration is deprecated since SOLR-1696 "
- + "and no longer supported from Solr 7.3 onwards. "
- + "Please configure via <searchComponent> instead.",
- (null == pluginInfo),
- core.getSolrConfig().luceneMatchVersion.onOrAfter(Version.LUCENE_7_3_0));
- if (pluginInfo != null) {
- solrConfigHighlighter = core.createInitInstance(pluginInfo, SolrHighlighter.class, null, DefaultSolrHighlighter.class.getName());
- } else {
- DefaultSolrHighlighter defHighlighter = new DefaultSolrHighlighter(core);
- defHighlighter.init(PluginInfo.EMPTY_INFO);
- solrConfigHighlighter = defHighlighter;
- }
+ DefaultSolrHighlighter defHighlighter = new DefaultSolrHighlighter(core);
+ defHighlighter.init(PluginInfo.EMPTY_INFO);
+ solrConfigHighlighter = defHighlighter;
} else {
solrConfigHighlighter = core.createInitInstance(children.get(0),SolrHighlighter.class,null, DefaultSolrHighlighter.class.getName());
}
[40/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8119: Remove
SimScorer.maxScore(float maxFreq).
Posted by da...@apache.org.
LUCENE-8119: Remove SimScorer.maxScore(float maxFreq).
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/838c604b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/838c604b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/838c604b
Branch: refs/heads/jira/solr-11702
Commit: 838c604b76534a8774c846dcf6b1e94532f0d62c
Parents: 6336ed4
Author: Adrien Grand <jp...@gmail.com>
Authored: Tue Jan 9 09:52:54 2018 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Tue Jan 9 14:42:16 2018 +0100
----------------------------------------------------------------------
.../org/apache/lucene/search/LeafSimScorer.java | 3 +-
.../apache/lucene/search/MultiPhraseQuery.java | 4 +-
.../org/apache/lucene/search/PhraseQuery.java | 4 +-
.../org/apache/lucene/search/SynonymQuery.java | 2 +-
.../lucene/search/similarities/Axiomatic.java | 6 ---
.../search/similarities/BM25Similarity.java | 6 ---
.../search/similarities/BooleanSimilarity.java | 5 ---
.../search/similarities/DFISimilarity.java | 6 ---
.../search/similarities/DFRSimilarity.java | 6 ---
.../search/similarities/IBSimilarity.java | 6 ---
.../similarities/LMDirichletSimilarity.java | 6 ---
.../similarities/LMJelinekMercerSimilarity.java | 6 ---
.../search/similarities/MultiSimilarity.java | 9 -----
.../lucene/search/similarities/Similarity.java | 42 +++++++++++++++-----
.../search/similarities/SimilarityBase.java | 12 ------
.../search/similarities/TFIDFSimilarity.java | 10 -----
.../apache/lucene/search/spans/SpanWeight.java | 4 +-
.../lucene/index/TestMaxTermFrequency.java | 4 --
.../search/TestBooleanQueryVisitSubscorers.java | 4 --
.../apache/lucene/search/TestConjunctions.java | 5 ---
.../lucene/search/TestSimilarityProvider.java | 11 -----
.../lucene/search/TestSubScorerFreqs.java | 5 ---
.../lucene/search/join/TestBlockJoin.java | 5 ---
.../function/valuesource/NormValueSource.java | 2 +-
.../lucene/search/TermAutomatonQuery.java | 2 +-
.../similarities/AssertingSimilarity.java | 13 ++----
.../similarities/BaseSimilarityTestCase.java | 6 ++-
27 files changed, 49 insertions(+), 145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java b/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java
index 46f9dc2..5de8295 100644
--- a/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/LeafSimScorer.java
@@ -37,7 +37,7 @@ public final class LeafSimScorer {
public LeafSimScorer(SimScorer scorer, LeafReader reader, boolean needsScores, float maxFreq) throws IOException {
this.scorer = scorer;
norms = needsScores ? reader.getNormValues(scorer.getField()) : null;
- maxScore = needsScores ? scorer.maxScore(maxFreq) : Integer.MAX_VALUE;
+ maxScore = needsScores ? scorer.score(maxFreq, 1) : Float.MAX_VALUE;
}
private long getNormValue(int doc) throws IOException {
@@ -66,7 +66,6 @@ public final class LeafSimScorer {
/**
* Return an upper bound of the score.
- * @see SimScorer#maxScore(float)
*/
public float maxScore() {
return maxScore;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
index c76f7ea..65d6631 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
@@ -295,7 +295,7 @@ public class MultiPhraseQuery extends Query {
scoreMode, totalMatchCost);
} else {
return new SloppyPhraseScorer(this, postingsFreqs, slop,
- new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Float.POSITIVE_INFINITY),
+ new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Float.MAX_VALUE),
scoreMode.needsScores(), totalMatchCost);
}
}
@@ -312,7 +312,7 @@ public class MultiPhraseQuery extends Query {
int newDoc = scorer.iterator().advance(doc);
if (newDoc == doc) {
float freq = slop == 0 ? ((ExactPhraseScorer)scorer).freq() : ((SloppyPhraseScorer)scorer).sloppyFreq();
- LeafSimScorer docScorer = new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Float.POSITIVE_INFINITY);
+ LeafSimScorer docScorer = new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Float.MAX_VALUE);
Explanation freqExplanation = Explanation.match(freq, "phraseFreq=" + freq);
Explanation scoreExplanation = docScorer.explain(doc, freqExplanation);
return Explanation.match(
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
index 011771e..ff15388 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
@@ -438,7 +438,7 @@ public class PhraseQuery extends Query {
scoreMode, totalMatchCost);
} else {
return new SloppyPhraseScorer(this, postingsFreqs, slop,
- new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Float.POSITIVE_INFINITY),
+ new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Float.MAX_VALUE),
scoreMode.needsScores(), totalMatchCost);
}
}
@@ -460,7 +460,7 @@ public class PhraseQuery extends Query {
int newDoc = scorer.iterator().advance(doc);
if (newDoc == doc) {
float freq = slop == 0 ? ((ExactPhraseScorer)scorer).freq() : ((SloppyPhraseScorer)scorer).sloppyFreq();
- LeafSimScorer docScorer = new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Float.POSITIVE_INFINITY);
+ LeafSimScorer docScorer = new LeafSimScorer(stats, context.reader(), scoreMode.needsScores(), Float.MAX_VALUE);
Explanation freqExplanation = Explanation.match(freq, "phraseFreq=" + freq);
Explanation scoreExplanation = docScorer.explain(doc, freqExplanation);
return Explanation.match(
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
index 10a5d42..d9335cf 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
@@ -174,7 +174,7 @@ public final class SynonymQuery extends Query {
assert scorer instanceof TermScorer;
freq = ((TermScorer)scorer).freq();
}
- LeafSimScorer docScorer = new LeafSimScorer(simWeight, context.reader(), true, Float.POSITIVE_INFINITY);
+ LeafSimScorer docScorer = new LeafSimScorer(simWeight, context.reader(), true, Float.MAX_VALUE);
Explanation freqExplanation = Explanation.match(freq, "termFreq=" + freq);
Explanation scoreExplanation = docScorer.explain(doc, freqExplanation);
return Explanation.match(
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
index 3865933..527c2fd 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
@@ -113,12 +113,6 @@ public abstract class Axiomatic extends SimilarityBase {
}
@Override
- protected double maxScore(BasicStats stats, double maxFreq) {
- // TODO: can we compute a better upper bound on the produced scores
- return Double.POSITIVE_INFINITY;
- }
-
- @Override
protected Explanation explain(
BasicStats stats, Explanation freq, double docLen) {
List<Explanation> subs = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
index 09bef40..bd20bf9 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
@@ -220,12 +220,6 @@ public class BM25Similarity extends Similarity {
}
@Override
- public float maxScore(float maxFreq) {
- // TODO: leverage maxFreq and the min norm from the cache
- return weight;
- }
-
- @Override
public Explanation explain(Explanation freq, long encodedNorm) throws IOException {
List<Explanation> subs = new ArrayList<>(explainConstantFactors());
Explanation tfExpl = explainTF(freq, encodedNorm);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java
index 2690365..6f7d26b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java
@@ -64,11 +64,6 @@ public class BooleanSimilarity extends Similarity {
}
@Override
- public float maxScore(float maxFreq) {
- return boost;
- }
-
- @Override
public Explanation explain(Explanation freq, long norm) throws IOException {
Explanation queryBoostExpl = Explanation.match(boost, "boost, query boost");
return Explanation.match(
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/similarities/DFISimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/DFISimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/DFISimilarity.java
index 44da93c..f7f3d52 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/DFISimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/DFISimilarity.java
@@ -64,12 +64,6 @@ public class DFISimilarity extends SimilarityBase {
return stats.getBoost() * log2(measure + 1);
}
- @Override
- protected double maxScore(BasicStats stats, double maxFreq) {
- // TODO: can we compute a better upper bound on the produced scores
- return Double.POSITIVE_INFINITY;
- }
-
/**
* Returns the measure of independence
*/
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java
index 1677168..cbe6773 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java
@@ -114,12 +114,6 @@ public class DFRSimilarity extends SimilarityBase {
}
@Override
- protected double maxScore(BasicStats stats, double maxFreq) {
- // TODO: can we compute a better upper bound on the produced scores
- return Double.POSITIVE_INFINITY;
- }
-
- @Override
protected void explain(List<Explanation> subs,
BasicStats stats, double freq, double docLen) {
if (stats.getBoost() != 1.0d) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java
index 231d554..d08bdab 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java
@@ -105,12 +105,6 @@ public class IBSimilarity extends SimilarityBase {
}
@Override
- protected double maxScore(BasicStats stats, double maxFreq) {
- // TODO: can we compute a better upper bound on the produced scores
- return Double.POSITIVE_INFINITY;
- }
-
- @Override
protected void explain(
List<Explanation> subs, BasicStats stats, double freq, double docLen) {
if (stats.getBoost() != 1.0d) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java
index 7522c17..a66871c 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java
@@ -78,12 +78,6 @@ public class LMDirichletSimilarity extends LMSimilarity {
}
@Override
- protected double maxScore(BasicStats stats, double maxFreq) {
- // TODO: can we compute a better upper bound on the produced scores
- return Double.POSITIVE_INFINITY;
- }
-
- @Override
protected void explain(List<Explanation> subs, BasicStats stats,
double freq, double docLen) {
if (stats.getBoost() != 1.0d) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java
index dde0650..3f4f41a 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java
@@ -68,12 +68,6 @@ public class LMJelinekMercerSimilarity extends LMSimilarity {
}
@Override
- protected double maxScore(BasicStats stats, double maxFreq) {
- // TODO: can we compute a better upper bound on the produced scores
- return Double.POSITIVE_INFINITY;
- }
-
- @Override
protected void explain(List<Explanation> subs, BasicStats stats,
double freq, double docLen) {
if (stats.getBoost() != 1.0d) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
index 3526db4..de2ea7f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
@@ -74,15 +74,6 @@ public class MultiSimilarity extends Similarity {
}
@Override
- public float maxScore(float freq) {
- float sumMaxScore = 0;
- for (SimScorer subScorer : subScorers) {
- sumMaxScore += subScorer.maxScore(freq);
- }
- return sumMaxScore;
- }
-
- @Override
public Explanation explain(Explanation freq, long norm) throws IOException {
List<Explanation> subs = new ArrayList<>();
for (SimScorer subScorer : subScorers) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
index a2ebe4a..7479fff 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
@@ -106,7 +106,17 @@ public abstract class Similarity {
* <p>Matches in longer fields are less precise, so implementations of this
* method usually set smaller values when <code>state.getLength()</code> is large,
* and larger values when <code>state.getLength()</code> is small.
- *
+ *
+ * <p>Note that for a given term-document frequency, greater unsigned norms
+ * must produce scores that are lower or equal, ie. for two encoded norms
+ * {@code n1} and {@code n2} so that
+ * {@code Long.compareUnsigned(n1, n2) > 0} then
+ * {@code SimScorer.score(freq, n1) <= SimScorer.score(freq, n2)}
+ * for any legal {@code freq}.
+ *
+ * <p>{@code 0} is not a legal norm, so {@code 1} is the norm that produces
+ * the highest scores.
+ *
* @lucene.experimental
*
* @param state current processing state for this field
@@ -148,21 +158,31 @@ public abstract class Similarity {
}
/**
- * Score a single document
- * @param freq sloppy term frequency
- * @param norm encoded normalization factor, as returned by {@link Similarity#computeNorm}, or {@code 1} if norms are disabled
+ * Score a single document. {@code freq} is the document-term sloppy
+ * frequency and must be finite and positive. {@code norm} is the
+ * encoded normalization factor as computed by
+ * {@link Similarity#computeNorm(FieldInvertState)} at index time, or
+ * {@code 1} if norms are disabled. {@code norm} is never {@code 0}.
+ * <p>
+ * Score must not decrease when {@code freq} increases, ie. if
+ * {@code freq1 > freq2}, then {@code score(freq1, norm) >=
+ * score(freq2, norm)} for any value of {@code norm} that may be produced
+ * by {@link Similarity#computeNorm(FieldInvertState)}.
+ * <p>
+ * Score must not increase when the unsigned {@code norm} increases, ie. if
+ * {@code Long.compareUnsigned(norm1, norm2) > 0} then
+ * {@code score(freq, norm1) <= score(freq, norm2)} for any legal
+ * {@code freq}.
+ * <p>
+ * As a consequence, the maximum score that this scorer can produce is bound
+ * by {@code score(Float.MAX_VALUE, 1)}.
+ * @param freq sloppy term frequency, must be finite and positive
+ * @param norm encoded normalization factor or {@code 1} if norms are disabled
* @return document's score
*/
public abstract float score(float freq, long norm) throws IOException;
/**
- * Return the maximum score that this scorer may produce for freqs in {@code ]0, maxFreq]}.
- * {@code Float.POSITIVE_INFINITY} is a fine return value if scores are not bounded.
- * @param maxFreq the maximum frequency
- */
- public abstract float maxScore(float maxFreq);
-
- /**
* Explain the score for a single document
* @param freq Explanation of how the sloppy term frequency was computed
* @param norm encoded normalization factor, as returned by {@link Similarity#computeNorm}, or {@code 1} if norms are disabled
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
index f750b12..f9dd636 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
@@ -125,13 +125,6 @@ public abstract class SimilarityBase extends Similarity {
protected abstract double score(BasicStats stats, double freq, double docLen);
/**
- * Return the maximum value that may be returned by {@link #score(BasicStats, double, double)}
- * for the given stats.
- * @see org.apache.lucene.search.similarities.Similarity.SimScorer#maxScore(float)
- */
- protected abstract double maxScore(BasicStats stats, double maxFreq);
-
- /**
* Subclasses should implement this method to explain the score. {@code expl}
* already contains the score, the name of the class and the doc id, as well
* as the term frequency and its explanation; subclasses can add additional
@@ -234,11 +227,6 @@ public abstract class SimilarityBase extends Similarity {
}
@Override
- public float maxScore(float maxFreq) {
- return (float) SimilarityBase.this.maxScore(stats, maxFreq);
- }
-
- @Override
public Explanation explain(Explanation freq, long norm) throws IOException {
return SimilarityBase.this.explain(stats, freq, getLengthValue(norm));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
index de317d0..9e4b5bf 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
@@ -550,16 +550,6 @@ public abstract class TFIDFSimilarity extends Similarity {
}
@Override
- public float maxScore(float maxFreq) {
- final float raw = tf(maxFreq) * queryWeight;
- float maxNormValue = Float.NEGATIVE_INFINITY;
- for (float norm : normTable) {
- maxNormValue = Math.max(maxNormValue, norm);
- }
- return raw * maxNormValue;
- }
-
- @Override
public Explanation explain(Explanation freq, long norm) throws IOException {
return explainScore(freq, norm, normTable);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
index e971610..25b58fd 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
@@ -140,7 +140,7 @@ public abstract class SpanWeight extends Weight {
* @throws IOException on error
*/
public LeafSimScorer getSimScorer(LeafReaderContext context) throws IOException {
- return simScorer == null ? null : new LeafSimScorer(simScorer, context.reader(), true, Float.POSITIVE_INFINITY);
+ return simScorer == null ? null : new LeafSimScorer(simScorer, context.reader(), true, Float.MAX_VALUE);
}
@Override
@@ -150,7 +150,7 @@ public abstract class SpanWeight extends Weight {
int newDoc = scorer.iterator().advance(doc);
if (newDoc == doc) {
float freq = scorer.sloppyFreq();
- LeafSimScorer docScorer = new LeafSimScorer(simScorer, context.reader(), true, Float.POSITIVE_INFINITY);
+ LeafSimScorer docScorer = new LeafSimScorer(simScorer, context.reader(), true, Float.MAX_VALUE);
Explanation freqExplanation = Explanation.match(freq, "phraseFreq=" + freq);
Explanation scoreExplanation = docScorer.explain(doc, freqExplanation);
return Explanation.match(scoreExplanation.getValue(),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java b/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
index cf29392..7cc39b3 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
@@ -117,10 +117,6 @@ public class TestMaxTermFrequency extends LuceneTestCase {
return 0;
}
- @Override
- public float maxScore(float maxFreq) {
- return 0;
- }
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java
index 37a1b57..0d7f4e6 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java
@@ -334,10 +334,6 @@ public class TestBooleanQueryVisitSubscorers extends LuceneTestCase {
public float score(float freq, long norm) throws IOException {
return freq;
}
- @Override
- public float maxScore(float maxFreq) {
- return maxFreq;
- }
};
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java b/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java
index 9b79675..4cfa4d3 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java
@@ -106,11 +106,6 @@ public class TestConjunctions extends LuceneTestCase {
public float score(float freq, long norm) {
return freq;
}
-
- @Override
- public float maxScore(float maxFreq) {
- return maxFreq;
- }
};
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
index f0e3b22..f99a674 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
@@ -119,11 +119,6 @@ public class TestSimilarityProvider extends LuceneTestCase {
public float score(float freq, long norm) throws IOException {
return 1;
}
-
- @Override
- public float maxScore(float maxFreq) {
- return 1;
- }
};
}
@@ -139,16 +134,10 @@ public class TestSimilarityProvider extends LuceneTestCase {
@Override
public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
return new SimScorer(collectionStats.field()) {
-
@Override
public float score(float freq, long norm) throws IOException {
return 10;
}
-
- @Override
- public float maxScore(float maxFreq) {
- return 10;
- }
};
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java b/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
index 1cc4d59..c3bfcb3 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
@@ -236,11 +236,6 @@ public class TestSubScorerFreqs extends LuceneTestCase {
public float score(float freq, long norm) throws IOException {
return freq;
}
-
- @Override
- public float maxScore(float maxFreq) {
- return maxFreq;
- }
};
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
index d70beaf..c90bfdc 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
@@ -1489,11 +1489,6 @@ public class TestBlockJoin extends LuceneTestCase {
protected double score(BasicStats stats, double freq, double docLen) {
return freq;
}
-
- @Override
- protected double maxScore(BasicStats stats, double maxFreq) {
- return maxFreq;
- }
};
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig().setSimilarity(sim));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
index 682c0c0..662f80d 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
@@ -71,7 +71,7 @@ public class NormValueSource extends ValueSource {
final SimScorer simScorer = similarity.scorer(1f,
new CollectionStatistics(field, 1, 1, 1, 1),
new TermStatistics(new BytesRef("bogus"), 1, 1));
- final LeafSimScorer leafSimScorer = new LeafSimScorer(simScorer, readerContext.reader(), true, Float.POSITIVE_INFINITY);
+ final LeafSimScorer leafSimScorer = new LeafSimScorer(simScorer, readerContext.reader(), true, Float.MAX_VALUE);
return new FloatDocValues(this) {
int lastDocID = -1;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
index b877976..42a5f74 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
@@ -397,7 +397,7 @@ public class TermAutomatonQuery extends Query {
}
if (any) {
- return new TermAutomatonScorer(this, enums, anyTermID, idToTerm, new LeafSimScorer(stats, context.reader(), true, Float.POSITIVE_INFINITY));
+ return new TermAutomatonScorer(this, enums, anyTermID, idToTerm, new LeafSimScorer(stats, context.reader(), true, Float.MAX_VALUE));
} else {
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/AssertingSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/similarities/AssertingSimilarity.java b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/AssertingSimilarity.java
index afc33ba..bedfe73 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/similarities/AssertingSimilarity.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/AssertingSimilarity.java
@@ -43,7 +43,9 @@ public class AssertingSimilarity extends Similarity {
assert state.getNumOverlap() < state.getLength();
assert state.getUniqueTermCount() > 0;
assert state.getUniqueTermCount() <= state.getLength();
- return delegate.computeNorm(state);
+ long norm = delegate.computeNorm(state);
+ assert norm != 0;
+ return norm;
}
@Override
@@ -78,19 +80,12 @@ public class AssertingSimilarity extends Similarity {
// result in bounds
float score = delegate.score(freq, norm);
assert Float.isFinite(score);
- assert score <= maxScore(freq);
+ assert score <= delegate.score(freq, 1);
assert score >= 0;
return score;
}
@Override
- public float maxScore(float maxFreq) {
- float maxScore = delegate.maxScore(maxFreq);
- assert Float.isNaN(maxScore) == false;
- return maxScore;
- }
-
- @Override
public Explanation explain(Explanation freq, long norm) throws IOException {
// freq in bounds
assert freq != null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/838c604b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/BaseSimilarityTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/similarities/BaseSimilarityTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/BaseSimilarityTestCase.java
index 348584e..dcd6534 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/similarities/BaseSimilarityTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/BaseSimilarityTestCase.java
@@ -346,12 +346,14 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {
boolean success = false;
SimScorer scorer = similarity.scorer(boost, corpus, term);
try {
+ float maxScore = scorer.score(Float.MAX_VALUE, 1);
+ assertFalse("maxScore is NaN", Float.isNaN(maxScore));
+
float score = scorer.score(freq, norm);
// check that score isn't infinite or negative
assertTrue("infinite/NaN score: " + score, Float.isFinite(score));
assertTrue("negative score: " + score, score >= 0);
- float maxScore = scorer.maxScore(freq);
- assertTrue("score > maxScore: " + score + " > " + maxScore, score <= maxScore);
+ assertTrue("greater than maxScore: " + score + ">" + maxScore, score <= maxScore);
// check explanation matches
Explanation explanation = scorer.explain(Explanation.match(freq, "freq, occurrences of term within document"), norm);
if (score != explanation.getValue().doubleValue()) {
[41/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11631: fix Solrj
tests
Posted by da...@apache.org.
SOLR-11631: fix Solrj tests
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e3f3cdd0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e3f3cdd0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e3f3cdd0
Branch: refs/heads/jira/solr-11702
Commit: e3f3cdd0851b7b8d681eb1aa47141a4278e2289c
Parents: 2432bde
Author: Steve Rowe <sa...@apache.org>
Authored: Tue Jan 9 11:55:30 2018 -0500
Committer: Steve Rowe <sa...@apache.org>
Committed: Tue Jan 9 11:55:30 2018 -0500
----------------------------------------------------------------------
.../solr/client/solrj/request/SchemaTest.java | 76 +++++++++++---------
1 file changed, 44 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e3f3cdd0/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java
index 4f64009..bab4a41 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java
@@ -28,7 +28,9 @@ import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.commons.io.FileUtils;
+import org.apache.solr.api.ApiBag;
import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.schema.AnalyzerDefinition;
import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition;
import org.apache.solr.client.solrj.request.schema.SchemaRequest;
@@ -37,6 +39,8 @@ import org.apache.solr.client.solrj.response.schema.FieldTypeRepresentation;
import org.apache.solr.client.solrj.response.schema.SchemaRepresentation;
import org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.util.RestTestBase;
import org.eclipse.jetty.servlet.ServletHolder;
import org.junit.After;
@@ -57,6 +61,15 @@ public class SchemaTest extends RestTestBase {
assertEquals("Response contained errors: " + schemaResponse.toString(), 0, schemaResponse.getStatus());
assertNull("Response contained errors: " + schemaResponse.toString(), schemaResponse.getResponse().get("errors"));
}
+
+ private static void assertFailedSchemaResponse(ThrowingRunnable runnable, String expectedErrorMessage) {
+ HttpSolrClient.RemoteExecutionException e = expectThrows(HttpSolrClient.RemoteExecutionException.class, runnable);
+ SimpleOrderedMap errorMap = (SimpleOrderedMap)e.getMetaData().get("error");
+ assertEquals("org.apache.solr.api.ApiBag$ExceptionWithErrObject",
+ ((NamedList)errorMap.get("metadata")).get("error-class"));
+ List details = (List)errorMap.get("details");
+ assertTrue(((List)((Map)details.get(0)).get("errorMessages")).get(0).toString().contains(expectedErrorMessage));
+ }
private static void createStoredStringField(String fieldName, SolrClient solrClient) throws Exception {
Map<String, Object> fieldAttributes = new LinkedHashMap<>();
@@ -283,16 +296,16 @@ public class SchemaTest extends RestTestBase {
@Test
public void addFieldShouldntBeCalledTwiceWithTheSameName() throws Exception {
Map<String, Object> fieldAttributes = new LinkedHashMap<>();
- fieldAttributes.put("name", "failureField");
+ String fieldName = "failureField";
+ fieldAttributes.put("name", fieldName);
fieldAttributes.put("type", "string");
SchemaRequest.AddField addFieldUpdateSchemaRequest =
new SchemaRequest.AddField(fieldAttributes);
SchemaResponse.UpdateResponse addFieldFirstResponse = addFieldUpdateSchemaRequest.process(getSolrClient());
assertValidSchemaResponse(addFieldFirstResponse);
- SchemaResponse.UpdateResponse addFieldSecondResponse = addFieldUpdateSchemaRequest.process(getSolrClient());
- assertEquals(0, addFieldSecondResponse.getStatus());
- assertNotNull(addFieldSecondResponse.getResponse().get("errors"));
+ assertFailedSchemaResponse(() -> addFieldUpdateSchemaRequest.process(getSolrClient()),
+ "Field '" + fieldName + "' already exists.");
}
@Test
@@ -326,11 +339,11 @@ public class SchemaTest extends RestTestBase {
}
@Test
- public void deletingAFieldThatDoesntExistInTheSchemaShouldFail() throws Exception {
- SchemaRequest.DeleteField deleteFieldRequest =
- new SchemaRequest.DeleteField("fieldToBeDeleted");
- SchemaResponse.UpdateResponse deleteFieldResponse = deleteFieldRequest.process(getSolrClient());
- assertNotNull(deleteFieldResponse.getResponse().get("errors"));
+ public void deletingAFieldThatDoesntExistInTheSchemaShouldFail() {
+ String fieldName = "fieldToBeDeleted";
+ SchemaRequest.DeleteField deleteFieldRequest = new SchemaRequest.DeleteField(fieldName);
+ assertFailedSchemaResponse(() -> deleteFieldRequest.process(getSolrClient()),
+ "The field '" + fieldName + "' is not present in this schema, and so cannot be deleted.");
}
@Test
@@ -406,7 +419,8 @@ public class SchemaTest extends RestTestBase {
@Test
public void addDynamicFieldShouldntBeCalledTwiceWithTheSameName() throws Exception {
Map<String, Object> fieldAttributes = new LinkedHashMap<>();
- fieldAttributes.put("name", "*_failure");
+ String dynamicFieldName = "*_failure";
+ fieldAttributes.put("name", dynamicFieldName);
fieldAttributes.put("type", "string");
SchemaRequest.AddDynamicField addDFieldUpdateSchemaRequest =
new SchemaRequest.AddDynamicField(fieldAttributes);
@@ -414,9 +428,8 @@ public class SchemaTest extends RestTestBase {
SchemaResponse.UpdateResponse addDFieldFirstResponse = addDFieldUpdateSchemaRequest.process(client);
assertValidSchemaResponse(addDFieldFirstResponse);
- SchemaResponse.UpdateResponse addDFieldSecondResponse = addDFieldUpdateSchemaRequest.process(getSolrClient());
- assertEquals(0, addDFieldSecondResponse.getStatus());
- assertNotNull(addDFieldSecondResponse.getResponse().get("errors"));
+ assertFailedSchemaResponse(() -> addDFieldUpdateSchemaRequest.process(getSolrClient()),
+ "[schema.xml] Duplicate DynamicField definition for '" + dynamicFieldName + "'");
}
@Test
@@ -453,10 +466,10 @@ public class SchemaTest extends RestTestBase {
@Test
public void deletingADynamicFieldThatDoesntExistInTheSchemaShouldFail() throws Exception {
- SchemaRequest.DeleteDynamicField deleteDynamicFieldRequest =
- new SchemaRequest.DeleteDynamicField("*_notexists");
- SchemaResponse.UpdateResponse deleteDynamicFieldResponse = deleteDynamicFieldRequest.process(getSolrClient());
- assertNotNull(deleteDynamicFieldResponse.getResponse().get("errors"));
+ String dynamicFieldName = "*_notexists";
+ SchemaRequest.DeleteDynamicField deleteDynamicFieldRequest = new SchemaRequest.DeleteDynamicField(dynamicFieldName);
+ assertFailedSchemaResponse(() -> deleteDynamicFieldRequest.process(getSolrClient()),
+ "The dynamic field '" + dynamicFieldName + "' is not present in this schema, and so cannot be deleted.");
}
@Test
@@ -635,7 +648,8 @@ public class SchemaTest extends RestTestBase {
@Test
public void addFieldTypeShouldntBeCalledTwiceWithTheSameName() throws Exception {
Map<String, Object> fieldTypeAttributes = new LinkedHashMap<>();
- fieldTypeAttributes.put("name", "failureInt");
+ String fieldName = "failureInt";
+ fieldTypeAttributes.put("name", fieldName);
fieldTypeAttributes.put("class", RANDOMIZED_NUMERIC_FIELDTYPES.get(Integer.class));
fieldTypeAttributes.put("omitNorms", true);
fieldTypeAttributes.put("positionIncrementGap", 0);
@@ -646,9 +660,8 @@ public class SchemaTest extends RestTestBase {
SchemaResponse.UpdateResponse addFieldTypeFirstResponse = addFieldTypeRequest.process(getSolrClient());
assertValidSchemaResponse(addFieldTypeFirstResponse);
- SchemaResponse.UpdateResponse addFieldTypeSecondResponse = addFieldTypeRequest.process(getSolrClient());
- assertEquals(0, addFieldTypeSecondResponse.getStatus());
- assertNotNull(addFieldTypeSecondResponse.getResponse().get("errors"));
+ assertFailedSchemaResponse(() -> addFieldTypeRequest.process(getSolrClient()),
+ "Field type '" + fieldName + "' already exists.");
}
@Test
@@ -689,10 +702,10 @@ public class SchemaTest extends RestTestBase {
@Test
public void deletingAFieldTypeThatDoesntExistInTheSchemaShouldFail() throws Exception {
- SchemaRequest.DeleteFieldType deleteFieldTypeRequest =
- new SchemaRequest.DeleteFieldType("fieldTypeToBeDeleted");
- SchemaResponse.UpdateResponse deleteFieldResponse = deleteFieldTypeRequest.process(getSolrClient());
- assertNotNull(deleteFieldResponse.getResponse().get("errors"));
+ String fieldType = "fieldTypeToBeDeleted";
+ SchemaRequest.DeleteFieldType deleteFieldTypeRequest = new SchemaRequest.DeleteFieldType(fieldType);
+ assertFailedSchemaResponse(() -> deleteFieldTypeRequest.process(getSolrClient()),
+ "The field type '" + fieldType + "' is not present in this schema, and so cannot be deleted.");
}
@Test
@@ -800,11 +813,10 @@ public class SchemaTest extends RestTestBase {
String srcFieldName = "srcnotexist";
String destFieldName1 = "destNotExist1", destFieldName2 = "destNotExist2";
- SchemaRequest.AddCopyField addCopyFieldRequest =
- new SchemaRequest.AddCopyField(srcFieldName,
- Arrays.asList(destFieldName1, destFieldName2));
- SchemaResponse.UpdateResponse addCopyFieldResponse = addCopyFieldRequest.process(getSolrClient());
- assertNotNull(addCopyFieldResponse.getResponse().get("errors"));
+ SchemaRequest.AddCopyField addCopyFieldRequest
+ = new SchemaRequest.AddCopyField(srcFieldName, Arrays.asList(destFieldName1, destFieldName2));
+ assertFailedSchemaResponse(() -> addCopyFieldRequest.process(getSolrClient()),
+ "copyField source :'" + srcFieldName + "' is not a glob and doesn't match any explicit field or dynamicField.");
}
@Test
@@ -838,8 +850,8 @@ public class SchemaTest extends RestTestBase {
SchemaRequest.DeleteCopyField deleteCopyFieldsRequest =
new SchemaRequest.DeleteCopyField(srcFieldName,
Arrays.asList(destFieldName1, destFieldName2));
- SchemaResponse.UpdateResponse deleteCopyFieldResponse = deleteCopyFieldsRequest.process(getSolrClient());
- assertNotNull(deleteCopyFieldResponse.getResponse().get("errors"));
+ assertFailedSchemaResponse(() -> deleteCopyFieldsRequest.process(getSolrClient()),
+ "Copy field directive not found: '" + srcFieldName + "' -> '" + destFieldName1 + "'");
}
@Test
[15/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11771: Overseer can
never process some last messages
Posted by da...@apache.org.
SOLR-11771: Overseer can never process some last messages
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b8b00786
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b8b00786
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b8b00786
Branch: refs/heads/jira/solr-11702
Commit: b8b00786f377af30e3572282e89dbbd32fbef742
Parents: 925733d
Author: Cao Manh Dat <da...@apache.org>
Authored: Sat Jan 6 09:05:14 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Sat Jan 6 09:05:14 2018 +0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
solr/core/src/java/org/apache/solr/cloud/Overseer.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8b00786/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 221d6ad..3da194b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -86,6 +86,8 @@ Bug Fixes
* SOLR-11555: If the query terms reduce to nothing, filter(clause) produces an NPE whereas
fq=clause does not (Erick Erickson)
+* SOLR-11771: Overseer can never process some last messages (Cao Manh Dat)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8b00786/solr/core/src/java/org/apache/solr/cloud/Overseer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index 3b9dd28..ee5fb18 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -201,7 +201,7 @@ public class Overseer implements SolrCloseable {
LinkedList<Pair<String, byte[]>> queue = null;
try {
// We do not need to filter any nodes here cause all processed nodes are removed once we flush clusterstate
- queue = new LinkedList<>(stateUpdateQueue.peekElements(1000, Long.MAX_VALUE, (x) -> true));
+ queue = new LinkedList<>(stateUpdateQueue.peekElements(1000, 3000L, (x) -> true));
} catch (KeeperException.SessionExpiredException e) {
log.warn("Solr cannot talk to ZK, exiting Overseer main queue loop", e);
return;
[49/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8121: UH switch
to SpanCollector API. Better accuracy. * Use the filtered freq in position
sensitive terms (better scores) * Refactored UH's OffsetsEnum * Improved test
randomization in TestUnifiedHighl
Posted by da...@apache.org.
LUCENE-8121: UH switch to SpanCollector API. Better accuracy.
* Use the filtered freq in position sensitive terms (better scores)
* Refactored UH's OffsetsEnum
* Improved test randomization in TestUnifiedHighlighter & MTQ
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/352ec01a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/352ec01a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/352ec01a
Branch: refs/heads/jira/solr-11702
Commit: 352ec01a6ef68bc81fdb84a7f72e81a6698f594c
Parents: 6a55def
Author: David Smiley <ds...@apache.org>
Authored: Wed Jan 10 22:49:40 2018 -0500
Committer: David Smiley <ds...@apache.org>
Committed: Wed Jan 10 22:49:40 2018 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 10 +
.../search/uhighlight/FieldHighlighter.java | 14 +-
.../search/uhighlight/FieldOffsetStrategy.java | 80 +--
.../lucene/search/uhighlight/OffsetsEnum.java | 132 +++--
.../lucene/search/uhighlight/Passage.java | 18 +
.../lucene/search/uhighlight/PhraseHelper.java | 584 +++++--------------
.../uhighlight/TokenStreamOffsetStrategy.java | 56 +-
.../uhighlight/TestUnifiedHighlighter.java | 87 ++-
.../uhighlight/TestUnifiedHighlighterMTQ.java | 55 +-
.../TestUnifiedHighlighterStrictPhrases.java | 88 +++
.../TestUnifiedHighlighterExtensibility.java | 4 +-
11 files changed, 518 insertions(+), 610 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/352ec01a/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c309a92..570da98 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -119,6 +119,10 @@ Improvements
* LUCENE-8094: TermInSetQuery.toString now returns "field:(A B C)" (Mike McCandless)
+* LUCENE-8121: UnifiedHighlighter passage relevancy is improved for terms that are
+ position sensitive (e.g. part of a phrase) by having an accurate freq.
+ (David Smiley)
+
Bug Fixes
* LUCENE-8077: Fixed bug in how CheckIndex verifies doc-value iterators.
@@ -127,6 +131,12 @@ Bug Fixes
* SOLR-11758: Fixed FloatDocValues.boolVal to correctly return true for all values != 0.0F
(Munendra S N via hossman)
+* LUCENE-8121: The UnifiedHighlighter would highlight some terms within some nested
+ SpanNearQueries at positions where it should not have. It's fixed in the UH by
+ switching to the SpanCollector API. The original Highlighter still has this
+ problem (LUCENE-2287, LUCENE-5455, LUCENE-6796). Some public but internal parts of
+ the UH were refactored. (David Smiley, Steve Davids)
+
Other
* LUCENE-8111: IndexOrDocValuesQuery Javadoc references outdated method name.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/352ec01a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldHighlighter.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldHighlighter.java
index cc9f318..a0e6d0a 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldHighlighter.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldHighlighter.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.text.BreakIterator;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;
@@ -136,13 +137,15 @@ public class FieldHighlighter {
BreakIterator breakIterator = this.breakIterator;
final int contentLength = breakIterator.getText().getEndIndex();
+ //TODO consider moving this part to an aggregate OffsetsEnum subclass so we have one enum that already has its weight
PriorityQueue<OffsetsEnum> offsetsEnumQueue = new PriorityQueue<>(offsetsEnums.size() + 1);
for (OffsetsEnum off : offsetsEnums) {
off.setWeight(scorer.weight(contentLength, off.freq()));
- off.nextPosition(); // go to first position
- offsetsEnumQueue.add(off);
+ if (off.nextPosition()) {// go to first position
+ offsetsEnumQueue.add(off);
+ }
}
- offsetsEnumQueue.add(new OffsetsEnum(null, EMPTY)); // a sentinel for termination
+ offsetsEnumQueue.add(new OffsetsEnum.OfPostings(new BytesRef(), EMPTY)); // a sentinel for termination
PriorityQueue<Passage> passageQueue = new PriorityQueue<>(Math.min(64, maxPassages + 1), (left, right) -> {
if (left.getScore() < right.getScore()) {
@@ -203,10 +206,9 @@ public class FieldHighlighter {
assert term != null;
passage.addMatch(start, end, term);
// see if there are multiple occurrences of this term in this passage. If so, add them.
- if (!off.hasMorePositions()) {
+ if (!off.nextPosition()) {
break; // No more in the entire text. Already removed from pq; move on
}
- off.nextPosition();
start = off.startOffset();
end = off.endOffset();
if (start >= passage.getEndOffset() || end > contentLength) { // it's beyond this passage
@@ -222,7 +224,7 @@ public class FieldHighlighter {
p.sort();
}
// sort in ascending order
- Arrays.sort(passages, (left, right) -> left.getStartOffset() - right.getStartOffset());
+ Arrays.sort(passages, Comparator.comparingInt(Passage::getStartOffset));
return passages;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/352ec01a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java
index 155f0a7..faef106 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java
@@ -20,14 +20,12 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
@@ -41,9 +39,9 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton;
public abstract class FieldOffsetStrategy {
protected final String field;
- protected final PhraseHelper phraseHelper; // Query: position-sensitive information TODO: rename
- protected final BytesRef[] terms; // Query: free-standing terms
- protected final CharacterRunAutomaton[] automata; // Query: free-standing wildcards (multi-term query)
+ protected final PhraseHelper phraseHelper; // Query: position-sensitive information
+ protected final BytesRef[] terms; // Query: all terms we extracted (some may be position sensitive)
+ protected final CharacterRunAutomaton[] automata; // Query: wildcards (i.e. multi-term query), not position sensitive
public FieldOffsetStrategy(String field, BytesRef[] queryTerms, PhraseHelper phraseHelper, CharacterRunAutomaton[] automata) {
this.field = field;
@@ -70,47 +68,50 @@ public abstract class FieldOffsetStrategy {
return Collections.emptyList();
}
- // For strict positions, get a Map of term to Spans:
- // note: ScriptPhraseHelper.NONE does the right thing for these method calls
- final Map<BytesRef, Spans> strictPhrasesTermToSpans =
- phraseHelper.getTermToSpans(leafReader, doc);
- // Usually simply wraps terms in a List; but if willRewrite() then can be expanded
- final List<BytesRef> sourceTerms =
- phraseHelper.expandTermsIfRewrite(terms, strictPhrasesTermToSpans);
-
- final List<OffsetsEnum> offsetsEnums = new ArrayList<>(sourceTerms.size() + automata.length);
-
- // Handle sourceTerms:
- if (!sourceTerms.isEmpty()) {
- TermsEnum termsEnum = termsIndex.iterator();//does not return null
- for (BytesRef term : sourceTerms) {
- if (termsEnum.seekExact(term)) {
- PostingsEnum postingsEnum = termsEnum.postings(null, PostingsEnum.OFFSETS);
-
- if (postingsEnum == null) {
- // no offsets or positions available
- throw new IllegalArgumentException("field '" + field + "' was indexed without offsets, cannot highlight");
- }
+ final List<OffsetsEnum> offsetsEnums = new ArrayList<>(terms.length + automata.length);
- if (doc == postingsEnum.advance(doc)) { // now it's positioned, although may be exhausted
- postingsEnum = phraseHelper.filterPostings(term, postingsEnum, strictPhrasesTermToSpans.get(term));
- if (postingsEnum != null) {
- offsetsEnums.add(new OffsetsEnum(term, postingsEnum));
- }
- }
- }
- }
+ // Handle position insensitive terms (a subset of this.terms field):
+ final BytesRef[] insensitiveTerms;
+ if (phraseHelper.hasPositionSensitivity()) {
+ insensitiveTerms = phraseHelper.getAllPositionInsensitiveTerms();
+ assert insensitiveTerms.length <= terms.length : "insensitive terms should be smaller set of all terms";
+ } else {
+ insensitiveTerms = terms;
+ }
+ if (insensitiveTerms.length > 0) {
+ createOffsetsEnumsForTerms(insensitiveTerms, termsIndex, doc, offsetsEnums);
+ }
+
+ // Handle spans
+ if (phraseHelper.hasPositionSensitivity()) {
+ phraseHelper.createOffsetsEnumsForSpans(leafReader, doc, offsetsEnums);
}
// Handle automata
if (automata.length > 0) {
- offsetsEnums.addAll(createAutomataOffsetsFromTerms(termsIndex, doc));
+ createOffsetsEnumsForAutomata(termsIndex, doc, offsetsEnums);
}
return offsetsEnums;
}
- protected List<OffsetsEnum> createAutomataOffsetsFromTerms(Terms termsIndex, int doc) throws IOException {
+ protected void createOffsetsEnumsForTerms(BytesRef[] sourceTerms, Terms termsIndex, int doc, List<OffsetsEnum> results) throws IOException {
+ TermsEnum termsEnum = termsIndex.iterator();//does not return null
+ for (BytesRef term : sourceTerms) {
+ if (termsEnum.seekExact(term)) {
+ PostingsEnum postingsEnum = termsEnum.postings(null, PostingsEnum.OFFSETS);
+ if (postingsEnum == null) {
+ // no offsets or positions available
+ throw new IllegalArgumentException("field '" + field + "' was indexed without offsets, cannot highlight");
+ }
+ if (doc == postingsEnum.advance(doc)) { // now it's positioned, although may be exhausted
+ results.add(new OffsetsEnum.OfPostings(term, postingsEnum));
+ }
+ }
+ }
+ }
+
+ protected void createOffsetsEnumsForAutomata(Terms termsIndex, int doc, List<OffsetsEnum> results) throws IOException {
List<List<PostingsEnum>> automataPostings = new ArrayList<>(automata.length);
for (int i = 0; i < automata.length; i++) {
automataPostings.add(new ArrayList<>());
@@ -118,6 +119,7 @@ public abstract class FieldOffsetStrategy {
TermsEnum termsEnum = termsIndex.iterator();
BytesRef term;
+
CharsRefBuilder refBuilder = new CharsRefBuilder();
while ((term = termsEnum.next()) != null) {
for (int i = 0; i < automata.length; i++) {
@@ -132,7 +134,6 @@ public abstract class FieldOffsetStrategy {
}
}
- List<OffsetsEnum> offsetsEnums = new ArrayList<>(automata.length); //will be at most this long
for (int i = 0; i < automata.length; i++) {
CharacterRunAutomaton automaton = automata[i];
List<PostingsEnum> postingsEnums = automataPostings.get(i);
@@ -140,14 +141,13 @@ public abstract class FieldOffsetStrategy {
if (size > 0) { //only add if we have offsets
BytesRef wildcardTerm = new BytesRef(automaton.toString());
if (size == 1) { //don't wrap in a composite if there's only one OffsetsEnum
- offsetsEnums.add(new OffsetsEnum(wildcardTerm, postingsEnums.get(0)));
+ results.add(new OffsetsEnum.OfPostings(wildcardTerm, postingsEnums.get(0)));
} else {
- offsetsEnums.add(new OffsetsEnum(wildcardTerm, new CompositeOffsetsPostingsEnum(postingsEnums)));
+ results.add(new OffsetsEnum.OfPostings(wildcardTerm, new CompositeOffsetsPostingsEnum(postingsEnums)));
}
}
}
- return offsetsEnums;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/352ec01a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/OffsetsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/OffsetsEnum.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/OffsetsEnum.java
index 708f5c3..f0a46a5 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/OffsetsEnum.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/OffsetsEnum.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.lucene.search.uhighlight;
import java.io.Closeable;
@@ -25,25 +26,19 @@ import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.util.BytesRef;
/**
- * Holds the term ({@link BytesRef}), {@link PostingsEnum}, offset iteration tracking.
- * It is advanced with the underlying postings and is placed in a priority queue by
+ * An enumeration/iterator of a term and its offsets for use by {@link FieldHighlighter}.
+ * It is advanced and is placed in a priority queue by
* {@link FieldHighlighter#highlightOffsetsEnums(List)} based on the start offset.
*
* @lucene.internal
*/
-public class OffsetsEnum implements Comparable<OffsetsEnum>, Closeable {
- private final BytesRef term;
- private final PostingsEnum postingsEnum; // with offsets
+public abstract class OffsetsEnum implements Comparable<OffsetsEnum>, Closeable {
private float weight; // set once in highlightOffsetsEnums
- private int posCounter = 0; // the occurrence counter of this term within the text being highlighted.
-
- public OffsetsEnum(BytesRef term, PostingsEnum postingsEnum) throws IOException {
- this.term = term; // can be null
- this.postingsEnum = Objects.requireNonNull(postingsEnum);
- }
// note: the ordering clearly changes as the postings enum advances
+ // note: would be neat to use some Comparator utilities with method
+ // references but our methods throw IOException
@Override
public int compareTo(OffsetsEnum other) {
try {
@@ -51,53 +46,41 @@ public class OffsetsEnum implements Comparable<OffsetsEnum>, Closeable {
if (cmp != 0) {
return cmp; // vast majority of the time we return here.
}
- if (this.term == null || other.term == null) {
- if (this.term == null && other.term == null) {
+ final BytesRef thisTerm = this.getTerm();
+ final BytesRef otherTerm = other.getTerm();
+ if (thisTerm == null || otherTerm == null) {
+ if (thisTerm == null && otherTerm == null) {
return 0;
- } else if (this.term == null) {
+ } else if (thisTerm == null) {
return 1; // put "this" (wildcard mtq enum) last
} else {
return -1;
}
}
- return term.compareTo(other.term);
+ return thisTerm.compareTo(otherTerm);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
- /** The term at this position; usually always the same. This term is a reference that is safe to continue to refer to,
- * even after we move to next position. */
- public BytesRef getTerm() throws IOException {
- // TODO TokenStreamOffsetStrategy could override OffsetsEnum; then remove this hack here
- return term != null ? term : postingsEnum.getPayload(); // abusing payload like this is a total hack!
- }
+ /**
+ * Advances to the next position and returns true, or if can't then returns false.
+ * Note that the initial state of this class is not positioned.
+ */
+ public abstract boolean nextPosition() throws IOException;
- public PostingsEnum getPostingsEnum() {
- return postingsEnum;
- }
-
- public int freq() throws IOException {
- return postingsEnum.freq();
- }
-
- public boolean hasMorePositions() throws IOException {
- return posCounter < postingsEnum.freq();
- }
+ /** An estimate of the number of occurrences of this term/OffsetsEnum. */
+ public abstract int freq() throws IOException;
- public void nextPosition() throws IOException {
- assert hasMorePositions();
- posCounter++;
- postingsEnum.nextPosition();
- }
+ /**
+ * The term at this position; usually always the same.
+ * This BytesRef is safe to continue to refer to, even after we move to the next position.
+ */
+ public abstract BytesRef getTerm() throws IOException;
- public int startOffset() throws IOException {
- return postingsEnum.startOffset();
- }
+ public abstract int startOffset() throws IOException;
- public int endOffset() throws IOException {
- return postingsEnum.endOffset();
- }
+ public abstract int endOffset() throws IOException;
public float getWeight() {
return weight;
@@ -109,9 +92,66 @@ public class OffsetsEnum implements Comparable<OffsetsEnum>, Closeable {
@Override
public void close() throws IOException {
- // TODO TokenStreamOffsetStrategy could override OffsetsEnum; then this base impl would be no-op.
- if (postingsEnum instanceof Closeable) {
- ((Closeable) postingsEnum).close();
+ }
+
+ @Override
+ public String toString() {
+ final String name = getClass().getSimpleName();
+ try {
+ return name + "(term:" + getTerm().utf8ToString() +")";
+ } catch (Exception e) {
+ return name;
}
}
+
+ /**
+ * Based on a {@link PostingsEnum} -- the typical/standard OE impl.
+ */
+ public static class OfPostings extends OffsetsEnum {
+ private final BytesRef term;
+ private final PostingsEnum postingsEnum; // with offsets
+
+ private int posCounter = 0; // the occurrence counter of this term within the text being highlighted.
+
+ public OfPostings(BytesRef term, PostingsEnum postingsEnum) throws IOException {
+ this.term = Objects.requireNonNull(term);
+ this.postingsEnum = Objects.requireNonNull(postingsEnum);
+ }
+
+ public PostingsEnum getPostingsEnum() {
+ return postingsEnum;
+ }
+
+ @Override
+ public boolean nextPosition() throws IOException {
+ if (posCounter < postingsEnum.freq()) {
+ posCounter++;
+ postingsEnum.nextPosition(); // note: we don't need to save the position
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int freq() throws IOException {
+ return postingsEnum.freq();
+ }
+
+ @Override
+ public BytesRef getTerm() throws IOException {
+ return term;
+ }
+
+ @Override
+ public int startOffset() throws IOException {
+ return postingsEnum.startOffset();
+ }
+
+ @Override
+ public int endOffset() throws IOException {
+ return postingsEnum.endOffset();
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/352ec01a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/Passage.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/Passage.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/Passage.java
index 3efb694..24b1015 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/Passage.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/Passage.java
@@ -98,6 +98,24 @@ public class Passage {
numMatches = 0;
}
+ /** For debugging. ex: Passage[0-22]{yin[0-3],yang[4-8],yin[10-13]}score=2.4964213 */
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder();
+ buf.append("Passage[").append(startOffset).append('-').append(endOffset).append(']');
+ buf.append('{');
+ for (int i = 0; i < numMatches; i++) {
+ if (i != 0) {
+ buf.append(',');
+ }
+ buf.append(matchTerms[i].utf8ToString());
+ buf.append('[').append(matchStarts[i] - startOffset).append('-').append(matchEnds[i] - startOffset).append(']');
+ }
+ buf.append('}');
+ buf.append("score=").append(score);
+ return buf.toString();
+ }
+
/**
* Start offset of this passage.
*
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/352ec01a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java
index cfb6570..f5205e4 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java
@@ -17,82 +17,58 @@
package org.apache.lucene.search.uhighlight;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import java.util.PriorityQueue;
import java.util.Set;
import java.util.TreeSet;
import java.util.function.Function;
import java.util.function.Predicate;
-import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.search.Weight;
import org.apache.lucene.search.highlight.WeightedSpanTerm;
import org.apache.lucene.search.highlight.WeightedSpanTermExtractor;
import org.apache.lucene.search.spans.SpanCollector;
import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
import org.apache.lucene.search.spans.SpanQuery;
-import org.apache.lucene.search.spans.SpanWeight;
+import org.apache.lucene.search.spans.SpanScorer;
import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.PriorityQueue;
/**
- * Helps the {@link FieldOffsetStrategy} with strict position highlighting (e.g. highlight phrases correctly).
+ * Helps the {@link FieldOffsetStrategy} with position sensitive queries (e.g. highlight phrases correctly).
* This is a stateful class holding information about the query, but it can (and is) re-used across highlighting
- * documents. Despite this state; it's immutable after construction. The approach taken in this class is very similar
- * to the standard Highlighter's {@link WeightedSpanTermExtractor} which is in fact re-used here. However, we ought to
- * completely rewrite it to use the SpanCollector interface to collect offsets directly. We'll get better
- * phrase accuracy.
+ * documents. Despite this state, it's immutable after construction.
*
* @lucene.internal
*/
+// TODO rename to SpanHighlighting ?
public class PhraseHelper {
public static final PhraseHelper NONE = new PhraseHelper(new MatchAllDocsQuery(), "_ignored_",
(s) -> false, spanQuery -> null, query -> null, true);
- //TODO it seems this ought to be a general thing on Spans?
- private static final Comparator<? super Spans> SPANS_COMPARATOR = (o1, o2) -> {
- int cmp = Integer.compare(o1.docID(), o2.docID());
- if (cmp != 0) {
- return cmp;
- }
- if (o1.docID() == DocIdSetIterator.NO_MORE_DOCS) {
- return 0; // don't ask for start/end position; not sure if we can even call those methods
- }
- cmp = Integer.compare(o1.startPosition(), o2.startPosition());
- if (cmp != 0) {
- return cmp;
- } else {
- return Integer.compare(o1.endPosition(), o2.endPosition());
- }
- };
-
private final String fieldName;
- private final Set<Term> positionInsensitiveTerms; // (TermQuery terms)
+ private final Set<BytesRef> positionInsensitiveTerms; // (TermQuery terms)
private final Set<SpanQuery> spanQueries;
private final boolean willRewrite;
private final Predicate<String> fieldMatcher;
@@ -114,13 +90,27 @@ public class PhraseHelper {
this.fieldName = field;
this.fieldMatcher = fieldMatcher;
// filter terms to those we want
- positionInsensitiveTerms = new FieldFilteringTermSet();
+ positionInsensitiveTerms = new HashSet<>();
spanQueries = new HashSet<>();
// TODO Have toSpanQuery(query) Function as an extension point for those with custom Query impls
boolean[] mustRewriteHolder = {false}; // boolean wrapped in 1-ary array so it's mutable from inner class
+ // When we call Weight.extractTerms, we do it on clauses that are NOT position sensitive.
+ // We only want the to track a Set of bytes for the Term, not Term class with field part.
+ Set<Term> extractPosInsensitiveTermsTarget = new TreeSet<Term>() {
+ @Override
+ public boolean add(Term term) {
+ // don't call super.add; we don't actually use the superclass
+ if (fieldMatcher.test(term.field())) {
+ return positionInsensitiveTerms.add(term.bytes());
+ } else {
+ return false;
+ }
+ }
+ };
+
// For TermQueries or other position insensitive queries, collect the Terms.
// For other Query types, WSTE will convert to an equivalent SpanQuery. NOT extracting position spans here.
new WeightedSpanTermExtractor(field) {
@@ -155,13 +145,15 @@ public class PhraseHelper {
return true; //TODO set to false and provide a hook to customize certain queries.
}
+ // called on Query types that are NOT position sensitive, e.g. TermQuery
@Override
protected void extractWeightedTerms(Map<String, WeightedSpanTerm> terms, Query query, float boost)
throws IOException {
query.createWeight(UnifiedHighlighter.EMPTY_INDEXSEARCHER, ScoreMode.COMPLETE_NO_SCORES, boost)
- .extractTerms(positionInsensitiveTerms);
+ .extractTerms(extractPosInsensitiveTermsTarget);
}
+ // called on SpanQueries. Some other position-sensitive queries like PhraseQuery are converted beforehand
@Override
protected void extractWeightedSpanTerms(Map<String, WeightedSpanTerm> terms, SpanQuery spanQuery,
float boost) throws IOException {
@@ -174,7 +166,6 @@ public class PhraseHelper {
}
}
- // TODO allow users to override the answer to mustRewriteQuery
boolean mustRewriteQuery = mustRewriteQuery(spanQuery);
if (ignoreQueriesNeedingRewrite && mustRewriteQuery) {
return;// ignore this query
@@ -194,14 +185,14 @@ public class PhraseHelper {
willRewrite = mustRewriteHolder[0];
}
- Set<SpanQuery> getSpanQueries() {
+ public Set<SpanQuery> getSpanQueries() {
return spanQueries;
}
/**
* If there is no position sensitivity then use of the instance of this class can be ignored.
*/
- boolean hasPositionSensitivity() {
+ public boolean hasPositionSensitivity() {
return spanQueries.isEmpty() == false;
}
@@ -210,335 +201,85 @@ public class PhraseHelper {
* custom things. When true, the resulting term list will probably be different than what it was known
* to be initially.
*/
- boolean willRewrite() {
+ public boolean willRewrite() {
return willRewrite;
}
- /**
- * Collect a list of pre-positioned {@link Spans} for each term, given a reader that has just one document.
- * It returns no mapping for query terms that occurs in a position insensitive way which therefore don't
- * need to be filtered.
- */
- Map<BytesRef, Spans> getTermToSpans(LeafReader leafReader, int doc)
- throws IOException {
- if (spanQueries.isEmpty()) {
- return Collections.emptyMap();
- }
- final LeafReader filteredReader = new SingleFieldFilterLeafReader(leafReader, fieldName);
- // for each SpanQuery, collect the member spans into a map.
- Map<BytesRef, Spans> result = new HashMap<>();
- for (SpanQuery spanQuery : spanQueries) {
- getTermToSpans(spanQuery, filteredReader.getContext(), doc, result);
- }
+ /** Returns the terms that are position-insensitive (sorted). */
+ public BytesRef[] getAllPositionInsensitiveTerms() {
+ BytesRef[] result = positionInsensitiveTerms.toArray(new BytesRef[positionInsensitiveTerms.size()]);
+ Arrays.sort(result);
return result;
}
- // code extracted & refactored from WSTE.extractWeightedSpanTerms()
- private void getTermToSpans(SpanQuery spanQuery, LeafReaderContext readerContext,
- int doc, Map<BytesRef, Spans> result)
- throws IOException {
- // note: in WSTE there was some field specific looping that seemed pointless so that isn't here.
- final IndexSearcher searcher = new IndexSearcher(readerContext.reader());
+ /** Given the internal SpanQueries, produce a number of OffsetsEnum into the {@code results} param. */
+ public void createOffsetsEnumsForSpans(LeafReader leafReader, int docId, List<OffsetsEnum> results) throws IOException {
+ leafReader = new SingleFieldWithOffsetsFilterLeafReader(leafReader, fieldName);
+ //TODO avoid searcher and do what it does to rewrite & get weight?
+ IndexSearcher searcher = new IndexSearcher(leafReader);
searcher.setQueryCache(null);
- if (willRewrite) {
- spanQuery = (SpanQuery) searcher.rewrite(spanQuery); // searcher.rewrite loops till done
- }
-
- // Get the underlying query terms
- TreeSet<Term> termSet = new FieldFilteringTermSet(); // sorted so we can loop over results in order shortly...
- searcher.createWeight(spanQuery, ScoreMode.COMPLETE_NO_SCORES, 1.0f).extractTerms(termSet);//needsScores==false
-
- // Get Spans by running the query against the reader
- // TODO it might make sense to re-use/cache the Spans instance, to advance forward between docs
- SpanWeight spanWeight = (SpanWeight) searcher.createNormalizedWeight(spanQuery, ScoreMode.COMPLETE_NO_SCORES);
- Spans spans = spanWeight.getSpans(readerContext, SpanWeight.Postings.POSITIONS);
- if (spans == null) {
- return;
- }
- TwoPhaseIterator twoPhaseIterator = spans.asTwoPhaseIterator();
- if (twoPhaseIterator != null) {
- if (twoPhaseIterator.approximation().advance(doc) != doc || !twoPhaseIterator.matches()) {
- return;
- }
- } else if (spans.advance(doc) != doc) { // preposition, and return doing nothing if find none
- return;
- }
-
- // Consume the Spans into a cache. This instance is used as a source for multiple cloned copies.
- // It's important we do this and not re-use the same original Spans instance since these will be iterated
- // independently later on; sometimes in ways that prevents sharing the original Spans.
- CachedSpans cachedSpansSource = new CachedSpans(spans); // consumes spans for this doc only and caches
- spans = null;// we don't use it below
-
- // Map terms to a Spans instance (aggregate if necessary)
- for (final Term queryTerm : termSet) {
- // note: we expect that at least one query term will pass these filters. This is because the collected
- // spanQuery list were already filtered by these conditions.
- if (positionInsensitiveTerms.contains(queryTerm)) {
- continue;
- }
- // copy-constructor refers to same data (shallow) but has iteration state from the beginning
- CachedSpans cachedSpans = new CachedSpans(cachedSpansSource);
- // Add the span to whatever span may or may not exist
- Spans existingSpans = result.get(queryTerm.bytes());
- if (existingSpans != null) {
- if (existingSpans instanceof MultiSpans) {
- ((MultiSpans) existingSpans).addSpans(cachedSpans);
- } else { // upgrade to MultiSpans
- MultiSpans multiSpans = new MultiSpans();
- multiSpans.addSpans(existingSpans);
- multiSpans.addSpans(cachedSpans);
- result.put(queryTerm.bytes(), multiSpans);
- }
- } else {
- result.put(queryTerm.bytes(), cachedSpans);
- }
- }
- }
-
- /**
- * Returns terms as a List, but expanded to any terms in phraseHelper' keySet if present. That can only
- * happen if willRewrite() is true.
- */
- List<BytesRef> expandTermsIfRewrite(BytesRef[] terms, Map<BytesRef, Spans> strictPhrasesTermToSpans) {
- if (willRewrite()) {
- Set<BytesRef> allTermSet = new LinkedHashSet<>(terms.length + strictPhrasesTermToSpans.size());
- Collections.addAll(allTermSet, terms);//FYI already sorted; will keep order
- if (allTermSet.addAll(strictPhrasesTermToSpans.keySet())) { // true if any were added
- List<BytesRef> sourceTerms = Arrays.asList(allTermSet.toArray(new BytesRef[allTermSet.size()]));
- sourceTerms.sort(Comparator.naturalOrder());
- return sourceTerms;
- }
- }
- return Arrays.asList(terms); // no rewrite; use original terms
- }
-
- /**
- * Returns a filtered postings where the position must be in the given Spans.
- * The Spans must be in a positioned state (not initial) and should not be shared between other terms.
- * {@code postingsEnum} should be positioned at the
- * document (the same one as the spans) but it hasn't iterated the positions yet.
- * The Spans should be the result of a simple
- * lookup from {@link #getTermToSpans(LeafReader, int)}, and so it could be null which could mean
- * either it's completely filtered or that there should be no filtering; this class knows what to do.
- * <p>
- * Due to limitations in filtering, the {@link PostingsEnum#freq()} is un-changed even if some positions
- * get filtered. So when {@link PostingsEnum#nextPosition()} is called or {@code startOffset} or {@code
- * endOffset} beyond the "real" positions, these methods returns {@link Integer#MAX_VALUE}.
- * <p>
- * <b>This will return null if it's completely filtered out (i.e. effectively has no postings).</b>
- */
- PostingsEnum filterPostings(BytesRef term, PostingsEnum postingsEnum, Spans spans)
- throws IOException {
- if (spans == null) {
- if (hasPositionSensitivity() == false || positionInsensitiveTerms.contains(new Term(fieldName, term))) {
- return postingsEnum; // no filtering
- } else {
- return null; // completely filtered out
- }
- }
- if (postingsEnum.docID() != spans.docID()) {
- throw new IllegalStateException("Spans & Postings doc ID misaligned or not positioned");
- }
-
- return new FilterLeafReader.FilterPostingsEnum(postingsEnum) {
- // freq() is max times nextPosition can be called. We'll set this var to -1 when exhausted.
- int remainingPositions = postingsEnum.freq();
-
- @Override
- public String toString() {
- String where;
- try {
- where = "[" + startOffset() + ":" + endOffset() + "]";
- } catch (IOException e) {
- where = "[" + e + "]";
- }
- return "'" + term.utf8ToString() + "'@" + where + " filtered by " + spans;
- }
-
- @Override
- public int nextDoc() throws IOException {
- throw new IllegalStateException("not expected"); // don't need to implement; just used on one doc
- }
-
- @Override
- public int advance(int target) throws IOException {
- throw new IllegalStateException("not expected"); // don't need to implement; just used on one doc
- }
-
- @Override
- public int nextPosition() throws IOException {
- // loop over posting positions...
- NEXT_POS_LOOP:
- while (remainingPositions > 0) {
- final int thisPos = super.nextPosition();
- remainingPositions--;
-
- // loop spans forward (if necessary) while the span end is behind thisPos
- while (spans.endPosition() <= thisPos) {
- if (spans.nextStartPosition() == Spans.NO_MORE_POSITIONS) { // advance
- break NEXT_POS_LOOP;
- }
- assert spans.docID() == postingsEnum.docID();
- }
-
- // is this position within the span?
- if (thisPos >= spans.startPosition()) {
- assert thisPos < spans.endPosition(); // guaranteed by previous loop
- return thisPos; // yay!
- }
- // else continue and try the next position
- }
- remainingPositions = -1; // signify done
- return Integer.MAX_VALUE;
- }
+ // for each SpanQuery, grab it's Spans and put it into a PriorityQueue
+ PriorityQueue<Spans> spansPriorityQueue = new PriorityQueue<Spans>(spanQueries.size()) {
@Override
- public int startOffset() throws IOException {
- return remainingPositions >= 0 ? super.startOffset() : Integer.MAX_VALUE;
- }
-
- @Override
- public int endOffset() throws IOException {
- return remainingPositions >= 0 ? super.endOffset() : Integer.MAX_VALUE;
+ protected boolean lessThan(Spans a, Spans b) {
+ return a.startPosition() <= b.startPosition();
}
};
- }
-
- /**
- * Simple TreeSet that filters out Terms not matching the provided predicate on {@code add()}.
- */
- private class FieldFilteringTermSet extends TreeSet<Term> {
- @Override
- public boolean add(Term term) {
- if (fieldMatcher.test(term.field())) {
- if (term.field().equals(fieldName)) {
- return super.add(term);
- } else {
- return super.add(new Term(fieldName, term.bytes()));
- }
- } else {
- return false;
- }
- }
- }
-
- /**
- * A single {@link Spans} view over multiple spans. At least one span is mandatory, but you should probably
- * supply more than one. Furthermore, the given spans are expected to be positioned to a document already
- * via a call to next or advance).
- */ // TODO move to Lucene core as a Spans utility class?
- static class MultiSpans extends Spans {
- final PriorityQueue<Spans> spansQueue = new PriorityQueue<>(SPANS_COMPARATOR);
- long cost;
-
- void addSpans(Spans spans) {
- if (spans.docID() < 0 || spans.docID() == NO_MORE_DOCS) {
- throw new IllegalArgumentException("Expecting given spans to be in a positioned state.");
- }
- spansQueue.add(spans);
- cost = Math.max(cost, spans.cost());
- }
-
- // DocIdSetIterator methods:
-
- @Override
- public int nextDoc() throws IOException {
- if (spansQueue.isEmpty()) {
- return NO_MORE_DOCS;
- }
- return advance(spansQueue.peek().docID() + 1);
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (spansQueue.isEmpty()) {
- return NO_MORE_DOCS;
+ for (Query query : spanQueries) {
+ Weight weight = searcher.createNormalizedWeight(query, ScoreMode.COMPLETE_NO_SCORES);
+ Scorer scorer = weight.scorer(leafReader.getContext());
+ if (scorer == null) {
+ continue;
}
- while (true) {
- Spans spans = spansQueue.peek();
- if (spans.docID() >= target) {
- return spans.docID();
- }
- spansQueue.remove(); // must remove before modify state
- if (spans.advance(target) != NO_MORE_DOCS) { // ... otherwise it's not re-added
- spansQueue.add(spans);
- } else if (spansQueue.isEmpty()) {
- return NO_MORE_DOCS;
+ TwoPhaseIterator twoPhaseIterator = scorer.twoPhaseIterator();
+ if (twoPhaseIterator != null) {
+ if (twoPhaseIterator.approximation().advance(docId) != docId || !twoPhaseIterator.matches()) {
+ continue;
}
+ } else if (scorer.iterator().advance(docId) != docId) { // preposition, and return doing nothing if find none
+ continue;
}
- }
- @Override
- public int docID() {
- if (spansQueue.isEmpty()) {
- return NO_MORE_DOCS;
+ Spans spans = ((SpanScorer) scorer).getSpans();
+ assert spans.docID() == docId;
+ if (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
+ spansPriorityQueue.add(spans);
}
- return spansQueue.peek().docID();
- }
-
- @Override
- public long cost() {
- return cost;
}
- // Spans methods:
-
- @Override
- public int nextStartPosition() throws IOException {
- // advance any spans at the initial position per document
- boolean atDocStart = false;
- while (spansQueue.peek().startPosition() == -1) {
- atDocStart = true;
- Spans headSpans = spansQueue.remove(); // remove because we will change state
- headSpans.nextStartPosition();
- spansQueue.add(headSpans);
- }
- if (!atDocStart) {
- Spans headSpans = spansQueue.remove(); // remove because we will change state
- headSpans.nextStartPosition();
- spansQueue.add(headSpans);
+ // Iterate the Spans in the PriorityQueue, collecting as we go. By using a PriorityQueue ordered by position,
+ // the underlying offsets in our collector will be mostly appended to the end of arrays (efficient).
+ // note: alternatively it'd interesting if we produced one OffsetsEnum that internally advanced
+ // this PriorityQueue when nextPosition is called; it would cap what we have to cache for large docs and
+ // exiting early (due to maxLen) is easy.
+ // But at least we have an accurate "freq" and it shouldn't be too much data to collect. Even SpanScorer
+ // navigates the spans fully to compute a good freq (and thus score)!
+ OffsetSpanCollector spanCollector = new OffsetSpanCollector();
+ while (spansPriorityQueue.size() > 0) {
+ Spans spans = spansPriorityQueue.top();
+ //TODO limit to a capped endOffset length somehow so we can break this loop early
+ spans.collect(spanCollector);
+
+ if (spans.nextStartPosition() == Spans.NO_MORE_POSITIONS) {
+ spansPriorityQueue.pop();
+ } else {
+ spansPriorityQueue.updateTop();
}
- return startPosition();
- }
-
- @Override
- public int startPosition() {
- return spansQueue.peek().startPosition();
- }
-
- @Override
- public int endPosition() {
- return spansQueue.peek().endPosition();
- }
-
- @Override
- public int width() {
- return spansQueue.peek().width();
- }
-
- @Override
- public void collect(SpanCollector collector) throws IOException {
- spansQueue.peek().collect(collector);
- }
-
- @Override
- public float positionsCost() {
- return 100f;// no idea; and we can't delegate due to not allowing to call it dependent on TwoPhaseIterator
}
+ results.addAll(spanCollector.termToOffsetsEnums.values());
}
- //TODO move up; it's currently inbetween other inner classes that are related
/**
* Needed to support the ability to highlight a query irrespective of the field a query refers to
* (aka requireFieldMatch=false).
* This reader will just delegate every call to a single field in the wrapped
* LeafReader. This way we ensure that all queries going through this reader target the same field.
*/
- static final class SingleFieldFilterLeafReader extends FilterLeafReader {
+ private static final class SingleFieldWithOffsetsFilterLeafReader extends FilterLeafReader {
final String fieldName;
- SingleFieldFilterLeafReader(LeafReader in, String fieldName) {
+ SingleFieldWithOffsetsFilterLeafReader(LeafReader in, String fieldName) {
super(in);
this.fieldName = fieldName;
}
@@ -550,22 +291,18 @@ public class PhraseHelper {
@Override
public Terms terms(String field) throws IOException {
- return super.terms(fieldName);
- }
-
- @Override
- public NumericDocValues getNumericDocValues(String field) throws IOException {
- return super.getNumericDocValues(fieldName);
- }
-
- @Override
- public BinaryDocValues getBinaryDocValues(String field) throws IOException {
- return super.getBinaryDocValues(fieldName);
- }
-
- @Override
- public SortedDocValues getSortedDocValues(String field) throws IOException {
- return super.getSortedDocValues(fieldName);
+ // ensure the underlying PostingsEnum returns offsets. It's sad we have to do this to use the SpanCollector.
+ return new FilterTerms(super.terms(fieldName)) {
+ @Override
+ public TermsEnum iterator() throws IOException {
+ return new FilterTermsEnum(in.iterator()) {
+ @Override
+ public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {
+ return super.postings(reuse, flags | PostingsEnum.OFFSETS);
+ }
+ };
+ }
+ };
}
@Override
@@ -584,99 +321,102 @@ public class PhraseHelper {
}
}
+ private class OffsetSpanCollector implements SpanCollector {
+ Map<BytesRef, SpanCollectedOffsetsEnum> termToOffsetsEnums = new HashMap<>();
- /**
- * A Spans based on a list of cached spans for one doc. It is pre-positioned to this doc.
- */
- private static class CachedSpans extends Spans {
-
- private static class CachedSpan {
- final int start;
- final int end;
-
- CachedSpan(int start, int end) {
- this.start = start;
- this.end = end;
+ @Override
+ public void collectLeaf(PostingsEnum postings, int position, Term term) throws IOException {
+ if (!fieldMatcher.test(term.field())) {
+ return;
}
- }
- final int docId;
- final ArrayList<CachedSpan> cachedSpanList;
- int index = -1;
-
- CachedSpans(Spans spans) throws IOException {
- this.docId = spans.docID();
- assert this.docId != -1;
- // Consume the spans for this doc into a list. There's always at least one; the first/current one.
- cachedSpanList = new ArrayList<>();
- while (spans.nextStartPosition() != NO_MORE_POSITIONS) {
- cachedSpanList.add(new CachedSpan(spans.startPosition(), spans.endPosition()));
+ SpanCollectedOffsetsEnum offsetsEnum = termToOffsetsEnums.get(term.bytes());
+ if (offsetsEnum == null) {
+ // If it's pos insensitive we handle it outside of PhraseHelper. term.field() is from the Query.
+ if (positionInsensitiveTerms.contains(term.bytes())) {
+ return;
+ }
+ offsetsEnum = new SpanCollectedOffsetsEnum(term.bytes(), postings.freq());
+ termToOffsetsEnums.put(term.bytes(), offsetsEnum);
}
- assert !cachedSpanList.isEmpty(); // bad Span impl?
- }
-
- /**
- * Clone; reset iteration state.
- */
- CachedSpans(CachedSpans cloneMe) {
- docId = cloneMe.docId;
- cachedSpanList = cloneMe.cachedSpanList;
- }
-
- @Override
- public int nextDoc() throws IOException {
- throw new UnsupportedOperationException("Not expected");
- }
-
- @Override
- public int advance(int target) throws IOException {
- throw new UnsupportedOperationException("Not expected");
- }
-
- @Override
- public int docID() {
- return docId;
+ offsetsEnum.add(postings.startOffset(), postings.endOffset());
}
@Override
- public long cost() {
- return 1;
+ public void reset() { // called when at a new position. We don't care.
}
+ }
- @Override
- public int nextStartPosition() throws IOException {
- index++;
- return startPosition();
+ private static class SpanCollectedOffsetsEnum extends OffsetsEnum {
+ // TODO perhaps optionally collect (and expose) payloads?
+ private final BytesRef term;
+ private final int[] startOffsets;
+ private final int[] endOffsets;
+ private int numPairs = 0;
+ private int enumIdx = -1;
+
+ private SpanCollectedOffsetsEnum(BytesRef term, int postingsFreq) {
+ this.term = term;
+ this.startOffsets = new int[postingsFreq]; // hopefully not wasteful? At least we needn't resize it.
+ this.endOffsets = new int[postingsFreq];
+ }
+
+ // called from collector before it's navigated
+ void add(int startOffset, int endOffset) {
+ assert enumIdx == -1 : "bad state";
+
+ // loop backwards since we expect a match at the end or close to it. We expect O(1) not O(N).
+ int pairIdx = numPairs - 1;
+ for (; pairIdx >= 0; pairIdx--) {
+ int iStartOffset = startOffsets[pairIdx];
+ int iEndOffset = endOffsets[pairIdx];
+ int cmp = Integer.compare(iStartOffset, startOffset);
+ if (cmp == 0) {
+ cmp = Integer.compare(iEndOffset, endOffset);
+ }
+ if (cmp == 0) {
+ return; // we already have this offset-pair for this term
+ } else if (cmp < 0) {
+ break; //we will insert offsetPair to the right of pairIdx
+ }
+ }
+ // pairIdx is now one position to the left of where we insert the new pair
+ // shift right any pairs by one to make room
+ final int shiftLen = numPairs - (pairIdx + 1);
+ if (shiftLen > 0) {
+ System.arraycopy(startOffsets, pairIdx + 2, startOffsets, pairIdx + 3, shiftLen);
+ System.arraycopy(endOffsets, pairIdx + 2, endOffsets, pairIdx + 3, shiftLen);
+ }
+ // now we can place the offset pair
+ startOffsets[pairIdx + 1] = startOffset;
+ endOffsets[pairIdx + 1] = endOffset;
+ numPairs++;
}
@Override
- public int startPosition() {
- return index < 0 ?
- -1 : index >= cachedSpanList.size() ?
- NO_MORE_POSITIONS : cachedSpanList.get(index).start;
+ public boolean nextPosition() throws IOException {
+ return ++enumIdx < numPairs;
}
@Override
- public int endPosition() {
- return index < 0 ?
- -1 : index >= cachedSpanList.size() ?
- NO_MORE_POSITIONS : cachedSpanList.get(index).end;
+ public int freq() throws IOException {
+ return startOffsets.length;
}
@Override
- public int width() {
- return endPosition() - startPosition();
+ public BytesRef getTerm() throws IOException {
+ return term;
}
@Override
- public void collect(SpanCollector collector) throws IOException {
- throw new UnsupportedOperationException("Not expected");
+ public int startOffset() throws IOException {
+ return startOffsets[enumIdx];
}
@Override
- public float positionsCost() {
- return 1f;
+ public int endOffset() throws IOException {
+ return endOffsets[enumIdx];
}
+ }
- } // class CachedSpans
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/352ec01a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TokenStreamOffsetStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TokenStreamOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TokenStreamOffsetStrategy.java
index 28eb6b1..5f47a5d 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TokenStreamOffsetStrategy.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TokenStreamOffsetStrategy.java
@@ -16,7 +16,6 @@
*/
package org.apache.lucene.search.uhighlight;
-import java.io.Closeable;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
@@ -26,7 +25,6 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.automaton.Automata;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
@@ -63,29 +61,20 @@ public class TokenStreamOffsetStrategy extends AnalysisOffsetStrategy {
@Override
public List<OffsetsEnum> getOffsetsEnums(IndexReader reader, int docId, String content) throws IOException {
- TokenStream tokenStream = tokenStream(content);
- PostingsEnum mtqPostingsEnum = new TokenStreamPostingsEnum(tokenStream, automata);
- mtqPostingsEnum.advance(docId);
- return Collections.singletonList(new OffsetsEnum(null, mtqPostingsEnum));
+ return Collections.singletonList(new TokenStreamOffsetsEnum(tokenStream(content), automata));
}
- // See class javadocs.
- // TODO: DWS perhaps instead OffsetsEnum could become abstract and this would be an impl? See TODOs in OffsetsEnum.
- private static class TokenStreamPostingsEnum extends PostingsEnum implements Closeable {
+ private static class TokenStreamOffsetsEnum extends OffsetsEnum {
TokenStream stream; // becomes null when closed
final CharacterRunAutomaton[] matchers;
final CharTermAttribute charTermAtt;
final OffsetAttribute offsetAtt;
- int currentDoc = -1;
int currentMatch = -1;
- int currentStartOffset = -1;
-
- int currentEndOffset = -1;
final BytesRef matchDescriptions[];
- TokenStreamPostingsEnum(TokenStream ts, CharacterRunAutomaton[] matchers) throws IOException {
+ TokenStreamOffsetsEnum(TokenStream ts, CharacterRunAutomaton[] matchers) throws IOException {
this.stream = ts;
this.matchers = matchers;
matchDescriptions = new BytesRef[matchers.length];
@@ -95,15 +84,13 @@ public class TokenStreamOffsetStrategy extends AnalysisOffsetStrategy {
}
@Override
- public int nextPosition() throws IOException {
+ public boolean nextPosition() throws IOException {
if (stream != null) {
while (stream.incrementToken()) {
for (int i = 0; i < matchers.length; i++) {
if (matchers[i].run(charTermAtt.buffer(), 0, charTermAtt.length())) {
- currentStartOffset = offsetAtt.startOffset();
- currentEndOffset = offsetAtt.endOffset();
currentMatch = i;
- return 0;
+ return true;
}
}
}
@@ -111,8 +98,7 @@ public class TokenStreamOffsetStrategy extends AnalysisOffsetStrategy {
close();
}
// exhausted
- currentStartOffset = currentEndOffset = Integer.MAX_VALUE;
- return Integer.MAX_VALUE;
+ return false;
}
@Override
@@ -122,46 +108,24 @@ public class TokenStreamOffsetStrategy extends AnalysisOffsetStrategy {
@Override
public int startOffset() throws IOException {
- assert currentStartOffset >= 0;
- return currentStartOffset;
+ return offsetAtt.startOffset();
}
@Override
public int endOffset() throws IOException {
- assert currentEndOffset >= 0;
- return currentEndOffset;
+ return offsetAtt.endOffset();
}
- // TOTAL HACK; used in OffsetsEnum.getTerm()
@Override
- public BytesRef getPayload() throws IOException {
+ public BytesRef getTerm() throws IOException {
if (matchDescriptions[currentMatch] == null) {
+ // these CharRunAutomata are subclassed so that toString() returns the query
matchDescriptions[currentMatch] = new BytesRef(matchers[currentMatch].toString());
}
return matchDescriptions[currentMatch];
}
@Override
- public int docID() {
- return currentDoc;
- }
-
- @Override
- public int nextDoc() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public int advance(int target) throws IOException {
- return currentDoc = target;
- }
-
- @Override
- public long cost() {
- return 0;
- }
-
- @Override
public void close() throws IOException {
if (stream != null) {
stream.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/352ec01a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
index 96ec155..086d7a0 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
@@ -23,11 +23,14 @@ import java.nio.charset.StandardCharsets;
import java.text.BreakIterator;
import java.util.Arrays;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.function.Predicate;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
@@ -49,6 +52,7 @@ import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.uhighlight.UnifiedHighlighter.HighlightFlag;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.After;
@@ -81,6 +85,36 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
dir.close();
}
+ static UnifiedHighlighter randomUnifiedHighlighter(IndexSearcher searcher, Analyzer indexAnalyzer) {
+ return randomUnifiedHighlighter(searcher, indexAnalyzer, EnumSet.noneOf(HighlightFlag.class));
+ }
+
+ static UnifiedHighlighter randomUnifiedHighlighter(IndexSearcher searcher, Analyzer indexAnalyzer,
+ EnumSet<HighlightFlag> mandatoryFlags) {
+ if (random().nextBoolean()) {
+ return new UnifiedHighlighter(searcher, indexAnalyzer);
+ } else {
+ final UnifiedHighlighter uh = new UnifiedHighlighter(searcher, indexAnalyzer) {
+ @Override
+ protected Set<HighlightFlag> getFlags(String field) {
+ final EnumSet<HighlightFlag> result = EnumSet.copyOf(mandatoryFlags);
+ int r = random().nextInt();
+ for (HighlightFlag highlightFlag : HighlightFlag.values()) {
+ if (((1 << highlightFlag.ordinal()) & r) == 0) {
+ result.add(highlightFlag);
+ }
+ }
+ return result;
+ }
+ };
+ uh.setCacheFieldValCharsThreshold(random().nextInt(100));
+ if (random().nextBoolean()) {
+ uh.setFieldMatcher(f -> true); // requireFieldMatch==false
+ }
+ return uh;
+ }
+ }
+
//
// Tests below were ported from the PostingsHighlighter. Possibly augmented. Far below are newer tests.
//
@@ -101,7 +135,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
Query query = new TermQuery(new Term("body", "highlighting"));
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.totalHits);
@@ -167,7 +201,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
assertEquals(1, topDocs.totalHits);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
highlighter.setMaxLength(maxLength);
String snippets[] = highlighter.highlight("body", query, topDocs);
@@ -191,7 +225,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
Query query = new TermQuery(new Term("body", "test"));
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
assertEquals(1, topDocs.totalHits);
@@ -219,7 +253,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
Query query = new TermQuery(new Term("body", "test"));
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.totalHits);
@@ -248,7 +282,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
highlighter.setMaxLength(value.length() * 2 + 1);
Query query = new TermQuery(new Term("body", "field"));
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
@@ -281,7 +315,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
BooleanQuery query = new BooleanQuery.Builder()
.add(new TermQuery(new Term("body", "highlighting")), BooleanClause.Occur.SHOULD)
.add(new TermQuery(new Term("title", "best")), BooleanClause.Occur.SHOULD)
@@ -313,7 +347,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
BooleanQuery query = new BooleanQuery.Builder()
.add(new TermQuery(new Term("body", "highlighting")), BooleanClause.Occur.SHOULD)
.add(new TermQuery(new Term("body", "just")), BooleanClause.Occur.SHOULD)
@@ -345,7 +379,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
Query query = new TermQuery(new Term("body", "test"));
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.totalHits);
@@ -382,7 +416,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
.build();
TopDocs topDocs = searcher.search(query, 10);
assertEquals(1, topDocs.totalHits);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
highlighter.setHighlightPhrasesStrictly(false);
String snippets[] = highlighter.highlight("body", query, topDocs, 2);
assertEquals(1, snippets.length);
@@ -410,7 +444,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
.build();
TopDocs topDocs = searcher.search(query, 10);
assertEquals(1, topDocs.totalHits);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
highlighter.setHighlightPhrasesStrictly(false);
String snippets[] = highlighter.highlight("body", query, topDocs, 2);
assertEquals(1, snippets.length);
@@ -438,7 +472,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
.build();
TopDocs topDocs = searcher.search(query, 10);
assertEquals(1, topDocs.totalHits);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
highlighter.setMaxLength(Integer.MAX_VALUE - 1);
String snippets[] = highlighter.highlight("body", query, topDocs, 2);
assertEquals(1, snippets.length);
@@ -461,7 +495,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
Query query = new TermQuery(new Term("body", "test"));
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
assertEquals(1, topDocs.totalHits);
@@ -494,7 +528,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
TopDocs topDocs = searcher.search(query, 10);
assertEquals(1, topDocs.totalHits);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
highlighter.setMaxLength(Integer.MAX_VALUE - 1);
String snippets[] = highlighter.highlight("body", query, topDocs, 2);
assertEquals(1, snippets.length);
@@ -549,7 +583,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
Query query = new TermQuery(new Term("body", "highlighting"));
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.totalHits);
@@ -623,7 +657,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
Query query = new TermQuery(new Term("body", "highlighting"));
int[] docIDs = new int[]{0};
String snippets[] = highlighter.highlightFields(new String[]{"body"}, query, docIDs, new int[]{2}).get("body");
@@ -652,7 +686,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
int docID = searcher.search(new TermQuery(new Term("id", "id")), 1).scoreDocs[0].doc;
Query query = new TermQuery(new Term("body", "highlighting"));
@@ -683,7 +717,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
highlighter.setMaxNoHighlightPassages(0);// don't want any default summary
Query query = new TermQuery(new Term("body", "highlighting"));
int[] docIDs = new int[]{0};
@@ -743,7 +777,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
Query query = new TermQuery(new Term("bogus", "highlighting"));
int[] docIDs = new int[]{0};
String snippets[] = highlighter.highlightFields(new String[]{"bogus"}, query, docIDs, new int[]{2}).get("bogus");
@@ -769,7 +803,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
int docID = searcher.search(new TermQuery(new Term("id", "id")), 1).scoreDocs[0].doc;
Query query = new TermQuery(new Term("body", "highlighting"));
@@ -798,7 +832,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
int docID = searcher.search(new TermQuery(new Term("id", "id")), 1).scoreDocs[0].doc;
Query query = new TermQuery(new Term("body", "highlighting"));
@@ -834,7 +868,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
highlighter.setCacheFieldValCharsThreshold(random().nextInt(10) * 10);// 0 thru 90 intervals of 10
Query query = new TermQuery(new Term("body", "answer"));
TopDocs hits = searcher.search(query, numDocs);
@@ -872,7 +906,7 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
BooleanQuery query = new BooleanQuery.Builder()
.add(new TermQuery(new Term("body", "test")), BooleanClause.Occur.SHOULD)
.add(new TermQuery(new Term("title", "test")), BooleanClause.Occur.SHOULD)
@@ -995,7 +1029,8 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
return (qf) -> true;
}
};
- UnifiedHighlighter highlighterFieldMatch = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighterFieldMatch = randomUnifiedHighlighter(searcher, indexAnalyzer);
+ highlighterFieldMatch.setFieldMatcher(null);//default
BooleanQuery.Builder queryBuilder =
new BooleanQuery.Builder()
.add(new TermQuery(new Term("text", "some")), BooleanClause.Occur.SHOULD)
@@ -1078,7 +1113,8 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
return (qf) -> true;
}
};
- UnifiedHighlighter highlighterFieldMatch = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighterFieldMatch = randomUnifiedHighlighter(searcher, indexAnalyzer, EnumSet.of(HighlightFlag.MULTI_TERM_QUERY));
+ highlighterFieldMatch.setFieldMatcher(null);//default
BooleanQuery.Builder queryBuilder =
new BooleanQuery.Builder()
.add(new FuzzyQuery(new Term("text", "sime"), 1), BooleanClause.Occur.SHOULD)
@@ -1161,7 +1197,8 @@ public class TestUnifiedHighlighter extends LuceneTestCase {
return (qf) -> true;
}
};
- UnifiedHighlighter highlighterFieldMatch = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighterFieldMatch = randomUnifiedHighlighter(searcher, indexAnalyzer, EnumSet.of(HighlightFlag.PHRASES));
+ highlighterFieldMatch.setFieldMatcher(null);//default
BooleanQuery.Builder queryBuilder =
new BooleanQuery.Builder()
.add(new PhraseQuery("title", "this", "is", "the", "title"), BooleanClause.Occur.SHOULD)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/352ec01a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterMTQ.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterMTQ.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterMTQ.java
index 8791b76..a9fadc0 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterMTQ.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterMTQ.java
@@ -20,6 +20,7 @@ package org.apache.lucene.search.uhighlight;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.List;
import java.util.Objects;
@@ -65,6 +66,7 @@ import org.apache.lucene.search.spans.SpanOrQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.search.spans.SpanWeight;
+import org.apache.lucene.search.uhighlight.UnifiedHighlighter.HighlightFlag;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -150,6 +152,11 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
ir.close();
}
+ private UnifiedHighlighter randomUnifiedHighlighter(IndexSearcher searcher, Analyzer indexAnalyzer) {
+ return TestUnifiedHighlighter.randomUnifiedHighlighter(searcher, indexAnalyzer,
+ EnumSet.of(HighlightFlag.MULTI_TERM_QUERY));
+ }
+
public void testOnePrefix() throws Exception {
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);
@@ -166,7 +173,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
// wrap in a BoostQuery to also show we see inside it
Query query = new BoostQuery(new PrefixQuery(new Term("body", "te")), 2.0f);
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
@@ -177,6 +184,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
// wrong field
+ highlighter.setFieldMatcher(null);//default
BooleanQuery bq = new BooleanQuery.Builder()
.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)
.add(new PrefixQuery(new Term("bogus", "te")), BooleanClause.Occur.SHOULD)
@@ -207,7 +215,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
Query query = new RegexpQuery(new Term("body", "te.*"));
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.totalHits);
@@ -217,6 +225,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
// wrong field
+ highlighter.setFieldMatcher(null);//default
BooleanQuery bq = new BooleanQuery.Builder()
.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)
.add(new RegexpQuery(new Term("bogus", "te.*")), BooleanClause.Occur.SHOULD)
@@ -247,7 +256,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
Query query = new FuzzyQuery(new Term("body", "tets"), 1);
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.totalHits);
@@ -266,6 +275,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
// wrong field
+ highlighter.setFieldMatcher(null);//default
BooleanQuery bq = new BooleanQuery.Builder()
.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)
.add(new FuzzyQuery(new Term("bogus", "tets"), 1), BooleanClause.Occur.SHOULD)
@@ -296,7 +306,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
Query query = TermRangeQuery.newStringRange("body", "ta", "tf", true, true);
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.totalHits);
@@ -366,6 +376,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
assertEquals("Test a one sentence document.", snippets[1]);
// wrong field
+ highlighter.setFieldMatcher(null);//default
bq = new BooleanQuery.Builder()
.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)
.add(TermRangeQuery.newStringRange("bogus", "ta", "tf", true, true), BooleanClause.Occur.SHOULD)
@@ -396,7 +407,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
BooleanQuery query = new BooleanQuery.Builder()
.add(new WildcardQuery(new Term("body", "te*")), BooleanClause.Occur.SHOULD)
.build();
@@ -438,7 +449,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
BooleanQuery query = new BooleanQuery.Builder()
.add(new WildcardQuery(new Term("body", "te*")), BooleanClause.Occur.MUST)
.add(new TermQuery(new Term("body", "test")), BooleanClause.Occur.FILTER)
@@ -469,7 +480,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
ConstantScoreQuery query = new ConstantScoreQuery(new WildcardQuery(new Term("body", "te*")));
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.totalHits);
@@ -497,7 +508,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
DisjunctionMaxQuery query = new DisjunctionMaxQuery(
Collections.singleton(new WildcardQuery(new Term("body", "te*"))), 0);
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
@@ -526,7 +537,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
// wrap in a SpanBoostQuery to also show we see inside it
Query query = new SpanBoostQuery(
new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*"))), 2.0f);
@@ -556,7 +567,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
Query query = new SpanOrQuery(new SpanQuery[]{childQuery});
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
@@ -585,7 +596,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
Query query = new SpanNearQuery(new SpanQuery[]{childQuery, childQuery}, 0, false);
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
@@ -614,7 +625,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
SpanQuery include = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
SpanQuery exclude = new SpanTermQuery(new Term("body", "bogus"));
Query query = new SpanNotQuery(include, exclude);
@@ -644,7 +655,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
Query query = new SpanFirstQuery(childQuery, 1000000);
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
@@ -675,7 +686,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
// use a variety of common MTQ types
BooleanQuery query = new BooleanQuery.Builder()
.add(new PrefixQuery(new Term("body", "te")), BooleanClause.Occur.SHOULD)
@@ -765,7 +776,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
highlighter.setMaxLength(25);//a little past first sentence
BooleanQuery query = new BooleanQuery.Builder()
@@ -798,7 +809,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
highlighter.setMaxLength(32);//a little past first sentence
BooleanQuery query = new BooleanQuery.Builder()
@@ -846,7 +857,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
};
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, buggyAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, buggyAnalyzer);
highlighter.setHandleMultiTermQuery(true);
if (rarely()) {
highlighter.setMaxLength(25);//a little past first sentence
@@ -903,7 +914,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
int docID = searcher.search(new TermQuery(new Term("id", "id")), 1).scoreDocs[0].doc;
Query query = new PrefixQuery(new Term("body", "nonexistent"));
@@ -934,7 +945,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
Query query = new PrefixQuery(new Term("body", "ab"));
TopDocs topDocs = searcher.search(query, 10);
@@ -956,7 +967,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.close();
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
int docID = searcher.search(new TermQuery(new Term("id", "id")), 1).scoreDocs[0].doc;
PhraseQuery pq = new PhraseQuery.Builder()
@@ -1076,7 +1087,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
IndexSearcher searcher = newSearcher(ir);
Query query = new PrefixQuery(new Term(field, "я"));
TopDocs topDocs = searcher.search(query, 1);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, analyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, analyzer);
String[] snippets = highlighter.highlight(field, query, topDocs);
assertEquals("[<b>я</b>]", Arrays.toString(snippets));
ir.close();
@@ -1100,7 +1111,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {
iw.commit();
try (IndexReader ir = iw.getReader()) {
IndexSearcher searcher = newSearcher(ir);
- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, analyzer);
+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, analyzer);
highlighter.setBreakIterator(WholeBreakIterator::new);
// Test PrefixQuery
[25/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11809:
QueryComponent.prepare rq parsing could fail under SOLR 7.2.0 - fix:
QueryComponent's rq parameter parsing no longer considers the defType
parameter. (Christine Poerschke and David Smiley in respon
Posted by da...@apache.org.
SOLR-11809: QueryComponent.prepare rq parsing could fail under SOLR 7.2.0 - fix:
QueryComponent's rq parameter parsing no longer considers the defType parameter.
(Christine Poerschke and David Smiley in response to bug report/analysis from Dariusz Wojtas and Diego Ceccarelli)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/28286568
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/28286568
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/28286568
Branch: refs/heads/jira/solr-11702
Commit: 2828656892114ab7bb4c7742eac9c4e6f49f69ab
Parents: a9fec9b
Author: Christine Poerschke <cp...@apache.org>
Authored: Mon Jan 8 19:44:05 2018 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Mon Jan 8 19:44:05 2018 +0000
----------------------------------------------------------------------
solr/CHANGES.txt | 7 ++++
.../solr/handler/component/QueryComponent.java | 2 +-
.../solr/search/TestReRankQParserPlugin.java | 41 ++++++++++++++++++++
3 files changed, 49 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/28286568/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 011766f..31197a0 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -60,6 +60,8 @@ Upgrade Notes
* SOLR-11798: The top-level <highlighting> syntax in solrconfig.xml is now formally
deprecated in favour of <searchComponent> equivalent syntax. See also SOLR-1696.
+* SOLR-11809: QueryComponent's rq parameter parsing no longer considers the defType parameter.
+
New Features
----------------------
* SOLR-11285: Simulation framework for autoscaling. (ab)
@@ -131,6 +133,11 @@ Bug Fixes
* SOLR-11771: Overseer can never process some last messages (Cao Manh Dat)
+* SOLR-11809: QueryComponent.prepare rq parsing could fail under SOLR 7.2.0 - fix:
+ QueryComponent's rq parameter parsing no longer considers the defType parameter.
+ (Christine Poerschke and David Smiley in response to bug report/analysis
+ from Dariusz Wojtas and Diego Ceccarelli)
+
================== 7.2.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/28286568/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
index 7dbd311..71ac9c0 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
@@ -167,7 +167,7 @@ public class QueryComponent extends SearchComponent
String rankQueryString = rb.req.getParams().get(CommonParams.RQ);
if(rankQueryString != null) {
- QParser rqparser = QParser.getParser(rankQueryString, defType, req);
+ QParser rqparser = QParser.getParser(rankQueryString, req);
Query rq = rqparser.getQuery();
if(rq instanceof RankQuery) {
RankQuery rankQuery = (RankQuery)rq;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/28286568/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java
index 0fd1a4a..93673d2 100644
--- a/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java
+++ b/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java
@@ -605,4 +605,45 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 {
}
}
+ @Test
+ public void testReRankQueriesWithDefType() throws Exception {
+
+ assertU(delQ("*:*"));
+ assertU(commit());
+
+ final String[] doc1 = {"id","1"};
+ assertU(adoc(doc1));
+ assertU(commit());
+ final String[] doc2 = {"id","2"};
+ assertU(adoc(doc2));
+ assertU(commit());
+
+ final String preferredDocId;
+ final String lessPreferrredDocId;
+ if (random().nextBoolean()) {
+ preferredDocId = "1";
+ lessPreferrredDocId = "2";
+ } else {
+ preferredDocId = "2";
+ lessPreferrredDocId = "1";
+ }
+
+ for (final String defType : new String[] {
+ null,
+ LuceneQParserPlugin.NAME,
+ ExtendedDismaxQParserPlugin.NAME
+ }) {
+ final ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("rq", "{!"+ReRankQParserPlugin.NAME+" "+ReRankQParserPlugin.RERANK_QUERY+"=id:"+preferredDocId+"}");
+ params.add("q", "*:*");
+ if (defType != null) {
+ params.add(QueryParsing.DEFTYPE, defType);
+ }
+ assertQ(req(params), "*[count(//doc)=2]",
+ "//result/doc[1]/str[@name='id'][.='"+preferredDocId+"']",
+ "//result/doc[2]/str[@name='id'][.='"+lessPreferrredDocId+"']"
+ );
+ }
+ }
+
}
[47/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-10995: Improve
ChaosMonkey logic to find Jetty running leader node
Posted by da...@apache.org.
SOLR-10995: Improve ChaosMonkey logic to find Jetty running leader node
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6a55def1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6a55def1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6a55def1
Branch: refs/heads/jira/solr-11702
Commit: 6a55def1ead44fcd3c3640ead7bb4040db86f8c7
Parents: 077e8b1
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Tue Jan 9 22:58:22 2018 -0800
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Tue Jan 9 22:58:22 2018 -0800
----------------------------------------------------------------------
.../src/java/org/apache/solr/cloud/ChaosMonkey.java | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6a55def1/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
index abb6255..71e1b43 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
@@ -418,11 +418,7 @@ public class ChaosMonkey {
CoreContainer cc = cjetty.jetty.getCoreContainer();
if (cc != null) {
try (SolrCore core = cc.getCore(leader.getStr(ZkStateReader.CORE_NAME_PROP))) {
- if (core == null) {
- monkeyLog("selected jetty not running correctly - skip");
- return null;
- }
- rtIsLeader = core.getCoreDescriptor().getCloudDescriptor().isLeader();
+ rtIsLeader = core != null && core.getCoreDescriptor().getCloudDescriptor().isLeader();
}
} else {
return null;
[50/50] [abbrv] lucene-solr:jira/solr-11702: Merge with master
Posted by da...@apache.org.
Merge with master
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ca364c72
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ca364c72
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ca364c72
Branch: refs/heads/jira/solr-11702
Commit: ca364c72946190aab1e0b0defabd9b715fb666d0
Parents: 116fe4a 352ec01
Author: Cao Manh Dat <da...@apache.org>
Authored: Thu Jan 11 15:00:19 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Thu Jan 11 15:00:19 2018 +0700
----------------------------------------------------------------------
build.xml | 1 +
dev-tools/scripts/reproduceJenkinsFailures.py | 215 +++++++
lucene/CHANGES.txt | 33 ++
lucene/MIGRATE.txt | 12 +-
.../analysis/commongrams/CommonGramsFilter.java | 3 -
.../commongrams/CommonGramsQueryFilter.java | 3 -
.../miscellaneous/FingerprintFilter.java | 3 -
.../miscellaneous/HyphenatedWordsFilter.java | 6 -
.../RemoveDuplicatesTokenFilter.java | 6 -
lucene/analysis/icu/src/data/uax29/Default.rbbi | 96 ++-
.../icu/src/data/utr30/DiacriticFolding.txt | 11 +-
.../icu/src/data/utr30/NativeDigitFolding.txt | 10 +
lucene/analysis/icu/src/data/utr30/nfc.txt | 13 +-
lucene/analysis/icu/src/data/utr30/nfkc.txt | 4 +-
lucene/analysis/icu/src/data/utr30/nfkc_cf.txt | 10 +-
.../analysis/icu/segmentation/ICUTokenizer.java | 8 +-
lucene/analysis/icu/src/java/overview.html | 2 +-
.../analysis/icu/segmentation/Default.brk | Bin 36768 -> 54488 bytes
.../icu/segmentation/MyanmarSyllable.brk | Bin 20744 -> 21976 bytes
.../org/apache/lucene/analysis/icu/utr30.nrm | Bin 55184 -> 59056 bytes
.../analysis/icu/GenerateUTR30DataFiles.java | 6 +-
.../ja/JapaneseIterationMarkCharFilter.java | 6 -
.../lucene/classification/BM25NBClassifier.java | 9 -
.../BooleanPerceptronClassifier.java | 9 -
.../classification/KNearestFuzzyClassifier.java | 11 +-
.../KNearestNeighborClassifier.java | 11 +-
.../SimpleNaiveBayesClassifier.java | 9 -
.../KNearestNeighborDocumentClassifier.java | 9 -
.../SimpleNaiveBayesDocumentClassifier.java | 9 -
.../classification/utils/NearestFuzzyQuery.java | 12 +-
.../codecs/blocktree/IntersectTermsEnum.java | 7 +-
.../org/apache/lucene/index/TermContext.java | 197 -------
.../org/apache/lucene/index/TermStates.java | 234 ++++++++
.../apache/lucene/search/BlendedTermQuery.java | 28 +-
.../org/apache/lucene/search/BooleanWeight.java | 2 +-
.../lucene/search/DisjunctionMaxQuery.java | 4 +-
.../lucene/search/DoubleValuesSource.java | 8 +-
.../apache/lucene/search/ExactPhraseScorer.java | 7 +-
.../org/apache/lucene/search/Explanation.java | 14 +-
.../org/apache/lucene/search/IndexSearcher.java | 49 +-
.../org/apache/lucene/search/LeafSimScorer.java | 73 +++
.../apache/lucene/search/MultiPhraseQuery.java | 51 +-
.../apache/lucene/search/MultiTermQuery.java | 8 +-
.../MultiTermQueryConstantScoreWrapper.java | 8 +-
.../org/apache/lucene/search/PhraseQuery.java | 31 +-
.../org/apache/lucene/search/QueryRescorer.java | 6 +-
.../apache/lucene/search/ScoringRewrite.java | 14 +-
.../lucene/search/SloppyPhraseScorer.java | 7 +-
.../org/apache/lucene/search/SortRescorer.java | 2 +-
.../org/apache/lucene/search/SynonymQuery.java | 49 +-
.../lucene/search/TermCollectingRewrite.java | 4 +-
.../apache/lucene/search/TermInSetQuery.java | 8 +-
.../org/apache/lucene/search/TermQuery.java | 71 +--
.../org/apache/lucene/search/TermScorer.java | 11 +-
.../apache/lucene/search/TopTermsRewrite.java | 10 +-
.../org/apache/lucene/search/package-info.java | 4 +-
.../lucene/search/similarities/Axiomatic.java | 27 +-
.../search/similarities/BM25Similarity.java | 140 ++---
.../lucene/search/similarities/BasicStats.java | 2 +-
.../search/similarities/BooleanSimilarity.java | 48 +-
.../search/similarities/DFISimilarity.java | 18 +-
.../search/similarities/DFRSimilarity.java | 16 +-
.../search/similarities/IBSimilarity.java | 18 +-
.../similarities/LMDirichletSimilarity.java | 18 +-
.../similarities/LMJelinekMercerSimilarity.java | 18 +-
.../search/similarities/LMSimilarity.java | 2 +-
.../search/similarities/MultiSimilarity.java | 46 +-
.../similarities/PerFieldSimilarityWrapper.java | 22 +-
.../lucene/search/similarities/Similarity.java | 139 ++---
.../search/similarities/SimilarityBase.java | 93 +--
.../search/similarities/TFIDFSimilarity.java | 127 ++--
.../lucene/search/spans/SpanContainQuery.java | 10 +-
.../search/spans/SpanContainingQuery.java | 10 +-
.../search/spans/SpanMultiTermQueryWrapper.java | 6 +-
.../lucene/search/spans/SpanNearQuery.java | 14 +-
.../lucene/search/spans/SpanNotQuery.java | 12 +-
.../apache/lucene/search/spans/SpanOrQuery.java | 12 +-
.../search/spans/SpanPositionCheckQuery.java | 12 +-
.../apache/lucene/search/spans/SpanQuery.java | 18 +-
.../apache/lucene/search/spans/SpanScorer.java | 6 +-
.../lucene/search/spans/SpanTermQuery.java | 36 +-
.../apache/lucene/search/spans/SpanWeight.java | 42 +-
.../lucene/search/spans/SpanWithinQuery.java | 10 +-
.../apache/lucene/search/spans/TermSpans.java | 4 +-
.../apache/lucene/index/TestCustomNorms.java | 7 +-
.../apache/lucene/index/TestCustomTermFreq.java | 9 +-
.../lucene/index/TestFieldInvertState.java | 8 +-
.../apache/lucene/index/TestIndexSorting.java | 9 +-
.../lucene/index/TestMaxTermFrequency.java | 16 +-
.../test/org/apache/lucene/index/TestNorms.java | 7 +-
.../lucene/index/TestUniqueTermCount.java | 8 +-
.../apache/lucene/search/JustCompileSearch.java | 7 +-
.../org/apache/lucene/search/TestBoolean2.java | 4 +-
.../apache/lucene/search/TestBooleanQuery.java | 2 +-
.../search/TestBooleanQueryVisitSubscorers.java | 16 +-
.../lucene/search/TestBooleanRewrites.java | 2 +-
.../apache/lucene/search/TestConjunctions.java | 17 +-
.../lucene/search/TestDocValuesScoring.java | 192 ------
.../lucene/search/TestMinShouldMatch2.java | 13 +-
.../apache/lucene/search/TestQueryRescorer.java | 4 +-
.../lucene/search/TestSimilarityProvider.java | 34 +-
.../lucene/search/TestSubScorerFreqs.java | 17 +-
.../org/apache/lucene/search/TestTermQuery.java | 6 +-
.../similarities/TestClassicSimilarity.java | 3 +-
.../search/similarities/TestSimilarity2.java | 2 +-
.../search/similarities/TestSimilarityBase.java | 11 +-
.../apache/lucene/search/spans/TestBasics.java | 54 +-
.../search/spans/TestFieldMaskingSpanQuery.java | 4 +-
.../search/spans/TestNearSpansOrdered.java | 2 +-
.../expressions/ExpressionValueSource.java | 4 +-
.../search/grouping/GroupingSearchTest.java | 2 +
.../highlight/WeightedSpanTermExtractor.java | 2 +-
.../search/uhighlight/FieldHighlighter.java | 14 +-
.../search/uhighlight/FieldOffsetStrategy.java | 80 +--
.../lucene/search/uhighlight/OffsetsEnum.java | 132 +++--
.../lucene/search/uhighlight/Passage.java | 18 +
.../lucene/search/uhighlight/PhraseHelper.java | 584 +++++--------------
.../uhighlight/TokenStreamOffsetStrategy.java | 56 +-
.../uhighlight/TestUnifiedHighlighter.java | 87 ++-
.../uhighlight/TestUnifiedHighlighterMTQ.java | 55 +-
.../TestUnifiedHighlighterStrictPhrases.java | 88 +++
.../TestUnifiedHighlighterExtensibility.java | 4 +-
lucene/ivy-versions.properties | 2 +-
.../search/join/ToParentBlockJoinQuery.java | 2 +-
.../lucene/search/join/TestBlockJoin.java | 9 +-
.../apache/lucene/search/join/TestJoinUtil.java | 2 +-
lucene/licenses/icu4j-59.1.jar.sha1 | 1 -
lucene/licenses/icu4j-60.2.jar.sha1 | 1 +
.../lucene/index/memory/TestMemoryIndex.java | 8 +-
.../lucene/misc/SweetSpotSimilarityTest.java | 2 +-
.../search/TestDiversifiedTopDocsCollector.java | 132 +++--
.../apache/lucene/queries/CommonTermsQuery.java | 40 +-
.../lucene/queries/function/FunctionQuery.java | 6 +-
.../queries/function/FunctionScoreQuery.java | 151 +++++
.../lucene/queries/function/ValueSource.java | 2 +-
.../function/docvalues/FloatDocValues.java | 5 +
.../function/valuesource/IDFValueSource.java | 2 +-
.../function/valuesource/NormValueSource.java | 10 +-
.../function/valuesource/TFValueSource.java | 2 +-
.../queries/payloads/PayloadScoreQuery.java | 17 +-
.../queries/payloads/SpanPayloadCheckQuery.java | 18 +-
.../lucene/queries/CommonTermsQueryTest.java | 6 +-
.../function/TestFunctionScoreExplanations.java | 10 +-
.../function/TestFunctionScoreQuery.java | 62 +-
.../function/TestLongNormValueSource.java | 2 +-
.../queries/function/TestValueSources.java | 8 +-
.../docvalues/TestBoolValOfNumericDVs.java | 76 +++
.../queries/payloads/TestPayloadCheckQuery.java | 2 +-
.../sandbox/queries/FuzzyLikeThisQuery.java | 8 +-
.../org/apache/lucene/search/CoveringQuery.java | 2 +-
.../lucene/search/TermAutomatonQuery.java | 26 +-
.../lucene/search/TermAutomatonScorer.java | 7 +-
.../spatial/bbox/BBoxSimilarityValueSource.java | 2 +-
.../util/ReciprocalDoubleValuesSource.java | 2 +-
.../spatial3d/geom/GeoExactCircleTest.java | 2 +-
.../lucene/index/BaseNormsFormatTestCase.java | 7 +-
.../org/apache/lucene/search/CheckHits.java | 4 +-
.../org/apache/lucene/search/QueryUtils.java | 10 +-
.../lucene/search/ShardSearchingTestBase.java | 8 +-
.../similarities/AssertingSimilarity.java | 94 ++-
.../similarities/BaseSimilarityTestCase.java | 154 +----
.../search/spans/AssertingSpanWeight.java | 10 +-
.../search/TestBaseExplanationTestCase.java | 2 +-
solr/CHANGES.txt | 53 ++
.../org/apache/solr/ltr/model/LinearModel.java | 2 +-
.../ltr/model/MultipleAdditiveTreesModel.java | 2 +-
.../org/apache/solr/ltr/norm/Normalizer.java | 2 +-
.../apache/solr/cloud/DeleteCollectionCmd.java | 13 +-
.../java/org/apache/solr/cloud/Overseer.java | 6 +-
.../cloud/OverseerCollectionMessageHandler.java | 3 +-
.../solr/cloud/OverseerTaskProcessor.java | 7 +-
.../cloud/RoutedAliasCreateCollectionCmd.java | 182 ++++++
.../cloud/autoscaling/ScheduledTriggers.java | 34 +-
.../handler/DocumentAnalysisRequestHandler.java | 3 -
.../handler/FieldAnalysisRequestHandler.java | 3 -
.../org/apache/solr/handler/SchemaHandler.java | 12 +-
.../org/apache/solr/handler/StreamHandler.java | 10 +-
.../solr/handler/admin/CollectionsHandler.java | 16 +-
.../handler/component/HighlightComponent.java | 52 +-
.../solr/handler/component/QueryComponent.java | 2 +-
.../handler/component/StatsValuesFactory.java | 60 --
.../solr/handler/component/TermsComponent.java | 26 +-
.../org/apache/solr/query/SolrRangeQuery.java | 8 +-
.../apache/solr/request/SolrRequestInfo.java | 12 +-
.../java/org/apache/solr/schema/LatLonType.java | 2 +-
.../apache/solr/search/BoostQParserPlugin.java | 21 +-
.../solr/search/ExtendedDismaxQParser.java | 5 +-
.../solr/search/GraphTermsQParserPlugin.java | 38 +-
.../java/org/apache/solr/search/Grouping.java | 51 --
.../apache/solr/search/SolrIndexSearcher.java | 6 +-
.../apache/solr/search/ValueSourceParser.java | 3 +-
.../apache/solr/search/facet/FacetRange.java | 10 +-
.../solr/search/facet/FacetRangeMerger.java | 2 +-
.../SearchGroupsRequestFactory.java | 3 -
.../TopGroupsShardRequestFactory.java | 3 -
.../SearchGroupShardResponseProcessor.java | 3 -
.../StoredFieldsShardResponseProcessor.java | 3 -
.../TopGroupsShardResponseProcessor.java | 3 -
.../SearchGroupsResultTransformer.java | 6 -
.../TopGroupsResultTransformer.java | 6 -
.../GroupedEndResultTransformer.java | 3 -
.../MainEndResultTransformer.java | 3 -
.../SimpleEndResultTransformer.java | 3 -
.../solr/search/stats/ExactStatsCache.java | 8 +-
.../apache/solr/search/stats/LRUStatsCache.java | 4 +-
.../solr/search/stats/LocalStatsSource.java | 4 +-
.../apache/solr/search/stats/StatsSource.java | 4 +-
.../solr/security/AuthenticationPlugin.java | 1 +
.../apache/solr/servlet/SolrDispatchFilter.java | 48 +-
.../TimeRoutedAliasUpdateProcessor.java | 228 +++++++-
.../org/apache/solr/util/TimeZoneUtils.java | 18 +
.../configuration/providers/package-info.java | 23 -
.../TestEmbeddedSolrServerSchemaAPI.java | 16 +-
.../apache/solr/cloud/AliasIntegrationTest.java | 160 ++++-
.../apache/solr/cloud/TestConfigSetsAPI.java | 3 +-
.../solr/cloud/autoscaling/TestPolicyCloud.java | 8 +-
.../autoscaling/TriggerIntegrationTest.java | 18 -
.../cloud/autoscaling/sim/SimCloudManager.java | 17 +
.../autoscaling/sim/SimSolrCloudTestCase.java | 11 +-
.../cloud/autoscaling/sim/TestLargeCluster.java | 116 +++-
.../cloud/autoscaling/sim/package-info.java | 79 ++-
.../solr/core/TestConfigSetImmutable.java | 6 +-
.../component/CustomHighlightComponentTest.java | 310 ++++++++++
.../solr/rest/schema/TestBulkSchemaAPI.java | 91 +--
.../solr/schema/SchemaApiFailureTest.java | 66 +++
.../org/apache/solr/schema/TestPointFields.java | 66 +--
.../solr/search/TestReRankQParserPlugin.java | 41 ++
.../solr/search/facet/TestJsonFacets.java | 42 ++
.../solr/search/function/TestFunctionQuery.java | 16 +-
.../similarities/BaseSimilarityTestCase.java | 2 +-
.../TestSweetSpotSimilarityFactory.java | 2 +-
.../solr/security/MockAuthorizationPlugin.java | 19 +-
.../PKIAuthenticationIntegrationTest.java | 15 +-
.../security/TestAuthorizationFramework.java | 35 +-
.../TimeRoutedAliasUpdateProcessorTest.java | 140 +++--
solr/licenses/icu4j-59.1.jar.sha1 | 1 -
solr/licenses/icu4j-60.2.jar.sha1 | 1 +
solr/solr-ref-guide/src/documents-screen.adoc | 52 +-
solr/solr-ref-guide/src/schema-api.adoc | 20 +-
.../src/solrcloud-autoscaling-api.adoc | 3 -
.../apache/solr/client/solrj/SolrRequest.java | 6 +-
.../apache/solr/client/solrj/SolrResponse.java | 2 +
.../cloud/autoscaling/AddReplicaSuggester.java | 9 +
.../cloud/autoscaling/MoveReplicaSuggester.java | 8 +
.../client/solrj/cloud/autoscaling/Policy.java | 16 +-
.../client/solrj/cloud/autoscaling/Row.java | 9 +-
.../solrj/cloud/autoscaling/Suggester.java | 12 +-
.../solrj/cloud/autoscaling/Suggestion.java | 11 +-
.../solrj/impl/SolrClientNodeStateProvider.java | 27 +-
.../client/solrj/io/eval/AkimaEvaluator.java | 71 +++
.../solr/client/solrj/io/eval/Attributes.java | 26 +
.../io/eval/ChiSquareDataSetEvaluator.java | 65 +++
.../solrj/io/eval/CorrelationEvaluator.java | 6 +-
.../eval/CorrelationSignificanceEvaluator.java | 2 +-
.../solrj/io/eval/GTestDataSetEvaluator.java | 64 ++
.../solrj/io/eval/GetAttributeEvaluator.java | 43 ++
.../solrj/io/eval/GetColumnLabelsEvaluator.java | 42 ++
.../solrj/io/eval/GetRowLabelsEvaluator.java | 42 ++
.../client/solrj/io/eval/LerpEvaluator.java | 71 +++
.../solr/client/solrj/io/eval/Matrix.java | 35 +-
.../client/solrj/io/eval/MeanEvaluator.java | 1 -
.../client/solrj/io/eval/SequenceEvaluator.java | 2 +-
.../solrj/io/eval/TermVectorsEvaluator.java | 177 ++++++
.../solrj/io/eval/TransposeEvaluator.java | 6 +-
.../solrj/request/DocumentAnalysisRequest.java | 3 -
.../solrj/request/FieldAnalysisRequest.java | 3 -
.../response/DocumentAnalysisResponse.java | 3 -
.../solrj/response/FieldAnalysisResponse.java | 3 -
.../solrj/response/schema/SchemaResponse.java | 39 --
.../java/org/apache/solr/common/MapWriter.java | 13 +
.../solr/common/cloud/rule/ImplicitSnitch.java | 1 +
.../solr/common/params/AutoScalingParams.java | 2 +
.../solr/common/params/CollectionParams.java | 1 +
.../java/org/apache/solr/common/util/Pair.java | 11 +-
.../client/solrj/impl/CloudSolrClientTest.java | 23 -
.../solrj/io/stream/StreamExpressionTest.java | 362 +++++++++++-
.../solr/client/solrj/request/SchemaTest.java | 75 ++-
.../java/org/apache/solr/cloud/ChaosMonkey.java | 6 +-
278 files changed, 5248 insertions(+), 3282 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ca364c72/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
[33/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-10716: Improve
error handling
Posted by da...@apache.org.
SOLR-10716: Improve error handling
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d189b587
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d189b587
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d189b587
Branch: refs/heads/jira/solr-11702
Commit: d189b587084dfc8ffb359061e43ec76761b5056d
Parents: 459ed85
Author: Joel Bernstein <jb...@apache.org>
Authored: Mon Jan 8 20:12:43 2018 -0500
Committer: Joel Bernstein <jb...@apache.org>
Committed: Mon Jan 8 20:12:43 2018 -0500
----------------------------------------------------------------------
.../client/solrj/io/eval/TermVectorsEvaluator.java | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d189b587/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TermVectorsEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TermVectorsEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TermVectorsEvaluator.java
index 343e65c..8bf050d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TermVectorsEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TermVectorsEvaluator.java
@@ -68,6 +68,20 @@ public class TermVectorsEvaluator extends RecursiveObjectEvaluator implements Ma
if (objects.length == 1) {
//Just docs
+ if(!(objects[0] instanceof List)) {
+ throw new IOException("The termVectors function expects a list of Tuples as a parameter.");
+ } else {
+ List list = (List)objects[0];
+ if(list.size() > 0) {
+ Object o = list.get(0);
+ if(!(o instanceof Tuple)) {
+ throw new IOException("The termVectors function expects a list of Tuples as a parameter.");
+ }
+ } else {
+ throw new IOException("Empty list was passed as a parameter to termVectors function.");
+ }
+ }
+
List<Tuple> tuples = (List<Tuple>) objects[0];
TreeMap<String, Integer> docFreqs = new TreeMap();
List<String> features = new ArrayList();
@@ -157,7 +171,7 @@ public class TermVectorsEvaluator extends RecursiveObjectEvaluator implements Ma
matrix.setAttribute("docFreqs", docFreqs);
return matrix;
} else {
- throw new IOException("The termVectors function a single positional parameter.");
+ throw new IOException("The termVectors function takes a single positional parameter.");
}
}
}
[02/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11813: Reuse a
NodeStateProvider in a session
Posted by da...@apache.org.
SOLR-11813: Reuse a NodeStateProvider in a session
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8836fda9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8836fda9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8836fda9
Branch: refs/heads/jira/solr-11702
Commit: 8836fda95f005294960ee4df807d55eafb41f68e
Parents: 94a680c
Author: Noble Paul <no...@apache.org>
Authored: Thu Jan 4 19:45:59 2018 +1100
Committer: Noble Paul <no...@apache.org>
Committed: Thu Jan 4 19:45:59 2018 +1100
----------------------------------------------------------------------
.../solr/client/solrj/cloud/autoscaling/Policy.java | 16 ++++++++++++----
.../solr/client/solrj/cloud/autoscaling/Row.java | 9 ++++-----
.../client/solrj/cloud/autoscaling/Suggester.java | 12 ++++++++++--
.../src/java/org/apache/solr/common/MapWriter.java | 13 +++++++++++++
.../src/java/org/apache/solr/common/util/Pair.java | 11 ++++++++++-
5 files changed, 49 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8836fda9/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index f11121d..2c4bf43 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -217,19 +217,23 @@ public class Policy implements MapWriter {
Set<String> collections = new HashSet<>();
List<Clause> expandedClauses;
List<Violation> violations = new ArrayList<>();
+ final NodeStateProvider nodeStateProvider;
final int znodeVersion;
private Session(List<String> nodes, SolrCloudManager cloudManager,
- List<Row> matrix, List<Clause> expandedClauses, int znodeVersion) {
+ List<Row> matrix, List<Clause> expandedClauses, int znodeVersion, NodeStateProvider nodeStateProvider) {
this.nodes = nodes;
this.cloudManager = cloudManager;
this.matrix = matrix;
this.expandedClauses = expandedClauses;
this.znodeVersion = znodeVersion;
+ this.nodeStateProvider = nodeStateProvider;
}
+
Session(SolrCloudManager cloudManager) {
ClusterState state = null;
+ this.nodeStateProvider = cloudManager.getNodeStateProvider();
try {
state = cloudManager.getClusterStateProvider().getClusterState();
LOG.trace("-- session created with cluster state: {}", state);
@@ -240,7 +244,7 @@ public class Policy implements MapWriter {
this.nodes = new ArrayList<>(cloudManager.getClusterStateProvider().getLiveNodes());
this.cloudManager = cloudManager;
for (String node : nodes) {
- collections.addAll(cloudManager.getNodeStateProvider().getReplicaInfo(node, Collections.emptyList()).keySet());
+ collections.addAll(nodeStateProvider.getReplicaInfo(node, Collections.emptyList()).keySet());
}
expandedClauses = clusterPolicy.stream()
@@ -255,7 +259,7 @@ public class Policy implements MapWriter {
Collections.sort(expandedClauses);
matrix = new ArrayList<>(nodes.size());
- for (String node : nodes) matrix.add(new Row(node, params, perReplicaAttributes,cloudManager));
+ for (String node : nodes) matrix.add(new Row(node, params, perReplicaAttributes,this));
applyRules();
}
@@ -272,7 +276,7 @@ public class Policy implements MapWriter {
}
Session copy() {
- return new Session(nodes, cloudManager, getMatrixCopy(), expandedClauses, znodeVersion);
+ return new Session(nodes, cloudManager, getMatrixCopy(), expandedClauses, znodeVersion, nodeStateProvider);
}
List<Row> getMatrixCopy() {
@@ -328,6 +332,10 @@ public class Policy implements MapWriter {
public List<Row> getSorted() {
return Collections.unmodifiableList(matrix);
}
+
+ public NodeStateProvider getNodeStateProvider() {
+ return nodeStateProvider;
+ }
}
static void setApproxValuesAndSortNodes(List<Preference> clusterPreferences, List<Row> matrix) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8836fda9/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
index e842fb9..69f81e6 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
@@ -43,14 +43,13 @@ public class Row implements MapWriter {
boolean anyValueMissing = false;
boolean isLive = true;
- public Row(String node, List<String> params, List<String> perReplicaAttributes, SolrCloudManager cloudManager) {
- NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider();
- collectionVsShardVsReplicas = nodeStateProvider.getReplicaInfo(node, perReplicaAttributes);
+ public Row(String node, List<String> params, List<String> perReplicaAttributes, Policy.Session session) {
+ collectionVsShardVsReplicas = session.nodeStateProvider.getReplicaInfo(node, perReplicaAttributes);
if (collectionVsShardVsReplicas == null) collectionVsShardVsReplicas = new HashMap<>();
this.node = node;
cells = new Cell[params.size()];
- isLive = cloudManager.getClusterStateProvider().getLiveNodes().contains(node);
- Map<String, Object> vals = isLive ? nodeStateProvider.getNodeValues(node, params) : Collections.emptyMap();
+ isLive = session.cloudManager.getClusterStateProvider().getLiveNodes().contains(node);
+ Map<String, Object> vals = isLive ? session.nodeStateProvider.getNodeValues(node, params) : Collections.emptyMap();
for (int i = 0; i < params.size(); i++) {
String s = params.get(i);
cells[i] = new Cell(i, s, Clause.validate(s,vals.get(s), false));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8836fda9/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index c501916..15d124d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -44,7 +44,7 @@ import org.apache.solr.common.util.Utils;
* b) it causes no new violations
*
*/
-public abstract class Suggester {
+public abstract class Suggester implements MapWriter {
protected final EnumMap<Hint, Object> hints = new EnumMap<>(Hint.class);
Policy.Session session;
SolrRequest operation;
@@ -105,7 +105,7 @@ public abstract class Suggester {
// the source node is dead so live nodes may not have it
for (String srcNode : srcNodes) {
if(session.matrix.stream().noneMatch(row -> row.node.equals(srcNode)))
- session.matrix.add(new Row(srcNode, session.getPolicy().params, session.getPolicy().perReplicaAttributes, session.cloudManager));
+ session.matrix.add(new Row(srcNode, session.getPolicy().params, session.getPolicy().perReplicaAttributes, session));
}
}
session.applyRules();
@@ -279,5 +279,13 @@ public abstract class Suggester {
}
+ @Override
+ public String toString() {
+ return jsonStr();
+ }
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put("hints", (MapWriter) ew1 -> hints.forEach((hint, o) -> ew1.putNoEx(hint.toString(), o)));
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8836fda9/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
index c8ef3fb..71e084c 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
@@ -24,6 +24,8 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import org.apache.solr.common.util.Utils;
+
/**
* Use this class to push all entries of a Map into an output.
* This avoids creating map instances and is supposed to be memory efficient.
@@ -31,6 +33,9 @@ import java.util.Map;
*/
public interface MapWriter extends MapSerializable {
+ default String jsonStr(){
+ return Utils.toJSONString(this);
+ }
@Override
default Map toMap(Map<String, Object> map) {
try {
@@ -83,6 +88,14 @@ public interface MapWriter extends MapSerializable {
* @param v The value can be any supported object
*/
EntryWriter put(String k, Object v) throws IOException;
+ default EntryWriter putNoEx(String k, Object v) {
+ try {
+ put(k,v);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return this;
+ }
default EntryWriter putIfNotNull(String k, Object v) throws IOException {
if(v != null) put(k,v);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8836fda9/solr/solrj/src/java/org/apache/solr/common/util/Pair.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Pair.java b/solr/solrj/src/java/org/apache/solr/common/util/Pair.java
index b6b16f4..b51edd0 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Pair.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Pair.java
@@ -16,13 +16,16 @@
*/
package org.apache.solr.common.util;
+import java.io.IOException;
import java.io.Serializable;
import java.util.Objects;
+import org.apache.solr.common.MapWriter;
+
import static org.apache.solr.common.util.Utils.makeMap;
import static org.apache.solr.common.util.Utils.toJSONString;
-public class Pair<T1, T2> implements Serializable {
+public class Pair<T1, T2> implements Serializable, MapWriter {
private final T1 first;
private final T2 second;
@@ -56,4 +59,10 @@ public class Pair<T1, T2> implements Serializable {
return Objects.hash(first, second);
}
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put("first", first);
+ ew.put("second", second);
+ }
+
}
\ No newline at end of file
[36/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11692: Constrain
cases where SolrDispatchFilter uses closeShield
Posted by da...@apache.org.
SOLR-11692: Constrain cases where SolrDispatchFilter uses closeShield
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7a375fda
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7a375fda
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7a375fda
Branch: refs/heads/jira/solr-11702
Commit: 7a375fda828015ab62702e2e0f07a1038aef40c6
Parents: f7e166e
Author: David Smiley <ds...@apache.org>
Authored: Mon Jan 8 22:39:17 2018 -0500
Committer: David Smiley <ds...@apache.org>
Committed: Mon Jan 8 22:39:17 2018 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../solr/security/AuthenticationPlugin.java | 1 +
.../apache/solr/servlet/SolrDispatchFilter.java | 48 ++++++++++----------
.../apache/solr/cloud/TestConfigSetsAPI.java | 3 +-
4 files changed, 29 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a375fda/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 533748f..d7eb033 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -129,6 +129,9 @@ Other Changes
* SOLR-11801: Support customisation of the "highlighting" query response element.
(Ramsey Haddad, Pranav Murugappan, David Smiley, Christine Poerschke)
+* SOLR-11692: SolrDispatchFilter's use of a "close shield" in tests should not be applied to
+ further servlet chain processing. (Jeff Miller, David Smiley)
+
================== 7.2.1 ==================
Bug Fixes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a375fda/solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java b/solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java
index d8f2ef2..a9d112a 100644
--- a/solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java
+++ b/solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java
@@ -48,6 +48,7 @@ public abstract class AuthenticationPlugin implements Closeable {
* the response and status code have already been sent.
* @throws Exception any exception thrown during the authentication, e.g. PrivilegedActionException
*/
+ //TODO redeclare params as HttpServletRequest & HttpServletResponse
public abstract boolean doAuthenticate(ServletRequest request, ServletResponse response,
FilterChain filterChain) throws Exception;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a375fda/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
index 8f8bda8..714d127 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
@@ -326,8 +326,10 @@ public class SolrDispatchFilter extends BaseSolrFilter {
doFilter(request, response, chain, false);
}
- public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain, boolean retry) throws IOException, ServletException {
- if (!(request instanceof HttpServletRequest)) return;
+ public void doFilter(ServletRequest _request, ServletResponse _response, FilterChain chain, boolean retry) throws IOException, ServletException {
+ if (!(_request instanceof HttpServletRequest)) return;
+ HttpServletRequest request = (HttpServletRequest)_request;
+ HttpServletResponse response = (HttpServletResponse)_response;
try {
@@ -343,28 +345,24 @@ public class SolrDispatchFilter extends BaseSolrFilter {
}
}
- AtomicReference<ServletRequest> wrappedRequest = new AtomicReference<>();
- if (!authenticateRequest(request, response, wrappedRequest)) { // the response and status code have already been
- // sent
+ AtomicReference<HttpServletRequest> wrappedRequest = new AtomicReference<>();
+ if (!authenticateRequest(request, response, wrappedRequest)) { // the response and status code have already been sent
return;
}
if (wrappedRequest.get() != null) {
request = wrappedRequest.get();
}
- request = closeShield(request, retry);
- response = closeShield(response, retry);
-
if (cores.getAuthenticationPlugin() != null) {
- log.debug("User principal: {}", ((HttpServletRequest) request).getUserPrincipal());
+ log.debug("User principal: {}", request.getUserPrincipal());
}
// No need to even create the HttpSolrCall object if this path is excluded.
if (excludePatterns != null) {
- String requestPath = ((HttpServletRequest) request).getServletPath();
- String extraPath = ((HttpServletRequest) request).getPathInfo();
- if (extraPath != null) { // In embedded mode, servlet path is empty - include all post-context path here for
- // testing
+ String requestPath = request.getServletPath();
+ String extraPath = request.getPathInfo();
+ if (extraPath != null) {
+ // In embedded mode, servlet path is empty - include all post-context path here for testing
requestPath += extraPath;
}
for (Pattern p : excludePatterns) {
@@ -376,7 +374,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
}
}
- HttpSolrCall call = getHttpSolrCall((HttpServletRequest) request, (HttpServletResponse) response, retry);
+ HttpSolrCall call = getHttpSolrCall(closeShield(request, retry), closeShield(response, retry), retry);
ExecutorUtil.setServerThreadFlag(Boolean.TRUE);
try {
Action result = call.call();
@@ -385,7 +383,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
chain.doFilter(request, response);
break;
case RETRY:
- doFilter(request, response, chain, true);
+ doFilter(request, response, chain, true); // RECURSION
break;
case FORWARD:
request.getRequestDispatcher(call.getPath()).forward(request, response);
@@ -396,7 +394,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
ExecutorUtil.setServerThreadFlag(null);
}
} finally {
- consumeInputFully((HttpServletRequest) request);
+ consumeInputFully(request);
}
}
@@ -430,7 +428,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
}
}
- private boolean authenticateRequest(ServletRequest request, ServletResponse response, final AtomicReference<ServletRequest> wrappedRequest) throws IOException {
+ private boolean authenticateRequest(HttpServletRequest request, HttpServletResponse response, final AtomicReference<HttpServletRequest> wrappedRequest) throws IOException {
boolean requestContinues = false;
final AtomicBoolean isAuthenticated = new AtomicBoolean(false);
AuthenticationPlugin authenticationPlugin = cores.getAuthenticationPlugin();
@@ -440,9 +438,9 @@ public class SolrDispatchFilter extends BaseSolrFilter {
// /admin/info/key must be always open. see SOLR-9188
// tests work only w/ getPathInfo
//otherwise it's just enough to have getServletPath()
- if (PKIAuthenticationPlugin.PATH.equals(((HttpServletRequest) request).getServletPath()) ||
- PKIAuthenticationPlugin.PATH.equals(((HttpServletRequest) request).getPathInfo())) return true;
- String header = ((HttpServletRequest) request).getHeader(PKIAuthenticationPlugin.HEADER);
+ if (PKIAuthenticationPlugin.PATH.equals(request.getServletPath()) ||
+ PKIAuthenticationPlugin.PATH.equals(request.getPathInfo())) return true;
+ String header = request.getHeader(PKIAuthenticationPlugin.HEADER);
if (header != null && cores.getPkiAuthenticationPlugin() != null)
authenticationPlugin = cores.getPkiAuthenticationPlugin();
try {
@@ -450,7 +448,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
// upon successful authentication, this should call the chain's next filter.
requestContinues = authenticationPlugin.doAuthenticate(request, response, (req, rsp) -> {
isAuthenticated.set(true);
- wrappedRequest.set(req);
+ wrappedRequest.set((HttpServletRequest) req);
});
} catch (Exception e) {
log.info("Error authenticating", e);
@@ -478,9 +476,9 @@ public class SolrDispatchFilter extends BaseSolrFilter {
* @param retry If this is an original request or a retry.
* @return A request object with an {@link InputStream} that will ignore calls to close.
*/
- private ServletRequest closeShield(ServletRequest request, boolean retry) {
+ private HttpServletRequest closeShield(HttpServletRequest request, boolean retry) {
if (testMode && !retry) {
- return new HttpServletRequestWrapper((HttpServletRequest) request) {
+ return new HttpServletRequestWrapper(request) {
ServletInputStream stream;
@Override
@@ -510,9 +508,9 @@ public class SolrDispatchFilter extends BaseSolrFilter {
* @param retry If this response corresponds to an original request or a retry.
* @return A response object with an {@link OutputStream} that will ignore calls to close.
*/
- private ServletResponse closeShield(ServletResponse response, boolean retry) {
+ private HttpServletResponse closeShield(HttpServletResponse response, boolean retry) {
if (testMode && !retry) {
- return new HttpServletResponseWrapper((HttpServletResponse) response) {
+ return new HttpServletResponseWrapper(response) {
ServletOutputStream stream;
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a375fda/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
index 374e27a..ddd8fd9 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
@@ -540,7 +540,8 @@ public class TestConfigSetsAPI extends SolrTestCaseJ4 {
m = (Map) ObjectBuilder.getVal(new JSONParser(
new StringReader(response)));
} catch (JSONParser.ParseException e) {
- fail(e.getMessage());
+ System.err.println("err response: " + response);
+ throw new AssertionError(e);
}
} finally {
httpPost.releaseConnection();
[16/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11771: Move CHANGES
entry to the right version
Posted by da...@apache.org.
SOLR-11771: Move CHANGES entry to the right version
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0d558117
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0d558117
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0d558117
Branch: refs/heads/jira/solr-11702
Commit: 0d5581177423aef95489351e07b2169150a9c102
Parents: b8b00786
Author: Cao Manh Dat <da...@apache.org>
Authored: Sat Jan 6 09:08:50 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Sat Jan 6 09:08:50 2018 +0700
----------------------------------------------------------------------
solr/CHANGES.txt | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d558117/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3da194b..cb830ce 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -86,8 +86,6 @@ Bug Fixes
* SOLR-11555: If the query terms reduce to nothing, filter(clause) produces an NPE whereas
fq=clause does not (Erick Erickson)
-* SOLR-11771: Overseer can never process some last messages (Cao Manh Dat)
-
Optimizations
----------------------
@@ -122,6 +120,13 @@ Other Changes
* SOLR-11801: Support customisation of the "highlighting" query response element.
(Ramsey Haddad, Pranav Murugappan, David Smiley, Christine Poerschke)
+================== 7.2.1 ==================
+
+Bug Fixes
+----------------------
+
+* SOLR-11771: Overseer can never process some last messages (Cao Manh Dat)
+
================== 7.2.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
[32/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-10716: Add
termVectors Stream Evaluator
Posted by da...@apache.org.
SOLR-10716: Add termVectors Stream Evaluator
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/459ed850
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/459ed850
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/459ed850
Branch: refs/heads/jira/solr-11702
Commit: 459ed85052a72219631f0dcdeb1b6650b632a8fa
Parents: 07407a5
Author: Joel Bernstein <jb...@apache.org>
Authored: Mon Jan 8 19:38:37 2018 -0500
Committer: Joel Bernstein <jb...@apache.org>
Committed: Mon Jan 8 19:39:43 2018 -0500
----------------------------------------------------------------------
.../org/apache/solr/handler/StreamHandler.java | 4 +
.../solr/client/solrj/io/eval/Attributes.java | 26 +++
.../solrj/io/eval/CorrelationEvaluator.java | 6 +-
.../eval/CorrelationSignificanceEvaluator.java | 2 +-
.../solrj/io/eval/GetAttributeEvaluator.java | 43 ++++
.../solrj/io/eval/GetColumnLabelsEvaluator.java | 42 ++++
.../solrj/io/eval/GetRowLabelsEvaluator.java | 42 ++++
.../solr/client/solrj/io/eval/Matrix.java | 35 ++-
.../solrj/io/eval/TermVectorsEvaluator.java | 163 +++++++++++++
.../solrj/io/eval/TransposeEvaluator.java | 6 +-
.../solrj/io/stream/StreamExpressionTest.java | 229 +++++++++++++++++++
11 files changed, 585 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/459ed850/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index 8f3c4d6..ee3a17b 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -292,6 +292,10 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("lerp", LerpEvaluator.class)
.withFunctionName("chiSquareDataSet", ChiSquareDataSetEvaluator.class)
.withFunctionName("gtestDataSet", GTestDataSetEvaluator.class)
+ .withFunctionName("termVectors", TermVectorsEvaluator.class)
+ .withFunctionName("getColumnLabels", GetColumnLabelsEvaluator.class)
+ .withFunctionName("getRowLabels", GetRowLabelsEvaluator.class)
+ .withFunctionName("getAttribute", GetAttributeEvaluator.class)
// Boolean Stream Evaluators
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/459ed850/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/Attributes.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/Attributes.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/Attributes.java
new file mode 100644
index 0000000..10f3a33
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/Attributes.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.util.Map;
+
+
+public interface Attributes {
+ Object getAttribute(String key);
+ void setAttribute(String key, Object value);
+ Map getAttributes();
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/459ed850/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationEvaluator.java
index a5065d4..866c5d0 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationEvaluator.java
@@ -108,21 +108,21 @@ public class CorrelationEvaluator extends RecursiveObjectEvaluator implements Ma
RealMatrix corrMatrix = pearsonsCorrelation.getCorrelationMatrix();
double[][] corrMatrixData = corrMatrix.getData();
Matrix realMatrix = new Matrix(corrMatrixData);
- realMatrix.addToContext("corr", pearsonsCorrelation);
+ realMatrix.setAttribute("corr", pearsonsCorrelation);
return realMatrix;
} else if (type.equals(CorrelationType.kendalls)) {
KendallsCorrelation kendallsCorrelation = new KendallsCorrelation(data);
RealMatrix corrMatrix = kendallsCorrelation.getCorrelationMatrix();
double[][] corrMatrixData = corrMatrix.getData();
Matrix realMatrix = new Matrix(corrMatrixData);
- realMatrix.addToContext("corr", kendallsCorrelation);
+ realMatrix.setAttribute("corr", kendallsCorrelation);
return realMatrix;
} else if (type.equals(CorrelationType.spearmans)) {
SpearmansCorrelation spearmansCorrelation = new SpearmansCorrelation(new Array2DRowRealMatrix(data));
RealMatrix corrMatrix = spearmansCorrelation.getCorrelationMatrix();
double[][] corrMatrixData = corrMatrix.getData();
Matrix realMatrix = new Matrix(corrMatrixData);
- realMatrix.addToContext("corr", spearmansCorrelation.getRankCorrelation());
+ realMatrix.setAttribute("corr", spearmansCorrelation.getRankCorrelation());
return realMatrix;
} else {
return null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/459ed850/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationSignificanceEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationSignificanceEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationSignificanceEvaluator.java
index 1726c9c..f534a8d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationSignificanceEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationSignificanceEvaluator.java
@@ -42,7 +42,7 @@ public class CorrelationSignificanceEvaluator extends RecursiveObjectEvaluator i
return null;
} else if(value instanceof Matrix) {
Matrix matrix = (Matrix) value;
- Object corr = matrix.getContextValue("corr");
+ Object corr = matrix.getAttribute("corr");
if(corr instanceof PearsonsCorrelation) {
PearsonsCorrelation pcorr = (PearsonsCorrelation)corr;
RealMatrix realMatrix = pcorr.getCorrelationPValues();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/459ed850/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetAttributeEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetAttributeEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetAttributeEvaluator.java
new file mode 100644
index 0000000..81eea23
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetAttributeEvaluator.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class GetAttributeEvaluator extends RecursiveObjectEvaluator implements TwoValueWorker {
+ private static final long serialVersionUID = 1;
+
+ public GetAttributeEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object value1, Object value2) throws IOException {
+ if(!(value1 instanceof Attributes)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for value, expecting an Attributes",toExpression(constructingFactory), value1.getClass().getSimpleName()));
+ } else {
+ Attributes attributes = (Attributes)value1;
+ String key = (String)value2;
+ return attributes.getAttribute(key.replace("\"", ""));
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/459ed850/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetColumnLabelsEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetColumnLabelsEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetColumnLabelsEvaluator.java
new file mode 100644
index 0000000..f2710d3
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetColumnLabelsEvaluator.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class GetColumnLabelsEvaluator extends RecursiveObjectEvaluator implements OneValueWorker {
+ private static final long serialVersionUID = 1;
+
+ public GetColumnLabelsEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object value) throws IOException {
+ if(!(value instanceof Matrix)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for value, expecting a Matrix",toExpression(constructingFactory), value.getClass().getSimpleName()));
+ } else {
+ Matrix matrix = (Matrix)value;
+ return matrix.getColumnLabels();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/459ed850/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetRowLabelsEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetRowLabelsEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetRowLabelsEvaluator.java
new file mode 100644
index 0000000..9af25b4
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetRowLabelsEvaluator.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class GetRowLabelsEvaluator extends RecursiveObjectEvaluator implements OneValueWorker {
+ private static final long serialVersionUID = 1;
+
+ public GetRowLabelsEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object value) throws IOException {
+ if(!(value instanceof Matrix)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for value, expecting a Matrix",toExpression(constructingFactory), value.getClass().getSimpleName()));
+ } else {
+ Matrix matrix = (Matrix)value;
+ return matrix.getRowLabels();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/459ed850/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/Matrix.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/Matrix.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/Matrix.java
index 365a018..7fcfca2 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/Matrix.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/Matrix.java
@@ -23,25 +23,44 @@ import java.util.ArrayList;
import java.util.Iterator;
-public class Matrix implements Iterable {
+public class Matrix implements Iterable, Attributes {
private double[][] data;
- private Map context = new HashMap();
+ private List<String> columnLabels;
+ private List<String> rowLabels;
+
+ private Map<String, Object> attributes = new HashMap();
public Matrix(double[][] data) {
this.data = data;
}
- public Map getContext() {
- return this.context;
+ public Map getAttributes() {
+ return this.attributes;
+ }
+
+ public void setAttribute(String key, Object value) {
+ this.attributes.put(key, value);
+ }
+
+ public Object getAttribute(String key) {
+ return this.attributes.get(key);
+ }
+
+ public List<String> getColumnLabels() {
+ return this.columnLabels;
+ }
+
+ public void setColumnLabels(List<String> columnLabels) {
+ this.columnLabels = columnLabels;
}
- public void addToContext(Object key, Object value) {
- this.context.put(key, value);
+ public List<String> getRowLabels() {
+ return rowLabels;
}
- public Object getContextValue(Object key) {
- return this.context.get(key);
+ public void setRowLabels(List<String> rowLables) {
+ this.rowLabels = rowLables;
}
public double[][] getData() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/459ed850/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TermVectorsEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TermVectorsEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TermVectorsEvaluator.java
new file mode 100644
index 0000000..343e65c
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TermVectorsEvaluator.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class TermVectorsEvaluator extends RecursiveObjectEvaluator implements ManyValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ private int minTermLength = 3;
+ private double minDocFreq = .05; // 5% of the docs min
+ private double maxDocFreq = .5; // 50% of the docs max
+
+ public TermVectorsEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+
+ List<StreamExpressionNamedParameter> namedParams = factory.getNamedOperands(expression);
+
+ for (StreamExpressionNamedParameter namedParam : namedParams) {
+ if (namedParam.getName().equals("minTermLength")) {
+ this.minTermLength = Integer.parseInt(namedParam.getParameter().toString().trim());
+ } else if (namedParam.getName().equals("minDocFreq")) {
+ this.minDocFreq = Double.parseDouble(namedParam.getParameter().toString().trim());
+ if (minDocFreq < 0 || minDocFreq > 1) {
+ throw new IOException("Doc frequency percentage must be between 0 and 1");
+ }
+ } else if (namedParam.getName().equals("maxDocFreq")) {
+ this.maxDocFreq = Double.parseDouble(namedParam.getParameter().toString().trim());
+ if (maxDocFreq < 0 || maxDocFreq > 1) {
+ throw new IOException("Doc frequency percentage must be between 0 and 1");
+ }
+ } else {
+ throw new IOException("Unexpected named parameter:" + namedParam.getName());
+ }
+ }
+ }
+
+ @Override
+ public Object doWork(Object... objects) throws IOException {
+
+ if (objects.length == 1) {
+ //Just docs
+ List<Tuple> tuples = (List<Tuple>) objects[0];
+ TreeMap<String, Integer> docFreqs = new TreeMap();
+ List<String> features = new ArrayList();
+ List<String> rowLabels = new ArrayList();
+
+ for (Tuple tuple : tuples) {
+
+ Set<String> docTerms = new HashSet();
+
+ if (tuple.get("terms") == null) {
+ throw new IOException("The document tuples must contain a terms field");
+ }
+
+ List<String> terms = (List<String>) tuple.get("terms");
+
+ String id = tuple.getString("id");
+ rowLabels.add(id);
+
+ for (String term : terms) {
+
+ if (term.length() < minTermLength) {
+ //Eliminate terms due to length
+ continue;
+ }
+
+ if (!docTerms.contains(term)) {
+ docTerms.add(term);
+ if (docFreqs.containsKey(term)) {
+ int count = docFreqs.get(term).intValue();
+ docFreqs.put(term, ++count);
+ } else {
+ docFreqs.put(term, 1);
+ }
+ }
+ }
+ }
+
+ //Eliminate terms based on frequency
+
+ int min = (int) (tuples.size() * minDocFreq);
+ int max = (int) (tuples.size() * maxDocFreq);
+
+ Set<Map.Entry<String, Integer>> entries = docFreqs.entrySet();
+ Iterator<Map.Entry<String, Integer>> it = entries.iterator();
+ while (it.hasNext()) {
+ Map.Entry<String, Integer> entry = it.next();
+ int count = entry.getValue().intValue();
+
+ if (count < min || count > max) {
+ it.remove();
+ }
+ }
+
+ int totalTerms = docFreqs.size();
+ Set<String> keys = docFreqs.keySet();
+ features.addAll(keys);
+ double[][] docVec = new double[tuples.size()][];
+ for (int t = 0; t < tuples.size(); t++) {
+ Tuple tuple = tuples.get(t);
+ List<String> terms = (List<String>) tuple.get("terms");
+ Map<String, Integer> termFreq = new HashMap();
+
+ for (String term : terms) {
+ if (docFreqs.containsKey(term)) {
+ if (termFreq.containsKey(term)) {
+ int count = termFreq.get(term).intValue();
+ termFreq.put(term, ++count);
+ } else {
+ termFreq.put(term, 1);
+ }
+ }
+ }
+
+ double[] termVec = new double[totalTerms];
+ for (int i = 0; i < totalTerms; i++) {
+ String feature = features.get(i);
+ int df = docFreqs.get(feature);
+ int tf = termFreq.containsKey(feature) ? termFreq.get(feature) : 0;
+ termVec[i] = Math.sqrt(tf) * (double) (Math.log((tuples.size() + 1) / (double) (df + 1)) + 1.0);
+ }
+ docVec[t] = termVec;
+ }
+
+ Matrix matrix = new Matrix(docVec);
+ matrix.setColumnLabels(features);
+ matrix.setRowLabels(rowLabels);
+ matrix.setAttribute("docFreqs", docFreqs);
+ return matrix;
+ } else {
+ throw new IOException("The termVectors function a single positional parameter.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/459ed850/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TransposeEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TransposeEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TransposeEvaluator.java
index b206cc5..dfc2346 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TransposeEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TransposeEvaluator.java
@@ -44,7 +44,11 @@ public class TransposeEvaluator extends RecursiveObjectEvaluator implements OneV
double[][] data = matrix.getData();
Array2DRowRealMatrix amatrix = new Array2DRowRealMatrix(data);
Array2DRowRealMatrix tmatrix = (Array2DRowRealMatrix)amatrix.transpose();
- return new Matrix(tmatrix.getData());
+ Matrix newMatrix = new Matrix(tmatrix.getData());
+ //Switch the row and column labels
+ newMatrix.setColumnLabels(matrix.getRowLabels());
+ newMatrix.setRowLabels(matrix.getColumnLabels());
+ return newMatrix;
} else {
throw new IOException("matrix parameter expected for transpose function");
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/459ed850/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 9d41c54..2a9df01 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -6632,6 +6632,235 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertEquals(sd, 3.5, 0.0);
}
+ @Test
+ public void testTermVectors() throws Exception {
+ // Test termVectors with only documents and default termVector settings
+ String cexpr = "let(echo=true," +
+ "a=select(list(tuple(id=\"1\", text=\"hello world\"), " +
+ "tuple(id=\"2\", text=\"hello steve\"), " +
+ "tuple(id=\"3\", text=\"hello jim jim\"), " +
+ "tuple(id=\"4\", text=\"hello jack\")), id, analyze(text, test_t) as terms)," +
+ " b=termVectors(a, minDocFreq=0, maxDocFreq=1)," +
+ " c=getRowLabels(b)," +
+ " d=getColumnLabels(b)," +
+ " e=getAttribute(b, docFreqs))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<List<Number>> termVectors = (List<List<Number>>)tuples.get(0).get("b");
+
+ assertEquals(termVectors.size(), 4);
+ List<Number> termVector = termVectors.get(0);
+ assertEquals(termVector.size(), 5);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(4).doubleValue(), 1.916290731874155, 0.0);
+
+ termVector = termVectors.get(1);
+ assertEquals(termVector.size(), 5);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 1.916290731874155, 0.0);
+ assertEquals(termVector.get(4).doubleValue(), 0.0, 0.0);
+
+ termVector = termVectors.get(2);
+ assertEquals(termVector.size(), 5);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 2.7100443424662948, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(4).doubleValue(), 0.0, 0.0);
+
+ termVector = termVectors.get(3);
+ assertEquals(termVector.size(), 5);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 1.916290731874155, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(4).doubleValue(), 0.0, 0.0);
+
+ List<String> rowLabels = (List<String>)tuples.get(0).get("c");
+ assertEquals(rowLabels.size(), 4);
+ assertEquals(rowLabels.get(0), "1");
+ assertEquals(rowLabels.get(1), "2");
+ assertEquals(rowLabels.get(2), "3");
+ assertEquals(rowLabels.get(3), "4");
+
+ List<String> columnLabels = (List<String>)tuples.get(0).get("d");
+ assertEquals(columnLabels.size(), 5);
+ assertEquals(columnLabels.get(0), "hello");
+ assertEquals(columnLabels.get(1), "jack");
+ assertEquals(columnLabels.get(2), "jim");
+ assertEquals(columnLabels.get(3), "steve");
+ assertEquals(columnLabels.get(4), "world");
+
+ Map<String, Number> docFreqs = (Map<String, Number>)tuples.get(0).get("e");
+
+ assertEquals(docFreqs.size(), 5);
+ assertEquals(docFreqs.get("hello").intValue(), 4);
+ assertEquals(docFreqs.get("jack").intValue(), 1);
+ assertEquals(docFreqs.get("jim").intValue(), 1);
+ assertEquals(docFreqs.get("steve").intValue(), 1);
+ assertEquals(docFreqs.get("world").intValue(), 1);
+
+ //Test minTermLength. This should drop off the term jim
+
+ cexpr = "let(echo=true," +
+ "a=select(list(tuple(id=\"1\", text=\"hello world\"), " +
+ "tuple(id=\"2\", text=\"hello steve\"), " +
+ "tuple(id=\"3\", text=\"hello jim jim\"), " +
+ "tuple(id=\"4\", text=\"hello jack\")), id, analyze(text, test_t) as terms)," +
+ " b=termVectors(a, minTermLength=4, minDocFreq=0, maxDocFreq=1)," +
+ " c=getRowLabels(b)," +
+ " d=getColumnLabels(b)," +
+ " e=getAttribute(b, docFreqs))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ solrStream = new SolrStream(url, paramsLoc);
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ termVectors = (List<List<Number>>)tuples.get(0).get("b");
+ assertEquals(termVectors.size(), 4);
+ termVector = termVectors.get(0);
+ assertEquals(termVector.size(), 4);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 1.916290731874155, 0.0);
+
+ termVector = termVectors.get(1);
+ assertEquals(termVector.size(), 4);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 1.916290731874155, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+
+ termVector = termVectors.get(2);
+ assertEquals(termVector.size(), 4);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+
+ termVector = termVectors.get(3);
+ assertEquals(termVector.size(), 4);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 1.916290731874155, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+
+ rowLabels = (List<String>)tuples.get(0).get("c");
+ assertEquals(rowLabels.size(), 4);
+ assertEquals(rowLabels.get(0), "1");
+ assertEquals(rowLabels.get(1), "2");
+ assertEquals(rowLabels.get(2), "3");
+ assertEquals(rowLabels.get(3), "4");
+
+ columnLabels = (List<String>)tuples.get(0).get("d");
+ assertEquals(columnLabels.size(), 4);
+ assertEquals(columnLabels.get(0), "hello");
+ assertEquals(columnLabels.get(1), "jack");
+ assertEquals(columnLabels.get(2), "steve");
+ assertEquals(columnLabels.get(3), "world");
+
+ docFreqs = (Map<String, Number>)tuples.get(0).get("e");
+
+ assertEquals(docFreqs.size(), 4);
+ assertEquals(docFreqs.get("hello").intValue(), 4);
+ assertEquals(docFreqs.get("jack").intValue(), 1);
+ assertEquals(docFreqs.get("steve").intValue(), 1);
+ assertEquals(docFreqs.get("world").intValue(), 1);
+
+
+ //Test minDocFreq attribute at .5. This should eliminate all but the term hello
+
+ cexpr = "let(echo=true," +
+ "a=select(list(tuple(id=\"1\", text=\"hello world\"), " +
+ "tuple(id=\"2\", text=\"hello steve\"), " +
+ "tuple(id=\"3\", text=\"hello jim jim\"), " +
+ "tuple(id=\"4\", text=\"hello jack\")), id, analyze(text, test_t) as terms)," +
+ " b=termVectors(a, minDocFreq=.5, maxDocFreq=1)," +
+ " c=getRowLabels(b)," +
+ " d=getColumnLabels(b)," +
+ " e=getAttribute(b, docFreqs))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ solrStream = new SolrStream(url, paramsLoc);
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ termVectors = (List<List<Number>>)tuples.get(0).get("b");
+
+ assertEquals(termVectors.size(), 4);
+ termVector = termVectors.get(0);
+ assertEquals(termVector.size(), 1);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+
+ termVector = termVectors.get(1);
+ assertEquals(termVector.size(), 1);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+
+ termVector = termVectors.get(2);
+ assertEquals(termVector.size(), 1);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+
+ termVector = termVectors.get(3);
+ assertEquals(termVector.size(), 1);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+
+ rowLabels = (List<String>)tuples.get(0).get("c");
+ assertEquals(rowLabels.size(), 4);
+ assertEquals(rowLabels.get(0), "1");
+ assertEquals(rowLabels.get(1), "2");
+ assertEquals(rowLabels.get(2), "3");
+ assertEquals(rowLabels.get(3), "4");
+
+ columnLabels = (List<String>)tuples.get(0).get("d");
+ assertEquals(columnLabels.size(), 1);
+ assertEquals(columnLabels.get(0), "hello");
+
+ docFreqs = (Map<String, Number>)tuples.get(0).get("e");
+
+ assertEquals(docFreqs.size(), 1);
+ assertEquals(docFreqs.get("hello").intValue(), 4);
+
+ //Test maxDocFreq attribute at 0. This should eliminate all terms
+
+ cexpr = "let(echo=true," +
+ "a=select(list(tuple(id=\"1\", text=\"hello world\"), " +
+ "tuple(id=\"2\", text=\"hello steve\"), " +
+ "tuple(id=\"3\", text=\"hello jim jim\"), " +
+ "tuple(id=\"4\", text=\"hello jack\")), id, analyze(text, test_t) as terms)," +
+ " b=termVectors(a, maxDocFreq=0)," +
+ " c=getRowLabels(b)," +
+ " d=getColumnLabels(b)," +
+ " e=getAttribute(b, docFreqs))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ solrStream = new SolrStream(url, paramsLoc);
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ termVectors = (List<List<Number>>)tuples.get(0).get("b");
+ assertEquals(termVectors.size(), 4);
+ assertEquals(termVectors.get(0).size(), 0);
+ }
@Test
public void testEBESubtract() throws Exception {
[42/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11631: fix another
test
Posted by da...@apache.org.
SOLR-11631: fix another test
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f6eae40a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f6eae40a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f6eae40a
Branch: refs/heads/jira/solr-11702
Commit: f6eae40a1b2304dd61e348b2d4af110f6e2a1a66
Parents: e3f3cdd
Author: Steve Rowe <sa...@apache.org>
Authored: Tue Jan 9 17:51:42 2018 -0500
Committer: Steve Rowe <sa...@apache.org>
Committed: Tue Jan 9 17:51:42 2018 -0500
----------------------------------------------------------------------
.../TestEmbeddedSolrServerSchemaAPI.java | 20 ++++++++++++++------
1 file changed, 14 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6eae40a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java
index f253831..0539c70 100644
--- a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java
+++ b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java
@@ -20,14 +20,19 @@ import java.io.IOException;
import java.nio.file.Path;
import java.util.Collections;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.api.ApiBag;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.schema.SchemaRequest;
import org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.apache.solr.client.solrj.response.schema.SchemaResponse.FieldResponse;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -98,14 +103,17 @@ public class TestEmbeddedSolrServerSchemaAPI extends SolrTestCaseJ4 {
}
@Test
- public void testSchemaAddFieldAndFailOnImmutable() throws Exception {
+ public void testSchemaAddFieldAndFailOnImmutable() {
assumeFalse("it needs a readonly schema", Boolean.getBoolean("managed.schema.mutable"));
- SchemaRequest.AddField addFieldUpdateSchemaRequest = new SchemaRequest.AddField(fieldAttributes);
- SchemaResponse.UpdateResponse addFieldResponse = addFieldUpdateSchemaRequest.process(server);
- // wt hell???? assertFalse(addFieldResponse.toString(), addFieldResponse.getStatus()==0);
- assertTrue((""+addFieldResponse).contains("schema is not editable"));
-
+ SchemaRequest.AddField addFieldUpdateSchemaRequest = new SchemaRequest.AddField(fieldAttributes);
+ assertFailedSchemaResponse(() -> addFieldUpdateSchemaRequest.process(server),
+ "schema is not editable");
}
+ private static void assertFailedSchemaResponse(ThrowingRunnable runnable, String expectedErrorMessage) {
+ ApiBag.ExceptionWithErrObject e = expectThrows(ApiBag.ExceptionWithErrObject.class, runnable);
+ String msg = e.getErrs().get(0).get("errorMessages").toString();
+ assertTrue(msg.contains(expectedErrorMessage));
+ }
}
[39/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8075: Removed
unnecessary null check in
core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java.
Posted by da...@apache.org.
LUCENE-8075: Removed unnecessary null check in core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java.
Closes #286
Signed-off-by: Adrien Grand <jp...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2432bdeb
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2432bdeb
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2432bdeb
Branch: refs/heads/jira/solr-11702
Commit: 2432bdebe36674010ea4c20e9eb8ff7402227395
Parents: 838c604
Author: Pulak Ghosh <gh...@gmail.com>
Authored: Wed Dec 6 12:17:05 2017 +0530
Committer: Adrien Grand <jp...@gmail.com>
Committed: Tue Jan 9 14:42:16 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../apache/lucene/codecs/blocktree/IntersectTermsEnum.java | 7 ++-----
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2432bdeb/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f3335ac..c309a92 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -137,6 +137,9 @@ Other
* LUCENE-8106: Add script (reproduceJenkinsFailures.py) to attempt to reproduce
failing tests from a Jenkins log. (Steve Rowe)
+* LUCENE-8075: Removed unnecessary null check in IntersectTermsEnum.
+ (Pulak Ghosh via Adrien Grand)
+
======================= Lucene 7.2.0 =======================
API Changes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2432bdeb/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java
index 19e56a4..7521763 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java
@@ -103,11 +103,8 @@ final class IntersectTermsEnum extends TermsEnum {
arcs[arcIdx] = new FST.Arc<>();
}
- if (fr.index == null) {
- fstReader = null;
- } else {
- fstReader = fr.index.getBytesReader();
- }
+
+ fstReader = fr.index.getBytesReader();
// TODO: if the automaton is "smallish" we really
// should use the terms index to seek at least to
[13/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8099: Add
equality tests for FunctionScoreQuery
Posted by da...@apache.org.
LUCENE-8099: Add equality tests for FunctionScoreQuery
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3980aea1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3980aea1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3980aea1
Branch: refs/heads/jira/solr-11702
Commit: 3980aea18ded0a526900dd649fd5926ae65c3897
Parents: d250a14
Author: Alan Woodward <ro...@apache.org>
Authored: Fri Jan 5 16:18:37 2018 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Fri Jan 5 16:18:37 2018 +0000
----------------------------------------------------------------------
.../function/TestFunctionScoreQuery.java | 39 ++++++++++++++++++++
1 file changed, 39 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3980aea1/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
index 4e82eb2..b865cb7 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
@@ -60,6 +60,45 @@ public class TestFunctionScoreQuery extends FunctionTestSetup {
reader.close();
}
+ public void testEqualities() {
+
+ Query q1 = new FunctionScoreQuery(new TermQuery(new Term(TEXT_FIELD, "a")), DoubleValuesSource.constant(1));
+ Query q2 = new FunctionScoreQuery(new TermQuery(new Term(TEXT_FIELD, "b")), DoubleValuesSource.constant(1));
+ Query q3 = new FunctionScoreQuery(new TermQuery(new Term(TEXT_FIELD, "b")), DoubleValuesSource.constant(2));
+ Query q4 = new FunctionScoreQuery(new TermQuery(new Term(TEXT_FIELD, "b")), DoubleValuesSource.constant(2));
+
+ QueryUtils.check(q1);
+ QueryUtils.checkUnequal(q1, q3);
+ QueryUtils.checkUnequal(q1, q2);
+ QueryUtils.checkUnequal(q2, q3);
+ QueryUtils.checkEqual(q3, q4);
+
+ Query bq1 = FunctionScoreQuery.boostByValue(new TermQuery(new Term(TEXT_FIELD, "a")), DoubleValuesSource.constant(2));
+ QueryUtils.check(bq1);
+ Query bq2 = FunctionScoreQuery.boostByValue(new TermQuery(new Term(TEXT_FIELD, "a")), DoubleValuesSource.constant(4));
+ QueryUtils.checkUnequal(bq1, bq2);
+ Query bq3 = FunctionScoreQuery.boostByValue(new TermQuery(new Term(TEXT_FIELD, "b")), DoubleValuesSource.constant(4));
+ QueryUtils.checkUnequal(bq1, bq3);
+ QueryUtils.checkUnequal(bq2, bq3);
+ Query bq4 = FunctionScoreQuery.boostByValue(new TermQuery(new Term(TEXT_FIELD, "b")), DoubleValuesSource.constant(4));
+ QueryUtils.checkEqual(bq3, bq4);
+
+ Query qq1 = FunctionScoreQuery.boostByQuery(new TermQuery(new Term(TEXT_FIELD, "a")), new TermQuery(new Term(TEXT_FIELD, "z")), 0.1f);
+ QueryUtils.check(qq1);
+ Query qq2 = FunctionScoreQuery.boostByQuery(new TermQuery(new Term(TEXT_FIELD, "a")), new TermQuery(new Term(TEXT_FIELD, "z")), 0.2f);
+ QueryUtils.checkUnequal(qq1, qq2);
+ Query qq3 = FunctionScoreQuery.boostByQuery(new TermQuery(new Term(TEXT_FIELD, "b")), new TermQuery(new Term(TEXT_FIELD, "z")), 0.1f);
+ QueryUtils.checkUnequal(qq1, qq3);
+ QueryUtils.checkUnequal(qq2, qq3);
+ Query qq4 = FunctionScoreQuery.boostByQuery(new TermQuery(new Term(TEXT_FIELD, "a")), new TermQuery(new Term(TEXT_FIELD, "zz")), 0.1f);
+ QueryUtils.checkUnequal(qq1, qq4);
+ QueryUtils.checkUnequal(qq2, qq4);
+ QueryUtils.checkUnequal(qq3, qq4);
+ Query qq5 = FunctionScoreQuery.boostByQuery(new TermQuery(new Term(TEXT_FIELD, "a")), new TermQuery(new Term(TEXT_FIELD, "z")), 0.1f);
+ QueryUtils.checkEqual(qq1, qq5);
+
+ }
+
// FunctionQuery equivalent
public void testSimpleSourceScore() throws Exception {
[23/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11830: PKI
authentication testcases do not check for null principal
Posted by da...@apache.org.
SOLR-11830: PKI authentication testcases do not check for null principal
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/72e68697
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/72e68697
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/72e68697
Branch: refs/heads/jira/solr-11702
Commit: 72e68697fc304ff0c42f6d422660146e7195f4b9
Parents: 0744fea
Author: Noble Paul <no...@apache.org>
Authored: Mon Jan 8 21:43:30 2018 +1100
Committer: Noble Paul <no...@apache.org>
Committed: Mon Jan 8 21:43:30 2018 +1100
----------------------------------------------------------------------
.../solr/security/MockAuthorizationPlugin.java | 19 ++++++-----
.../PKIAuthenticationIntegrationTest.java | 15 +++------
.../security/TestAuthorizationFramework.java | 35 ++++++++++++--------
3 files changed, 36 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/72e68697/solr/core/src/test/org/apache/solr/security/MockAuthorizationPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/security/MockAuthorizationPlugin.java b/solr/core/src/test/org/apache/solr/security/MockAuthorizationPlugin.java
index 17091ab..8eb93c8 100644
--- a/solr/core/src/test/org/apache/solr/security/MockAuthorizationPlugin.java
+++ b/solr/core/src/test/org/apache/solr/security/MockAuthorizationPlugin.java
@@ -30,6 +30,7 @@ public class MockAuthorizationPlugin implements AuthorizationPlugin {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
static final HashSet<String> denyUsers = new HashSet<>();
+ static final HashSet<String> protectedResources = new HashSet<>();
static Predicate<AuthorizationContext> predicate;
@Override
@@ -42,15 +43,17 @@ public class MockAuthorizationPlugin implements AuthorizationPlugin {
} catch (SolrException e) {
return new AuthorizationResponse(e.code());
}
+ } else {
+ if (!protectedResources.contains(context.getResource())) {
+ return new AuthorizationResponse(200);
+ }
+ if (uname == null) uname = context.getParams().get("uname");
+ log.info("User request: " + uname);
+ if (uname == null || denyUsers.contains(uname))
+ return new AuthorizationResponse(403);
+ else
+ return new AuthorizationResponse(200);
}
-
-
- if (uname == null) uname = context.getParams().get("uname");
- log.info("User request: " + uname);
- if (denyUsers.contains(uname))
- return new AuthorizationResponse(403);
- else
- return new AuthorizationResponse(200);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/72e68697/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java
index bc4f4e5..1f85f3a 100644
--- a/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java
@@ -16,12 +16,10 @@
*/
package org.apache.solr.security;
-import javax.servlet.ServletRequest;
import javax.servlet.http.HttpServletRequest;
import java.lang.invoke.MethodHandles;
import java.security.Principal;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Predicate;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -77,9 +75,7 @@ public class PKIAuthenticationIntegrationTest extends SolrCloudTestCase {
final AtomicInteger count = new AtomicInteger();
- MockAuthorizationPlugin.predicate = new Predicate<AuthorizationContext>() {
- @Override
- public boolean test(AuthorizationContext context) {
+ MockAuthorizationPlugin.predicate = context -> {
if ("/select".equals(context.getResource())) {
Principal principal = context.getUserPrincipal();
log.info("principalIs : {}", principal);
@@ -88,22 +84,19 @@ public class PKIAuthenticationIntegrationTest extends SolrCloudTestCase {
}
}
return true;
- }
};
- MockAuthenticationPlugin.predicate = new Predicate<ServletRequest>() {
- @Override
- public boolean test(ServletRequest servletRequest) {
+ MockAuthenticationPlugin.predicate = servletRequest -> {
String s = ((HttpServletRequest) servletRequest).getQueryString();
if (s != null && s.contains("__user=solr") && s.contains("__pwd=SolrRocks")) {
servletRequest.setAttribute(Principal.class.getName(), "solr");
}
return true;
- }
};
QueryRequest query = new QueryRequest(params);
query.process(cluster.getSolrClient(), "collection");
- assertTrue("all nodes must get the user solr , no:of nodes got solr : " + count.get(),count.get() > 2);
+ assertTrue("all nodes must get the user solr , no:of nodes got solr : " + count.get(), count.get() > 2);
+
}
@After
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/72e68697/solr/core/src/test/org/apache/solr/security/TestAuthorizationFramework.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/security/TestAuthorizationFramework.java b/solr/core/src/test/org/apache/solr/security/TestAuthorizationFramework.java
index acdf578..c3d6f60 100644
--- a/solr/core/src/test/org/apache/solr/security/TestAuthorizationFramework.java
+++ b/solr/core/src/test/org/apache/solr/security/TestAuthorizationFramework.java
@@ -17,7 +17,6 @@
package org.apache.solr.security;
import java.lang.invoke.MethodHandles;
-
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
@@ -59,22 +58,30 @@ public class TestAuthorizationFramework extends AbstractFullDistribZkTestBase {
public void authorizationFrameworkTest() throws Exception {
MockAuthorizationPlugin.denyUsers.add("user1");
MockAuthorizationPlugin.denyUsers.add("user1");
- waitForThingsToLevelOut(10);
- String baseUrl = jettys.get(0).getBaseUrl().toString();
- verifySecurityStatus(cloudClient.getLbClient().getHttpClient(), baseUrl + "/admin/authorization", "authorization/class", MockAuthorizationPlugin.class.getName(), 20);
- log.info("Starting test");
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.add("q", "*:*");
- // This should work fine.
- cloudClient.query(params);
- // This user is blacklisted in the mock. The request should return a 403.
- params.add("uname", "user1");
try {
+ waitForThingsToLevelOut(10);
+ String baseUrl = jettys.get(0).getBaseUrl().toString();
+ verifySecurityStatus(cloudClient.getLbClient().getHttpClient(), baseUrl + "/admin/authorization", "authorization/class", MockAuthorizationPlugin.class.getName(), 20);
+ log.info("Starting test");
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ // This should work fine.
cloudClient.query(params);
- fail("This should have failed");
- } catch (Exception e) {}
- log.info("Ending test");
+ MockAuthorizationPlugin.protectedResources.add("/select");
+
+ // This user is blacklisted in the mock. The request should return a 403.
+ params.add("uname", "user1");
+ try {
+ cloudClient.query(params);
+ fail("This should have failed");
+ } catch (Exception e) {}
+ log.info("Ending test");
+ } finally {
+ MockAuthorizationPlugin.denyUsers.clear();
+ MockAuthorizationPlugin.protectedResources.clear();
+
+ }
}
@Override
[10/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8099: Update
MIGRATE.txt
Posted by da...@apache.org.
LUCENE-8099: Update MIGRATE.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0c18acb0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0c18acb0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0c18acb0
Branch: refs/heads/jira/solr-11702
Commit: 0c18acb00f3083553e782c916638dfd6bb15cfe9
Parents: 79bd05d
Author: Alan Woodward <ro...@apache.org>
Authored: Fri Jan 5 10:12:03 2018 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Fri Jan 5 10:12:03 2018 +0000
----------------------------------------------------------------------
lucene/MIGRATE.txt | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c18acb0/lucene/MIGRATE.txt
----------------------------------------------------------------------
diff --git a/lucene/MIGRATE.txt b/lucene/MIGRATE.txt
index 9d02db8..d486a21 100644
--- a/lucene/MIGRATE.txt
+++ b/lucene/MIGRATE.txt
@@ -19,12 +19,14 @@ FunctionScoreQuery maps negative values to 0.
## CustomScoreQuery, BoostedQuery and BoostingQuery removed (LUCENE-8099) ##
-Instead use FunctionScoreQuery and a DoubleValuesSource implementation. For example,
-to replace the functionality of BoostedQuery, you could do the following, using
-the lucene-expressions module:
+Instead use FunctionScoreQuery and a DoubleValuesSource implementation. BoostedQuery
+and BoostingQuery may be replaced by calls to FunctionScoreQuery.boostByValue() and
+FunctionScoreQuery.boostByQuery(). To replace more complex calculations in
+CustomScoreQuery, use the lucene-expressions module:
SimpleBindings bindings = new SimpleBindings();
bindings.add("score", DoubleValuesSource.SCORES);
-bindings.add("boost", DoubleValuesSource.fromIntField("myboostfield"));
-Expression expr = JavascriptCompiler.compile("score * boost");
+bindings.add("boost1", DoubleValuesSource.fromIntField("myboostfield"));
+bindings.add("boost2", DoubleValuesSource.fromIntField("myotherboostfield"));
+Expression expr = JavascriptCompiler.compile("score * (boost1 + ln(boost2))");
FunctionScoreQuery q = new FunctionScoreQuery(inputQuery, expr.getDoubleValuesSource(bindings));
[17/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11758: Fixed
FloatDocValues.boolVal to correctly return true for all values != 0.0F
Posted by da...@apache.org.
SOLR-11758: Fixed FloatDocValues.boolVal to correctly return true for all values != 0.0F
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d03cb44d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d03cb44d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d03cb44d
Branch: refs/heads/jira/solr-11702
Commit: d03cb44de73bb29e16a3ab927ee57d6eb916789a
Parents: 0d55811
Author: Chris Hostetter <ho...@apache.org>
Authored: Sat Jan 6 12:54:05 2018 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Sat Jan 6 12:54:05 2018 -0700
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../function/docvalues/FloatDocValues.java | 5 ++
.../docvalues/TestBoolValOfNumericDVs.java | 76 ++++++++++++++++++++
.../solr/search/function/TestFunctionQuery.java | 16 ++++-
4 files changed, 97 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d03cb44d/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 89afdfd..2b02b87 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -124,6 +124,9 @@ Bug Fixes
* LUCENE-8077: Fixed bug in how CheckIndex verifies doc-value iterators.
(Xiaoshan Sun via Adrien Grand)
+* SOLR-11758: Fixed FloatDocValues.boolVal to correctly return true for all values != 0.0F
+ (Munendra S N via hossman)
+
Other
* LUCENE-8111: IndexOrDocValuesQuery Javadoc references outdated method name.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d03cb44d/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java
index 8b9e942..72798d6 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java
@@ -58,6 +58,11 @@ public abstract class FloatDocValues extends FunctionValues {
}
@Override
+ public boolean boolVal(int doc) throws IOException {
+ return floatVal(doc) != 0.0f;
+ }
+
+ @Override
public double doubleVal(int doc) throws IOException {
return (double)floatVal(doc);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d03cb44d/lucene/queries/src/test/org/apache/lucene/queries/function/docvalues/TestBoolValOfNumericDVs.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/docvalues/TestBoolValOfNumericDVs.java b/lucene/queries/src/test/org/apache/lucene/queries/function/docvalues/TestBoolValOfNumericDVs.java
new file mode 100644
index 0000000..a604674
--- /dev/null
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/docvalues/TestBoolValOfNumericDVs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.queries.function.docvalues;
+
+import java.io.IOException;
+
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.util.LuceneTestCase;
+
+/**
+ * <p>
+ * Sanity check that {@link FunctionValues#boolVal} behaves as expected for trivial subclasses of the various
+ * (Numeric) DocValue implementations.
+ * </p>
+ * <p>
+ * Any "non-zero" value should result in "true"
+ * </p>
+ */
+public class TestBoolValOfNumericDVs extends LuceneTestCase {
+
+ public void test() throws IOException {
+ check(true);
+ check(false);
+ }
+
+ public void check(final boolean expected) throws IOException {
+
+ // create "constant" based instances of each superclass that should returned the expected value based on
+ // the constant used
+ final FunctionValues[] values = new FunctionValues[] {
+ new FloatDocValues(null) {
+ @Override
+ public float floatVal(int doc) throws IOException {
+ return expected ? Float.MIN_VALUE : 0.0F;
+ }
+ },
+ new DoubleDocValues(null) {
+ @Override
+ public double doubleVal(int doc) throws IOException {
+ return expected ? Double.MIN_VALUE : 0.0D;
+ }
+ },
+ new IntDocValues(null) {
+ @Override
+ public int intVal(int doc) throws IOException {
+ return expected ? 1 : 0;
+ }
+ },
+ new LongDocValues(null) {
+ @Override
+ public long longVal(int doc) throws IOException {
+ return expected ? 1L : 0L;
+ }
+ },
+ };
+
+ for (FunctionValues fv : values) {
+ // docId is irrelevant since all of our FunctionValues return a constant value.
+ assertEquals(fv.getClass().getSuperclass().toString(), expected, fv.boolVal(123));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d03cb44d/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
index 53e1ea7..32f603a 100644
--- a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
+++ b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
@@ -800,7 +800,7 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
public void testBooleanFunctions() throws Exception {
clearIndex();
- assertU(adoc("id", "1", "text", "hello", "foo_s","A", "foo_ti", "0", "foo_tl","0"));
+ assertU(adoc("id", "1", "text", "hello", "foo_s","A", "foo_ti", "0", "foo_tl","0", "foo_tf", "0.00001"));
assertU(adoc("id", "2" , "foo_ti","10", "foo_tl","11"));
assertU(commit());
@@ -819,6 +819,10 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
// test if()
assertJQ(req("q", "id:1", "fl", "a1:if(true,'A','B')", "fl","b1:if(false,'A',testfunc('B'))")
, "/response/docs/[0]=={'a1':'A', 'b1':'B'}");
+ // queries with positive scores < 1 should still evaluate to 'true' in boolean context
+ assertJQ(req("q", "id:1", "nested", "*:*^=0.00001",
+ "fl", "a1:if(query($nested),'A','B')", "fl","b1:if(not(query($nested)),'A','B')")
+ , "/response/docs/[0]=={'a1':'A', 'b1':'B'}");
// test boolean operators
assertJQ(req("q", "id:1", "fl", "t1:and(testfunc(true),true)", "fl","f1:and(true,false)", "fl","f2:and(false,true)", "fl","f3:and(false,false)")
@@ -830,6 +834,12 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
assertJQ(req("q", "id:1", "fl", "t:not(testfunc(false)),f:not(true)")
, "/response/docs/[0]=={'t':true, 'f':false}");
+ // test fields evaluated as booleans in wrapping functions
+ assertJQ(req("q", "id:1", "fl", "a:not(foo_ti), b:if(foo_tf,'TT','FF'), c:and(true,foo_tf)")
+ , "/response/docs/[0]=={'a':true, 'b':'TT', 'c':true}");
+ assertJQ(req("q", "id:2", "fl", "a:not(foo_ti), b:if(foo_tf,'TT','FF'), c:and(true,foo_tf)")
+ , "/response/docs/[0]=={'a':false, 'b':'FF', 'c':false}");
+
// def(), the default function that returns the first value that exists
assertJQ(req("q", "id:1", "fl", "x:def(id,testfunc(123)), y:def(foo_f,234.0)")
@@ -838,8 +848,8 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
, "/response/docs/[0]=={'x':'A', 'y':'W'}");
// test constant conversion to boolean
- assertJQ(req("q", "id:1", "fl", "a:not(0), b:not(1), c:not(0.0), d:not(1.1), e:not('A')")
- , "/response/docs/[0]=={'a':true, 'b':false, 'c':true, 'd':false, 'e':false}");
+ assertJQ(req("q", "id:1", "fl", "a:not(0), b:not(1), c:not(0.0), d:not(1.1), e:not('A'), f:not(0.001)")
+ , "/response/docs/[0]=={'a':true, 'b':false, 'c':true, 'd':false, 'e':false, 'f':false}");
}
[28/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8115: remove one
TODO-on-its-own javadoc.
Posted by da...@apache.org.
LUCENE-8115: remove one TODO-on-its-own javadoc.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bd69d64a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bd69d64a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bd69d64a
Branch: refs/heads/jira/solr-11702
Commit: bd69d64ad04fb0fe6f17f68dcc1fa685e15a9317
Parents: ad6e8b8
Author: Christine Poerschke <cp...@apache.org>
Authored: Mon Jan 8 20:04:58 2018 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Mon Jan 8 20:04:58 2018 +0000
----------------------------------------------------------------------
.../apache/lucene/classification/KNearestNeighborClassifier.java | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd69d64a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
index 1bc53b0..b12bb5c 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
@@ -124,9 +124,6 @@ public class KNearestNeighborClassifier implements Classifier<BytesRef> {
return classifyFromTopDocs(knnSearch(text));
}
- /**
- * TODO
- */
protected ClassificationResult<BytesRef> classifyFromTopDocs(TopDocs knnResults) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = buildListFromTopDocs(knnResults);
ClassificationResult<BytesRef> assignedClass = null;
[18/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11824: Fixed bucket
ordering in distributed json.facet type:range when mincount>0
Posted by da...@apache.org.
SOLR-11824: Fixed bucket ordering in distributed json.facet type:range when mincount>0
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d598517b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d598517b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d598517b
Branch: refs/heads/jira/solr-11702
Commit: d598517b965be2762d63dacf523352e393988e36
Parents: d03cb44
Author: Chris Hostetter <ho...@apache.org>
Authored: Sat Jan 6 14:23:45 2018 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Sat Jan 6 14:23:45 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../apache/solr/search/facet/FacetRange.java | 10 +++--
.../solr/search/facet/FacetRangeMerger.java | 2 +-
.../solr/search/facet/TestJsonFacets.java | 42 ++++++++++++++++++++
4 files changed, 53 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d598517b/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index cb830ce..4b4820c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -86,6 +86,9 @@ Bug Fixes
* SOLR-11555: If the query terms reduce to nothing, filter(clause) produces an NPE whereas
fq=clause does not (Erick Erickson)
+* SOLR-11824: Fixed bucket ordering in distributed json.facet type:range when mincount>0 (hossman)
+
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d598517b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java
index b99b4b8..1176a77 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java
@@ -93,9 +93,13 @@ class FacetRangeProcessor extends FacetProcessor<FacetRange> {
super.process();
// Under the normal mincount=0, each shard will need to return 0 counts since we don't calculate buckets at the top level.
- // But if mincount>0 then our sub mincount can be set to 1.
-
- effectiveMincount = fcontext.isShard() ? (freq.mincount > 0 ? 1 : 0) : freq.mincount;
+ // If mincount>0 then we could *potentially* set our sub mincount to 1...
+ // ...but that would require sorting the buckets (by their val) at the top level
+ //
+ // Tather then do that, which could be complicated by non trivial field types, we'll force the sub-shard effectiveMincount
+ // to be 0, ensuring that we can trivially merge all the buckets from every shard
+ // (we have to filter the merged buckets by the original mincount either way)
+ effectiveMincount = fcontext.isShard() ? 0 : freq.mincount;
sf = fcontext.searcher.getSchema().getField(freq.field);
response = getRangeCounts();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d598517b/solr/core/src/java/org/apache/solr/search/facet/FacetRangeMerger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRangeMerger.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRangeMerger.java
index 5fae6c6..6ddc05e 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetRangeMerger.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRangeMerger.java
@@ -45,7 +45,7 @@ public class FacetRangeMerger extends FacetRequestSortedMerger<FacetRange> {
@Override
public void sortBuckets() {
- // TODO: mincount>0 will mess up order?
+ // regardless of mincount, every shard returns a consistent set of buckets which are already in the correct order
sortedBuckets = new ArrayList<>( buckets.values() );
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d598517b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
index bf709cf..753c7dc 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
@@ -1028,6 +1028,48 @@ public class TestJsonFacets extends SolrTestCaseHS {
",between:{count:3,x:0.0, ny:{count:2}}" +
" } }"
);
+
+ // sparse range facet (with sub facets and stats), with "other:all"
+ client.testJQ(params(p, "q", "*:*", "json.facet",
+ "{f:{range:{field:${num_d}, start:-5, end:10, gap:1, other:all, "+
+ " facet:{ x:'sum(${num_i})', ny:{query:'${where_s}:NY'}} }}}"
+ )
+ , "facets=={count:6, f:{buckets:[ {val:-5.0,count:1, x:-5.0,ny:{count:1}}, "+
+ " {val:-4.0,count:0 /* ,x:0.0,ny:{count:0} */} ,"+
+ " {val:-3.0,count:0 /* ,x:0.0,ny:{count:0} */} ,"+
+ " {val:-2.0,count:0 /* ,x:0.0,ny:{count:0} */} ,"+
+ " {val:-1.0,count:0 /* ,x:0.0,ny:{count:0} */} ,"+
+ " {val: 0.0,count:0 /* ,x:0.0,ny:{count:0} */} ,"+
+ " {val: 1.0,count:0 /* ,x:0.0,ny:{count:0} */} ,"+
+ " {val: 2.0,count:1, x:3.0,ny:{count:0}} , "+
+ " {val: 3.0,count:0 /* ,x:0.0,ny:{count:0} */} ,"+
+ " {val: 4.0,count:1, x:2.0,ny:{count:1}} , "+
+ " {val: 5.0,count:0 /* ,x:0.0,ny:{count:0} */} ,"+
+ " {val: 6.0,count:0 /* ,x:0.0,ny:{count:0} */} ,"+
+ " {val: 7.0,count:0 /* ,x:0.0,ny:{count:0} */} ,"+
+ " {val: 8.0,count:0 /* ,x:0.0,ny:{count:0} */} ,"+
+ " {val: 9.0,count:0 /* ,x:0.0,ny:{count:0} */}"+
+ " ]" +
+ " ,before: {count:1,x:-5.0,ny:{count:0}}" +
+ " ,after: {count:1,x:7.0, ny:{count:0}}" +
+ " ,between:{count:3,x:0.0, ny:{count:2}}" +
+ " } }"
+ );
+
+ // sparse range facet (with sub facets and stats), with "other:all" & mincount==1
+ client.testJQ(params(p, "q", "*:*", "json.facet",
+ "{f:{range:{field:${num_d}, start:-5, end:10, gap:1, other:all, mincount:1, "+
+ " facet:{ x:'sum(${num_i})', ny:{query:'${where_s}:NY'}} }}}"
+ )
+ , "facets=={count:6, f:{buckets:[ {val:-5.0,count:1, x:-5.0,ny:{count:1}}, "+
+ " {val: 2.0,count:1, x:3.0,ny:{count:0}} , "+
+ " {val: 4.0,count:1, x:2.0,ny:{count:1}} "+
+ " ]" +
+ " ,before: {count:1,x:-5.0,ny:{count:0}}" +
+ " ,after: {count:1,x:7.0, ny:{count:0}}" +
+ " ,between:{count:3,x:0.0, ny:{count:2}}" +
+ " } }"
+ );
// range facet with sub facets and stats, with "other:all", on subset
client.testJQ(params(p, "q", "id:(3 4 6)"
[24/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11730: Collect more
stats in the benchmark. Add simulation framework package docs.
Posted by da...@apache.org.
SOLR-11730: Collect more stats in the benchmark. Add simulation framework package docs.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a9fec9bf
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a9fec9bf
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a9fec9bf
Branch: refs/heads/jira/solr-11702
Commit: a9fec9bf7caee2620d09086efde4a29b245aab7b
Parents: 72e6869
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Mon Jan 8 15:08:17 2018 +0100
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Mon Jan 8 15:08:17 2018 +0100
----------------------------------------------------------------------
.../cloud/autoscaling/sim/SimCloudManager.java | 17 +++++
.../autoscaling/sim/SimSolrCloudTestCase.java | 15 +---
.../cloud/autoscaling/sim/TestLargeCluster.java | 68 ++++++++++++++---
.../cloud/autoscaling/sim/package-info.java | 79 +++++++++++++++++++-
4 files changed, 152 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a9fec9bf/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
index cd9f177..a842a87 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
@@ -26,10 +26,12 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.solr.client.solrj.SolrClient;
@@ -305,6 +307,21 @@ public class SimCloudManager implements SolrCloudManager {
return systemColl;
}
+ public Map<String, Map<String, AtomicInteger>> simGetEventCounts() {
+ TreeMap<String, Map<String, AtomicInteger>> counts = new TreeMap<>();
+ synchronized (systemColl) {
+ for (SolrInputDocument d : systemColl) {
+ if (!"autoscaling_event".equals(d.getFieldValue("type"))) {
+ continue;
+ }
+ counts.computeIfAbsent((String)d.getFieldValue("event.source_s"), s -> new TreeMap<>())
+ .computeIfAbsent((String)d.getFieldValue("stage_s"), s -> new AtomicInteger())
+ .incrementAndGet();
+ }
+ }
+ return counts;
+ }
+
/**
* Get a {@link SolrClient} implementation where calls are forwarded to this
* instance of the cluster.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a9fec9bf/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
index 48d46b6..122ff2c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
@@ -35,7 +35,6 @@ import java.util.function.Predicate;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
-import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.CollectionStatePredicate;
import org.apache.solr.common.cloud.DocCollection;
@@ -137,19 +136,7 @@ public class SimSolrCloudTestCase extends SolrTestCaseJ4 {
log.info("######### Final Solr op counts ##########");
cluster.simGetOpCounts().forEach((k, cnt) -> log.info("##\t\t- " + String.format(Locale.ROOT, "%-14s %4d", k, cnt.get())));
log.info("######### Autoscaling event counts ###########");
- TreeMap<String, Map<String, AtomicInteger>> counts = new TreeMap<>();
-
- List<SolrInputDocument> solrInputDocuments = cluster.simGetSystemCollection();
- synchronized (solrInputDocuments) {
- for (SolrInputDocument d : solrInputDocuments) {
- if (!"autoscaling_event".equals(d.getFieldValue("type"))) {
- continue;
- }
- counts.computeIfAbsent((String)d.getFieldValue("event.source_s"), s -> new TreeMap<>())
- .computeIfAbsent((String)d.getFieldValue("stage_s"), s -> new AtomicInteger())
- .incrementAndGet();
- }
- }
+ Map<String, Map<String, AtomicInteger>> counts = cluster.simGetEventCounts();
counts.forEach((trigger, map) -> {
log.info("## * Trigger: " + trigger);
map.forEach((s, cnt) -> log.info("##\t\t- " + String.format(Locale.ROOT, "%-11s %4d", s, cnt.get())));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a9fec9bf/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java
index 15cb226..3adf652 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java
@@ -30,6 +30,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
@@ -61,6 +62,7 @@ import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAut
/**
*
*/
+@TimeoutSuite(millis = 4 * 3600 * 1000)
@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
public class TestLargeCluster extends SimSolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -315,6 +317,11 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
private static final int[] renard5x = new int[] {
1, 2, 3, 4, 6,
10, 16, 25, 40, 63,
+ 100
+ };
+ private static final int[] renard5xx = new int[] {
+ 1, 2, 3, 4, 6,
+ 10, 16, 25, 40, 63,
100, 158, 251, 398, 631,
1000, 1585, 2512, 3981, 6310,
10000
@@ -330,12 +337,18 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
100
};
+ private static final AtomicInteger ZERO = new AtomicInteger(0);
+
//@Test
public void benchmarkNodeLost() throws Exception {
List<String> results = new ArrayList<>();
- for (int wait : renard5) {
- for (int delay : renard5) {
- SummaryStatistics stat = new SummaryStatistics();
+ for (int wait : renard5x) {
+ for (int delay : renard5x) {
+ SummaryStatistics totalTime = new SummaryStatistics();
+ SummaryStatistics ignoredOurEvents = new SummaryStatistics();
+ SummaryStatistics ignoredOtherEvents = new SummaryStatistics();
+ SummaryStatistics startedOurEvents = new SummaryStatistics();
+ SummaryStatistics startedOtherEvents = new SummaryStatistics();
for (int i = 0; i < 5; i++) {
if (cluster != null) {
cluster.close();
@@ -344,14 +357,29 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
setUp();
setupTest();
long total = doTestNodeLost(wait, delay * 1000, 0);
- stat.addValue(total);
+ totalTime.addValue(total);
+ // get event counts
+ Map<String, Map<String, AtomicInteger>> counts = cluster.simGetEventCounts();
+ Map<String, AtomicInteger> map = counts.remove("node_lost_trigger");
+ startedOurEvents.addValue(map.getOrDefault("STARTED", ZERO).get());
+ ignoredOurEvents.addValue(map.getOrDefault("IGNORED", ZERO).get());
+ int otherStarted = 0;
+ int otherIgnored = 0;
+ for (Map<String, AtomicInteger> m : counts.values()) {
+ otherStarted += m.getOrDefault("STARTED", ZERO).get();
+ otherIgnored += m.getOrDefault("IGNORED", ZERO).get();
+ }
+ startedOtherEvents.addValue(otherStarted);
+ ignoredOtherEvents.addValue(otherIgnored);
}
- results.add(String.format(Locale.ROOT, "%d\t%d\t%6.0f\t%6.0f\t%6.0f\t%6.0f\t%6.0f", wait, delay,
- stat.getMin(), stat.getMax(), stat.getMean(), stat.getVariance(), stat.getStandardDeviation()));
+ results.add(String.format(Locale.ROOT, "%d\t%d\t%4.0f\t%4.0f\t%4.0f\t%4.0f\t%6.0f\t%6.0f\t%6.0f\t%6.0f\t%6.0f",
+ wait, delay, startedOurEvents.getMean(), ignoredOurEvents.getMean(),
+ startedOtherEvents.getMean(), ignoredOtherEvents.getMean(),
+ totalTime.getMin(), totalTime.getMax(), totalTime.getMean(), totalTime.getStandardDeviation(), totalTime.getVariance()));
}
}
log.info("===== RESULTS ======");
- log.info("waitFor\tkillDelay\tmin\tmax\tmean\tvar\tstdev");
+ log.info("waitFor\tdelay\tSTRT\tIGN\toSTRT\toIGN\tmin\tmax\tmean\tstdev\tvar");
results.forEach(s -> log.info(s));
}
@@ -364,6 +392,7 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
"'waitFor' : '" + waitFor + "s'," +
"'enabled' : true," +
"'actions' : [" +
+ "{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}," +
"{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
"{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}" +
"]" +
@@ -391,6 +420,11 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
cluster.simRemoveNode(nodes.get(i), false);
cluster.getTimeSource().sleep(killDelay);
}
+ // wait for the trigger to fire
+ boolean await = triggerFiredLatch.await(10 * waitFor * 1000 / SPEED, TimeUnit.MILLISECONDS);
+ assertTrue("trigger did not fire within timeout, " +
+ "waitFor=" + waitFor + ", killDelay=" + killDelay + ", minIgnored=" + minIgnored,
+ await);
List<SolrInputDocument> systemColl = cluster.simGetSystemCollection();
int startedEventPos = -1;
for (int i = 0; i < systemColl.size(); i++) {
@@ -404,7 +438,9 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
break;
}
}
- assertTrue("no STARTED event: " + systemColl, startedEventPos > -1);
+ assertTrue("no STARTED event: " + systemColl + ", " +
+ "waitFor=" + waitFor + ", killDelay=" + killDelay + ", minIgnored=" + minIgnored,
+ startedEventPos > -1);
SolrInputDocument startedEvent = systemColl.get(startedEventPos);
int ignored = 0;
int lastIgnoredPos = startedEventPos;
@@ -420,9 +456,13 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
}
}
}
- assertTrue("should be at least " + minIgnored + " IGNORED events", ignored >= minIgnored);
+ assertTrue("should be at least " + minIgnored + " IGNORED events, " +
+ "waitFor=" + waitFor + ", killDelay=" + killDelay + ", minIgnored=" + minIgnored,
+ ignored >= minIgnored);
// make sure some replicas have been moved
- assertTrue("no MOVEREPLICA ops?", cluster.simGetOpCount("MOVEREPLICA") > 0);
+ assertTrue("no MOVEREPLICA ops? " +
+ "waitFor=" + waitFor + ", killDelay=" + killDelay + ", minIgnored=" + minIgnored,
+ cluster.simGetOpCount("MOVEREPLICA") > 0);
log.info("Ready after " + waitForState(collectionName, 20 * NUM_NODES, TimeUnit.SECONDS, clusterShape(NUM_NODES / 5, NUM_NODES / 10)) + " ms");
@@ -450,12 +490,16 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
}
}
- assertTrue("did not finish processing changes", finishedEvent != null);
+ assertTrue("did not finish processing changes, " +
+ "waitFor=" + waitFor + ", killDelay=" + killDelay + ", minIgnored=" + minIgnored,
+ finishedEvent != null);
long delta = (Long)finishedEvent.getFieldValue("event.time_l") - (Long)startedEvent.getFieldValue("event.time_l");
delta = TimeUnit.NANOSECONDS.toMillis(delta);
log.info("#### System stabilized after " + delta + " ms");
long ops = cluster.simGetOpCount("MOVEREPLICA");
- assertTrue("unexpected number of MOVEREPLICA ops: " + ops, ops >= 40);
+ assertTrue("unexpected number of MOVEREPLICA ops: " + ops + ", " +
+ "waitFor=" + waitFor + ", killDelay=" + killDelay + ", minIgnored=" + minIgnored,
+ ops >= 40);
return delta;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a9fec9bf/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/package-info.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/package-info.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/package-info.java
index 0b412cb..8c1ee86 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/package-info.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/package-info.java
@@ -15,7 +15,84 @@
* limitations under the License.
*/
/**
- * Simulated environment for autoscaling tests.
+ * <h1>Simulated environment for autoscaling tests.</h1>
+ *
+ * <h2>Goals</h2>
+ * <ul>
+ * <li>Use the actual unchanged autoscaling code for cluster state monitoring and autoscaling plan execution.</li>
+ * <li>Support testing large clusters (> 100 nodes).</li>
+ * <li>Support fast testing using accelerated time (eg. 100x faster).</li>
+ * <li>Support enough of other Solr functionality for the test results to be meaningful.</li>
+ * </ul>
+ *
+ * <h2>Simulated SolrCloudManager - {@link org.apache.solr.cloud.autoscaling.sim.SimCloudManager}</h2>
+ * This implementation of {@link org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudManager}
+ * uses the following simulated components:
+ * <ul>
+ * <li>{@link org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager} - in-memory ZK look-alike, with support for Watcher-s, ephemeral and sequential nodes.</li>
+ * <li>{@link org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider} - manages collection, replica infos, states and replica metrics.</li>
+ * <li>{@link org.apache.solr.cloud.autoscaling.sim.SimNodeStateProvider} - manages node metrics.</li>
+ * <li>{@link org.apache.solr.cloud.autoscaling.sim.GenericDistributedQueue} - DistributedQueue that uses SimDistribStateManager.</li>
+ * </ul>
+ * SimCloudManager also maintains an up-to-date /live_nodes in SimDistribStateManager, provides a SolrClient instance for use in tests,
+ * and provides several convenience methods for setting up simulated clusters, populating node and replica metrics, collecting
+ * autoscaling-related event history, collecting autoscaling event statistics, etc.
+ *
+ * SimCloudManager runs actual {@link org.apache.solr.cloud.autoscaling.OverseerTriggerThread} so that it
+ * uses real trigger and trigger action implementations, as well as real event scheduling and processing code.
+ * It also provides methods for simulating Overseer leader change.
+ *
+ * An important part of the SimCloudManager is also a request handler that processes common autoscaling
+ * and collection admin requests. Autoscaling requests are processes by an instance of
+ * {@link org.apache.solr.cloud.autoscaling.AutoScalingHandler} (and result in changes in respective
+ * data stored in {@link org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager}). Collection
+ * admin commands are simulated, ie. they don't use actual {@link org.apache.solr.handler.admin.CollectionsHandler}
+ * due to the complex dependencies on real components.
+ *
+ * <h2>{@link org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider}</h2>
+ * This components maintains collection and replica states:
+ * <ul>
+ * <li>Simulates delays between request and the actual cluster state changes</li>
+ * <li>Marks replicas as down when a node goes down (optionally preserving the replica metrics in order to simulate a node coming back), and keeps track of per-node cores and disk space.</li>
+ * <li>Runs a shard leader election look-alike on collection state updates.</li>
+ * <li>Maintains up-to-date /clusterstate.json and /clusterprops.json in SimDistribStateManager (which in turn notifies Watcher-s about collection updates).
+ * Currently for simplicity it uses the old single /clusterstate.json format for representing ClusterState.</li>
+ * </ul>
+ *
+ * <h2>{@link org.apache.solr.cloud.autoscaling.sim.SimNodeStateProvider}</h2>
+ * This component maintains node metrics. When a simulated cluster is set up using eg.
+ * {@link org.apache.solr.cloud.autoscaling.sim.SimCloudManager#createCluster(int, org.apache.solr.common.util.TimeSource)}
+ * method, each simulated node is initialized with some basic metrics that are expected by the autoscaling
+ * framework, such as node name, fake system load average, heap usage and disk usage.
+ *
+ * The number of cores and disk space metrics may be used in autoscaling calculations, so they are
+ * tracked and adjusted by {@link org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider} according
+ * to the currently active replicas located on each node.
+ *
+ * <h2>Limitations of the simulation framework</h2>
+ * Currently the simulation framework is limited to testing the core autoscaling API in a single JVM.
+ * Using it for other purposes would require extensive modifications in Solr and in the framework code.
+ *
+ * Specifically, the framework supports testing the following autoscaling components:
+ * <ul>
+ * <li>OverseerTriggerThread and components that it uses.</li>
+ * <li>Autoscaling config, triggers, trigger listeners, ScheduledTriggers, trigger event queues, ComputePlanAction / ExecutePlanAction, etc.</li>
+ * </ul>
+ * Overseer and CollectionsHandler Cmd implementations are NOT used, so cannot be properly tested - some of their functionality is simulated.
+ * Other SolrCloud components make too many direct references to ZkStateReader, or direct HTTP requests, or rely on too many other components and require much more complex functionality - they may be refactored later but the effort may be too high.
+ *
+ * Simulation framework definitely does not support the following functionality:
+ * <ul>
+ * <li>Solr searching and indexing</li>
+ * <li>Any component that uses ZkController (eg. CoreContainer)</li>
+ * <li>Any component that uses ShardHandler (eg. CollectionsHandler Cmd-s)</li>
+ * </ul>
+ *
+ * <h2>Testing framework</h2>
+ * A base class {@link org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase} is provided, which
+ * provides similar helper methods to the ones in the {@link org.apache.solr.cloud.SolrCloudTestCase}.
+ *
*/
package org.apache.solr.cloud.autoscaling.sim;
+
[09/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8099: Add some
static methods to FunctionScoreQuery to replace Boosting/BoostedQuery
Posted by da...@apache.org.
LUCENE-8099: Add some static methods to FunctionScoreQuery to replace Boosting/BoostedQuery
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/79bd05da
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/79bd05da
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/79bd05da
Branch: refs/heads/jira/solr-11702
Commit: 79bd05da4dcbb584fffaf62bea683c00d3ac432c
Parents: 5a08fa8
Author: Alan Woodward <ro...@apache.org>
Authored: Wed Jan 3 15:28:23 2018 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Fri Jan 5 09:32:05 2018 +0000
----------------------------------------------------------------------
.../queries/function/FunctionScoreQuery.java | 151 +++++++++++++++++++
.../function/TestFunctionScoreQuery.java | 19 +--
2 files changed, 156 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/79bd05da/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
index d264267..09a592b 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
@@ -63,6 +63,38 @@ public final class FunctionScoreQuery extends Query {
return in;
}
+ /**
+ * Returns a FunctionScoreQuery where the scores of a wrapped query are multiplied by
+ * the value of a DoubleValuesSource.
+ *
+ * If the source has no value for a particular document, the score for that document
+ * is preserved as-is.
+ *
+ * @param in the query to boost
+ * @param boost a {@link DoubleValuesSource} containing the boost values
+ */
+ public static FunctionScoreQuery boostByValue(Query in, DoubleValuesSource boost) {
+ return new FunctionScoreQuery(in, new MultiplicativeBoostValuesSource(boost));
+ }
+
+ /**
+ * Returns a FunctionScoreQuery where the scores of a wrapped query are multiplied by
+ * a boost factor if the document being scored also matches a separate boosting query.
+ *
+ * Documents that do not match the boosting query have their scores preserved.
+ *
+ * This may be used to 'demote' documents that match the boosting query, by passing in
+ * a boostValue between 0 and 1.
+ *
+ * @param in the query to boost
+ * @param boostMatch the boosting query
+ * @param boostValue the amount to boost documents which match the boosting query
+ */
+ public static FunctionScoreQuery boostByQuery(Query in, Query boostMatch, float boostValue) {
+ return new FunctionScoreQuery(in,
+ new MultiplicativeBoostValuesSource(new QueryBoostValuesSource(DoubleValuesSource.fromQuery(boostMatch), boostValue)));
+ }
+
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
Weight inner = in.createWeight(searcher, scoreMode.needsScores() && source.needsScores() ? scoreMode : ScoreMode.COMPLETE_NO_SCORES, 1f);
@@ -189,4 +221,123 @@ public final class FunctionScoreQuery extends Query {
}
}
+
+ private static class MultiplicativeBoostValuesSource extends DoubleValuesSource {
+
+ private final DoubleValuesSource boost;
+
+ private MultiplicativeBoostValuesSource(DoubleValuesSource boost) {
+ this.boost = boost;
+ }
+
+ @Override
+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ DoubleValues in = DoubleValues.withDefault(boost.getValues(ctx, scores), 1);
+ return new DoubleValues() {
+ @Override
+ public double doubleValue() throws IOException {
+ return scores.doubleValue() * in.doubleValue();
+ }
+
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ return in.advanceExact(doc);
+ }
+ };
+ }
+
+ @Override
+ public boolean needsScores() {
+ return true;
+ }
+
+ @Override
+ public DoubleValuesSource rewrite(IndexSearcher reader) throws IOException {
+ return new MultiplicativeBoostValuesSource(boost.rewrite(reader));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ MultiplicativeBoostValuesSource that = (MultiplicativeBoostValuesSource) o;
+ return Objects.equals(boost, that.boost);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(boost);
+ }
+
+ @Override
+ public String toString() {
+ return "boost(" + boost.toString() + ")";
+ }
+
+ @Override
+ public boolean isCacheable(LeafReaderContext ctx) {
+ return boost.isCacheable(ctx);
+ }
+ }
+
+ private static class QueryBoostValuesSource extends DoubleValuesSource {
+
+ private final DoubleValuesSource query;
+ private final float boost;
+
+ QueryBoostValuesSource(DoubleValuesSource query, float boost) {
+ this.query = query;
+ this.boost = boost;
+ }
+
+ @Override
+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ DoubleValues in = query.getValues(ctx, null);
+ return DoubleValues.withDefault(new DoubleValues() {
+ @Override
+ public double doubleValue() {
+ return boost;
+ }
+
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ return in.advanceExact(doc);
+ }
+ }, 1);
+ }
+
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
+
+ @Override
+ public DoubleValuesSource rewrite(IndexSearcher reader) throws IOException {
+ return new QueryBoostValuesSource(query.rewrite(reader), boost);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ QueryBoostValuesSource that = (QueryBoostValuesSource) o;
+ return Float.compare(that.boost, boost) == 0 &&
+ Objects.equals(query, that.query);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(query, boost);
+ }
+
+ @Override
+ public String toString() {
+ return "queryboost(" + query + ")^" + boost;
+ }
+
+ @Override
+ public boolean isCacheable(LeafReaderContext ctx) {
+ return query.isCacheable(ctx);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/79bd05da/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
index c0560bc..4e82eb2 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
@@ -80,18 +80,13 @@ public class TestFunctionScoreQuery extends FunctionTestSetup {
// CustomScoreQuery and BoostedQuery equivalent
public void testScoreModifyingSource() throws Exception {
- SimpleBindings bindings = new SimpleBindings();
- bindings.add("score", DoubleValuesSource.SCORES);
- bindings.add("iii", DoubleValuesSource.fromIntField("iii"));
- Expression expr = JavascriptCompiler.compile("score * iii");
-
BooleanQuery bq = new BooleanQuery.Builder()
.add(new TermQuery(new Term(TEXT_FIELD, "first")), BooleanClause.Occur.SHOULD)
.add(new TermQuery(new Term(TEXT_FIELD, "text")), BooleanClause.Occur.SHOULD)
.build();
TopDocs plain = searcher.search(bq, 1);
- FunctionScoreQuery fq = new FunctionScoreQuery(bq, expr.getDoubleValuesSource(bindings));
+ FunctionScoreQuery fq = FunctionScoreQuery.boostByValue(bq, DoubleValuesSource.fromIntField("iii"));
QueryUtils.check(random(), fq, searcher, rarely());
@@ -108,20 +103,16 @@ public class TestFunctionScoreQuery extends FunctionTestSetup {
// BoostingQuery equivalent
public void testCombiningMultipleQueryScores() throws Exception {
- SimpleBindings bindings = new SimpleBindings();
- bindings.add("score", DoubleValuesSource.SCORES);
- bindings.add("testquery", DoubleValuesSource.fromQuery(new TermQuery(new Term(TEXT_FIELD, "rechecking"))));
- Expression expr = JavascriptCompiler.compile("score + (testquery * 100)");
-
TermQuery q = new TermQuery(new Term(TEXT_FIELD, "text"));
TopDocs plain = searcher.search(q, 1);
- FunctionScoreQuery fq = new FunctionScoreQuery(q, expr.getDoubleValuesSource(bindings));
+ FunctionScoreQuery fq
+ = FunctionScoreQuery.boostByQuery(q, new TermQuery(new Term(TEXT_FIELD, "rechecking")), 100f);
QueryUtils.check(random(), fq, searcher, rarely());
- int[] expectedDocs = new int[]{ 6, 1, 0, 2, 8 };
- TopDocs docs = searcher.search(fq, 5);
+ int[] expectedDocs = new int[]{ 6, 1, 0, 2, 8 };
+ TopDocs docs = searcher.search(fq, 20);
assertEquals(plain.totalHits, docs.totalHits);
for (int i = 0; i < expectedDocs.length; i++) {
assertEquals(expectedDocs[i], docs.scoreDocs[i].doc);
[29/50] [abbrv] lucene-solr:jira/solr-11702: Revert "SOLR-10783: add
(partial) package-info.java to fix precommit"
Posted by da...@apache.org.
Revert "SOLR-10783: add (partial) package-info.java to fix precommit"
This reverts commit a864c6289a8132988fc51cc711db79238ed2ce04.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/144616b4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/144616b4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/144616b4
Branch: refs/heads/jira/solr-11702
Commit: 144616b42469c2d815a657b3c05cbff99ce20387
Parents: bd69d64
Author: Christine Poerschke <cp...@apache.org>
Authored: Mon Jan 8 20:11:25 2018 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Mon Jan 8 20:11:25 2018 +0000
----------------------------------------------------------------------
.../configuration/providers/package-info.java | 23 --------------------
1 file changed, 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/144616b4/solr/core/src/java/org/apache/solr/util/configuration/providers/package-info.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/providers/package-info.java b/solr/core/src/java/org/apache/solr/util/configuration/providers/package-info.java
deleted file mode 100644
index 7b5e8f8..0000000
--- a/solr/core/src/java/org/apache/solr/util/configuration/providers/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * TODO
- */
-package org.apache.solr.util.configuration.providers;
-
-
[44/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11218: Fail and
return an error when attempting to delete a collection that's part of an
alias
Posted by da...@apache.org.
SOLR-11218: Fail and return an error when attempting to delete a collection that's part of an alias
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4471c1b7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4471c1b7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4471c1b7
Branch: refs/heads/jira/solr-11702
Commit: 4471c1b77cbfa9d2cd7f804232655dc56fc859c2
Parents: e538792
Author: Erick Erickson <er...@apache.org>
Authored: Tue Jan 9 17:27:12 2018 -0800
Committer: Erick Erickson <er...@apache.org>
Committed: Tue Jan 9 17:27:12 2018 -0800
----------------------------------------------------------------------
solr/CHANGES.txt | 4 +-
.../apache/solr/cloud/DeleteCollectionCmd.java | 13 +-
.../apache/solr/cloud/AliasIntegrationTest.java | 160 ++++++++++++++++++-
.../client/solrj/impl/CloudSolrClientTest.java | 23 ---
4 files changed, 172 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4471c1b7/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 14f80a3..4ba50b9 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -95,7 +95,7 @@ Bug Fixes
* SOLR-11821: ConcurrentModificationException in SimSolrCloudTestCase.tearDown (shalin)
* SOLR-11631: The Schema API should return non-zero status when there are failures.
- (Noble Paul, Steve Rowe)
+ (Noble Paul, Steve Rowe)
Optimizations
----------------------
@@ -134,6 +134,8 @@ Other Changes
* SOLR-11692: SolrDispatchFilter's use of a "close shield" in tests should not be applied to
further servlet chain processing. (Jeff Miller, David Smiley)
+* SOLR-11218: Fail and return an error when attempting to delete a collection that's part of an alias (Erick Erickson)
+
================== 7.2.1 ==================
Bug Fixes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4471c1b7/solr/core/src/java/org/apache/solr/cloud/DeleteCollectionCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/DeleteCollectionCmd.java
index dc91905..8ee0168 100644
--- a/solr/core/src/java/org/apache/solr/cloud/DeleteCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/DeleteCollectionCmd.java
@@ -21,14 +21,15 @@ package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.solr.common.NonExistentCoreException;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -60,9 +61,15 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
@Override
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
ZkStateReader zkStateReader = ocmh.zkStateReader;
+ Aliases aliases = zkStateReader.getAliases();
final String collection = message.getStr(NAME);
- DocCollection coll = state.getCollectionOrNull(collection);
- String policy = coll == null ? null : coll.getPolicyName();
+ for (Map.Entry<String, List<String>> ent : aliases.getCollectionAliasListMap().entrySet()) {
+ if (ent.getValue().contains(collection)) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Collection : " + collection + " is part of alias " + ent.getKey() + " remove or modify the alias before removing this collection.");
+ }
+ }
+
try {
// Remove the snapshots meta-data for this collection in ZK. Deleting actual index files
// should be taken care of as part of collection delete operation.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4471c1b7/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
index d0f0e80..1c20b30 100644
--- a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
@@ -32,6 +32,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -176,7 +177,163 @@ public class AliasIntegrationTest extends SolrCloudTestCase {
}
}
}
-
+
+ // Rather a long title, but it's common to recommend when people need to re-index for any reason that they:
+ // 1> create a new collection
+ // 2> index the corpus to the new collection and verify it
+ // 3> create an alias pointing to the new collection WITH THE SAME NAME as their original collection
+ // 4> delete the old collection.
+ //
+ // They may or may not have an alias already pointing to the old collection that's being replaced.
+ // If they don't already have an alias, this leaves them with:
+ //
+ // > a collection named old_collection
+ // > a collection named new_collection
+ // > an alias old_collection->new_collection
+ //
+ // What happens when they delete old_collection now?
+ //
+ // Current behavior is that delete "does the right thing" and deletes old_collection rather than new_collection,
+ // but if this behavior changes it could be disastrous for users so this test insures that this behavior.
+ //
+ @Test
+ public void testDeleteAliasWithExistingCollectionName() throws Exception {
+ CollectionAdminRequest.createCollection("collection_old", "conf", 2, 1).process(cluster.getSolrClient());
+ CollectionAdminRequest.createCollection("collection_new", "conf", 1, 1).process(cluster.getSolrClient());
+ waitForState("Expected collection_old to be created with 2 shards and 1 replica", "collection_old", clusterShape(2, 1));
+ waitForState("Expected collection_new to be created with 1 shard and 1 replica", "collection_new", clusterShape(1, 1));
+
+ new UpdateRequest()
+ .add("id", "6", "a_t", "humpty dumpy sat on a wall")
+ .add("id", "7", "a_t", "humpty dumpy3 sat on a walls")
+ .add("id", "8", "a_t", "humpty dumpy2 sat on a walled")
+ .commit(cluster.getSolrClient(), "collection_old");
+
+ new UpdateRequest()
+ .add("id", "1", "a_t", "humpty dumpy sat on an unfortunate wall")
+ .commit(cluster.getSolrClient(), "collection_new");
+
+ QueryResponse res = cluster.getSolrClient().query("collection_old", new SolrQuery("*:*"));
+ assertEquals(3, res.getResults().getNumFound());
+
+ // Let's insure we have a "handle" to the old collection
+ CollectionAdminRequest.createAlias("collection_old_reserve", "collection_old").process(cluster.getSolrClient());
+
+ // This is the critical bit. The alias uses the _old collection name.
+ CollectionAdminRequest.createAlias("collection_old", "collection_new").process(cluster.getSolrClient());
+
+ // aliases: collection_old->collection_new, collection_old_reserve -> collection_old -> collection_new
+ // collections: collection_new and collection_old
+
+ // Now we should only see the doc in collection_new through the collection_old alias
+ res = cluster.getSolrClient().query("collection_old", new SolrQuery("*:*"));
+ assertEquals(1, res.getResults().getNumFound());
+
+ // Now we should still transitively see collection_new
+ res = cluster.getSolrClient().query("collection_old_reserve", new SolrQuery("*:*"));
+ assertEquals(1, res.getResults().getNumFound());
+
+ // Now delete the old collection. This should fail since the collection_old_reserve points to collection_old
+ RequestStatusState delResp = CollectionAdminRequest.deleteCollection("collection_old").processAndWait(cluster.getSolrClient(), 60);
+ assertEquals("Should have failed to delete collection: ", delResp, RequestStatusState.FAILED);
+
+ // assure ourselves that the old colletion is, indeed, still there.
+ assertNotNull("collection_old should exist!", cluster.getSolrClient().getZkStateReader().getClusterState().getCollectionOrNull("collection_old"));
+
+ // Now we should still succeed using the alias collection_old which points to collection_new
+ // aliase: collection_old -> collection_new, collection_old_reserve -> collection_old -> collection_new
+ // collections: collection_old, collection_new
+ res = cluster.getSolrClient().query("collection_old", new SolrQuery("*:*"));
+ assertEquals(1, res.getResults().getNumFound());
+
+ Aliases aliases = cluster.getSolrClient().getZkStateReader().getAliases();
+ assertTrue("collection_old should point to collection_new", aliases.resolveAliases("collection_old").contains("collection_new"));
+ assertTrue("collection_old_reserve should point to collection_new", aliases.resolveAliases("collection_old_reserve").contains("collection_new"));
+
+ // Clean up
+ CollectionAdminRequest.deleteAlias("collection_old_reserve").processAndWait(cluster.getSolrClient(), 60);
+ CollectionAdminRequest.deleteAlias("collection_old").processAndWait(cluster.getSolrClient(), 60);
+ CollectionAdminRequest.deleteCollection("collection_new").processAndWait(cluster.getSolrClient(), 60);
+ CollectionAdminRequest.deleteCollection("collection_old").processAndWait(cluster.getSolrClient(), 60);
+ // collection_old already deleted as well as collection_old_reserve
+
+ assertNull("collection_old_reserve should be gone", cluster.getSolrClient().getZkStateReader().getAliases().getCollectionAliasMap().get("collection_old_reserve"));
+ assertNull("collection_old should be gone", cluster.getSolrClient().getZkStateReader().getAliases().getCollectionAliasMap().get("collection_old"));
+
+ assertFalse("collection_new should be gone",
+ cluster.getSolrClient().getZkStateReader().getClusterState().hasCollection("collection_new"));
+
+ assertFalse("collection_old should be gone",
+ cluster.getSolrClient().getZkStateReader().getClusterState().hasCollection("collection_old"));
+ }
+
+ // While writing the above test I wondered what happens when an alias points to two collections and one of them
+ // is deleted.
+ @Test
+ public void testDeleteOneOfTwoCollectionsAliased() throws Exception {
+ CollectionAdminRequest.createCollection("collection_one", "conf", 2, 1).process(cluster.getSolrClient());
+ CollectionAdminRequest.createCollection("collection_two", "conf", 1, 1).process(cluster.getSolrClient());
+ waitForState("Expected collection_one to be created with 2 shards and 1 replica", "collection_one", clusterShape(2, 1));
+ waitForState("Expected collection_two to be created with 1 shard and 1 replica", "collection_two", clusterShape(1, 1));
+
+ new UpdateRequest()
+ .add("id", "1", "a_t", "humpty dumpy sat on a wall")
+ .commit(cluster.getSolrClient(), "collection_one");
+
+
+ new UpdateRequest()
+ .add("id", "10", "a_t", "humpty dumpy sat on a high wall")
+ .add("id", "11", "a_t", "humpty dumpy sat on a low wall")
+ .commit(cluster.getSolrClient(), "collection_two");
+
+ // Create an alias pointing to both
+ CollectionAdminRequest.createAlias("collection_alias_pair", "collection_one,collection_two").process(cluster.getSolrClient());
+
+ QueryResponse res = cluster.getSolrClient().query("collection_alias_pair", new SolrQuery("*:*"));
+ assertEquals(3, res.getResults().getNumFound());
+
+ // Now delete one of the collections, should fail since an alias points to it.
+ RequestStatusState delResp = CollectionAdminRequest.deleteCollection("collection_one").processAndWait(cluster.getSolrClient(), 60);
+
+ assertEquals("Should have failed to delete collection: ", delResp, RequestStatusState.FAILED);
+
+ // Now redefine the alias to only point to colletion two
+ CollectionAdminRequest.createAlias("collection_alias_pair", "collection_two").process(cluster.getSolrClient());
+
+ //Delete collection_one.
+ delResp = CollectionAdminRequest.deleteCollection("collection_one").processAndWait(cluster.getSolrClient(), 60);
+
+ assertEquals("Should not have failed to delete collection, it was removed from the alias: ", delResp, RequestStatusState.COMPLETED);
+
+ // Should only see two docs now in second collection
+ res = cluster.getSolrClient().query("collection_alias_pair", new SolrQuery("*:*"));
+ assertEquals(2, res.getResults().getNumFound());
+
+ // We shouldn't be able to ping the deleted collection directly as
+ // was deleted (and, assuming that it only points to collection_old).
+ try {
+ cluster.getSolrClient().query("collection_one", new SolrQuery("*:*"));
+ } catch (SolrServerException se) {
+ assertTrue(se.getMessage().contains("No live SolrServers"));
+ }
+
+ // Clean up
+ CollectionAdminRequest.deleteAlias("collection_alias_pair").processAndWait(cluster.getSolrClient(), 60);
+ CollectionAdminRequest.deleteCollection("collection_two").processAndWait(cluster.getSolrClient(), 60);
+ // collection_one already deleted
+
+ assertNull("collection_alias_pair should be gone",
+ cluster.getSolrClient().getZkStateReader().getAliases().getCollectionAliasMap().get("collection_alias_pair"));
+
+ assertFalse("collection_one should be gone",
+ cluster.getSolrClient().getZkStateReader().getClusterState().hasCollection("collection_one"));
+
+ assertFalse("collection_two should be gone",
+ cluster.getSolrClient().getZkStateReader().getClusterState().hasCollection("collection_two"));
+
+ }
+
+
@Test
public void test() throws Exception {
CollectionAdminRequest.createCollection("collection1", "conf", 2, 1).process(cluster.getSolrClient());
@@ -332,6 +489,7 @@ public class AliasIntegrationTest extends SolrCloudTestCase {
}
}
+ @Test
public void testErrorChecks() throws Exception {
CollectionAdminRequest.createCollection("testErrorChecks-collection", "conf", 2, 1).process(cluster.getSolrClient());
waitForState("Expected testErrorChecks-collection to be created with 2 shards and 1 replica", "testErrorChecks-collection", clusterShape(2, 1));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4471c1b7/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
index c0580ed..9539846 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
@@ -220,29 +220,6 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
}
@Test
- public void testHandlingOfStaleAlias() throws Exception {
- CloudSolrClient client = getRandomClient();
-
- CollectionAdminRequest.createCollection("nemesis", "conf", 2, 1).process(client);
- CollectionAdminRequest.createAlias("misconfigured-alias", "nemesis").process(client);
- CollectionAdminRequest.deleteCollection("nemesis").process(client);
-
- List<SolrInputDocument> docs = new ArrayList<>();
-
- SolrInputDocument doc = new SolrInputDocument();
- doc.addField(id, Integer.toString(1));
- docs.add(doc);
-
- try {
- client.add("misconfigured-alias", docs);
- fail("Alias points to non-existing collection, add should fail");
- } catch (SolrException e) {
- assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code());
- assertTrue("Unexpected exception", e.getMessage().contains("Collection not found"));
- }
- }
-
- @Test
public void testRouting() throws Exception {
AbstractUpdateRequest request = new UpdateRequest()
[03/50] [abbrv] lucene-solr:jira/solr-11702: LUCENE-8116: SimScorer
now only takes a frequency and a norm as per-document scoring factors.
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/test-framework/src/java/org/apache/lucene/search/similarities/BaseSimilarityTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/similarities/BaseSimilarityTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/BaseSimilarityTestCase.java
index 91e64c0..348584e 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/similarities/BaseSimilarityTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/BaseSimilarityTestCase.java
@@ -17,24 +17,18 @@
package org.apache.lucene.search.similarities;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Random;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.CheckHits;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.Similarity.SimScorer;
-import org.apache.lucene.search.similarities.Similarity.SimWeight;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -54,119 +48,28 @@ import org.junit.BeforeClass;
* test fails to catch then this test needs to be improved! */
public abstract class BaseSimilarityTestCase extends LuceneTestCase {
- static LeafReader WITHOUT_NORM;
- static Directory WITHOUT_NORM_DIR;
-
- static LeafReader WITH_NORM_BASE;
- static Directory WITH_NORM_DIR;
- static List<LeafReader> NORM_VALUES;
+ static LeafReader READER;
+ static Directory DIR;
@BeforeClass
public static void beforeClass() throws Exception {
- // without norms
- WITHOUT_NORM_DIR = newDirectory();
- RandomIndexWriter writer = new RandomIndexWriter(random(), WITHOUT_NORM_DIR);
- Document doc = new Document();
- doc.add(newTextField("field", "value", Field.Store.NO));
- writer.addDocument(doc);
- WITHOUT_NORM = getOnlyLeafReader(writer.getReader());
- writer.close();
-
// with norms
- WITH_NORM_DIR = newDirectory();
- writer = new RandomIndexWriter(random(), WITH_NORM_DIR);
- doc = new Document();
+ DIR = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), DIR);
+ Document doc = new Document();
FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
fieldType.setOmitNorms(true);
doc.add(newField("field", "value", fieldType));
writer.addDocument(doc);
- WITH_NORM_BASE = getOnlyLeafReader(writer.getReader());
+ READER = getOnlyLeafReader(writer.getReader());
writer.close();
-
- // all possible norm values for the doc
- NORM_VALUES = new ArrayList<>();
- NORM_VALUES.add(WITHOUT_NORM);
- for (int i = 1; i < 256; i++) {
- final long value = i;
- NORM_VALUES.add(new FilterLeafReader(WITH_NORM_BASE) {
- @Override
- public CacheHelper getCoreCacheHelper() {
- return null;
- }
-
- @Override
- public CacheHelper getReaderCacheHelper() {
- return null;
- }
-
- @Override
- public NumericDocValues getNormValues(String field) throws IOException {
- if (field.equals("field")) {
- return new CannedNorm(value);
- } else {
- return super.getNormValues(field);
- }
- }
- });
- }
}
@AfterClass
public static void afterClass() throws Exception {
- IOUtils.close(WITH_NORM_BASE, WITH_NORM_DIR, WITHOUT_NORM, WITHOUT_NORM_DIR);
- WITH_NORM_BASE = WITHOUT_NORM = null;
- WITH_NORM_DIR = WITHOUT_NORM_DIR = null;
- NORM_VALUES = null;
- }
-
- /** 1-document norms impl of the given value */
- static class CannedNorm extends NumericDocValues {
- int docID = -1;
- final long value;
-
- CannedNorm(long value) {
- this.value = value;
- }
-
- @Override
- public long longValue() throws IOException {
- return value;
- }
-
- @Override
- public boolean advanceExact(int target) throws IOException {
- assert target == 0;
- docID = target;
- return true;
- }
-
- @Override
- public int docID() {
- return docID;
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (docID == -1) {
- return docID = 0;
- } else {
- return docID = NO_MORE_DOCS;
- }
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target == 0) {
- return docID = 0;
- } else {
- return docID = NO_MORE_DOCS;
- }
- }
-
- @Override
- public long cost() {
- return 0;
- }
+ IOUtils.close(READER, DIR);
+ READER = null;
+ DIR = null;
}
/**
@@ -354,7 +257,7 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {
Similarity similarity = getSimilarity(random);
for (int j = 0; j < 10; j++) {
// for each norm value...
- for (int k = 0; k < NORM_VALUES.size(); k++) {
+ for (int k = 1; k < 256; k++) {
CollectionStatistics corpus = newCorpus(random, k);
for (int l = 0; l < 10; l++) {
TermStatistics term = newTerm(random, corpus);
@@ -441,17 +344,16 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {
/** runs for a single test case, so that if you hit a test failure you can write a reproducer just for that scenario */
private static void doTestScoring(Similarity similarity, CollectionStatistics corpus, TermStatistics term, float boost, float freq, int norm) throws IOException {
boolean success = false;
- SimWeight weight = similarity.computeWeight(boost, corpus, term);
- SimScorer scorer = similarity.simScorer(weight, NORM_VALUES.get(norm).getContext());
+ SimScorer scorer = similarity.scorer(boost, corpus, term);
try {
- float score = scorer.score(0, freq);
+ float score = scorer.score(freq, norm);
// check that score isn't infinite or negative
assertTrue("infinite/NaN score: " + score, Float.isFinite(score));
assertTrue("negative score: " + score, score >= 0);
float maxScore = scorer.maxScore(freq);
assertTrue("score > maxScore: " + score + " > " + maxScore, score <= maxScore);
// check explanation matches
- Explanation explanation = scorer.explain(0, Explanation.match(freq, "freq, occurrences of term within document"));
+ Explanation explanation = scorer.explain(Explanation.match(freq, "freq, occurrences of term within document"), norm);
if (score != explanation.getValue().doubleValue()) {
fail("expected: " + score + ", got: " + explanation);
}
@@ -467,12 +369,12 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {
prevFreq = Math.nextDown(freq);
}
- float prevScore = scorer.score(0, prevFreq);
+ float prevScore = scorer.score(prevFreq, norm);
// check that score isn't infinite or negative
assertTrue(Float.isFinite(prevScore));
assertTrue(prevScore >= 0);
// check explanation matches
- Explanation prevExplanation = scorer.explain(0, Explanation.match(prevFreq, "freq, occurrences of term within document"));
+ Explanation prevExplanation = scorer.explain(Explanation.match(prevFreq, "freq, occurrences of term within document"), norm);
if (prevScore != prevExplanation.getValue().doubleValue()) {
fail("expected: " + prevScore + ", got: " + prevExplanation);
}
@@ -486,13 +388,12 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {
// check score(norm-1), given the same freq it should be >= score(norm) [scores non-decreasing as docs get shorter]
if (norm > 1) {
- SimScorer prevNormScorer = similarity.simScorer(weight, NORM_VALUES.get(norm - 1).getContext());
- float prevNormScore = prevNormScorer.score(0, freq);
+ float prevNormScore = scorer.score(freq, norm - 1);
// check that score isn't infinite or negative
assertTrue(Float.isFinite(prevNormScore));
assertTrue(prevNormScore >= 0);
// check explanation matches
- Explanation prevNormExplanation = prevNormScorer.explain(0, Explanation.match(freq, "freq, occurrences of term within document"));
+ Explanation prevNormExplanation = scorer.explain(Explanation.match(freq, "freq, occurrences of term within document"), norm - 1);
if (prevNormScore != prevNormExplanation.getValue().doubleValue()) {
fail("expected: " + prevNormScore + ", got: " + prevNormExplanation);
}
@@ -508,14 +409,13 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {
// check score(term-1), given the same freq/norm it should be >= score(term) [scores non-decreasing as terms get rarer]
if (term.docFreq() > 1 && freq < term.totalTermFreq()) {
TermStatistics prevTerm = new TermStatistics(term.term(), term.docFreq() - 1, term.totalTermFreq() - 1);
- SimWeight prevWeight = similarity.computeWeight(boost, corpus, term);
- SimScorer prevTermScorer = similarity.simScorer(prevWeight, NORM_VALUES.get(norm).getContext());
- float prevTermScore = prevTermScorer.score(0, freq);
+ SimScorer prevTermScorer = similarity.scorer(boost, corpus, term);
+ float prevTermScore = prevTermScorer.score(freq, norm);
// check that score isn't infinite or negative
assertTrue(Float.isFinite(prevTermScore));
assertTrue(prevTermScore >= 0);
// check explanation matches
- Explanation prevTermExplanation = prevTermScorer.explain(0, Explanation.match(freq, "freq, occurrences of term within document"));
+ Explanation prevTermExplanation = prevTermScorer.explain(Explanation.match(freq, "freq, occurrences of term within document"), norm);
if (prevTermScore != prevTermExplanation.getValue().doubleValue()) {
fail("expected: " + prevTermScore + ", got: " + prevTermExplanation);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpanWeight.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpanWeight.java b/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpanWeight.java
index a9956b3..65f9599 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpanWeight.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpanWeight.java
@@ -25,7 +25,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.LeafSimScorer;
/**
* Wraps a SpanWeight with additional asserts
@@ -58,7 +58,7 @@ public class AssertingSpanWeight extends SpanWeight {
}
@Override
- public Similarity.SimScorer getSimScorer(LeafReaderContext context) throws IOException {
+ public LeafSimScorer getSimScorer(LeafReaderContext context) throws IOException {
return in.getSimScorer(context);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8fd7ead9/solr/core/src/test/org/apache/solr/search/similarities/BaseSimilarityTestCase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/similarities/BaseSimilarityTestCase.java b/solr/core/src/test/org/apache/solr/search/similarities/BaseSimilarityTestCase.java
index d782c2f..6ce4331 100644
--- a/solr/core/src/test/org/apache/solr/search/similarities/BaseSimilarityTestCase.java
+++ b/solr/core/src/test/org/apache/solr/search/similarities/BaseSimilarityTestCase.java
@@ -29,7 +29,7 @@ public abstract class BaseSimilarityTestCase extends SolrTestCaseJ4 {
protected Similarity getSimilarity(String field) {
SolrCore core = h.getCore();
RefCounted<SolrIndexSearcher> searcher = core.getSearcher();
- Similarity sim = searcher.get().getSimilarity(true);
+ Similarity sim = searcher.get().getSimilarity();
searcher.decref();
while (sim instanceof PerFieldSimilarityWrapper) {
sim = ((PerFieldSimilarityWrapper)sim).get(field);
[14/50] [abbrv] lucene-solr:jira/solr-11702: SOLR-11653:
TimeRoutedAlias URP now auto-creates collections using new
RoutedAliasCreateCollectionCmd
Posted by da...@apache.org.
SOLR-11653: TimeRoutedAlias URP now auto-creates collections using new RoutedAliasCreateCollectionCmd
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/925733d1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/925733d1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/925733d1
Branch: refs/heads/jira/solr-11702
Commit: 925733d1ef3ac6fbabc450804511c65a4c6424ac
Parents: 3980aea
Author: David Smiley <ds...@apache.org>
Authored: Fri Jan 5 13:53:26 2018 -0500
Committer: David Smiley <ds...@apache.org>
Committed: Fri Jan 5 13:53:26 2018 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 4 +
.../java/org/apache/solr/cloud/Overseer.java | 4 +
.../cloud/OverseerCollectionMessageHandler.java | 3 +-
.../solr/cloud/OverseerTaskProcessor.java | 7 +-
.../cloud/RoutedAliasCreateCollectionCmd.java | 182 +++++++++++++++
.../solr/handler/admin/CollectionsHandler.java | 16 +-
.../apache/solr/request/SolrRequestInfo.java | 12 +-
.../TimeRoutedAliasUpdateProcessor.java | 228 +++++++++++++++++--
.../org/apache/solr/util/TimeZoneUtils.java | 18 ++
.../TimeRoutedAliasUpdateProcessorTest.java | 140 +++++++++---
.../solr/common/params/CollectionParams.java | 1 +
11 files changed, 539 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925733d1/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 61551e0..221d6ad 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -74,6 +74,10 @@ New Features
* SOLR-11201: Implement autoscaling trigger for arbitrary metrics that creates events when
a given metric breaches a threshold (shalin)
+* SOLR-11653: TimeRoutedAlias URP now auto-creates new collections on the fly according to alias metadata
+ rules that sets the time interval for each collection. An internal Overseer command "ROUTEDALIAS_CREATECOLL"
+ was created to facilitate this. (David Smiley)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925733d1/solr/core/src/java/org/apache/solr/cloud/Overseer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index d1bb13a..3b9dd28 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -39,6 +39,7 @@ import org.apache.solr.cloud.overseer.SliceMutator;
import org.apache.solr.cloud.overseer.ZkStateWriter;
import org.apache.solr.cloud.overseer.ZkWriteCommand;
import org.apache.solr.common.SolrCloseable;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -267,6 +268,9 @@ public class Overseer implements SolrCloseable {
private ClusterState processQueueItem(ZkNodeProps message, ClusterState clusterState, ZkStateWriter zkStateWriter, boolean enableBatching, ZkStateWriter.ZkWriteCallback callback) throws Exception {
final String operation = message.getStr(QUEUE_OPERATION);
+ if (operation == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Message missing " + QUEUE_OPERATION + ":" + message);
+ }
List<ZkWriteCommand> zkWriteCommands = null;
final Timer.Context timerContext = stats.time(operation);
try {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925733d1/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
index abfecab..426c879 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
@@ -219,6 +219,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
.put(DELETE, new DeleteCollectionCmd(this))
.put(CREATEALIAS, new CreateAliasCmd(this))
.put(DELETEALIAS, new DeleteAliasCmd(this))
+ .put(ROUTEDALIAS_CREATECOLL, new RoutedAliasCreateCollectionCmd(this))
.put(OVERSEERSTATUS, new OverseerStatusCmd(this))
.put(DELETESHARD, new DeleteShardCmd(this))
.put(DELETEREPLICA, new DeleteReplicaCmd(this))
@@ -232,7 +233,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
@Override
@SuppressWarnings("unchecked")
public SolrResponse processMessage(ZkNodeProps message, String operation) {
- log.debug("OverseerCollectionMessageHandler.processMessage : "+ operation + " , "+ message.toString());
+ log.debug("OverseerCollectionMessageHandler.processMessage : {} , {}", operation, message);
NamedList results = new NamedList();
try {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925733d1/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
index d014fc4..86e3564 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
@@ -257,7 +257,6 @@ public class OverseerTaskProcessor implements Runnable, Closeable {
}
if (runningZKTasks.contains(head.getId())) continue;
final ZkNodeProps message = ZkNodeProps.load(head.getBytes());
- OverseerMessageHandler messageHandler = selector.selectOverseerMessageHandler(message);
final String asyncId = message.getStr(ASYNC);
if (hasLeftOverItems) {
if (head.getId().equals(oldestItemInWorkQueue))
@@ -269,6 +268,12 @@ public class OverseerTaskProcessor implements Runnable, Closeable {
}
}
String operation = message.getStr(Overseer.QUEUE_OPERATION);
+ if (operation == null) {
+ log.error("Msg does not have required " + Overseer.QUEUE_OPERATION + ": {}", message);
+ workQueue.remove(head);
+ continue;
+ }
+ OverseerMessageHandler messageHandler = selector.selectOverseerMessageHandler(message);
OverseerMessageHandler.Lock lock = messageHandler.lockTask(message, taskBatch);
if (lock == null) {
log.debug("Exclusivity check failed for [{}]", message.toString());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925733d1/solr/core/src/java/org/apache/solr/cloud/RoutedAliasCreateCollectionCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/RoutedAliasCreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/RoutedAliasCreateCollectionCmd.java
new file mode 100644
index 0000000..607588c
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/RoutedAliasCreateCollectionCmd.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.lang.invoke.MethodHandles;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.Aliases;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.handler.admin.CollectionsHandler;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor;
+import org.apache.solr.util.TimeZoneUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF;
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor.ROUTER_FIELD_METADATA;
+import static org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor.ROUTER_INTERVAL_METADATA;
+
+/**
+ * For "routed aliases", creates another collection and adds it to the alias. In some cases it will not
+ * add a new collection.
+ * If a collection is created, then collection creation info is returned.
+ *
+ * Note: this logic is within an Overseer because we want to leverage the mutual exclusion
+ * property afforded by the lock it obtains on the alias name.
+ * @since 7.3
+ */
+public class RoutedAliasCreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final String IF_MOST_RECENT_COLL_NAME = "ifMostRecentCollName";
+
+ public static final String COLL_METAPREFIX = "collection-create.";
+
+ private final OverseerCollectionMessageHandler ocmh;
+
+ public RoutedAliasCreateCollectionCmd(OverseerCollectionMessageHandler ocmh) {
+ this.ocmh = ocmh;
+ }
+
+ /* TODO:
+ There are a few classes related to time routed alias processing. We need to share some logic better.
+ */
+
+
+ @Override
+ public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception {
+ //---- PARSE PRIMARY MESSAGE PARAMS
+ // important that we use NAME for the alias as that is what the Overseer will get a lock on before calling us
+ final String aliasName = message.getStr(NAME);
+ // the client believes this is the mostRecent collection name. We assert this if provided.
+ final String ifMostRecentCollName = message.getStr(IF_MOST_RECENT_COLL_NAME); // optional
+
+ // TODO collection param (or intervalDateMath override?), useful for data capped collections
+
+ //---- PARSE ALIAS INFO FROM ZK
+ final ZkStateReader.AliasesManager aliasesHolder = ocmh.zkStateReader.aliasesHolder;
+ final Aliases aliases = aliasesHolder.getAliases();
+ final Map<String, String> aliasMetadata = aliases.getCollectionAliasMetadata(aliasName);
+ if (aliasMetadata == null) {
+ throw newAliasMustExistException(aliasName); // if it did exist, we'd have a non-null map
+ }
+
+ String routeField = aliasMetadata.get(ROUTER_FIELD_METADATA);
+ if (routeField == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "This command only works on time routed aliases. Expected alias metadata not found.");
+ }
+ String intervalDateMath = aliasMetadata.getOrDefault(ROUTER_INTERVAL_METADATA, "+1DAY");
+ TimeZone intervalTimeZone = TimeZoneUtils.parseTimezone(aliasMetadata.get(CommonParams.TZ));
+
+ //TODO this is ugly; how can we organize the code related to this feature better?
+ final List<Map.Entry<Instant, String>> parsedCollections =
+ TimeRoutedAliasUpdateProcessor.parseCollections(aliasName, aliases, () -> newAliasMustExistException(aliasName));
+
+ //---- GET MOST RECENT COLL
+ final Map.Entry<Instant, String> mostRecentEntry = parsedCollections.get(0);
+ final Instant mostRecentCollTimestamp = mostRecentEntry.getKey();
+ final String mostRecentCollName = mostRecentEntry.getValue();
+ if (ifMostRecentCollName != null) {
+ if (!mostRecentCollName.equals(ifMostRecentCollName)) {
+ // Possibly due to race conditions in URPs on multiple leaders calling us at the same time
+ String msg = IF_MOST_RECENT_COLL_NAME + " expected " + ifMostRecentCollName + " but it's " + mostRecentCollName;
+ if (parsedCollections.stream().map(Map.Entry::getValue).noneMatch(ifMostRecentCollName::equals)) {
+ msg += ". Furthermore this collection isn't in the list of collections referenced by the alias.";
+ }
+ log.info(msg);
+ results.add("message", msg);
+ return;
+ }
+ } else if (mostRecentCollTimestamp.isAfter(Instant.now())) {
+ final String msg = "Most recent collection is in the future, so we won't create another.";
+ log.info(msg);
+ results.add("message", msg);
+ return;
+ }
+
+ //---- COMPUTE NEXT COLLECTION NAME
+ final Instant nextCollTimestamp = TimeRoutedAliasUpdateProcessor.computeNextCollTimestamp(mostRecentCollTimestamp, intervalDateMath, intervalTimeZone);
+ assert nextCollTimestamp.isAfter(mostRecentCollTimestamp);
+ final String createCollName = TimeRoutedAliasUpdateProcessor.formatCollectionNameFromInstant(aliasName, nextCollTimestamp);
+
+ //---- CREATE THE COLLECTION
+ // Map alias metadata starting with a prefix to a create-collection API request
+ final ModifiableSolrParams createReqParams = new ModifiableSolrParams();
+ for (Map.Entry<String, String> e : aliasMetadata.entrySet()) {
+ if (e.getKey().startsWith(COLL_METAPREFIX)) {
+ createReqParams.set(e.getKey().substring(COLL_METAPREFIX.length()), e.getValue());
+ }
+ }
+ if (createReqParams.get(COLL_CONF) == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "We require an explicit " + COLL_CONF );
+ }
+ createReqParams.set(NAME, createCollName);
+ createReqParams.set("property." + TimeRoutedAliasUpdateProcessor.TIME_PARTITION_ALIAS_NAME_CORE_PROP, aliasName);
+ // a CollectionOperation reads params and produces a message (Map) that is supposed to be sent to the Overseer.
+ // Although we could create the Map without it, there are a fair amount of rules we don't want to reproduce.
+ final Map<String, Object> createMsgMap = CollectionsHandler.CollectionOperation.CREATE_OP.execute(
+ new LocalSolrQueryRequest(null, createReqParams),
+ null,
+ ocmh.overseer.getCoreContainer().getCollectionsHandler());
+ createMsgMap.put(Overseer.QUEUE_OPERATION, "create");
+ // Since we are running in the Overseer here, send the message directly to the Overseer CreateCollectionCmd
+ ocmh.commandMap.get(CollectionParams.CollectionAction.CREATE).call(clusterState, new ZkNodeProps(createMsgMap), results);
+
+ CollectionsHandler.waitForActiveCollection(createCollName, null, ocmh.overseer.getCoreContainer(), new OverseerSolrResponse(results));
+
+ //TODO delete some of the oldest collection(s) ?
+
+ //---- UPDATE THE ALIAS
+ aliasesHolder.applyModificationAndExportToZk(curAliases -> {
+ final List<String> curTargetCollections = curAliases.getCollectionAliasListMap().get(aliasName);
+ if (curTargetCollections.contains(createCollName)) {
+ return curAliases;
+ } else {
+ List<String> newTargetCollections = new ArrayList<>(curTargetCollections.size() + 1);
+ // prepend it on purpose (thus reverse sorted). Solr alias resolution defaults to the first collection in a list
+ newTargetCollections.add(createCollName);
+ newTargetCollections.addAll(curTargetCollections);
+ return curAliases.cloneWithCollectionAlias(aliasName, StrUtils.join(newTargetCollections, ','));
+ }
+ });
+
+ }
+
+ private SolrException newAliasMustExistException(String aliasName) {
+ return new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Alias " + aliasName + " does not exist.");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925733d1/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index d339f27..74d4764 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -260,16 +260,19 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
public static long DEFAULT_COLLECTION_OP_TIMEOUT = 180*1000;
- void handleResponse(String operation, ZkNodeProps m,
+ //TODO rename to submitToOverseerRPC
+ public void handleResponse(String operation, ZkNodeProps m,
SolrQueryResponse rsp) throws KeeperException, InterruptedException {
handleResponse(operation, m, rsp, DEFAULT_COLLECTION_OP_TIMEOUT);
}
- private SolrResponse handleResponse(String operation, ZkNodeProps m,
+ //TODO rename to submitToOverseerRPC
+ public SolrResponse handleResponse(String operation, ZkNodeProps m,
SolrQueryResponse rsp, long timeout) throws KeeperException, InterruptedException {
- long time = System.nanoTime();
-
- if (m.containsKey(ASYNC) && m.get(ASYNC) != null) {
+ if (!m.containsKey(QUEUE_OPERATION)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "missing key " + QUEUE_OPERATION);
+ }
+ if (m.get(ASYNC) != null) {
String asyncId = m.getStr(ASYNC);
@@ -297,6 +300,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
return response;
}
+ long time = System.nanoTime();
QueueEvent event = coreContainer.getZkController()
.getOverseerCollectionQueue()
.offer(Utils.toJSON(m), timeout);
@@ -1031,7 +1035,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
}
}
- private static void waitForActiveCollection(String collectionName, ZkNodeProps message, CoreContainer cc, SolrResponse response)
+ public static void waitForActiveCollection(String collectionName, ZkNodeProps message, CoreContainer cc, SolrResponse response)
throws KeeperException, InterruptedException {
if (response.getResponse().get("exception") != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925733d1/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java b/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java
index f759c91..f1a718d 100644
--- a/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java
+++ b/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java
@@ -101,17 +101,9 @@ public class SolrRequestInfo {
}
/** The TimeZone specified by the request, or null if none was specified */
- public TimeZone getClientTimeZone() {
-
+ public TimeZone getClientTimeZone() {
if (tz == null) {
- String tzStr = req.getParams().get(CommonParams.TZ);
- if (tzStr != null) {
- tz = TimeZoneUtils.getTimeZone(tzStr);
- if (null == tz) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Solr JVM does not support TZ: " + tzStr);
- }
- }
+ tz = TimeZoneUtils.parseTimezone(req.getParams().get(CommonParams.TZ));
}
return tz;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925733d1/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
index 9148912..bc242ba 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
@@ -19,6 +19,7 @@ package org.apache.solr.update.processor;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.text.ParseException;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
@@ -29,22 +30,34 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.TimeZone;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
+import org.apache.solr.cloud.Overseer;
+import org.apache.solr.cloud.RoutedAliasCreateCollectionCmd;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
@@ -52,14 +65,18 @@ import org.apache.solr.update.CommitUpdateCommand;
import org.apache.solr.update.DeleteUpdateCommand;
import org.apache.solr.update.SolrCmdDistributor;
import org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase;
+import org.apache.solr.util.DateMathParser;
+import org.apache.solr.util.TimeZoneUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT;
import static org.apache.solr.update.processor.DistributedUpdateProcessor.DISTRIB_FROM;
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
/**
- * Distributes update requests to rolling series of collections partitioned by a timestamp field.
+ * Distributes update requests to a rolling series of collections partitioned by a timestamp field. Issues
+ * requests to create new collections on-demand.
*
* Depends on this core having a special core property that points to the alias name that this collection is a part of.
* And further requires certain metadata on the Alias.
@@ -69,16 +86,15 @@ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactor
public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
//TODO do we make this more generic to others who want to partition collections using something else?
- // TODO auto add new collection partitions when cross a timestamp boundary. That needs to be coordinated to avoid
- // race conditions, remembering that even the lead collection might have multiple instances of this URP
- // (multiple shards or perhaps just multiple streams thus instances of this URP)
-
public static final String ALIAS_DISTRIB_UPDATE_PARAM = "alias." + DISTRIB_UPDATE_PARAM; // param
public static final String TIME_PARTITION_ALIAS_NAME_CORE_PROP = "timePartitionAliasName"; // core prop
- public static final String ROUTER_FIELD_METADATA = "router.field"; // alias metadata
+ // alias metadata:
+ public static final String ROUTER_FIELD_METADATA = "router.field";
+ public static final String ROUTER_MAX_FUTURE_TIME_METADATA = "router.maxFutureMs";
+ public static final String ROUTER_INTERVAL_METADATA = "router.interval";
// This format must be compatible with collection name limitations
- private static final DateTimeFormatter DATE_TIME_FORMATTER = new DateTimeFormatterBuilder()
+ public static final DateTimeFormatter DATE_TIME_FORMATTER = new DateTimeFormatterBuilder()
.append(DateTimeFormatter.ISO_LOCAL_DATE).appendPattern("[_HH[_mm[_ss]]]") //brackets mean optional
.parseDefaulting(ChronoField.HOUR_OF_DAY, 0)
.parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
@@ -87,18 +103,26 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ // used to limit unnecessary concurrent collection creation requests
+ private static ConcurrentHashMap<String, Semaphore> aliasToSemaphoreMap = new ConcurrentHashMap<>(4);
+
private final String thisCollection;
private final String aliasName;
private final String routeField;
+ private final long maxFutureMs;
+ private final String intervalDateMath;
+ private final TimeZone intervalTimeZone;
- private final SolrCmdDistributor cmdDistrib;
private final ZkController zkController;
+ private final SolrCmdDistributor cmdDistrib;
+ private final CollectionsHandler collHandler;
private final SolrParams outParamsToLeader;
private List<Map.Entry<Instant, String>> parsedCollectionsDesc; // k=timestamp (start), v=collection. Sorted descending
private Aliases parsedCollectionsAliases; // a cached reference to the source of what we parse into parsedCollectionsDesc
public static UpdateRequestProcessor wrap(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ //TODO get from "Collection property"
final String timePartitionAliasName = req.getCore().getCoreDescriptor()
.getCoreProperty(TIME_PARTITION_ALIAS_NAME_CORE_PROP, null);
final DistribPhase shardDistribPhase =
@@ -126,12 +150,21 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
CoreContainer cc = core.getCoreContainer();
zkController = cc.getZkController();
cmdDistrib = new SolrCmdDistributor(cc.getUpdateShardHandler());
+ collHandler = cc.getCollectionsHandler();
final Map<String, String> aliasMetadata = zkController.getZkStateReader().getAliases().getCollectionAliasMetadata(aliasName);
if (aliasMetadata == null) {
throw newAliasMustExistException(); // if it did exist, we'd have a non-null map
}
routeField = aliasMetadata.get(ROUTER_FIELD_METADATA);
+ intervalDateMath = aliasMetadata.getOrDefault(ROUTER_INTERVAL_METADATA, "+1DAY");
+ String futureTimeStr = aliasMetadata.get(ROUTER_MAX_FUTURE_TIME_METADATA);
+ if (futureTimeStr != null) {
+ maxFutureMs = Long.parseLong(futureTimeStr);
+ } else {
+ maxFutureMs = TimeUnit.MINUTES.toMillis(10);
+ }
+ intervalTimeZone = TimeZoneUtils.parseTimezone(aliasMetadata.get(CommonParams.TZ));
ModifiableSolrParams outParams = new ModifiableSolrParams(req.getParams());
// Don't distribute these params; they will be distributed from the local processCommit separately.
@@ -153,11 +186,59 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
@Override
public void processAdd(AddUpdateCommand cmd) throws IOException {
final Object routeValue = cmd.getSolrInputDocument().getFieldValue(routeField);
- final String targetCollection = findTargetCollectionGivenRouteKey(routeValue);
- if (targetCollection == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Doc " + cmd.getPrintableId() + " couldn't be routed with " + routeField + "=" + routeValue);
- }
+ final Instant routeTimestamp = parseRouteKey(routeValue);
+
+ updateParsedCollectionAliases();
+ String targetCollection;
+ do {
+ targetCollection = findTargetCollectionGivenTimestamp(routeTimestamp);
+
+ if (targetCollection == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Doc " + cmd.getPrintableId() + " couldn't be routed with " + routeField + "=" + routeTimestamp);
+ }
+
+ // Note: the following rule is tempting but not necessary and is not compatible with
+ // only using this URP when the alias distrib phase is NONE; otherwise a doc may be routed to from a non-recent
+ // collection to the most recent only to then go there directly instead of realizing a new collection is needed.
+ // // If it's going to some other collection (not "this") then break to just send it there
+ // if (!thisCollection.equals(targetCollection)) {
+ // break;
+ // }
+ // Also tempting but not compatible: check that we're the leader, if not then break
+
+ // If the doc goes to the most recent collection then do some checks below, otherwise break the loop.
+ final Instant mostRecentCollTimestamp = parsedCollectionsDesc.get(0).getKey();
+ final String mostRecentCollName = parsedCollectionsDesc.get(0).getValue();
+ if (!mostRecentCollName.equals(targetCollection)) {
+ break;
+ }
+
+ // Check the doc isn't too far in the future
+ final Instant maxFutureTime = Instant.now().plusMillis(maxFutureMs);
+ if (routeTimestamp.isAfter(maxFutureTime)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "The document's time routed key of " + routeValue + " is too far in the future given " +
+ ROUTER_MAX_FUTURE_TIME_METADATA + "=" + maxFutureMs);
+ }
+
+ // Create a new collection?
+ final Instant nextCollTimestamp = computeNextCollTimestamp(mostRecentCollTimestamp, intervalDateMath, intervalTimeZone);
+ if (routeTimestamp.isBefore(nextCollTimestamp)) {
+ break; // thus we don't need another collection
+ }
+
+ createCollectionAfter(mostRecentCollName); // *should* throw if fails for some reason but...
+ final boolean updated = updateParsedCollectionAliases();
+ if (!updated) { // thus we didn't make progress...
+ // this is not expected, even in known failure cases, but we check just in case
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "We need to create a new time routed collection but for unknown reasons were unable to do so.");
+ }
+ // then retry the loop ...
+ } while(true);
+ assert targetCollection != null;
+
if (thisCollection.equals(targetCollection)) {
// pass on through; we've reached the right collection
super.processAdd(cmd);
@@ -168,7 +249,23 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
}
}
- protected String findTargetCollectionGivenRouteKey(Object routeKey) {
+ /** Computes the timestamp of the next collection given the timestamp of the one before. */
+ public static Instant computeNextCollTimestamp(Instant fromTimestamp, String intervalDateMath, TimeZone intervalTimeZone) {
+ //TODO overload DateMathParser.parseMath to take tz and "now"
+ final DateMathParser dateMathParser = new DateMathParser(intervalTimeZone);
+ dateMathParser.setNow(Date.from(fromTimestamp));
+ final Instant nextCollTimestamp;
+ try {
+ nextCollTimestamp = dateMathParser.parseMath(intervalDateMath).toInstant();
+ } catch (ParseException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Invalid Date Math String:'" + intervalDateMath +'\'', e);
+ }
+ assert nextCollTimestamp.isAfter(fromTimestamp);
+ return nextCollTimestamp;
+ }
+
+ private Instant parseRouteKey(Object routeKey) {
final Instant docTimestamp;
if (routeKey instanceof Instant) {
docTimestamp = (Instant) routeKey;
@@ -179,15 +276,30 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
} else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unexpected type of routeKey: " + routeKey);
}
+ return docTimestamp;
+ }
+
+ /**
+ * Ensure {@link #parsedCollectionsAliases} is up to date. If it was modified, return true.
+ * Note that this will return true if some other alias was modified or if metadata was modified. These
+ * are spurious and the caller should be written to be tolerant of no material changes.
+ */
+ private boolean updateParsedCollectionAliases() {
final Aliases aliases = zkController.getZkStateReader().getAliases(); // note: might be different from last request
if (this.parsedCollectionsAliases != aliases) {
if (this.parsedCollectionsAliases != null) {
- log.info("Observing possibly updated alias {}", aliasName);
+ log.debug("Observing possibly updated alias: {}", aliasName);
}
- this.parsedCollectionsDesc = doParseCollections(aliases);
+ this.parsedCollectionsDesc = parseCollections(aliasName, aliases, this::newAliasMustExistException);
this.parsedCollectionsAliases = aliases;
+ return true;
}
- // iterates in reverse chronological order
+ return false;
+ }
+
+ /** Given the route key, finds the collection. Returns null if too old to go in last one. */
+ private String findTargetCollectionGivenTimestamp(Instant docTimestamp) {
+ // Lookup targetCollection given route key. Iterates in reverse chronological order.
// We're O(N) here but N should be small, the loop is fast, and usually looking for 1st.
for (Map.Entry<Instant, String> entry : parsedCollectionsDesc) {
Instant colStartTime = entry.getKey();
@@ -195,16 +307,77 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
return entry.getValue(); //found it
}
}
- return null;
+ return null; //not found
+ }
+
+ private void createCollectionAfter(String mostRecentCollName) {
+ // Invoke ROUTEDALIAS_CREATECOLL (in the Overseer, locked by alias name). It will create the collection
+ // and update the alias contingent on the most recent collection name being the same as
+ // what we think so here, otherwise it will return (without error).
+ // To avoid needless concurrent communication with the Overseer from this JVM, we
+ // maintain a Semaphore from an alias name keyed ConcurrentHashMap.
+ // Alternatively a Lock or CountDownLatch could have been used but they didn't seem
+ // to make it any easier.
+
+ final Semaphore semaphore = aliasToSemaphoreMap.computeIfAbsent(aliasName, n -> new Semaphore(1));
+ if (semaphore.tryAcquire()) {
+ try {
+ final String operation = CollectionParams.CollectionAction.ROUTEDALIAS_CREATECOLL.toLower();
+ Map<String, Object> msg = new HashMap<>();
+ msg.put(Overseer.QUEUE_OPERATION, operation);
+ msg.put(CollectionParams.NAME, aliasName);
+ msg.put(RoutedAliasCreateCollectionCmd.IF_MOST_RECENT_COLL_NAME, mostRecentCollName);
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ try {
+ this.collHandler.handleResponse(
+ operation,
+ new ZkNodeProps(msg),
+ rsp);
+ if (rsp.getException() != null) {
+ throw rsp.getException();
+ } // otherwise don't care about the response. It's possible no collection was created because
+ // of a race and that's okay... we'll ultimately retry any way.
+
+ // Ensure our view of the aliases has updated. If we didn't do this, our zkStateReader might
+ // not yet know about the new alias (thus won't see the newly added collection to it), and we might think
+ // we failed.
+ zkController.getZkStateReader().aliasesHolder.update();
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ } finally {
+ semaphore.release(); // to signal we're done to anyone waiting on it
+ }
+
+ } else {
+ // Failed to acquire permit because another URP instance on this JVM is creating a collection.
+ // So wait till it's available
+ log.debug("Collection creation is already in progress so we'll wait then try again.");
+ try {
+ if (semaphore.tryAcquire(DEFAULT_COLLECTION_OP_TIMEOUT, TimeUnit.MILLISECONDS)) {
+ semaphore.release(); // we don't actually want a permit so give it back
+ // return to continue...
+ } else {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Waited too long for another update thread to be done with collection creation.");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Interrupted waiting on collection creation.", e); // if we were interrupted, give up.
+ }
+ }
}
- /** Parses the timestamp from the collection list and returns them in reverse sorted order (newest 1st) */
- private List<Map.Entry<Instant,String>> doParseCollections(Aliases aliases) {
+ /** Parses the timestamp from the collection list and returns them in reverse sorted order (most recent 1st) */
+ public static List<Map.Entry<Instant,String>> parseCollections(String aliasName, Aliases aliases, Supplier<SolrException> aliasNotExist) {
final List<String> collections = aliases.getCollectionAliasListMap().get(aliasName);
if (collections == null) {
- throw newAliasMustExistException();
+ throw aliasNotExist.get();
}
- // note: I considered TreeMap but didn't like the log(N) just to grab the head when we use it later
+ // note: I considered TreeMap but didn't like the log(N) just to grab the most recent when we use it later
List<Map.Entry<Instant,String>> result = new ArrayList<>(collections.size());
for (String collection : collections) {
Instant colStartTime = parseInstantFromCollectionName(aliasName, collection);
@@ -225,6 +398,17 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
return DATE_TIME_FORMATTER.parse(dateTimePart, Instant::from);
}
+ public static String formatCollectionNameFromInstant(String aliasName, Instant timestamp) {
+ String nextCollName = TimeRoutedAliasUpdateProcessor.DATE_TIME_FORMATTER.format(timestamp);
+ for (int i = 0; i < 3; i++) { // chop off seconds, minutes, hours
+ if (nextCollName.endsWith("_00")) {
+ nextCollName = nextCollName.substring(0, nextCollName.length()-3);
+ }
+ }
+ assert TimeRoutedAliasUpdateProcessor.DATE_TIME_FORMATTER.parse(nextCollName, Instant::from).equals(timestamp);
+ return aliasName + "_" + nextCollName;
+ }
+
@Override
public void processDelete(DeleteUpdateCommand cmd) throws IOException {
final List<SolrCmdDistributor.Node> nodes = lookupShardLeadersOfCollections();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925733d1/solr/core/src/java/org/apache/solr/util/TimeZoneUtils.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/TimeZoneUtils.java b/solr/core/src/java/org/apache/solr/util/TimeZoneUtils.java
index 9d11f81..0600a83 100644
--- a/solr/core/src/java/org/apache/solr/util/TimeZoneUtils.java
+++ b/solr/core/src/java/org/apache/solr/util/TimeZoneUtils.java
@@ -25,6 +25,8 @@ import java.util.Arrays;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
+import org.apache.solr.common.SolrException;
+
/**
* Simple utilities for working with TimeZones
* @see java.util.TimeZone
@@ -82,4 +84,20 @@ public final class TimeZoneUtils {
private static Pattern CUSTOM_ID_REGEX = Pattern.compile("GMT(?:\\+|\\-)(\\d{1,2})(?::?(\\d{2}))?");
+ /**
+ * Parse the specified timezone ID. If null input then return UTC. If we can't resolve it then
+ * throw an exception.
+ */
+ public static TimeZone parseTimezone(String tzStr) {
+ if (tzStr != null) {
+ TimeZone tz = getTimeZone(tzStr);
+ if (null == tz) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Solr JVM does not support TZ: " + tzStr);
+ }
+ return tz;
+ } else {
+ return DateMathParser.UTC; //TODO move to TimeZoneUtils
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925733d1/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
index f7f200f..db4b877 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
@@ -19,16 +19,20 @@ package org.apache.solr.update.processor;
import java.io.IOException;
import java.time.Instant;
+import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Date;
import java.util.List;
-import java.util.function.UnaryOperator;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.ConfigSetAdminRequest;
import org.apache.solr.client.solrj.request.V2Request;
@@ -39,12 +43,14 @@ import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.util.DefaultSolrThreadFactory;
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -52,7 +58,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
static final String configName = "timeConfig";
static final String alias = "myalias";
- static final String timeField = "timestamp";
+ static final String timeField = "timestamp_dt";
static final String intField = "integer_i";
static SolrClient solrClient;
@@ -71,6 +77,14 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
IOUtils.close(solrClient);
}
+ //TODO this is necessary when -Dtests.iters but why? Some other tests aren't affected
+ @Before
+ public void doBefore() throws Exception {
+ for (String col : CollectionAdminRequest.listCollections(solrClient)) {
+ CollectionAdminRequest.deleteCollection(col).process(solrClient);
+ }
+ }
+
@Test
public void test() throws Exception {
// First create a config using REST API. To do this, we create a collection with the name of the eventual config.
@@ -91,18 +105,21 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
" 'fieldName':'" + intField + "'" +
" }," +
"}").build()));
+ // only sometimes test with "tolerant" URP
+ final String urpNames = "inc" + (random().nextBoolean() ? ",tolerant" : "");
checkNoError(solrClient.request(new V2Request.Builder("/collections/" + configName + "/config/params")
.withMethod(SolrRequest.METHOD.POST)
.withPayload("{" +
" 'set' : {" +
- " '_UPDATE' : {'processor':'inc,tolerant'}" +
+ " '_UPDATE' : {'processor':'" + urpNames + "'}" +
" }" +
"}").build()));
CollectionAdminRequest.deleteCollection(configName).process(solrClient);
// start with one collection and an alias for it
final String col23rd = alias + "_2017-10-23";
- CollectionAdminRequest.createCollection(col23rd, configName, 1, 1)
+ CollectionAdminRequest.createCollection(col23rd, configName, 2, 2)
+ .setMaxShardsPerNode(2)
.withProperty(TimeRoutedAliasUpdateProcessor.TIME_PARTITION_ALIAS_NAME_CORE_PROP, alias)
.process(solrClient);
@@ -112,30 +129,29 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
CollectionAdminRequest.createAlias(alias, col23rd).process(solrClient);
//TODO use SOLR-11617 client API to set alias metadata
final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
- UnaryOperator<Aliases> op = a -> a.cloneWithCollectionAliasMetadata(alias, TimeRoutedAliasUpdateProcessor.ROUTER_FIELD_METADATA, timeField);
- zkStateReader.aliasesHolder.applyModificationAndExportToZk(op);
+ zkStateReader.aliasesHolder.applyModificationAndExportToZk(a ->
+ a.cloneWithCollectionAliasMetadata(alias, TimeRoutedAliasUpdateProcessor.ROUTER_FIELD_METADATA, timeField)
+ .cloneWithCollectionAliasMetadata(alias, "collection-create.collection.configName", configName)
+ .cloneWithCollectionAliasMetadata(alias, "collection-create.numShards", "1")
+ .cloneWithCollectionAliasMetadata(alias, "collection-create.replicationFactor", "1")
+ .cloneWithCollectionAliasMetadata(alias, "router.interval", "+1DAY"));
// now we index a document
- solrClient.add(alias, newDoc(Instant.parse("2017-10-23T00:00:00Z")));
+ assertUpdateResponse(solrClient.add(alias, newDoc(Instant.parse("2017-10-23T00:00:00Z"))));
solrClient.commit(alias);
//assertDocRoutedToCol(lastDocId, col23rd);
- assertInvariants();
+ assertInvariants(col23rd);
- // a document that is too old (throws exception... if we have a TolerantUpdateProcessor then we see it there)
- try {
- final UpdateResponse resp = solrClient.add(alias, newDoc(Instant.parse("2017-10-01T00:00:00Z")));
- final Object errors = resp.getResponseHeader().get("errors");
- assertTrue(errors != null && errors.toString().contains("couldn't be routed"));
- } catch (SolrException e) {
- assertTrue(e.getMessage().contains("couldn't be routed"));
- }
- numDocsDeletedOrFailed++;
+ // a document that is too old
+ testFailedDocument(Instant.parse("2017-10-01T00:00:00Z"), "couldn't be routed");
+
+ // a document which is too far into the future
+ testFailedDocument(Instant.now().plus(30, ChronoUnit.MINUTES), "too far in the future");
// add another collection, add to alias (soonest comes first)
final String col24th = alias + "_2017-10-24";
- CollectionAdminRequest.createCollection(col24th, configName, 2, 2) // more shards and replicas now
- .setMaxShardsPerNode(2)
+ CollectionAdminRequest.createCollection(col24th, configName, 1, 1) // more shards and replicas now
.withProperty("timePartitionAliasName", alias)
.process(solrClient);
CollectionAdminRequest.createAlias(alias, col24th + "," + col23rd).process(solrClient);
@@ -146,7 +162,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
newDoc(Instant.parse("2017-10-24T01:00:00Z")),
newDoc(Instant.parse("2017-10-24T02:00:00Z"))
);
- assertInvariants();
+ assertInvariants(col24th, col23rd);
// assert that the IncrementURP has updated all '0' to '1'
final SolrDocumentList checkIncResults = solrClient.query(alias, params("q", "NOT " + intField + ":1")).getResults();
@@ -154,16 +170,45 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
//delete a random document id; ensure we don't find it
int idToDelete = 1 + random().nextInt(lastDocId);
- if (idToDelete == 2) { // #2 didn't make it
- idToDelete++;
+ if (idToDelete == 2 || idToDelete == 3) { // these didn't make it
+ idToDelete = 4;
}
- solrClient.deleteById(alias, Integer.toString(idToDelete));
- solrClient.commit(alias);
+ assertUpdateResponse(solrClient.deleteById(alias, Integer.toString(idToDelete)));
+ assertUpdateResponse(solrClient.commit(alias));
numDocsDeletedOrFailed++;
- assertInvariants();
+ assertInvariants(col24th, col23rd);
+
+ // delete the Oct23rd (save memory)...
+ // make sure we track that we are effectively deleting docs there
+ numDocsDeletedOrFailed += solrClient.query(col23rd, params("q", "*:*", "rows", "0")).getResults().getNumFound();
+ // remove from alias
+ CollectionAdminRequest.createAlias(alias, col24th).process(solrClient);
+ // delete the collection
+ CollectionAdminRequest.deleteCollection(col23rd).process(solrClient);
+
+ // now we're going to add documents that will trigger more collections to be created
+ // for 25th & 26th
+ addDocsAndCommit(
+ newDoc(Instant.parse("2017-10-24T03:00:00Z")),
+ newDoc(Instant.parse("2017-10-25T04:00:00Z")),
+ newDoc(Instant.parse("2017-10-26T05:00:00Z"))
+ );
+ assertInvariants(alias + "_2017-10-26", alias + "_2017-10-25", col24th);
}
- private void checkNoError(NamedList<Object> response) {
+ private void testFailedDocument(Instant timestamp, String errorMsg) throws SolrServerException, IOException {
+ try {
+ final UpdateResponse resp = solrClient.add(alias, newDoc(timestamp));
+ // if we have a TolerantUpdateProcessor then we see it there)
+ final Object errors = resp.getResponseHeader().get("errors"); // Tolerant URP
+ assertTrue(errors != null && errors.toString().contains(errorMsg));
+ } catch (SolrException e) {
+ assertTrue(e.getMessage().contains(errorMsg));
+ }
+ numDocsDeletedOrFailed++;
+ }
+
+ private void checkNoError(NamedList<Object> response) { //TODO rename
Object errors = response.get("errorMessages");
assertNull("" + errors, errors);
}
@@ -171,7 +216,8 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
/** Adds these documents and commits, returning when they are committed.
* We randomly go about this in different ways. */
private void addDocsAndCommit(SolrInputDocument... solrInputDocuments) throws Exception {
- // we assume these are not old docs!
+ // we assume all docs will be added (none too old/new to cause exception)
+ Collections.shuffle(Arrays.asList(solrInputDocuments), random());
// this is a list of the collections & the alias name. Use to pick randomly where to send.
// (it doesn't matter where we send docs since the alias is honored at the URP level)
@@ -182,15 +228,27 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
int commitWithin = random().nextBoolean() ? -1 : 500; // if -1, we commit explicitly instead
int numDocsBefore = queryNumDocs();
if (random().nextBoolean()) {
- // send in separate requests
- for (SolrInputDocument solrInputDocument : solrInputDocuments) {
- String col = collections.get(random().nextInt(collections.size()));
- solrClient.add(col, solrInputDocument, commitWithin);
+ // Send in separate threads. Choose random collection & solrClient
+ try (CloudSolrClient solrClient = getCloudSolrClient(cluster)) {
+ ExecutorService exec = ExecutorUtil.newMDCAwareFixedThreadPool(1 + random().nextInt(2),
+ new DefaultSolrThreadFactory(getTestName()));
+ List<Future<UpdateResponse>> futures = new ArrayList<>(solrInputDocuments.length);
+ for (SolrInputDocument solrInputDocument : solrInputDocuments) {
+ String col = collections.get(random().nextInt(collections.size()));
+ futures.add(exec.submit(() -> solrClient.add(col, solrInputDocument, commitWithin)));
+ }
+ for (Future<UpdateResponse> future : futures) {
+ assertUpdateResponse(future.get());
+ }
+ // at this point there shouldn't be any tasks running
+ assertEquals(0, exec.shutdownNow().size());
}
} else {
// send in a batch.
String col = collections.get(random().nextInt(collections.size()));
- solrClient.add(col, Arrays.asList(solrInputDocuments), commitWithin);
+ try (CloudSolrClient solrClient = getCloudSolrClient(cluster)) {
+ assertUpdateResponse(solrClient.add(col, Arrays.asList(solrInputDocuments), commitWithin));
+ }
}
String col = collections.get(random().nextInt(collections.size()));
if (commitWithin == -1) {
@@ -210,21 +268,30 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
}
}
+ private void assertUpdateResponse(UpdateResponse rsp) {
+ // use of TolerantUpdateProcessor can cause non-thrown "errors" that we need to check for
+ List errors = (List) rsp.getResponseHeader().get("errors");
+ assertTrue("Expected no errors: " + errors,errors == null || errors.isEmpty());
+ }
+
private int queryNumDocs() throws SolrServerException, IOException {
return (int) solrClient.query(alias, params("q", "*:*", "rows", "0")).getResults().getNumFound();
}
- private void assertInvariants() throws IOException, SolrServerException {
+ private void assertInvariants(String... expectedColls) throws IOException, SolrServerException {
final int expectNumFound = lastDocId - numDocsDeletedOrFailed; //lastDocId is effectively # generated docs
final List<String> cols = new CollectionAdminRequest.ListAliases().process(solrClient).getAliasesAsLists().get(alias);
assert !cols.isEmpty();
+ assertArrayEquals("expected reverse sorted",
+ cols.stream().sorted(Collections.reverseOrder()).toArray(),
+ cols.toArray());
+
int totalNumFound = 0;
Instant colEndInstant = null; // exclusive end
- for (String col : cols) {
+ for (String col : cols) { // ASSUMPTION: reverse sorted order
final Instant colStartInstant = TimeRoutedAliasUpdateProcessor.parseInstantFromCollectionName(alias, col);
- //TODO do this in parallel threads
final QueryResponse colStatsResp = solrClient.query(col, params(
"q", "*:*",
"rows", "0",
@@ -243,6 +310,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
colEndInstant = colStartInstant; // next older segment will max out at our current start time
}
assertEquals(expectNumFound, totalNumFound);
+ assertArrayEquals(expectedColls, cols.toArray());
}
private SolrInputDocument newDoc(Instant timestamp) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925733d1/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
index 77dd454..9d5fc36 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
@@ -78,6 +78,7 @@ public interface CollectionParams {
CREATEALIAS(true, LockLevel.COLLECTION),
DELETEALIAS(true, LockLevel.COLLECTION),
LISTALIASES(false, LockLevel.NONE),
+ ROUTEDALIAS_CREATECOLL(true, LockLevel.COLLECTION),
SPLITSHARD(true, LockLevel.SHARD),
DELETESHARD(true, LockLevel.SHARD),
CREATESHARD(true, LockLevel.COLLECTION),