You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by tf...@apache.org on 2023/06/23 18:18:18 UTC
[solr] branch main updated: SOLR-16846: Use IndexOrDocValuesQuery for PointFields on Set queries (#1706)
This is an automated email from the ASF dual-hosted git repository.
tflobbe pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 3b22f62b0df SOLR-16846: Use IndexOrDocValuesQuery for PointFields on Set queries (#1706)
3b22f62b0df is described below
commit 3b22f62b0df8e407955d5de10a309f5d84e1ef96
Author: Tomas Eduardo Fernandez Lobbe <tf...@apache.org>
AuthorDate: Fri Jun 23 11:18:13 2023 -0700
SOLR-16846: Use IndexOrDocValuesQuery for PointFields on Set queries (#1706)
---
solr/CHANGES.txt | 3 +
.../org/apache/solr/bench/MiniClusterState.java | 151 +++++++----
.../org/apache/solr/bench/search/JsonFaceting.java | 12 +-
.../apache/solr/bench/search/NumericSearch.java | 302 +++++++++++++++++++++
.../configs/cloud-minimal/conf/schema.xml | 25 +-
.../org/apache/solr/schema/DatePointField.java | 7 +-
.../org/apache/solr/schema/DoublePointField.java | 7 +-
.../org/apache/solr/schema/FloatPointField.java | 7 +-
.../java/org/apache/solr/schema/IntPointField.java | 7 +-
.../org/apache/solr/schema/LongPointField.java | 7 +-
.../org/apache/solr/schema/TestPointFields.java | 24 +-
.../apache/solr/search/TestSolrQueryParser.java | 31 ++-
12 files changed, 499 insertions(+), 84 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 9a472b633eb..24cbb415067 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -200,6 +200,9 @@ Optimizations
* SOLR-16273: Improve the performance of the Prometheus Metric Exporter scraping metrics (Matthew Biscocho via David Smiley & Christine Poerschke
+* SOLR-16846: Use IndexOrDocValuesQuery for PointFields on Set queries. This improves the performance in some scenarios when
+ the set query is combined with other selective queries. (Tomás Fernández Löbbe, Alex Deparvu)
+
Bug Fixes
---------------------
diff --git a/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java b/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java
index cbc30cb5a11..a35dd07bbcc 100755
--- a/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java
+++ b/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java
@@ -40,6 +40,7 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
@@ -331,6 +332,11 @@ public class MiniClusterState {
this.useHttp1 = useHttp1;
}
+ @SuppressForbidden(reason = "This module does not need to deal with logging context")
+ public void index(String collection, Docs docs, int docCount) throws Exception {
+ index(collection, docs, docCount, true);
+ }
+
/**
* Index.
*
@@ -339,64 +345,15 @@ public class MiniClusterState {
* @param docCount the doc count
* @throws Exception the exception
*/
- @SuppressForbidden(reason = "This module does not need to deal with logging context")
- public void index(String collection, Docs docs, int docCount) throws Exception {
+ public void index(String collection, Docs docs, int docCount, boolean parallel)
+ throws Exception {
if (createCollectionAndIndex) {
-
log("indexing data for benchmark...");
- Meter meter = new Meter();
- ExecutorService executorService =
- Executors.newFixedThreadPool(
- Runtime.getRuntime().availableProcessors(),
- new SolrNamedThreadFactory("SolrJMH Indexer"));
- ScheduledExecutorService scheduledExecutor =
- Executors.newSingleThreadScheduledExecutor(
- new SolrNamedThreadFactory("SolrJMH Indexer Progress"));
- scheduledExecutor.scheduleAtFixedRate(
- () -> {
- if (meter.getCount() == docCount) {
- scheduledExecutor.shutdown();
- } else {
- log(meter.getCount() + " docs at " + meter.getMeanRate() + " doc/s");
- }
- },
- 10,
- 10,
- TimeUnit.SECONDS);
- for (int i = 0; i < docCount; i++) {
- executorService.submit(
- new Runnable() {
- final SplittableRandom threadRandom = random.split();
-
- @Override
- public void run() {
- UpdateRequest updateRequest = new UpdateRequest();
- updateRequest.setBasePath(
- nodes.get(threadRandom.nextInt(cluster.getJettySolrRunners().size())));
- SolrInputDocument doc = docs.inputDocument();
- // log("add doc " + doc);
- updateRequest.add(doc);
- meter.mark();
-
- try {
- client.request(updateRequest, collection);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- });
- }
-
- log("done adding docs, waiting for executor to terminate...");
-
- executorService.shutdown();
- boolean result = false;
- while (!result) {
- result = executorService.awaitTermination(600, TimeUnit.MINUTES);
+ if (parallel) {
+ indexParallel(collection, docs, docCount);
+ } else {
+ indexBatch(collection, docs, docCount, 10000);
}
-
- scheduledExecutor.shutdown();
-
log("done indexing data for benchmark");
log("committing data ...");
@@ -422,6 +379,90 @@ public class MiniClusterState {
dumpCoreInfo();
}
+ @SuppressForbidden(reason = "This module does not need to deal with logging context")
+ private void indexParallel(String collection, Docs docs, int docCount)
+ throws InterruptedException {
+ Meter meter = new Meter();
+ ExecutorService executorService =
+ Executors.newFixedThreadPool(
+ Runtime.getRuntime().availableProcessors(),
+ new SolrNamedThreadFactory("SolrJMH Indexer"));
+ ScheduledExecutorService scheduledExecutor =
+ Executors.newSingleThreadScheduledExecutor(
+ new SolrNamedThreadFactory("SolrJMH Indexer Progress"));
+ scheduledExecutor.scheduleAtFixedRate(
+ () -> {
+ if (meter.getCount() == docCount) {
+ scheduledExecutor.shutdown();
+ } else {
+ log(meter.getCount() + " docs at " + meter.getMeanRate() + " doc/s");
+ }
+ },
+ 10,
+ 10,
+ TimeUnit.SECONDS);
+ for (int i = 0; i < docCount; i++) {
+ executorService.submit(
+ new Runnable() {
+ final SplittableRandom threadRandom = random.split();
+
+ @Override
+ public void run() {
+ UpdateRequest updateRequest = new UpdateRequest();
+ updateRequest.setBasePath(
+ nodes.get(threadRandom.nextInt(cluster.getJettySolrRunners().size())));
+ SolrInputDocument doc = docs.inputDocument();
+ // log("add doc " + doc);
+ updateRequest.add(doc);
+ meter.mark();
+
+ try {
+ client.request(updateRequest, collection);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
+
+ log("done adding docs, waiting for executor to terminate...");
+
+ executorService.shutdown();
+ boolean result = false;
+ while (!result) {
+ result = executorService.awaitTermination(600, TimeUnit.MINUTES);
+ }
+
+ scheduledExecutor.shutdown();
+ }
+
+ private void indexBatch(String collection, Docs docs, int docCount, int batchSize)
+ throws SolrServerException, IOException {
+ Meter meter = new Meter();
+ List<SolrInputDocument> batch = new ArrayList<>(batchSize);
+ for (int i = 0; i < docCount; i++) {
+ batch.add(docs.inputDocument());
+ if (i % batchSize == 0) {
+ UpdateRequest updateRequest = new UpdateRequest();
+ updateRequest.setBasePath(nodes.get(0));
+ updateRequest.add(batch);
+ client.request(updateRequest, collection);
+ meter.mark(batch.size());
+ batch.clear();
+ log(meter.getCount() + " docs at " + meter.getMeanRate() + " doc/s");
+ }
+ }
+ if (!batch.isEmpty()) {
+ UpdateRequest updateRequest = new UpdateRequest();
+ updateRequest.setBasePath(nodes.get(0));
+ updateRequest.add(batch);
+ client.request(updateRequest, collection);
+ meter.mark(batch.size());
+ batch = null;
+ }
+ log(meter.getCount() + " docs at " + meter.getMeanRate() + " doc/s");
+ }
+
/**
* Wait for merges.
*
diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java b/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java
index 61f0e352952..498faed5814 100755
--- a/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java
+++ b/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java
@@ -140,10 +140,10 @@ public class JsonFaceting {
"json.facet",
"{f1:{method:'"
+ fm
- + "', type:terms, field:'facet_s', sort:'x desc', facet:{x:'min(int3_i)'} }"
+ + "', type:terms, field:'facet_s', sort:'x desc', facet:{x:'min(int3_i_dv)'} }"
+ " , f2:{method:'"
+ fm
- + "',, type:terms, field:'facet_s', sort:'x desc', facet:{x:'max(int3_i)'} } "
+ + "',, type:terms, field:'facet_s', sort:'x desc', facet:{x:'max(int3_i_dv)'} } "
+ " , f3:{method:'"
+ fm
+ "', type:terms, field:'facet_s', sort:'x desc', facet:{x:'unique(facet2_s)'} } "
@@ -152,10 +152,10 @@ public class JsonFaceting {
+ "', type:terms, field:'facet_s', sort:'x desc', facet:{x:'hll(facet2_s)'} } "
+ " , f5:{method:'"
+ fm
- + "', type:terms, field:'facet_s', sort:'x desc', facet:{x:'variance(int3_i)'} } "
- + " , f6:{type:terms, field:'int3_i', limit:1, sort:'x desc', facet:{x:'hll(int2_i)'} } "
- + " , f7:{type:terms, field:'facet_s', limit:2, sort:'x desc', facet:{x:'missing(int4_i)'} } "
- + " , f8:{type:terms, field:'facet_s', limit:2, sort:'x desc', facet:{x:'countvals(int4_i)'} } "
+ + "', type:terms, field:'facet_s', sort:'x desc', facet:{x:'variance(int3_i_dv)'} } "
+ + " , f6:{type:terms, field:'int3_i_dv', limit:1, sort:'x desc', facet:{x:'hll(int2_i_dv)'} } "
+ + " , f7:{type:terms, field:'facet_s', limit:2, sort:'x desc', facet:{x:'missing(int4_i_dv)'} } "
+ + " , f8:{type:terms, field:'facet_s', limit:2, sort:'x desc', facet:{x:'countvals(int4_i_dv)'} } "
+ '}');
// MiniClusterState.log("params: " + params + "\n");
diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java
new file mode 100644
index 00000000000..a96a714ebef
--- /dev/null
+++ b/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.bench.search;
+
+import static org.apache.solr.bench.Docs.docs;
+import static org.apache.solr.bench.generators.SourceDSL.integers;
+import static org.apache.solr.bench.generators.SourceDSL.strings;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import org.apache.solr.bench.Docs;
+import org.apache.solr.bench.MiniClusterState;
+import org.apache.solr.bench.generators.SolrGen;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@Fork(value = 1)
+@Warmup(time = 5, iterations = 5)
+@Measurement(time = 5, iterations = 5)
+@Threads(value = 1)
+public class NumericSearch {
+
+ static final String COLLECTION = "c1";
+
+ @State(Scope.Benchmark)
+ public static class BenchState {
+
+ int setQuerySize = 20; // TODO: Params
+ String termQueryField = "term_low_s";
+ String basePath;
+ SolrGen<Integer> setValues;
+ SolrGen<String> lowCardinalityTerms;
+ SolrGen<String> highCardinalityTerms;
+ Iterator<String> lowCardTerms;
+ Iterator<String> highCardTerms;
+ Iterator<String> queries;
+
+ @Setup(Level.Trial)
+ public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState)
+ throws Exception {
+ miniClusterState.setUseHttp1(true);
+ miniClusterState.startMiniCluster(1);
+ miniClusterState.createCollection(COLLECTION, 1, 1);
+ int maxCardinality = 10000;
+ int numDocs = 2000000;
+ setValues = integers().allWithMaxCardinality(maxCardinality);
+ lowCardinalityTerms = strings().wordList().ofOne();
+ highCardinalityTerms = strings().wordList().multi(5);
+ Docs docs =
+ docs()
+ .field("id", integers().incrementing())
+ .field("numbers_i_dv", setValues)
+ .field("numbers_i", setValues)
+ .field("numbers_d", setValues)
+ .field("numbers_d_dv", setValues)
+ .field("numbers_l", setValues)
+ .field("numbers_l_dv", setValues)
+ .field("numbers_f", setValues)
+ .field("numbers_f_dv", setValues)
+ .field("term_low_s", lowCardinalityTerms)
+ .field("term_high_s", highCardinalityTerms);
+ // .field("numbers_dt", setValues);
+
+ miniClusterState.index(COLLECTION, docs, numDocs, false);
+ basePath = miniClusterState.nodes.get(0);
+ SolrQuery q = new SolrQuery("*:*");
+ q.setParam("facet", "true");
+ q.setParam("rows", "0");
+ q.setParam("facet.field", "numbers_i_dv", "term_low_s", "term_high_s");
+ q.setParam("facet.limit", String.valueOf(maxCardinality));
+ QueryRequest req = new QueryRequest(q);
+ req.setBasePath(basePath);
+ QueryResponse response = req.process(miniClusterState.client, COLLECTION);
+ Set<String> numbers =
+ response.getFacetField("numbers_i_dv").getValues().stream()
+ .map(FacetField.Count::getName)
+ .collect(Collectors.toSet());
+ Set<String> lowCardinalityStrings =
+ response.getFacetField("term_low_s").getValues().stream()
+ .map(FacetField.Count::getName)
+ .collect(Collectors.toSet());
+ Set<String> highCardinalityStrings =
+ response.getFacetField("term_high_s").getValues().stream()
+ .map(FacetField.Count::getName)
+ .collect(Collectors.toSet());
+
+ lowCardTerms = new CircularIterator<>(lowCardinalityStrings);
+ highCardTerms = new CircularIterator<>(highCardinalityStrings);
+ queries = new CircularIterator<>(buildSetQueries(numbers));
+ }
+
+ private Set<String> buildSetQueries(Set<String> numbers) {
+ Iterator<String> valueIterator = numbers.iterator();
+ Set<String> setQueries = new HashSet<>();
+ StringBuilder builder = new StringBuilder();
+ while (valueIterator.hasNext()) {
+ for (int i = 0; i < setQuerySize; i++) {
+ if (!valueIterator.hasNext()) {
+ break;
+ }
+ builder.append(valueIterator.next()).append(",");
+ }
+ builder.setLength(builder.length() - 1);
+ setQueries.add(builder.toString());
+ builder.setLength(0);
+ }
+ return setQueries;
+ }
+
+ @Setup(Level.Iteration)
+ public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState)
+ throws SolrServerException, IOException {
+ // Reload the collection/core to drop existing caches
+ CollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION);
+ reload.setBasePath(miniClusterState.nodes.get(0));
+ miniClusterState.client.request(reload);
+ }
+
+ public QueryRequest intSetQuery(boolean dvs) {
+ return setQuery("numbers_i" + (dvs ? "_dv" : ""));
+ }
+
+ public QueryRequest longSetQuery(boolean dvs) {
+ return setQuery("numbers_l" + (dvs ? "_dv" : ""));
+ }
+
+ public QueryRequest doubleSetQuery(boolean dvs) {
+ return setQuery("numbers_d" + (dvs ? "_dv" : ""));
+ }
+
+ public QueryRequest floatSetQuery(boolean dvs) {
+ return setQuery("numbers_f" + (dvs ? "_dv" : ""));
+ }
+
+ QueryRequest setQuery(String field) {
+ QueryRequest q =
+ new QueryRequest(
+ new SolrQuery(
+ "q",
+ termQueryField + ":" + lowCardTerms.next(),
+ "fq",
+ "{!terms cache=false f='" + field + "'}" + queries.next()));
+ q.setBasePath(basePath);
+ return q;
+ }
+
+ private static class CircularIterator<T> implements Iterator<T> {
+
+ private final Object[] collection;
+ private final AtomicInteger idx;
+
+ CircularIterator(Collection<T> collection) {
+ this.collection = Objects.requireNonNull(collection).toArray();
+ if (this.collection.length == 0) {
+ throw new IllegalArgumentException("This iterator doesn't support empty collections");
+ }
+ this.idx = new AtomicInteger();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return true;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T next() {
+ return (T) collection[idx.incrementAndGet() % collection.length];
+ }
+ }
+ }
+
+ @Benchmark
+ public Object intSet(
+ Blackhole blackhole,
+ BenchState benchState,
+ MiniClusterState.MiniClusterBenchState miniClusterState)
+ throws SolrServerException, IOException {
+ QueryResponse response =
+ benchState.intSetQuery(false).process(miniClusterState.client, COLLECTION);
+ blackhole.consume(response);
+ return response;
+ }
+
+ @Benchmark
+ public Object longSet(
+ Blackhole blackhole,
+ BenchState benchState,
+ MiniClusterState.MiniClusterBenchState miniClusterState)
+ throws SolrServerException, IOException {
+ QueryResponse response =
+ benchState.longSetQuery(false).process(miniClusterState.client, COLLECTION);
+ blackhole.consume(response);
+ return response;
+ }
+
+ @Benchmark
+ public Object floatSet(
+ Blackhole blackhole,
+ BenchState benchState,
+ MiniClusterState.MiniClusterBenchState miniClusterState)
+ throws SolrServerException, IOException {
+ QueryResponse response =
+ benchState.floatSetQuery(false).process(miniClusterState.client, COLLECTION);
+ blackhole.consume(response);
+ return response;
+ }
+
+ @Benchmark
+ public Object doubleSet(
+ Blackhole blackhole,
+ BenchState benchState,
+ MiniClusterState.MiniClusterBenchState miniClusterState)
+ throws SolrServerException, IOException {
+ QueryResponse response =
+ benchState.doubleSetQuery(false).process(miniClusterState.client, COLLECTION);
+ blackhole.consume(response);
+ return response;
+ }
+
+ @Benchmark
+ public Object intDvSet(
+ Blackhole blackhole,
+ BenchState benchState,
+ MiniClusterState.MiniClusterBenchState miniClusterState)
+ throws SolrServerException, IOException {
+ QueryResponse response =
+ benchState.intSetQuery(true).process(miniClusterState.client, COLLECTION);
+ blackhole.consume(response);
+ return response;
+ }
+
+ @Benchmark
+ public Object longDvSet(
+ Blackhole blackhole,
+ BenchState benchState,
+ MiniClusterState.MiniClusterBenchState miniClusterState)
+ throws SolrServerException, IOException {
+ QueryResponse response =
+ benchState.longSetQuery(true).process(miniClusterState.client, COLLECTION);
+ blackhole.consume(response);
+ return response;
+ }
+
+ @Benchmark
+ public Object floatDvSet(
+ Blackhole blackhole,
+ BenchState benchState,
+ MiniClusterState.MiniClusterBenchState miniClusterState)
+ throws SolrServerException, IOException {
+ QueryResponse response =
+ benchState.floatSetQuery(true).process(miniClusterState.client, COLLECTION);
+ blackhole.consume(response);
+ return response;
+ }
+
+ @Benchmark
+ public Object doubleDvSet(
+ Blackhole blackhole,
+ BenchState benchState,
+ MiniClusterState.MiniClusterBenchState miniClusterState)
+ throws SolrServerException, IOException {
+ QueryResponse response =
+ benchState.doubleSetQuery(true).process(miniClusterState.client, COLLECTION);
+ blackhole.consume(response);
+ return response;
+ }
+}
diff --git a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml
index bc4676c4079..d950c8db5db 100644
--- a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml
+++ b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml
@@ -18,15 +18,15 @@
<schema name="minimal" version="1.1">
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
<fieldType name="string" class="solr.StrField" docValues="true"/>
- <fieldType name="int" class="org.apache.solr.schema.IntPointField" docValues="true" omitNorms="true"
+ <fieldType name="int" class="org.apache.solr.schema.IntPointField" docValues="false" omitNorms="true"
positionIncrementGap="0"/>
- <fieldType name="long" class="org.apache.solr.schema.LongPointField" docValues="true" omitNorms="true"
+ <fieldType name="long" class="org.apache.solr.schema.LongPointField" docValues="false" omitNorms="true"
positionIncrementGap="0"/>
- <fieldType name="float" class="org.apache.solr.schema.FloatPointField" docValues="true" omitNorms="true"
+ <fieldType name="float" class="org.apache.solr.schema.FloatPointField" docValues="false" omitNorms="true"
positionIncrementGap="0"/>
- <fieldType name="double" class="org.apache.solr.schema.DoublePointField" docValues="true" omitNorms="true"
+ <fieldType name="double" class="org.apache.solr.schema.DoublePointField" docValues="false" omitNorms="true"
positionIncrementGap="0"/>
- <fieldType name="date" class="org.apache.solr.schema.DatePointField" docValues="true" omitNorms="true"
+ <fieldType name="date" class="org.apache.solr.schema.DatePointField" docValues="false" omitNorms="true"
positionIncrementGap="0"/>
<fieldType name="text" class="solr.TextField">
<analyzer>
@@ -44,11 +44,16 @@
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
<dynamicField name="*_s" type="string" indexed="true" stored="false"/>
<dynamicField name="*_t" type="text" indexed="true" stored="false"/>
- <dynamicField name="*_i" type="int" indexed="false" stored="false"/>
- <dynamicField name="*_l" type="long" indexed="false" stored="false"/>
- <dynamicField name="*_f" type="float" indexed="false" stored="false"/>
- <dynamicField name="*_d" type="double" indexed="false" stored="false"/>
- <dynamicField name="*_dt" type="date" indexed="false" stored="false"/>
+ <dynamicField name="*_i" type="int" indexed="true" stored="false"/>
+ <dynamicField name="*_i_dv" type="int" indexed="true" docValues="true" stored="false"/>
+ <dynamicField name="*_l" type="long" indexed="true" stored="false"/>
+ <dynamicField name="*_l_dv" type="long" indexed="true" docValues="true" stored="false"/>
+ <dynamicField name="*_f" type="float" indexed="true" stored="false"/>
+ <dynamicField name="*_f_dv" type="float" indexed="true" docValues="true" stored="false"/>
+ <dynamicField name="*_d" type="double" indexed="true" stored="false"/>
+ <dynamicField name="*_d_dv" type="double" indexed="true" docValues="true" stored="false"/>
+ <dynamicField name="*_dt" type="date" indexed="true" stored="false"/>
+ <dynamicField name="*_dt_dv" type="date" indexed="true" docValues="true" stored="false"/>
<uniqueKey>id</uniqueKey>
</schema>
diff --git a/solr/core/src/java/org/apache/solr/schema/DatePointField.java b/solr/core/src/java/org/apache/solr/schema/DatePointField.java
index 5e938e86131..83e3ac91699 100644
--- a/solr/core/src/java/org/apache/solr/schema/DatePointField.java
+++ b/solr/core/src/java/org/apache/solr/schema/DatePointField.java
@@ -22,6 +22,7 @@ import java.time.Instant;
import java.util.Collection;
import java.util.Date;
import java.util.Map;
+import org.apache.lucene.document.LongField;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexableField;
@@ -174,7 +175,11 @@ public class DatePointField extends PointField implements DateValueFieldType {
values[i] = DateMathParser.parseMath(null, val).getTime();
i++;
}
- return LongPoint.newSetQuery(field.getName(), values);
+ if (field.hasDocValues()) {
+ return LongField.newSetQuery(field.getName(), values);
+ } else {
+ return LongPoint.newSetQuery(field.getName(), values);
+ }
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/schema/DoublePointField.java b/solr/core/src/java/org/apache/solr/schema/DoublePointField.java
index 50bc8833116..430d3ff968d 100644
--- a/solr/core/src/java/org/apache/solr/schema/DoublePointField.java
+++ b/solr/core/src/java/org/apache/solr/schema/DoublePointField.java
@@ -18,6 +18,7 @@
package org.apache.solr.schema;
import java.util.Collection;
+import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DocValuesType;
@@ -125,7 +126,11 @@ public class DoublePointField extends PointField implements DoubleValueFieldType
values[i] = parseDoubleFromUser(field.getName(), val);
i++;
}
- return DoublePoint.newSetQuery(field.getName(), values);
+ if (field.hasDocValues()) {
+ return DoubleField.newSetQuery(field.getName(), values);
+ } else {
+ return DoublePoint.newSetQuery(field.getName(), values);
+ }
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/schema/FloatPointField.java b/solr/core/src/java/org/apache/solr/schema/FloatPointField.java
index 4fcdf8b7165..e1b7786f8ac 100644
--- a/solr/core/src/java/org/apache/solr/schema/FloatPointField.java
+++ b/solr/core/src/java/org/apache/solr/schema/FloatPointField.java
@@ -18,6 +18,7 @@
package org.apache.solr.schema;
import java.util.Collection;
+import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DocValuesType;
@@ -125,7 +126,11 @@ public class FloatPointField extends PointField implements FloatValueFieldType {
values[i] = parseFloatFromUser(field.getName(), val);
i++;
}
- return FloatPoint.newSetQuery(field.getName(), values);
+ if (field.hasDocValues()) {
+ return FloatField.newSetQuery(field.getName(), values);
+ } else {
+ return FloatPoint.newSetQuery(field.getName(), values);
+ }
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/schema/IntPointField.java b/solr/core/src/java/org/apache/solr/schema/IntPointField.java
index aa5ad6cefad..5f44195286b 100644
--- a/solr/core/src/java/org/apache/solr/schema/IntPointField.java
+++ b/solr/core/src/java/org/apache/solr/schema/IntPointField.java
@@ -18,6 +18,7 @@
package org.apache.solr.schema;
import java.util.Collection;
+import org.apache.lucene.document.IntField;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexableField;
@@ -119,7 +120,11 @@ public class IntPointField extends PointField implements IntValueFieldType {
values[i] = parseIntFromUser(field.getName(), val);
i++;
}
- return IntPoint.newSetQuery(field.getName(), values);
+ if (field.hasDocValues()) {
+ return IntField.newSetQuery(field.getName(), values);
+ } else {
+ return IntPoint.newSetQuery(field.getName(), values);
+ }
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/schema/LongPointField.java b/solr/core/src/java/org/apache/solr/schema/LongPointField.java
index 328a6ec1575..05a014ae4e3 100644
--- a/solr/core/src/java/org/apache/solr/schema/LongPointField.java
+++ b/solr/core/src/java/org/apache/solr/schema/LongPointField.java
@@ -18,6 +18,7 @@
package org.apache.solr.schema;
import java.util.Collection;
+import org.apache.lucene.document.LongField;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexableField;
@@ -119,7 +120,11 @@ public class LongPointField extends PointField implements LongValueFieldType {
values[i] = parseLongFromUser(field.getName(), val);
i++;
}
- return LongPoint.newSetQuery(field.getName(), values);
+ if (field.hasDocValues()) {
+ return LongField.newSetQuery(field.getName(), values);
+ } else {
+ return LongPoint.newSetQuery(field.getName(), values);
+ }
}
@Override
diff --git a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
index 0fe2cdc8a87..b1e7e6b8162 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
@@ -59,6 +59,7 @@ import org.apache.lucene.index.PointValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.PointRangeQuery;
+import org.apache.lucene.search.Query;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
@@ -749,7 +750,9 @@ public class TestPointFields extends SolrTestCaseJ4 {
@Test
public void testIntPointSetQuery() {
doTestSetQueries("number_p_i", toStringArray(getRandomInts(20, false)), false);
+ doTestSetQueries("number_p_i_dv", toStringArray(getRandomInts(20, false)), false);
doTestSetQueries("number_p_i_mv", toStringArray(getRandomInts(20, false)), true);
+ doTestSetQueries("number_p_i_mv_dv", toStringArray(getRandomInts(20, false)), true);
doTestSetQueries("number_p_i_ni_dv", toStringArray(getRandomInts(20, false)), false);
}
@@ -1363,7 +1366,9 @@ public class TestPointFields extends SolrTestCaseJ4 {
@Test
public void testDoublePointSetQuery() {
doTestSetQueries("number_p_d", toStringArray(getRandomDoubles(20, false)), false);
+ doTestSetQueries("number_p_d_dv", toStringArray(getRandomDoubles(20, false)), false);
doTestSetQueries("number_p_d_mv", toStringArray(getRandomDoubles(20, false)), true);
+ doTestSetQueries("number_p_d_mv_dv", toStringArray(getRandomDoubles(20, false)), true);
doTestSetQueries("number_p_d_ni_dv", toStringArray(getRandomDoubles(20, false)), false);
}
@@ -1916,7 +1921,9 @@ public class TestPointFields extends SolrTestCaseJ4 {
@Test
public void testFloatPointSetQuery() {
doTestSetQueries("number_p_f", toStringArray(getRandomFloats(20, false)), false);
+ doTestSetQueries("number_p_f_dv", toStringArray(getRandomFloats(20, false)), false);
doTestSetQueries("number_p_f_mv", toStringArray(getRandomFloats(20, false)), true);
+ doTestSetQueries("number_p_f_mv_dv", toStringArray(getRandomFloats(20, false)), true);
doTestSetQueries("number_p_f_ni_dv", toStringArray(getRandomFloats(20, false)), false);
}
@@ -2477,7 +2484,9 @@ public class TestPointFields extends SolrTestCaseJ4 {
@Test
public void testLongPointSetQuery() {
doTestSetQueries("number_p_l", toStringArray(getRandomLongs(20, false)), false);
+ doTestSetQueries("number_p_l_dv", toStringArray(getRandomLongs(20, false)), false);
doTestSetQueries("number_p_l_mv", toStringArray(getRandomLongs(20, false)), true);
+ doTestSetQueries("number_p_l_mv_dv", toStringArray(getRandomLongs(20, false)), true);
doTestSetQueries("number_p_l_ni_dv", toStringArray(getRandomLongs(20, false)), false);
}
@@ -3113,7 +3122,9 @@ public class TestPointFields extends SolrTestCaseJ4 {
@Test
public void testDatePointSetQuery() {
doTestSetQueries("number_p_dt", toStringArray(getRandomInstants(20, false)), false);
+ doTestSetQueries("number_p_dt_dv", toStringArray(getRandomInstants(20, false)), false);
doTestSetQueries("number_p_dt_mv", toStringArray(getRandomInstants(20, false)), true);
+ doTestSetQueries("number_p_dt_mv_dv", toStringArray(getRandomInstants(20, false)), true);
doTestSetQueries("number_p_dt_ni_dv", toStringArray(getRandomInstants(20, false)), false);
}
@@ -5319,9 +5330,9 @@ public class TestPointFields extends SolrTestCaseJ4 {
"fl",
"id," + fieldName),
"//*[@numFound='" + numTerms + "']",
- "//*[@name='parsed_filter_queries']/str[.='("
+ "//*[@name='parsed_filter_queries']/str[.='"
+ getSetQueryToString(fieldName, values, numTerms)
- + ")']");
+ + "']");
} else {
// Won't use PointInSetQuery if the field is not indexed, but should match the same docs
assertQ(
@@ -5376,9 +5387,12 @@ public class TestPointFields extends SolrTestCaseJ4 {
private String getSetQueryToString(String fieldName, String[] values, int numTerms) {
SchemaField sf = h.getCore().getLatestSchema().getField(fieldName);
- return sf.getType()
- .getSetQuery(null, sf, Arrays.asList(Arrays.copyOf(values, numTerms)))
- .toString();
+ Query setQuery =
+ sf.getType().getSetQuery(null, sf, Arrays.asList(Arrays.copyOf(values, numTerms)));
+ if (sf.indexed() && sf.hasDocValues()) {
+ return IndexOrDocValuesQuery.class.getSimpleName() + "(" + setQuery.toString() + ")";
+ }
+ return "(" + setQuery.toString() + ")";
}
private void doTestDatePointFieldExactQuery(final String field, final String baseDate)
diff --git a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
index e2e79f9fe1d..9dfd8557e99 100644
--- a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
+++ b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
@@ -36,6 +36,7 @@ import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.NormsFieldExistsQuery;
import org.apache.lucene.search.PointInSetQuery;
@@ -374,7 +375,24 @@ public class TestSolrQueryParser extends SolrTestCaseJ4 {
qParser.setParams(params);
q = qParser.getQuery();
if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) {
- assertEquals(20, ((PointInSetQuery) q).getPackedPoints().size());
+ if (Boolean.getBoolean(NUMERIC_DOCVALUES_SYSPROP)) {
+ assertTrue(req.getCore().getLatestSchema().getField("foo_ti").hasDocValues());
+ assertEquals(
+ "Expecting IndexOrDocValuesQuery when type is IntPointField AND docValues are enabled",
+ IndexOrDocValuesQuery.class,
+ q.getClass());
+ assertEquals(
+ 20,
+ ((PointInSetQuery) ((IndexOrDocValuesQuery) q).getIndexQuery())
+ .getPackedPoints()
+ .size());
+ } else {
+ assertFalse(req.getCore().getLatestSchema().getField("foo_ti").hasDocValues());
+ assertEquals(
+ "Expecting PointInSetQuery when type is IntPointField AND docValues are disabled",
+ 20,
+ ((PointInSetQuery) q).getPackedPoints().size());
+ }
} else {
assertEquals(20, ((TermInSetQuery) q).getTermData().size());
}
@@ -385,8 +403,15 @@ public class TestSolrQueryParser extends SolrTestCaseJ4 {
qParser.setIsFilter(true); // this may change in the future
qParser.setParams(params);
q = qParser.getQuery();
- assertTrue(q instanceof PointInSetQuery);
- assertEquals(20, ((PointInSetQuery) q).getPackedPoints().size());
+
+ assertTrue(req.getCore().getLatestSchema().getField("foo_pi").hasDocValues());
+ assertEquals(
+ "Expecting IndexOrDocValuesQuery when type is IntPointField AND docValues are enabled",
+ IndexOrDocValuesQuery.class,
+ q.getClass());
+ assertEquals(
+ 20,
+ ((PointInSetQuery) ((IndexOrDocValuesQuery) q).getIndexQuery()).getPackedPoints().size());
// a filter() clause inside a relevancy query should be able to use a TermsQuery
qParser =