You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ds...@apache.org on 2026/06/25 02:37:54 UTC
(solr) branch branch_10x updated (6ebaac6b4f3 -> 605dfc1e878)
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a change to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
from 6ebaac6b4f3 SOLR-18233 Strengthen Basic Authentication password policy and harden template users created by bin/solr auth enable (take2) (#4534)
new c5c6ac846e1 SOLR-14070: Deprecate CloudSolrClient ZkHost constructor (#4533)
new 99c70a67a01 Add CloudSolrClient.getHttpClient (#4523)
new f271f86c68c SOLR-18188: Fix ForceLeaderTest
new d51ed7c0967 HttpSolrClient: improve error when text/html (#4526)
new adf4048a86c Replace ConcurrentLRUCache and Cache interface with Caffeine (#4516)
new 16f6dd4bd0d TestDistributedTracing goldfile approach (#4529)
new 605dfc1e878 SolrJ: simplify ResponseParser.getContentType processing (#4528)
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../PR#4526-HttpSolrClient-error-message.yml | 9 +
.../PR#4528-ResponseParser_getContentType.yml | 8 +
...domizedtesting.randomizedtesting-runner.all.txt | 1 -
.../solr/cloud/api/collections/SplitShardCmd.java | 3 +-
.../java/org/apache/solr/schema/IndexSchema.java | 10 +-
.../processor/TemplateUpdateProcessorFactory.java | 31 +-
.../org/apache/solr/util/ConcurrentLRUCache.java | 974 -------
.../org/apache/solr/cloud/ForceLeaderTest.java | 11 +-
.../solr/cloud/TestAuthenticationFramework.java | 13 +-
.../solr/cloud/api/collections/ShardSplitTest.java | 3 +-
.../org/apache/solr/search/TestSmileRequest.java | 8 +-
.../solr/security/BasicAuthIntegrationTest.java | 2 +-
.../solr/servlet/TestRequestRateLimiter.java | 17 +-
solr/modules/opentelemetry/build.gradle | 2 +
.../solr/tracing/TestDistributedTracing/test.json | 111 +
.../TestDistributedTracing/testAdminApi.json | 21 +
.../tracing/TestDistributedTracing/testV2Api.json | 143 +
.../solr/opentelemetry/GoldFileTraceVerifier.java | 478 ++++
.../solr/opentelemetry/TestDistributedTracing.java | 158 +-
.../modules/deployment-guide/pages/solrj.adoc | 4 +-
.../client/solrj/jetty/HttpJettySolrClient.java | 28 -
.../solrj/impl/SolrClientNodeStateProvider.java | 12 +-
.../client/solrj/impl/CloudHttp2SolrClient.java | 1 +
.../solr/client/solrj/impl/CloudSolrClient.java | 2938 ++++++++++----------
.../solr/client/solrj/impl/HttpJdkSolrClient.java | 33 -
.../solr/client/solrj/impl/HttpSolrClient.java | 49 +-
.../solrj/response/JavaBinResponseParser.java | 8 +-
.../solr/client/solrj/response/ResponseParser.java | 33 +-
.../client/solrj/response/XMLResponseParser.java | 4 +-
.../json/JacksonDataBindResponseParser.java | 9 +-
.../solrj/response/json/JsonMapResponseParser.java | 7 +-
.../java/org/apache/solr/common/util/Cache.java | 44 -
.../org/apache/solr/common/util/JavaBinCodec.java | 19 +-
.../apache/solr/common/util/MapBackedCache.java | 60 -
.../org/apache/solr/common/util/ObjectCache.java | 27 +-
.../solr/client/solrj/SolrExampleCborTest.java | 5 +-
.../solrj/impl/CloudSolrClientCacheTest.java | 5 +
.../client/solrj/impl/HttpJdkSolrClientTest.java | 23 -
.../apache/solr/common/util/TestJavaBinCodec.java | 77 +-
.../AbstractCloudBackupRestoreTestCase.java | 6 +-
40 files changed, 2510 insertions(+), 2885 deletions(-)
create mode 100644 changelog/unreleased/PR#4526-HttpSolrClient-error-message.yml
create mode 100644 changelog/unreleased/PR#4528-ResponseParser_getContentType.yml
delete mode 100644 gradle/validation/forbidden-apis/com.carrotsearch.randomizedtesting.randomizedtesting-runner.all.txt
delete mode 100644 solr/core/src/java/org/apache/solr/util/ConcurrentLRUCache.java
create mode 100644 solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/test.json
create mode 100644 solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testAdminApi.json
create mode 100644 solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testV2Api.json
create mode 100644 solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/GoldFileTraceVerifier.java
delete mode 100644 solr/solrj/src/java/org/apache/solr/common/util/Cache.java
delete mode 100644 solr/solrj/src/java/org/apache/solr/common/util/MapBackedCache.java
(solr) 06/07: TestDistributedTracing goldfile approach (#4529)
Posted by ds...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit 16f6dd4bd0d4deefb23601a55ae700ee6e039642
Author: David Smiley <ds...@apache.org>
AuthorDate: Tue Jun 23 13:55:41 2026 -0400
TestDistributedTracing goldfile approach (#4529)
Goal: make it easy & insightful to maintain TestDistributedTracing
Risk: determinism is hard
(cherry picked from commit 55cd95d28e7a4800365ef622ef243bc45abc3ce7)
---
...domizedtesting.randomizedtesting-runner.all.txt | 1 -
solr/modules/opentelemetry/build.gradle | 2 +
.../solr/tracing/TestDistributedTracing/test.json | 111 +++++
.../TestDistributedTracing/testAdminApi.json | 21 +
.../tracing/TestDistributedTracing/testV2Api.json | 143 ++++++
.../solr/opentelemetry/GoldFileTraceVerifier.java | 478 +++++++++++++++++++++
.../solr/opentelemetry/TestDistributedTracing.java | 158 +++----
7 files changed, 817 insertions(+), 97 deletions(-)
diff --git a/gradle/validation/forbidden-apis/com.carrotsearch.randomizedtesting.randomizedtesting-runner.all.txt b/gradle/validation/forbidden-apis/com.carrotsearch.randomizedtesting.randomizedtesting-runner.all.txt
deleted file mode 100644
index fd54b9ea1f7..00000000000
--- a/gradle/validation/forbidden-apis/com.carrotsearch.randomizedtesting.randomizedtesting-runner.all.txt
+++ /dev/null
@@ -1 +0,0 @@
-com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded seeds
diff --git a/solr/modules/opentelemetry/build.gradle b/solr/modules/opentelemetry/build.gradle
index 2d9990f3465..9e0345ad2b5 100644
--- a/solr/modules/opentelemetry/build.gradle
+++ b/solr/modules/opentelemetry/build.gradle
@@ -47,6 +47,8 @@ dependencies {
compileOnly libs.apache.tomcat.annotationsapi
testImplementation project(':solr:test-framework')
+ testImplementation project(':solr:solrj-jetty')
+ testImplementation libs.carrotsearch.randomizedtesting.runner
testImplementation libs.junit.junit
testImplementation libs.opentelemetry.sdk.trace
testImplementation libs.opentelemetry.sdk.testing
diff --git a/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/test.json b/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/test.json
new file mode 100644
index 00000000000..8cd2d28437a
--- /dev/null
+++ b/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/test.json
@@ -0,0 +1,111 @@
+{"phases":[
+ {
+ "description":"phase 0",
+ "spans":[{
+ "name":"post:/{collection}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1/update",
+ "http.params":"commit=true&wt=javabin",
+ "children":[
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard1_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER&wt=javabin"},
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard1_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER"},
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=TOLEADER&wt=javabin",
+ "children":[{
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER&wt=javabin"}]},
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=TOLEADER",
+ "children":[{
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER"}]}]}]},
+ {
+ "description":"phase 1",
+ "spans":[{
+ "name":"get:/{collection}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1",
+ "db.type":"solr",
+ "http.request.method":"GET",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1/select",
+ "http.params":"q=*%3A*&wt=javabin",
+ "children":[
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard1_replica_nN/select"},
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard1_replica_nN/select"},
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/select"},
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/select"}]}]}]}
\ No newline at end of file
diff --git a/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testAdminApi.json b/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testAdminApi.json
new file mode 100644
index 00000000000..479f3cebb78
--- /dev/null
+++ b/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testAdminApi.json
@@ -0,0 +1,21 @@
+{"phases":[
+ {
+ "description":"phase 0",
+ "spans":[{
+ "name":"get:/admin/metrics",
+ "kind":"SERVER",
+ "db.type":"solr",
+ "http.request.method":"GET",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/admin/metrics",
+ "http.params":"wt=prometheus"}]},
+ {
+ "description":"phase 1",
+ "spans":[{
+ "name":"list:/admin/collections",
+ "kind":"SERVER",
+ "db.type":"solr",
+ "http.request.method":"GET",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/admin/collections",
+ "http.params":"action=LIST&wt=javabin"}]}]}
\ No newline at end of file
diff --git a/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testV2Api.json b/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testV2Api.json
new file mode 100644
index 00000000000..975aac13532
--- /dev/null
+++ b/solr/modules/opentelemetry/src/test-files/solr/tracing/TestDistributedTracing/testV2Api.json
@@ -0,0 +1,143 @@
+{"phases":[
+ {
+ "description":"phase 0",
+ "spans":[
+ {
+ "name":"ReloadCollectionCmd",
+ "kind":"CLIENT",
+ "db.instance":"collection1",
+ "children":[
+ {
+ "name":"post:/admin/cores",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/admin/cores"},
+ {
+ "name":"post:/admin/cores",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/admin/cores"},
+ {
+ "name":"post:/admin/cores",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/admin/cores"},
+ {
+ "name":"post:/admin/cores",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/admin/cores"}]},
+ {
+ "name":"post:/collections/{collection}/reload",
+ "kind":"SERVER",
+ "db.instance":"collection1",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/____v2/collections/collection1/reload",
+ "http.params":"wt=javabin"}]},
+ {
+ "description":"phase 1",
+ "spans":[{
+ "name":"post:/c/{collection}/update/json",
+ "kind":"SERVER",
+ "db.instance":"collection1",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/____v2/c/collection1/update/json",
+ "http.params":"commit=true&wt=javabin",
+ "children":[
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard1_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER"},
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=TOLEADER&wt=javabin",
+ "children":[{
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER&wt=javabin"}]},
+ {
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=TOLEADER",
+ "children":[{
+ "name":"post:/{core}/update",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/update",
+ "http.params":"distrib.from=http%3A%2F%2FNORMALIZED&update.distrib=FROMLEADER"}]}]}]},
+ {
+ "description":"phase 2",
+ "spans":[{
+ "name":"get:/c/{collection}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1",
+ "db.type":"solr",
+ "http.request.method":"GET",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/____v2/c/collection1/select",
+ "http.params":"q=id%3A9&wt=javabin",
+ "children":[
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard1_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard1_replica_nN/select"},
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/select"},
+ {
+ "name":"post:/{core}/select",
+ "kind":"SERVER",
+ "db.instance":"collection1_shard2_replica_nN",
+ "db.type":"solr",
+ "http.request.method":"POST",
+ "http.response.status_code":200,
+ "http.url":"http://NORMALIZED/solr/collection1_shard2_replica_nN/select"}]}]}]}
\ No newline at end of file
diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/GoldFileTraceVerifier.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/GoldFileTraceVerifier.java
new file mode 100644
index 00000000000..d3667009605
--- /dev/null
+++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/GoldFileTraceVerifier.java
@@ -0,0 +1,478 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.opentelemetry;
+
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.data.SpanData;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.util.RetryUtil;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.util.tracing.TraceUtils;
+import org.junit.Assert;
+
+/**
+ * Verifies collected OpenTelemetry spans against a committed gold file.
+ *
+ * <p>Spans are normalized (IDs/timestamps stripped, URLs scrubbed) and arranged into a tree by
+ * parent-child relationships. The resulting JSON is compared against a gold file committed in
+ * {@code src/test-files/solr/tracing/}.
+ *
+ * <h2>Usage</h2>
+ *
+ * <pre>
+ * var verifier = new GoldFileTraceVerifier("testV2Api");
+ * // ... perform operations ...
+ * verifier.verifyPhase(); // waits for spans, normalizes, compares
+ * // ... more operations ...
+ * verifier.verifyPhase();
+ * verifier.done();
+ * </pre>
+ *
+ * <h2>Regenerating Gold Files</h2>
+ *
+ * <p>Requires {@code -Ptests.useSecurityManager=false} since it writes to the source tree.
+ *
+ * <pre>
+ * gradlew :solr:modules:opentelemetry:test --tests TestDistributedTracing.testV2Api \
+ * "-Ptests.jvmargs=-Dregenerate.golden.files=true" -Ptests.useSecurityManager=false
+ * </pre>
+ *
+ * <h2>Dumping Raw Traces (no normalization, all attributes)</h2>
+ *
+ * <p>Also requires {@code -Ptests.useSecurityManager=false} (writes to source tree). Dump files are
+ * written alongside gold files as {@code {testMethod}-phase{N}-dump.json}. Output paths are logged
+ * to stdout (visible in {@code build/test-results/test/outputs/OUTPUT-*.txt}).
+ *
+ * <pre>
+ * gradlew :solr:modules:opentelemetry:test --tests TestDistributedTracing.testV2Api \
+ * "-Ptests.jvmargs=-Ddump.traces=true" -Ptests.useSecurityManager=false
+ * </pre>
+ */
+public class GoldFileTraceVerifier {
+
+ private static final boolean REGENERATE = Boolean.getBoolean("regenerate.golden.files");
+ private static final boolean DUMP_TRACES = Boolean.getBoolean("dump.traces");
+ private static final Pattern URL_HOST_PORT = Pattern.compile("https?://[^/]+/");
+ private static final Pattern REPLICA_SUFFIX = Pattern.compile("_replica_n\\d+");
+ private static final Pattern ENCODED_URL_IN_PARAMS = Pattern.compile("https?%3A%2F%2F[^&=]+");
+
+ private static final Comparator<Map<String, Object>> SPAN_COMPARATOR =
+ Comparator.comparing((Map<String, Object> s) -> (String) s.get("name"))
+ .thenComparing(s -> s.getOrDefault(TraceUtils.TAG_DB.getKey(), "").toString());
+
+ private final String testName;
+ private final Path goldFilePath;
+ private final Map<String, Object> goldFile; // parsed gold file (phases list)
+ private final List<List<Map<String, Object>>> recordedPhases = new ArrayList<>();
+ private int currentPhaseIndex = 0;
+
+ public GoldFileTraceVerifier(Class<?> testClass, String testMethodName) {
+ this.testName = testMethodName;
+ this.goldFilePath = resolveGoldFilePath(testClass, testMethodName);
+ if (REGENERATE) {
+ this.goldFile = null;
+ } else {
+ this.goldFile = loadGoldFile(goldFilePath);
+ }
+ }
+
+ /**
+ * Waits for the expected number of spans (from the gold file), collects them, normalizes into a
+ * tree, and compares against the gold file's current phase.
+ */
+ @SuppressWarnings("unchecked")
+ public void verifyPhase() {
+ InMemorySpanExporter exporter = CustomTestOtelTracerConfigurator.getInMemorySpanExporter();
+
+ List<SpanData> spans;
+ if (REGENERATE) {
+ spans = waitForStableSpans(exporter);
+ } else {
+ List<Map<String, Object>> phases = (List<Map<String, Object>>) goldFile.get("phases");
+ Map<String, Object> phase = phases.get(currentPhaseIndex);
+ List<Map<String, Object>> expectedSpans = (List<Map<String, Object>>) phase.get("spans");
+ int expectedCount = countSpansRecursive(expectedSpans);
+ spans = waitForSpans(exporter, expectedCount);
+ }
+
+ exporter.reset();
+
+ if (DUMP_TRACES) {
+ dumpRawSpans(spans);
+ }
+
+ List<Map<String, Object>> tree = buildTree(spans);
+ recordedPhases.add(tree);
+
+ if (!REGENERATE) {
+ comparePhase(currentPhaseIndex, tree);
+ }
+ currentPhaseIndex++;
+ }
+
+ /** Call after all phases are verified. In regenerate mode, writes the gold file. */
+ @SuppressWarnings("unchecked")
+ public void done() {
+ if (REGENERATE) {
+ writeGoldFile();
+ } else {
+ List<Map<String, Object>> phases = (List<Map<String, Object>>) goldFile.get("phases");
+ Assert.assertEquals(
+ "Unverified phases remain in gold file for " + testName,
+ phases.size(),
+ currentPhaseIndex);
+ }
+ }
+
+ // --- Span collection ---
+
+ private List<SpanData> waitForSpans(InMemorySpanExporter exporter, int expectedCount) {
+ try {
+ RetryUtil.retryUntil(
+ "Timed out waiting for " + expectedCount + " span(s) in phase " + currentPhaseIndex,
+ 500,
+ 20,
+ TimeUnit.MILLISECONDS,
+ () -> exporter.getFinishedSpanItems().size() >= expectedCount);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ return new ArrayList<>(exporter.getFinishedSpanItems());
+ }
+
+ private List<SpanData> waitForStableSpans(InMemorySpanExporter exporter) {
+ try {
+ RetryUtil.retryUntil(
+ "Timed out waiting for any spans in phase " + currentPhaseIndex,
+ 500,
+ 20,
+ TimeUnit.MILLISECONDS,
+ () -> !exporter.getFinishedSpanItems().isEmpty());
+ int lastCount = -1;
+ for (int i = 0; i < 10; i++) {
+ TimeUnit.MILLISECONDS.sleep(500);
+ int current = exporter.getFinishedSpanItems().size();
+ if (current == lastCount) {
+ break;
+ }
+ lastCount = current;
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ return new ArrayList<>(exporter.getFinishedSpanItems());
+ }
+
+ // --- Raw dump (for debugging, no normalization) ---
+
+ private void dumpRawSpans(List<SpanData> spans) {
+ // Sort by start timestamp for a chronological view
+ List<SpanData> sorted = new ArrayList<>(spans);
+ sorted.sort(Comparator.comparingLong(SpanData::getStartEpochNanos));
+
+ // Build tree from raw data (no normalization)
+ Map<String, Map<String, Object>> nodesBySpanId = new HashMap<>();
+ List<Map<String, Object>> roots = new ArrayList<>();
+
+ for (SpanData span : sorted) {
+ Map<String, Object> node = new LinkedHashMap<>();
+ node.put("name", span.getName());
+ node.put("kind", span.getKind().name());
+ node.put("traceId", span.getSpanContext().getTraceId());
+ node.put("spanId", span.getSpanContext().getSpanId());
+ if (span.getParentSpanContext().isValid()) {
+ node.put("parentSpanId", span.getParentSpanContext().getSpanId());
+ }
+ node.put("startEpochNanos", span.getStartEpochNanos());
+ node.put("endEpochNanos", span.getEndEpochNanos());
+ // All attributes
+ Map<String, Object> attrs = new LinkedHashMap<>();
+ span.getAttributes().forEach((key, value) -> attrs.put(key.getKey(), value));
+ if (!attrs.isEmpty()) {
+ node.put("attributes", attrs);
+ }
+ nodesBySpanId.put(span.getSpanContext().getSpanId(), node);
+ }
+
+ for (SpanData span : sorted) {
+ Map<String, Object> node = nodesBySpanId.get(span.getSpanContext().getSpanId());
+ String parentId = span.getParentSpanContext().getSpanId();
+ if (span.getParentSpanContext().isValid() && nodesBySpanId.containsKey(parentId)) {
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> children =
+ (List<Map<String, Object>>)
+ nodesBySpanId.get(parentId).computeIfAbsent("children", k -> new ArrayList<>());
+ children.add(node);
+ } else {
+ roots.add(node);
+ }
+ }
+
+ String json = Utils.toJSONString(roots, 2);
+ Path dumpDir = goldFilePath.getParent();
+ Path dumpFile = dumpDir.resolve(testName + "-phase" + currentPhaseIndex + "-dump.json");
+ try {
+ Files.createDirectories(dumpDir);
+ Files.writeString(dumpFile, json);
+ System.out.println("Dumped raw traces: " + dumpFile.toAbsolutePath());
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to write trace dump", e);
+ }
+ }
+
+ // --- Tree building ---
+
+ private List<Map<String, Object>> buildTree(List<SpanData> spans) {
+ Map<String, Map<String, Object>> nodesBySpanId = new HashMap<>();
+ List<Map<String, Object>> roots = new ArrayList<>();
+
+ for (SpanData span : spans) {
+ nodesBySpanId.put(span.getSpanContext().getSpanId(), normalize(span));
+ }
+
+ for (SpanData span : spans) {
+ Map<String, Object> node = nodesBySpanId.get(span.getSpanContext().getSpanId());
+ String parentId = span.getParentSpanContext().getSpanId();
+ if (span.getParentSpanContext().isValid() && nodesBySpanId.containsKey(parentId)) {
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> children =
+ (List<Map<String, Object>>) nodesBySpanId.get(parentId).get("children");
+ if (children == null) {
+ children = new ArrayList<>();
+ nodesBySpanId.get(parentId).put("children", children);
+ }
+ children.add(node);
+ } else {
+ roots.add(node);
+ }
+ }
+
+ sortTree(roots);
+ return roots;
+ }
+
+ @SuppressWarnings("unchecked")
+ private void sortTree(List<Map<String, Object>> nodes) {
+ nodes.sort(SPAN_COMPARATOR);
+ for (Map<String, Object> node : nodes) {
+ List<Map<String, Object>> children = (List<Map<String, Object>>) node.get("children");
+ if (children != null && !children.isEmpty()) {
+ sortTree(children);
+ }
+ }
+ }
+
+ private Map<String, Object> normalize(SpanData span) {
+ // Use LinkedHashMap for consistent key ordering in JSON output
+ Map<String, Object> node = new LinkedHashMap<>();
+ node.put("name", span.getName());
+ if (span.getKind() != SpanKind.INTERNAL) {
+ node.put("kind", span.getKind().name());
+ }
+
+ String dbInstance = normalizeDbInstance(span.getAttributes().get(TraceUtils.TAG_DB));
+ if (dbInstance != null) {
+ node.put(TraceUtils.TAG_DB.getKey(), dbInstance);
+ }
+
+ String dbType = span.getAttributes().get(TraceUtils.TAG_DB_TYPE);
+ if (dbType != null) {
+ node.put(TraceUtils.TAG_DB_TYPE.getKey(), dbType);
+ }
+
+ String httpMethod = span.getAttributes().get(TraceUtils.TAG_HTTP_METHOD);
+ if (httpMethod != null) {
+ node.put(TraceUtils.TAG_HTTP_METHOD.getKey(), httpMethod);
+ }
+
+ Long statusCode = span.getAttributes().get(TraceUtils.TAG_HTTP_STATUS);
+ if (statusCode != null) {
+ node.put(TraceUtils.TAG_HTTP_STATUS.getKey(), statusCode.intValue());
+ }
+
+ String httpUrl = span.getAttributes().get(TraceUtils.TAG_HTTP_URL);
+ if (httpUrl != null) {
+ node.put(TraceUtils.TAG_HTTP_URL.getKey(), normalizeUrl(httpUrl));
+ }
+
+ String httpParams = span.getAttributes().get(TraceUtils.TAG_HTTP_PARAMS);
+ if (httpParams != null) {
+ node.put(TraceUtils.TAG_HTTP_PARAMS.getKey(), normalizeParams(httpParams));
+ }
+
+ // children will be added later during tree building if needed
+ return node;
+ }
+
+ private String normalizeUrl(String url) {
+ url = URL_HOST_PORT.matcher(url).replaceFirst("http://NORMALIZED/");
+ url = REPLICA_SUFFIX.matcher(url).replaceAll("_replica_nN");
+ return url;
+ }
+
+ private String normalizeParams(String params) {
+ // Scrub embedded URLs (distrib.from=http%3A%2F%2F...) that contain ports
+ params = ENCODED_URL_IN_PARAMS.matcher(params).replaceAll("http%3A%2F%2FNORMALIZED");
+ String[] parts = params.split("&");
+ Arrays.sort(parts);
+ return String.join("&", parts);
+ }
+
+ private String normalizeDbInstance(String dbInstance) {
+ if (dbInstance == null) return null;
+ return REPLICA_SUFFIX.matcher(dbInstance).replaceAll("_replica_nN");
+ }
+
+ // --- Comparison ---
+
+ @SuppressWarnings("unchecked")
+ private void comparePhase(int phaseIndex, List<Map<String, Object>> actual) {
+ List<Map<String, Object>> phases = (List<Map<String, Object>>) goldFile.get("phases");
+ Map<String, Object> phase = phases.get(phaseIndex);
+ List<Map<String, Object>> expectedSpans = (List<Map<String, Object>>) phase.get("spans");
+ String description = (String) phase.get("description");
+
+ String expectedJson = Utils.toJSONString(expectedSpans, 2);
+ String actualJson = Utils.toJSONString(actual, 2);
+
+ if (!expectedJson.equals(actualJson)) {
+ Path tempFile = writeTempJson(Utils.toJSONString(actual, 2), phaseIndex);
+ String message =
+ String.format(
+ Locale.getDefault(),
+ """
+
+ Trace spans mismatch in phase %d%s
+
+ Expected (gold file):
+ %s
+
+ Actual output (written to temp file):
+ %s
+
+ To compare:
+ diff %s %s
+
+ If intentional, regenerate:
+ gradlew :solr:modules:opentelemetry:test --tests TestDistributedTracing.%s "-Ptests.jvmargs=-Dregenerate.golden.files=true" -Ptests.useSecurityManager=false
+ """,
+ phaseIndex,
+ description != null ? " \"" + description + "\"" : "",
+ goldFilePath.toAbsolutePath(),
+ tempFile.toAbsolutePath(),
+ goldFilePath.toAbsolutePath(),
+ tempFile.toAbsolutePath(),
+ testName);
+ Assert.assertEquals(message, expectedJson, actualJson);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private int countSpansRecursive(List<Map<String, Object>> nodes) {
+ int count = 0;
+ for (Map<String, Object> node : nodes) {
+ count++;
+ List<Map<String, Object>> children = (List<Map<String, Object>>) node.get("children");
+ if (children != null && !children.isEmpty()) {
+ count += countSpansRecursive(children);
+ }
+ }
+ return count;
+ }
+
+ // --- Gold file I/O ---
+
+ private static Path resolveGoldFilePath(Class<?> testClass, String testMethodName) {
+ // Gold files live at: {test-files}/solr/tracing/{TestClassName}/{testMethod}.json
+ // SolrTestCaseJ4.getFile() may resolve to the build output; walk up to find src/test-files.
+ Path solrDir = SolrTestCaseJ4.getFile("solr");
+ Path testFilesDir = solrDir.getParent();
+ // If we're in build output (e.g. build/resources/test/solr), find the source equivalent
+ String testFilesStr = testFilesDir.toString();
+ if (testFilesStr.contains("/build/")) {
+ testFilesDir =
+ Path.of(testFilesStr.substring(0, testFilesStr.indexOf("/build/")))
+ .resolve("src/test-files");
+ }
+ return testFilesDir
+ .resolve("solr/tracing")
+ .resolve(testClass.getSimpleName())
+ .resolve(testMethodName + ".json");
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map<String, Object> loadGoldFile(Path path) {
+ if (!Files.exists(path)) {
+ Assert.fail(
+ "Gold file not found: "
+ + path.toAbsolutePath()
+ + "\nRegenerate with: gradlew ... \"-Ptests.jvmargs=-Dregenerate.golden.files=true\" -Ptests.useSecurityManager=false");
+ }
+ try {
+ byte[] bytes = Files.readAllBytes(path);
+ return (Map<String, Object>) Utils.fromJSON(bytes);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to read gold file: " + path, e);
+ }
+ }
+
+ private void writeGoldFile() {
+ Map<String, Object> output = new LinkedHashMap<>();
+ List<Map<String, Object>> phases = new ArrayList<>();
+ for (int i = 0; i < recordedPhases.size(); i++) {
+ Map<String, Object> phase = new LinkedHashMap<>();
+ phase.put("description", "phase " + i);
+ phase.put("spans", recordedPhases.get(i));
+ phases.add(phase);
+ }
+ output.put("phases", phases);
+ try {
+ Files.createDirectories(goldFilePath.getParent());
+ String json = Utils.toJSONString(output, 2);
+ Files.writeString(goldFilePath, json);
+ System.out.println("Regenerated gold file: " + goldFilePath.toAbsolutePath());
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to write gold file: " + goldFilePath, e);
+ }
+ }
+
+ private Path writeTempJson(String json, int phaseIndex) {
+ try {
+ Path tempFile =
+ Path.of(System.getProperty("java.io.tmpdir"), testName + "-phase" + phaseIndex + ".json");
+ Files.writeString(tempFile, json);
+ return tempFile;
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to write temp file", e);
+ }
+ }
+}
diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java
index 5433c11fd1d..f73d4706003 100644
--- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java
+++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java
@@ -17,11 +17,11 @@
package org.apache.solr.opentelemetry;
+import com.carrotsearch.randomizedtesting.annotations.Seed;
import io.opentelemetry.api.GlobalOpenTelemetry;
import io.opentelemetry.api.trace.TracerProvider;
import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
import io.opentelemetry.sdk.trace.data.SpanData;
-import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
@@ -30,11 +30,10 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.MetricsRequest;
import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.InputStreamResponseParser;
@@ -50,6 +49,7 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+@Seed("0") // don't want randomization when testing observability
public class TestDistributedTracing extends SolrCloudTestCase {
private static final String COLLECTION = "collection1";
@@ -58,6 +58,8 @@ public class TestDistributedTracing extends SolrCloudTestCase {
public static void setupCluster() throws Exception {
// force early init
CustomTestOtelTracerConfigurator.prepareForTest();
+ // HTTP 2 clients can do things more asynchronously, leading to less determinism in what we test
+ System.setProperty("solr.http1", "true");
configureCluster(4)
.addConfig("config", TEST_PATH().resolve("collection1").resolve("conf"))
@@ -71,7 +73,28 @@ public class TestDistributedTracing extends SolrCloudTestCase {
TracerProvider.noop(),
GlobalOpenTelemetry.get().getTracerProvider());
- CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 2)
+ // Create collection with explicit replica placement for deterministic leader assignment.
+ // First replica on each shard becomes the leader.
+ String node0 = cluster.getJettySolrRunner(0).getNodeName();
+ String node1 = cluster.getJettySolrRunner(1).getNodeName();
+ String node2 = cluster.getJettySolrRunner(2).getNodeName();
+ String node3 = cluster.getJettySolrRunner(3).getNodeName();
+ CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 1)
+ .setCreateNodeSet("EMPTY")
+ .process(cluster.getSolrClient());
+ // shard1: node0 = leader, node1 = follower
+ CollectionAdminRequest.addReplicaToShard(COLLECTION, "shard1")
+ .setNode(node0)
+ .process(cluster.getSolrClient());
+ CollectionAdminRequest.addReplicaToShard(COLLECTION, "shard1")
+ .setNode(node1)
+ .process(cluster.getSolrClient());
+ // shard2: node2 = leader, node3 = follower
+ CollectionAdminRequest.addReplicaToShard(COLLECTION, "shard2")
+ .setNode(node2)
+ .process(cluster.getSolrClient());
+ CollectionAdminRequest.addReplicaToShard(COLLECTION, "shard2")
+ .setNode(node3)
.process(cluster.getSolrClient());
cluster.waitForActiveCollection(COLLECTION, 2, 4);
}
@@ -87,121 +110,76 @@ public class TestDistributedTracing extends SolrCloudTestCase {
}
@Test
- public void test() throws IOException, SolrServerException {
- // TODO it would be clearer if we could compare the complete Span tree between reality
- // and what we assert it looks like in a structured visual way.
- CloudSolrClient cloudClient = cluster.getSolrClient();
+ public void test() throws Exception {
+ var verifier = new GoldFileTraceVerifier(getClass(), "test");
+ // TODO use a CloudSolrClient. However it's not yet deterministic due to use of random not
+ // aligned to the test seed.
+ var client = cluster.getJettySolrRunner(0).getSolrClient();
// Indexing
- cloudClient.add(COLLECTION, sdoc("id", "1"));
- var finishedSpans = getAndClearSpans(1);
- finishedSpans.removeIf(
- span ->
- span.getAttributes().get(TraceUtils.TAG_HTTP_URL) == null
- || !span.getAttributes().get(TraceUtils.TAG_HTTP_URL).endsWith("/update"));
- assertEquals(2, finishedSpans.size());
- assertOneSpanIsChildOfAnother(finishedSpans);
- // core because cloudClient routes to core
- assertEquals("post:/{core}/update", finishedSpans.get(0).getName());
- assertCoreName(finishedSpans.get(0), COLLECTION);
-
- cloudClient.add(COLLECTION, sdoc("id", "2"));
- cloudClient.add(COLLECTION, sdoc("id", "3"));
- cloudClient.add(COLLECTION, sdoc("id", "4"));
- cloudClient.commit(COLLECTION);
- getAndClearSpans();
+ new UpdateRequest()
+ .add(List.of(sdoc("id", "1"), sdoc("id", "2"), sdoc("id", "3"), sdoc("id", "4")))
+ .commit(client, COLLECTION);
+ verifier.verifyPhase();
// Searching
- cloudClient.query(COLLECTION, new SolrQuery("*:*"));
- finishedSpans = getAndClearSpans(1);
- finishedSpans.removeIf(
- span ->
- span.getAttributes().get(TraceUtils.TAG_HTTP_URL) == null
- || !span.getAttributes().get(TraceUtils.TAG_HTTP_URL).endsWith("/select"));
- // one from client to server, 2 for execute query, 2 for fetching documents
- assertEquals(5, finishedSpans.size());
- var parentTraceId = getRootTraceId(finishedSpans);
- for (var span : finishedSpans) {
- if (isRootSpan(span)) {
- continue;
- }
- assertEquals(span.getParentSpanContext().getTraceId(), parentTraceId);
- assertEquals(span.getTraceId(), parentTraceId);
- }
- assertEquals("get:/{core}/select", finishedSpans.get(0).getName());
- assertCoreName(finishedSpans.get(0), COLLECTION);
+ client.query(COLLECTION, new SolrQuery("*:*"));
+ verifier.verifyPhase();
+
+ verifier.done();
}
@Test
public void testAdminApi() throws Exception {
- CloudSolrClient cloudClient = cluster.getSolrClient();
+ var verifier = new GoldFileTraceVerifier(getClass(), "testAdminApi");
+ // TODO use a CloudSolrClient. However it's not yet deterministic due to use of random not
+ // aligned to the test seed.
+ var client = cluster.getJettySolrRunner(0).getSolrClient();
MetricsRequest request = new MetricsRequest();
request.setResponseParser(new InputStreamResponseParser(MetricUtils.PROMETHEUS_METRICS_WT));
- NamedList<Object> rsp = cloudClient.request(request);
+ NamedList<Object> rsp = client.request(request);
((InputStream) rsp.get("stream")).close();
- var finishedSpans = getAndClearSpans(1);
- assertEquals("get:/admin/metrics", finishedSpans.get(0).getName());
+ verifier.verifyPhase();
+
+ CollectionAdminRequest.listCollections(client);
+ verifier.verifyPhase();
- CollectionAdminRequest.listCollections(cloudClient);
- finishedSpans = getAndClearSpans(1);
- assertEquals("list:/admin/collections", finishedSpans.get(0).getName());
+ verifier.done();
}
@Test
public void testV2Api() throws Exception {
- CloudSolrClient cloudClient = cluster.getSolrClient();
+ var verifier = new GoldFileTraceVerifier(getClass(), "testV2Api");
+ // TODO use a CloudSolrClient. However it's not yet deterministic due to use of random not
+ // aligned to the test seed.
+ var client = cluster.getJettySolrRunner(0).getSolrClient();
new V2Request.Builder("/collections/" + COLLECTION + "/reload")
.withMethod(SolrRequest.METHOD.POST)
.withPayload("{}")
.build()
- .process(cloudClient);
- var finishedSpans = getAndClearSpans(1);
- assertEquals("post:/collections/{collection}/reload", finishedSpans.get(0).getName());
- assertCollectionName(finishedSpans.get(0), COLLECTION);
+ .process(client);
+ verifier.verifyPhase();
new V2Request.Builder("/c/" + COLLECTION + "/update/json")
.withMethod(SolrRequest.METHOD.POST)
- .withPayload("{\n" + " \"id\" : \"9\"\n" + "}")
+ .withPayload("{\"id\":\"9\"}")
.withParams(params("commit", "true"))
.build()
- .process(cloudClient);
- finishedSpans = getAndClearSpans(1);
- assertEquals("post:/c/{collection}/update/json", finishedSpans.get(0).getName());
- assertCollectionName(finishedSpans.get(0), COLLECTION);
+ .process(client);
+ verifier.verifyPhase();
final V2Response v2Response =
new V2Request.Builder("/c/" + COLLECTION + "/select")
.withMethod(SolrRequest.METHOD.GET)
.withParams(params("q", "id:9"))
.build()
- .process(cloudClient);
- finishedSpans = getAndClearSpans(1);
- assertEquals("get:/c/{collection}/select", finishedSpans.get(0).getName());
- assertCollectionName(finishedSpans.get(0), COLLECTION);
+ .process(client);
+ verifier.verifyPhase();
assertEquals(1, ((SolrDocumentList) v2Response.getResponse().get("response")).getNumFound());
- }
- /**
- * Best effort test of the apache http client tracing. the test assumes the request uses the http
- * client but there is no way to enforce it, so when the api will be rewritten this test will
- * become obsolete
- */
- @Test
- public void testApacheClient() throws Exception {
- CollectionAdminRequest.ColStatus a1 = CollectionAdminRequest.collectionStatus(COLLECTION);
- CollectionAdminResponse r1 = a1.process(cluster.getSolrClient());
- assertEquals(0, r1.getStatus());
- var finishedSpans = getAndClearSpans(1);
- var parentTraceId = getRootTraceId(finishedSpans);
- for (var span : finishedSpans) {
- if (isRootSpan(span)) {
- continue;
- }
- assertEquals(span.getParentSpanContext().getTraceId(), parentTraceId);
- assertEquals(span.getTraceId(), parentTraceId);
- }
+ verifier.done();
}
@Test
@@ -316,18 +294,6 @@ public class TestDistributedTracing extends SolrCloudTestCase {
assertTrue(span.getAttributes().get(TraceUtils.TAG_DB).startsWith(collection + "_"));
}
- private void assertOneSpanIsChildOfAnother(List<SpanData> finishedSpans) {
- SpanData child = finishedSpans.get(0);
- SpanData parent = finishedSpans.get(1);
- if (isRootSpan(child)) {
- var temp = parent;
- parent = child;
- child = temp;
- }
- assertEquals(child.getParentSpanContext().getTraceId(), parent.getTraceId());
- assertEquals(child.getTraceId(), parent.getTraceId());
- }
-
static List<SpanData> getAndClearSpans() {
return getAndClearSpans(0);
}
(solr) 07/07: SolrJ: simplify ResponseParser.getContentType processing (#4528)
Posted by ds...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit 605dfc1e8785a754802597c84dcc0840f751c7ba
Author: David Smiley <ds...@apache.org>
AuthorDate: Tue Jun 23 14:48:32 2026 -0400
SolrJ: simplify ResponseParser.getContentType processing (#4528)
ResponseParser.getContentType must now return a Set, statically defined. Enforced by RP's constructor.
(cherry picked from commit f43171e145a3562509a9dadeb57c50adbd0a271b)
---
.../PR#4528-ResponseParser_getContentType.yml | 8 ++++++
.../org/apache/solr/search/TestSmileRequest.java | 8 ++++--
.../client/solrj/jetty/HttpJettySolrClient.java | 28 ------------------
.../solr/client/solrj/impl/HttpJdkSolrClient.java | 33 ----------------------
.../solr/client/solrj/impl/HttpSolrClient.java | 25 ++++------------
.../solrj/response/JavaBinResponseParser.java | 8 ++++--
.../solr/client/solrj/response/ResponseParser.java | 33 +++++++++++++++++++---
.../client/solrj/response/XMLResponseParser.java | 4 ++-
.../json/JacksonDataBindResponseParser.java | 9 +++---
.../solrj/response/json/JsonMapResponseParser.java | 7 +++--
.../solr/client/solrj/SolrExampleCborTest.java | 5 +++-
.../client/solrj/impl/HttpJdkSolrClientTest.java | 23 ---------------
12 files changed, 68 insertions(+), 123 deletions(-)
diff --git a/changelog/unreleased/PR#4528-ResponseParser_getContentType.yml b/changelog/unreleased/PR#4528-ResponseParser_getContentType.yml
new file mode 100644
index 00000000000..e9de829b06d
--- /dev/null
+++ b/changelog/unreleased/PR#4528-ResponseParser_getContentType.yml
@@ -0,0 +1,8 @@
+title: >
+ Simplify SolrJ ResponseParser.getContentType processing. getContentType must now return a Set.
+type: other
+authors:
+ - name: David Smiley
+links:
+ - name: PR#4528
+ url: https://github.com/apache/solr/pull/4528
diff --git a/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java b/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java
index 79c9e0a35c3..a3bb792496b 100644
--- a/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java
+++ b/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java
@@ -18,7 +18,6 @@ package org.apache.solr.search;
import java.io.IOException;
import java.io.InputStream;
-import java.util.Collection;
import java.util.Map;
import java.util.Set;
import org.apache.solr.JSONTestUtil;
@@ -106,9 +105,12 @@ public class TestSmileRequest extends SolrTestCaseJ4 {
return new NamedList(m);
}
+ private static final Set<String> CONTENT_TYPES =
+ Set.of("application/x-jackson-smile", "application/octet-stream");
+
@Override
- public Collection<String> getContentTypes() {
- return Set.of("application/x-jackson-smile", "application/octet-stream");
+ public Set<String> getContentTypes() {
+ return CONTENT_TYPES;
}
}
}
diff --git a/solr/solrj-jetty/src/java/org/apache/solr/client/solrj/jetty/HttpJettySolrClient.java b/solr/solrj-jetty/src/java/org/apache/solr/client/solrj/jetty/HttpJettySolrClient.java
index d0d8e034c22..fbc276cc992 100644
--- a/solr/solrj-jetty/src/java/org/apache/solr/client/solrj/jetty/HttpJettySolrClient.java
+++ b/solr/solrj-jetty/src/java/org/apache/solr/client/solrj/jetty/HttpJettySolrClient.java
@@ -26,7 +26,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
@@ -37,7 +36,6 @@ import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
import org.apache.solr.client.api.util.SolrVersion;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
@@ -161,7 +159,6 @@ public class HttpJettySolrClient extends HttpSolrClient {
this.listenerFactory = new ArrayList<>(0);
}
- updateDefaultMimeTypeForParser();
this.idleTimeoutMillis = builder.getIdleTimeoutMillis();
try {
@@ -807,31 +804,6 @@ public class HttpJettySolrClient extends HttpSolrClient {
return httpClient.isFollowRedirects();
}
- @Override
- protected boolean processorAcceptsMimeType(
- Collection<String> processorSupportedContentTypes, String mimeType) {
-
- return processorSupportedContentTypes.stream()
- .map(ct -> MimeTypes.getContentTypeWithoutCharset(ct).trim())
- .anyMatch(mimeType::equalsIgnoreCase);
- }
-
- @Override
- protected void updateDefaultMimeTypeForParser() {
- defaultParserMimeTypes =
- parser.getContentTypes().stream()
- .map(ct -> MimeTypes.getContentTypeWithoutCharset(ct).trim().toLowerCase(Locale.ROOT))
- .collect(Collectors.toSet());
- }
-
- @Override
- protected String allProcessorSupportedContentTypesCommaDelimited(
- Collection<String> processorSupportedContentTypes) {
- return processorSupportedContentTypes.stream()
- .map(ct -> MimeTypes.getContentTypeWithoutCharset(ct).trim().toLowerCase(Locale.ROOT))
- .collect(Collectors.joining(", "));
- }
-
/**
* An HttpJettySolrClient that doesn't close or cleanup any resources
*
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpJdkSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpJdkSolrClient.java
index 5489cac7058..1f88adc519f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpJdkSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpJdkSolrClient.java
@@ -37,7 +37,6 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
-import java.util.Objects;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
@@ -46,7 +45,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.stream.Collectors;
import javax.net.ssl.SSLContext;
import org.apache.solr.client.api.util.SolrVersion;
import org.apache.solr.client.solrj.SolrRequest;
@@ -139,7 +137,6 @@ public class HttpJdkSolrClient extends HttpSolrClient {
ProxySelector.of(new InetSocketAddress(builder.getProxyHost(), builder.getProxyPort())));
}
this.httpClient = b.build();
- updateDefaultMimeTypeForParser();
assert ObjectReleaseTracker.track(this);
}
@@ -537,36 +534,6 @@ public class HttpJdkSolrClient extends HttpSolrClient {
return httpClient.followRedirects() != HttpClient.Redirect.NEVER;
}
- @Override
- protected boolean processorAcceptsMimeType(
- Collection<String> processorSupportedContentTypes, String mimeType) {
- return processorSupportedContentTypes.stream()
- .map(this::contentTypeToMimeType)
- .filter(Objects::nonNull)
- .map(String::trim)
- .anyMatch(mimeType::equalsIgnoreCase);
- }
-
- @Override
- protected void updateDefaultMimeTypeForParser() {
- defaultParserMimeTypes =
- parser.getContentTypes().stream()
- .map(this::contentTypeToMimeType)
- .filter(Objects::nonNull)
- .map(s -> s.toLowerCase(Locale.ROOT).trim())
- .collect(Collectors.toSet());
- }
-
- @Override
- protected String allProcessorSupportedContentTypesCommaDelimited(
- Collection<String> processorSupportedContentTypes) {
- return processorSupportedContentTypes.stream()
- .map(this::contentTypeToMimeType)
- .filter(Objects::nonNull)
- .map(s -> s.toLowerCase(Locale.ROOT).trim())
- .collect(Collectors.joining(", "));
- }
-
@Override
protected BuilderBase<?, ?> toBuilder(String baseUrl) {
return new HttpJdkSolrClient.Builder(baseUrl).withHttpClient(this);
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
index d1e69255bab..bcf5eb6b3c1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
@@ -27,6 +27,7 @@ import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Collection;
+import java.util.Locale;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
@@ -72,11 +73,8 @@ public abstract class HttpSolrClient extends SolrClient {
protected RequestWriter requestWriter = new JavaBinRequestWriter();
- // updating parser instance needs to go via the setter to ensure update of defaultParserMimeTypes
protected ResponseParser parser = new JavaBinResponseParser();
- protected Set<String> defaultParserMimeTypes;
-
protected final String basicAuthAuthorizationStr;
protected HttpSolrClient(String serverBaseUrl, BuilderBase<?, ?> builder) {
@@ -266,12 +264,6 @@ public abstract class HttpSolrClient extends SolrClient {
return processor == null || processor instanceof InputStreamResponseParser;
}
- protected abstract boolean processorAcceptsMimeType(
- Collection<String> processorSupportedContentTypes, String mimeType);
-
- protected abstract String allProcessorSupportedContentTypesCommaDelimited(
- Collection<String> processorSupportedContentTypes);
-
/**
* Validates that the content type in the response can be processed by the Response Parser. Throws
* a {@code RemoteSolrException} if not.
@@ -283,19 +275,15 @@ public abstract class HttpSolrClient extends SolrClient {
String encoding,
int httpStatus,
String urlExceptionMessage) {
- if (mimeType == null
- || (processor == this.parser && defaultParserMimeTypes.contains(mimeType))) {
- // Shortcut the default scenario
+ if (mimeType == null) {
return;
}
final Collection<String> processorSupportedContentTypes = processor.getContentTypes();
if (!processorSupportedContentTypes.isEmpty()) {
- boolean processorAcceptsMimeType =
- processorAcceptsMimeType(processorSupportedContentTypes, mimeType);
- if (!processorAcceptsMimeType) {
+ final String normalizedMimeType = mimeType.toLowerCase(Locale.ROOT).trim();
+ if (!processorSupportedContentTypes.contains(normalizedMimeType)) {
// unexpected mime type
- final String allSupportedTypes =
- allProcessorSupportedContentTypesCommaDelimited(processorSupportedContentTypes);
+ final String allSupportedTypes = String.join(", ", processorSupportedContentTypes);
String prefix =
"Expected mime type in [" + allSupportedTypes + "] but got " + mimeType + ". ";
String exceptionEncoding = encoding != null ? encoding : FALLBACK_CHARSET.name();
@@ -322,11 +310,8 @@ public abstract class HttpSolrClient extends SolrClient {
protected void setParser(ResponseParser parser) {
this.parser = parser;
- updateDefaultMimeTypeForParser();
}
- protected abstract void updateDefaultMimeTypeForParser();
-
/**
* Executes a SolrRequest using the provided URL to temporarily override any "base URL" currently
* used by this client
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/JavaBinResponseParser.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/JavaBinResponseParser.java
index 0fd83fa1554..ded08f6d3c3 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/JavaBinResponseParser.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/JavaBinResponseParser.java
@@ -18,7 +18,6 @@ package org.apache.solr.client.solrj.response;
import java.io.IOException;
import java.io.InputStream;
-import java.util.Collection;
import java.util.Set;
import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.common.util.NamedList;
@@ -52,8 +51,11 @@ public class JavaBinResponseParser extends ResponseParser {
return new JavaBinCodec(null, stringCache);
}
+ private static final Set<String> CONTENT_TYPES =
+ Set.of(JAVABIN_CONTENT_TYPE, JAVABIN_CONTENT_TYPE_V2);
+
@Override
- public Collection<String> getContentTypes() {
- return Set.of(JAVABIN_CONTENT_TYPE, JAVABIN_CONTENT_TYPE_V2);
+ public Set<String> getContentTypes() {
+ return CONTENT_TYPES;
}
}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/ResponseParser.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/ResponseParser.java
index 9d2fbc7a67e..9884d8a1ef5 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/ResponseParser.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/ResponseParser.java
@@ -19,6 +19,8 @@ package org.apache.solr.client.solrj.response;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
+import java.util.Locale;
+import java.util.Set;
import org.apache.solr.common.util.NamedList;
/**
@@ -28,6 +30,23 @@ import org.apache.solr.common.util.NamedList;
*/
public abstract class ResponseParser {
+ protected ResponseParser() {
+ assert validateContentTypes();
+ }
+
+ private boolean validateContentTypes() {
+ Collection<String> contentTypes = getContentTypes();
+ assert contentTypes == getContentTypes()
+ : getClass().getName() + ".getContentTypes() must return the same instance on every call";
+ for (String ct : contentTypes) {
+ assert !ct.contains(";") && ct.equals(ct.toLowerCase(Locale.ROOT))
+ : getClass().getName()
+ + ".getContentTypes() must return lowercase MIME types without semicolons, got: "
+ + ct;
+ }
+ return true;
+ }
+
/** The writer type placed onto the request as the {@code wt} param. */
public abstract String getWriterType(); // for example: wt=XML, JSON, etc
@@ -35,10 +54,16 @@ public abstract class ResponseParser {
throws IOException;
/**
- * A well-behaved ResponseParser will return the content-types it supports.
+ * Returns the MIME types this parser supports. Return an empty set to disable. Implementations
+ * must:
+ *
+ * <ul>
+ * <li>Returns the same instance (same reference on every call).
+ * <li>Use only lowercase MIME types without charset or other parameters (no semicolons)
+ * <li>
+ * </ul>
*
- * @return the content-type values that this parser is capable of parsing. Never null. Empty means
- * no enforcement.
+ * @return the MIME types that this parser is capable of parsing. Never null.
*/
- public abstract Collection<String> getContentTypes();
+ public abstract Set<String> getContentTypes();
}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/XMLResponseParser.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/XMLResponseParser.java
index 8e378989668..5bc358b4f5e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/XMLResponseParser.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/XMLResponseParser.java
@@ -80,9 +80,11 @@ public class XMLResponseParser extends ResponseParser {
return "xml";
}
+ private static final Set<String> CONTENT_TYPES = Set.of("application/xml");
+
@Override
public Set<String> getContentTypes() {
- return Set.of(XML_CONTENT_TYPE);
+ return CONTENT_TYPES;
}
public NamedList<Object> processResponse(Reader in) {
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/json/JacksonDataBindResponseParser.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/json/JacksonDataBindResponseParser.java
index 954db55f10c..df363449fba 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/json/JacksonDataBindResponseParser.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/json/JacksonDataBindResponseParser.java
@@ -20,9 +20,8 @@ package org.apache.solr.client.solrj.response.json;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.util.Collection;
-import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.solr.client.api.model.SolrJerseyResponse;
import org.apache.solr.client.solrj.request.json.JacksonContentWriter;
import org.apache.solr.client.solrj.response.ResponseParser;
@@ -47,9 +46,11 @@ public class JacksonDataBindResponseParser<T> extends ResponseParser {
return "json";
}
+ private static final Set<String> CONTENT_TYPES = Set.of("application/json");
+
@Override
- public Collection<String> getContentTypes() {
- return List.of("application/json");
+ public Set<String> getContentTypes() {
+ return CONTENT_TYPES;
}
// TODO it'd be nice if the ResponseParser could receive the mime type so it can parse
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/json/JsonMapResponseParser.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/json/JsonMapResponseParser.java
index 8410612e0a8..e1848ba44ff 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/json/JsonMapResponseParser.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/json/JsonMapResponseParser.java
@@ -20,7 +20,6 @@ package org.apache.solr.client.solrj.response.json;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.util.Collection;
import java.util.Map;
import java.util.Set;
import org.apache.solr.client.solrj.response.ResponseParser;
@@ -55,8 +54,10 @@ public class JsonMapResponseParser extends ResponseParser {
return list;
}
+ private static final Set<String> CONTENT_TYPES = Set.of("application/json");
+
@Override
- public Collection<String> getContentTypes() {
- return Set.of("application/json");
+ public Set<String> getContentTypes() {
+ return CONTENT_TYPES;
}
}
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleCborTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleCborTest.java
index dda3a57426d..ff2d325579b 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleCborTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleCborTest.java
@@ -281,9 +281,12 @@ public class SolrExampleCborTest extends SolrExampleTests {
return "cbor";
}
+ private static final Set<String> CONTENT_TYPES =
+ Set.of("application/cbor", "application/octet-stream");
+
@Override
public Set<String> getContentTypes() {
- return Set.of("application/cbor", "application/octet-stream");
+ return CONTENT_TYPES;
}
@Override
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpJdkSolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpJdkSolrClientTest.java
index fe159e4ad3f..73ff00474bb 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpJdkSolrClientTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpJdkSolrClientTest.java
@@ -437,29 +437,6 @@ public class HttpJdkSolrClientTest extends HttpSolrClientTestBase {
}
}
- @Test
- public void testProcessorMimeTypes() throws Exception {
- ResponseParser rp = new XMLResponseParser();
-
- try (HttpJdkSolrClient client =
- builder(solrTestRule.getBaseUrl()).withResponseParser(rp).build()) {
- assertTrue(client.processorAcceptsMimeType(rp.getContentTypes(), "application/xml"));
- assertFalse(client.processorAcceptsMimeType(rp.getContentTypes(), "application/json"));
- queryToHelpJdkReleaseThreads(client);
- }
-
- rp = new JavaBinResponseParser();
- try (HttpJdkSolrClient client =
- builder(solrTestRule.getBaseUrl()).withResponseParser(rp).build()) {
- assertTrue(
- client.processorAcceptsMimeType(
- rp.getContentTypes(), "application/vnd.apache.solr.javabin"));
- assertTrue(client.processorAcceptsMimeType(rp.getContentTypes(), "application/octet-stream"));
- assertFalse(client.processorAcceptsMimeType(rp.getContentTypes(), "application/xml"));
- queryToHelpJdkReleaseThreads(client);
- }
- }
-
@Test
public void testContentTypeToEncoding() throws Exception {
try (HttpJdkSolrClient client = builder(solrTestRule.getBaseUrl()).build()) {
(solr) 02/07: Add CloudSolrClient.getHttpClient (#4523)
Posted by ds...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit 99c70a67a01261574fde1fa364898a4c040bc173
Author: David Smiley <ds...@apache.org>
AuthorDate: Mon Jun 22 22:02:08 2026 -0400
Add CloudSolrClient.getHttpClient (#4523)
The subclasses had this but now we can define this at the root abstraction. The CloudLegacySolrClient had prevented this from being added.
(cherry picked from commit dc910ce85803e292a9279dba80e394f24b4b964f)
---
.../org/apache/solr/cloud/api/collections/SplitShardCmd.java | 3 +--
.../apache/solr/cloud/api/collections/ShardSplitTest.java | 3 +--
.../solr-ref-guide/modules/deployment-guide/pages/solrj.adoc | 4 ++--
.../solr/client/solrj/impl/SolrClientNodeStateProvider.java | 12 ++++--------
.../apache/solr/client/solrj/impl/CloudHttp2SolrClient.java | 1 +
.../org/apache/solr/client/solrj/impl/CloudSolrClient.java | 3 +++
.../solr/client/solrj/impl/CloudSolrClientCacheTest.java | 5 +++++
.../api/collections/AbstractCloudBackupRestoreTestCase.java | 6 +++---
8 files changed, 20 insertions(+), 17 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
index 200c9a6607d..6740c5ba046 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
@@ -43,7 +43,6 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.VersionedData;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
import org.apache.solr.client.solrj.impl.NodeValueFetcher;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.MetricsRequest;
@@ -879,7 +878,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
var req = new MetricsRequest(params);
req.setResponseParser(new InputStreamResponseParser("prometheus"));
- var cloudClient = (CloudHttp2SolrClient) cloudManager.getSolrClient();
+ var cloudClient = cloudManager.getSolrClient();
var httpClient = cloudClient.getHttpClient();
NamedList<Object> resp =
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
index b52177d006f..9b7053adda5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
@@ -41,7 +41,6 @@ import org.apache.solr.client.solrj.RemoteSolrException;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.jetty.CloudJettySolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.SolrQuery;
@@ -157,7 +156,7 @@ public class ShardSplitTest extends BasicDistributedZkTest {
builder
.withDefaultCollection(collectionName)
.sendUpdatesOnlyToShardLeaders()
- .withHttpClient(((CloudJettySolrClient) cloudClient).getHttpClient())
+ .withHttpClient(cloudClient.getHttpClient())
.build()) {
StoppableIndexingThread thread =
new StoppableIndexingThread(controlClient, client, "i1", true);
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/solrj.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/solrj.adoc
index decefc3f36f..aabf9d73fc0 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/solrj.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/solrj.adoc
@@ -183,8 +183,8 @@ processing time of the largest update request.
=== Cloud Request Routing
-The SolrJ `CloudSolrClient` implementations (`CloudSolrClient` and `CloudHttp2SolrClient`) respect the xref:solrcloud-distributed-requests.adoc#shards-preference-parameter[shards.preference parameter].
-Therefore requests sent to single-sharded collections, using either of the above clients, will route requests the same way that distributed requests are routed to individual shards.
+SolrJ `CloudSolrClient` respects the xref:solrcloud-distributed-requests.adoc#shards-preference-parameter[shards.preference parameter].
+Therefore requests sent to single-sharded collections will route requests the same way that distributed requests are routed to individual shards.
If no `shards.preference` parameter is provided, the clients will default to sorting replicas randomly.
For update requests, while the replicas are sorted in the order defined by the request, leader replicas will always be sorted first.
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 44e4babce6d..a0fadf60590 100644
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -53,7 +53,7 @@ import org.slf4j.LoggerFactory;
public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private final CloudHttp2SolrClient solrClient;
+ private final CloudSolrClient solrClient;
protected final Map<String, Map<String, Map<String, List<Replica>>>>
nodeVsCollectionVsShardVsReplicaInfo = new HashMap<>();
@@ -61,11 +61,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
private Map<String, Map> nodeVsTags = new HashMap<>();
public SolrClientNodeStateProvider(CloudSolrClient solrClient) {
- if (!(solrClient instanceof CloudHttp2SolrClient)) {
- throw new IllegalArgumentException(
- "The passed-in CloudSolrClient must be a " + CloudHttp2SolrClient.class);
- }
- this.solrClient = (CloudHttp2SolrClient) solrClient;
+ this.solrClient = solrClient;
try {
readReplicaDetails();
} catch (IOException e) {
@@ -242,7 +238,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
static class RemoteCallCtx {
ZkClientClusterStateProvider zkClientClusterStateProvider;
- CloudHttp2SolrClient cloudSolrClient;
+ CloudSolrClient cloudSolrClient;
public final Map<String, Object> tags = new HashMap<>();
private final String node;
public Map<String, Object> session;
@@ -254,7 +250,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
return true;
}
- public RemoteCallCtx(String node, CloudHttp2SolrClient cloudSolrClient) {
+ public RemoteCallCtx(String node, CloudSolrClient cloudSolrClient) {
this.node = node;
this.cloudSolrClient = cloudSolrClient;
this.zkClientClusterStateProvider =
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java
index 0118b680428..067b26cf373 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java
@@ -102,6 +102,7 @@ public class CloudHttp2SolrClient extends CloudSolrClient {
return stateProvider;
}
+ @Override
public HttpSolrClient getHttpClient() {
return myClient;
}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
index b9b52442297..220f1a72f64 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
@@ -220,6 +220,9 @@ public abstract class CloudSolrClient extends SolrClient {
}
}
+ /** The HttpSolrClient that underlies this one. It has no baseUrl or default collection. */
+ public abstract HttpSolrClient getHttpClient();
+
public ResponseParser getParser() {
return getLbClient().getParser();
}
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java
index 706e334f1d1..18a279cbcd2 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java
@@ -424,6 +424,11 @@ public class CloudSolrClientCacheTest extends SolrTestCaseJ4 {
return provider;
}
+ @Override
+ public HttpSolrClient getHttpClient() {
+ throw new UnsupportedOperationException();
+ }
+
@FunctionalInterface
interface Invocation {
NamedList<Object> invoke(SolrRequest<?> request, List<String> inputCollections)
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
index 3124d5c1451..f569f3f12e0 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
@@ -32,7 +32,6 @@ import java.util.UUID;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.jetty.CloudJettySolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest.ClusterProp;
import org.apache.solr.client.solrj.request.QueryRequest;
@@ -467,12 +466,13 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
public static Map<String, Integer> getShardToDocCountMap(
CloudSolrClient client, DocCollection docCollection) throws SolrServerException, IOException {
Map<String, Integer> shardToDocCount = new TreeMap<>();
- var jettySolrClient = ((CloudJettySolrClient) client).getHttpClient();
for (Slice slice : docCollection.getActiveSlices()) {
long docsInShard =
new QueryRequest("/select", params("q", "*:*", "distrib", "false"))
.processWithBaseUrl(
- jettySolrClient, slice.getLeader().getBaseUrl(), slice.getLeader().getCoreName())
+ client.getHttpClient(),
+ slice.getLeader().getBaseUrl(),
+ slice.getLeader().getCoreName())
.getResults()
.getNumFound();
shardToDocCount.put(slice.getName(), (int) docsInShard);
(solr) 01/07: SOLR-14070: Deprecate CloudSolrClient ZkHost constructor (#4533)
Posted by ds...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit c5c6ac846e145d0e26735b3bac75516816447992
Author: David Smiley <ds...@apache.org>
AuthorDate: Thu Jun 18 10:44:17 2026 -0400
SOLR-14070: Deprecate CloudSolrClient ZkHost constructor (#4533)
-- again; was accidentally un-deprecated for 10.0. Oops.
(cherry picked from commit 86bc6f292245e0566511a9d9c3fb8aaba933e564)
---
.../solr/client/solrj/impl/CloudSolrClient.java | 2935 ++++++++++----------
1 file changed, 1466 insertions(+), 1469 deletions(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
index 69e5c8d5809..b9b52442297 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
@@ -92,25 +92,12 @@ import org.slf4j.MDC;
*/
public abstract class CloudSolrClient extends SolrClient {
+ public static final String STATE_VERSION = "_stateVer_";
+ static final int DEFAULT_STATE_REFRESH_PARALLELISM = 5;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
// no of times collection state to be reloaded if stale state error is received
private static final int MAX_STALE_RETRIES =
Integer.parseInt(System.getProperty("solr.solrj.cloud.max.stale.retries", "5"));
- static final int DEFAULT_STATE_REFRESH_PARALLELISM = 5;
- private final Random rand = new Random();
-
- private final boolean updatesToLeaders;
- private final boolean directUpdatesToLeadersOnly;
- private final RequestReplicaListTransformerGenerator requestRLTGenerator;
- private final boolean parallelUpdates;
- private final ExecutorService threadPool =
- ExecutorUtil.newMDCAwareCachedThreadPool(
- new SolrNamedThreadFactory("CloudSolrClient ThreadPool"));
-
- public static final String STATE_VERSION = "_stateVer_";
- protected long retryExpiryTimeNano =
- TimeUnit.NANOSECONDS.convert(3, TimeUnit.SECONDS); // 3 seconds or 3 million nanos
private static final Set<String> NON_ROUTABLE_PARAMS =
Set.of(
UpdateParams.EXPUNGE_DELETES,
@@ -125,1713 +112,1672 @@ public abstract class CloudSolrClient extends SolrClient {
// Not supported via SolrCloud
// UpdateParams.ROLLBACK
);
-
+ protected final StateCache collectionStateCache = new StateCache();
+ private final Random rand = new Random();
+ private final boolean updatesToLeaders;
+ private final boolean directUpdatesToLeadersOnly;
+ private final RequestReplicaListTransformerGenerator requestRLTGenerator;
+ private final boolean parallelUpdates;
+ private final ExecutorService threadPool =
+ ExecutorUtil.newMDCAwareCachedThreadPool(
+ new SolrNamedThreadFactory("CloudSolrClient ThreadPool"));
private final ConcurrentHashMap<String, CompletableFuture<DocCollection>> collectionRefreshes =
new ConcurrentHashMap<>();
private final Semaphore stateRefreshSemaphore;
private final int stateRefreshParallelism;
+ protected long retryExpiryTimeNano =
+ TimeUnit.NANOSECONDS.convert(3, TimeUnit.SECONDS); // 3 seconds or 3 million nanos
private volatile boolean closed;
- /**
- * Constructs {@link CloudSolrClient} instances from provided configuration. It will use a Jetty
- * based {@code HttpClient} if available, or will otherwise use the JDK.
- */
- public static class Builder {
+ protected CloudSolrClient(
+ boolean updatesToLeaders, boolean parallelUpdates, boolean directUpdatesToLeadersOnly) {
+ this(
+ updatesToLeaders,
+ parallelUpdates,
+ directUpdatesToLeadersOnly,
+ DEFAULT_STATE_REFRESH_PARALLELISM);
+ }
- protected Collection<String> zkHosts = new ArrayList<>();
- protected List<String> solrUrls = new ArrayList<>();
- protected String zkChroot;
- protected HttpSolrClient httpClient;
- protected boolean shardLeadersOnly = true;
- protected boolean directUpdatesToLeadersOnly = false;
- protected boolean parallelUpdates = true;
- protected ClusterStateProvider stateProvider;
- protected HttpSolrClient.BuilderBase<?, ?> internalClientBuilder;
- protected RequestWriter requestWriter;
- protected ResponseParser responseParser;
- protected long retryExpiryTimeNano =
- TimeUnit.NANOSECONDS.convert(3, TimeUnit.SECONDS); // 3 seconds or 3 million nanos
+ protected CloudSolrClient(
+ boolean updatesToLeaders,
+ boolean parallelUpdates,
+ boolean directUpdatesToLeadersOnly,
+ int stateRefreshThreads) {
+ this.updatesToLeaders = updatesToLeaders;
+ this.parallelUpdates = parallelUpdates;
+ this.directUpdatesToLeadersOnly = directUpdatesToLeadersOnly;
+ this.requestRLTGenerator = new RequestReplicaListTransformerGenerator();
+ this.stateRefreshParallelism = Math.max(1, stateRefreshThreads);
+ this.stateRefreshSemaphore = new Semaphore(this.stateRefreshParallelism);
+ }
- protected String defaultCollection;
- protected long timeToLiveSeconds = 60;
- protected int parallelCacheRefreshesLocks = DEFAULT_STATE_REFRESH_PARALLELISM;
- protected int zkConnectTimeout = SolrZkClientTimeout.DEFAULT_ZK_CONNECT_TIMEOUT;
- protected int zkClientTimeout = SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT;
- protected boolean canUseZkACLs = true;
+ /**
+ * Determines whether an UpdateRequest contains sufficient routing information to identify shard
+ * leaders for direct updates when directUpdatesToLeadersOnly is enabled.
+ */
+ private static boolean hasInfoToFindLeaders(UpdateRequest updateRequest, String idField) {
+ final Map<SolrInputDocument, Map<String, Object>> documents = updateRequest.getDocumentsMap();
+ final Map<String, Map<String, Object>> deleteById = updateRequest.getDeleteByIdMap();
- /**
- * Provide a series of Solr URLs to be used when configuring {@link CloudSolrClient} instances.
- * The solr client will use these urls to understand the cluster topology, which solr nodes are
- * active etc.
- *
- * <p>Provided Solr URLs are expected to point to the root Solr path
- * ("http://hostname:8983/solr"); they should not include any collections, cores, or other path
- * components.
- *
- * <p>Usage example:
- *
- * <pre>
- * final List<String> solrBaseUrls = new ArrayList<String>();
- * solrBaseUrls.add("http://solr1:8983/solr"); solrBaseUrls.add("http://solr2:8983/solr"); solrBaseUrls.add("http://solr3:8983/solr");
- * final SolrClient client = new CloudSolrClient.Builder(solrBaseUrls).build();
- * </pre>
- */
- public Builder(List<String> solrUrls) {
- this.solrUrls = solrUrls;
+ final boolean hasNoDocuments = (documents == null || documents.isEmpty());
+ final boolean hasNoDeleteById = (deleteById == null || deleteById.isEmpty());
+ if (hasNoDocuments && hasNoDeleteById) {
+ // no documents and no delete-by-id, so no info to find leader(s)
+ return false;
}
- /**
- * Provide a series of ZK hosts which will be used when configuring {@link CloudSolrClient}
- * instances.
- *
- * <p>Usage example when Solr stores data at the ZooKeeper root ('/'):
- *
- * <pre>
- * final List<String> zkServers = new ArrayList<String>();
- * zkServers.add("zookeeper1:2181"); zkServers.add("zookeeper2:2181"); zkServers.add("zookeeper3:2181");
- * final SolrClient client = new CloudSolrClient.Builder(zkServers, Optional.empty()).build();
- * </pre>
- *
- * Usage example when Solr data is stored in a ZooKeeper chroot:
- *
- * <pre>
- * final List<String> zkServers = new ArrayList<String>();
- * zkServers.add("zookeeper1:2181"); zkServers.add("zookeeper2:2181"); zkServers.add("zookeeper3:2181");
- * final SolrClient client = new CloudSolrClient.Builder(zkServers, Optional.of("/solr")).build();
- * </pre>
- *
- * @param zkHosts a List of at least one ZooKeeper host and port (e.g. "zookeeper1:2181")
- * @param zkChroot the path to the root ZooKeeper node containing Solr data. Provide {@code
- * java.util.Optional.empty()} if no ZK chroot is used.
- */
- public Builder(List<String> zkHosts, Optional<String> zkChroot) {
- this.zkHosts = zkHosts;
- if (zkChroot.isPresent()) this.zkChroot = zkChroot.get();
+ if (documents != null) {
+ for (final Map.Entry<SolrInputDocument, Map<String, Object>> entry : documents.entrySet()) {
+ final SolrInputDocument doc = entry.getKey();
+ final Object fieldValue = doc.getFieldValue(idField);
+ if (fieldValue == null) {
+ // a document with no id field value, so can't find leader for it
+ return false;
+ }
+ }
}
- /** for an expert use-case */
- public Builder(ClusterStateProvider stateProvider) {
- this.stateProvider = stateProvider;
+ if (deleteById != null) {
+ for (final Map.Entry<String, Map<String, Object>> entry : deleteById.entrySet()) {
+ final Map<String, Object> params = entry.getValue();
+ if (params == null || params.get(ShardParams._ROUTE_) == null) {
+ // deleteById entry lacks explicit route parameter, can't find leader for it
+ return false;
+ }
+ }
}
- /**
- * Creates a client builder based on a connection string of 2 possible formats:
- *
- * <ul>
- * <li>ZooKeeper connection string (optionally with chroot), e.g. {@code
- * zk1:2181,zk2:2181,zk3:2181/solr}
- * <li>Comma-separated list of Solr node base URLs (HTTP or HTTPS), e.g. {@code
- * http://solr1:8983/solr,http://solr2:8983/solr}
- * </ul>
- *
- * @param connectionString a string specifying either ZooKeeper connection string or HTTP(S)
- * Solr URLs
- * @throws IllegalArgumentException if string is null, empty, or malformed
- */
- public Builder(String connectionString) {
- this(CloudSolrClientConnection.parse(connectionString));
- }
+ return true;
+ }
- /**
- * Creates a client builder from a {@link CloudSolrClientConnection}.
- *
- * @param connection instance of {@link CloudSolrClientConnection}, which can be obtained from
- * the solr connection string or created via the constructor
- */
- public Builder(CloudSolrClientConnection connection) {
- if (connection.isZookeeper()) {
- this.zkHosts = connection.quorumItems();
- this.zkChroot = connection.zkChroot();
- } else {
- this.solrUrls = connection.quorumItems();
- }
- }
+ protected abstract LBSolrClient getLbClient();
- /** Whether to use the default ZK ACLs when building a ZK Client. */
- public Builder canUseZkACLs(boolean canUseZkACLs) {
- this.canUseZkACLs = canUseZkACLs;
- return this;
- }
+ public abstract ClusterStateProvider getClusterStateProvider();
- /**
- * Tells {@link Builder} that created clients should be configured such that {@link
- * CloudSolrClient#isUpdatesToLeaders} returns <code>true</code>.
- *
- * @see #sendUpdatesToAnyReplica
- * @see CloudSolrClient#isUpdatesToLeaders
- */
- public Builder sendUpdatesOnlyToShardLeaders() {
- shardLeadersOnly = true;
- return this;
- }
+ /**
+ * @deprecated problematic as a 'get' method, since one implementation will do a remote request
+ * each time this is called, potentially return lots of data that isn't even needed.
+ */
+ @Deprecated
+ public ClusterState getClusterState() {
+ // The future of "ClusterState" isn't clear. Could make it more of a cache instead of a
+ // snapshot, so we un-deprecate. Or we avoid it and maybe make the ClusterStateProvider as that
+ // cache. SOLR-17604 is related.
+ return getClusterStateProvider().getClusterState();
+ }
- /**
- * Tells {@link Builder} that created clients should be configured such that {@link
- * CloudSolrClient#isUpdatesToLeaders} returns <code>false</code>.
- *
- * @see #sendUpdatesOnlyToShardLeaders
- * @see CloudSolrClient#isUpdatesToLeaders
- */
- public Builder sendUpdatesToAnyReplica() {
- shardLeadersOnly = false;
- return this;
- }
+ /** Is this a communication error? We will retry if so. */
+ protected boolean wasCommError(Throwable t) {
+ return t instanceof SocketException || t instanceof UnknownHostException;
+ }
- /**
- * Tells {@link CloudSolrClient.Builder} that created clients should send direct updates to
- * shard leaders only.
- *
- * <p>UpdateRequests whose leaders cannot be found will "fail fast" on the client side with a
- * {@link SolrException}
- *
- * @see #sendDirectUpdatesToAnyShardReplica
- * @see CloudSolrClient#isDirectUpdatesToLeadersOnly
- */
- public Builder sendDirectUpdatesToShardLeadersOnly() {
- directUpdatesToLeadersOnly = true;
- return this;
+ @Override
+ public void close() {
+ closed = true;
+ collectionRefreshes.clear();
+ if (!ExecutorUtil.isShutdown(this.threadPool)) {
+ ExecutorUtil.shutdownAndAwaitTermination(this.threadPool);
}
+ }
- /**
- * Tells {@link CloudSolrClient.Builder} that created clients can send updates to any shard
- * replica (shard leaders and non-leaders).
- *
- * <p>Shard leaders are still preferred, but the created clients will fall back to using other
- * replicas if a leader cannot be found.
- *
- * @see #sendDirectUpdatesToShardLeadersOnly
- * @see CloudSolrClient#isDirectUpdatesToLeadersOnly
- */
- public Builder sendDirectUpdatesToAnyShardReplica() {
- directUpdatesToLeadersOnly = false;
- return this;
- }
-
- /** Provides a {@link RequestWriter} for created clients to use when handing requests. */
- public Builder withRequestWriter(RequestWriter requestWriter) {
- this.requestWriter = requestWriter;
- return this;
- }
-
- /** Provides a {@link ResponseParser} for created clients to use when handling requests. */
- public Builder withResponseParser(ResponseParser responseParser) {
- this.responseParser = responseParser;
- return this;
- }
-
- /**
- * Tells {@link CloudSolrClient.Builder} whether created clients should send shard updates
- * serially or in parallel
- *
- * <p>When an {@link UpdateRequest} affects multiple shards, {@link CloudSolrClient} splits it
- * up and sends a request to each affected shard. This setting chooses whether those
- * sub-requests are sent serially or in parallel.
- *
- * <p>If not set, this defaults to 'true' and sends sub-requests in parallel.
- */
- public Builder withParallelUpdates(boolean parallelUpdates) {
- this.parallelUpdates = parallelUpdates;
- return this;
- }
+ public ResponseParser getParser() {
+ return getLbClient().getParser();
+ }
- /**
- * Configures how many collection state refresh operations may run in parallel using a dedicated
- * thread pool. This controls the maximum number of concurrent ZooKeeper/cluster state lookups.
- *
- * <p>Defaults to 5.
- */
- public Builder withParallelCacheRefreshes(int parallelCacheRefreshesLocks) {
- this.parallelCacheRefreshesLocks = parallelCacheRefreshesLocks;
- return this;
- }
+ public RequestWriter getRequestWriter() {
+ return getLbClient().getRequestWriter();
+ }
- /**
- * This is the time to wait to re-fetch the state after getting the same state version from ZK
- */
- public Builder withRetryExpiryTime(long expiryTime, TimeUnit unit) {
- this.retryExpiryTimeNano = TimeUnit.NANOSECONDS.convert(expiryTime, unit);
- return this;
- }
+ /** Gets whether direct updates are sent in parallel */
+ public boolean isParallelUpdates() {
+ return parallelUpdates;
+ }
- /** Sets the default collection for request. */
- public Builder withDefaultCollection(String defaultCollection) {
- this.defaultCollection = defaultCollection;
- return this;
- }
+ /**
+ * Connect to the zookeeper ensemble. This is an optional method that may be used to force a
+ * connection before any other requests are sent.
+ *
+ * @deprecated Call {@link ClusterStateProvider#getLiveNodes()} instead.
+ */
+ @Deprecated
+ public void connect() {
+ getClusterStateProvider().connect();
+ }
- /**
- * Sets the cache ttl for DocCollection Objects cached.
- *
- * @param timeToLive ttl value
- */
- public Builder withCollectionCacheTtl(long timeToLive, TimeUnit unit) {
- assert timeToLive > 0;
- this.timeToLiveSeconds = TimeUnit.SECONDS.convert(timeToLive, unit);
- return this;
+ /**
+ * Connect to a cluster. If the cluster is not ready, retry connection up to a given timeout.
+ *
+ * @param duration the timeout
+ * @param timeUnit the units of the timeout
+ * @throws TimeoutException if the cluster is not ready after the timeout
+ * @throws InterruptedException if the wait is interrupted
+ */
+ @Deprecated
+ public void connect(long duration, TimeUnit timeUnit)
+ throws TimeoutException, InterruptedException {
+ if (log.isInfoEnabled()) {
+ log.info(
+ "Waiting for {} {} for cluster at {} to be ready",
+ duration,
+ timeUnit,
+ getClusterStateProvider());
}
-
- /**
- * Set the internal Solr HTTP client.
- *
- * <p>Note: closing the client instance is the responsibility of the caller.
- *
- * @return this
- */
- public Builder withHttpClient(HttpSolrClient httpSolrClient) {
- if (this.internalClientBuilder != null) {
- throw new IllegalStateException(
- "The builder can't accept an httpClient AND an internalClientBuilder, only one of those can be provided");
+ long timeout = System.nanoTime() + timeUnit.toNanos(duration);
+ while (System.nanoTime() < timeout) {
+ try {
+ connect();
+ if (log.isInfoEnabled()) {
+ log.info("Cluster at {} ready", getClusterStateProvider());
+ }
+ return;
+ } catch (RuntimeException e) {
+ // not ready yet, then...
}
- this.httpClient = httpSolrClient;
- return this;
+ TimeUnit.MILLISECONDS.sleep(250);
}
+ throw new TimeoutException("Timed out waiting for cluster");
+ }
- /**
- * If provided, the CloudSolrClient will build it's internal client using this builder (instead
- * of the empty default one). Providing this builder allows users to configure the internal
- * clients (authentication, timeouts, etc.).
- *
- * @param internalClientBuilder the builder to use for creating the internal http client.
- * @return this
- */
- public Builder withHttpClientBuilder(HttpSolrClient.BuilderBase<?, ?> internalClientBuilder) {
- if (this.httpClient != null) {
- throw new IllegalStateException(
- "The builder can't accept an httpClient AND an internalClientBuilder, only one of those can be provided");
- }
- this.internalClientBuilder = internalClientBuilder;
- return this;
- }
+ @SuppressWarnings({"unchecked"})
+ private NamedList<Object> directUpdate(UpdateRequest request, String collection)
+ throws SolrServerException {
+ SolrParams params = request.getParams();
+ ModifiableSolrParams routableParams = new ModifiableSolrParams();
+ ModifiableSolrParams nonRoutableParams = new ModifiableSolrParams();
- @Deprecated(since = "9.10")
- public Builder withInternalClientBuilder(
- HttpSolrClient.BuilderBase<?, ?> internalClientBuilder) {
- return withHttpClientBuilder(internalClientBuilder);
+ if (params != null) {
+ nonRoutableParams.add(params);
+ routableParams.add(params);
+ for (String param : NON_ROUTABLE_PARAMS) {
+ routableParams.remove(param);
+ }
+ } else {
+ params = new ModifiableSolrParams();
}
- /**
- * Sets the Zk connection timeout
- *
- * @param zkConnectTimeout timeout value
- * @param unit time unit
- */
- public Builder withZkConnectTimeout(int zkConnectTimeout, TimeUnit unit) {
- this.zkConnectTimeout = Math.toIntExact(unit.toMillis(zkConnectTimeout));
- return this;
+ if (collection == null) {
+ throw new SolrServerException(
+ "No collection param specified on request and no default collection has been set.");
}
- /**
- * Sets the Zk client session timeout
- *
- * @param zkClientTimeout timeout value
- * @param unit time unit
- */
- public Builder withZkClientTimeout(int zkClientTimeout, TimeUnit unit) {
- this.zkClientTimeout = Math.toIntExact(unit.toMillis(zkClientTimeout));
- return this;
+ // Check to see if the collection is an alias. Updates to multi-collection aliases are ok as
+ // long as they are routed aliases
+ List<String> aliasedCollections = new ArrayList<>(resolveAliases(List.of(collection)));
+ if (aliasedCollections.size() == 1 || getClusterStateProvider().isRoutedAlias(collection)) {
+ collection = aliasedCollections.get(0); // pick 1st (consistent with HttpSolrCall behavior)
+ } else {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Update request to non-routed multi-collection alias not supported: "
+ + collection
+ + " -> "
+ + aliasedCollections);
}
- /** Create a {@link CloudSolrClient} based on the provided configuration. */
- public CloudHttp2SolrClient build() {
- int providedOptions = 0;
- if (!zkHosts.isEmpty()) providedOptions++;
- if (!solrUrls.isEmpty()) providedOptions++;
- if (stateProvider != null) providedOptions++;
+ DocCollection col = getDocCollection(collection, null);
- if (providedOptions > 1) {
- throw new IllegalArgumentException(
- "Only one of zkHost(s), solrUrl(s), or stateProvider should be specified.");
- } else if (providedOptions == 0) {
- throw new IllegalArgumentException(
- "One of zkHosts, solrUrls, or stateProvider must be specified.");
- }
+ DocRouter router = col.getRouter();
- return new CloudHttp2SolrClient(this);
+ if (router instanceof ImplicitDocRouter) {
+ // short circuit as optimization
+ return null;
}
- protected HttpSolrClient createOrGetHttpClient() {
- if (httpClient != null) {
- return httpClient;
- } else if (internalClientBuilder != null) {
- return internalClientBuilder.build();
+ ReplicaListTransformer replicaListTransformer =
+ requestRLTGenerator.getReplicaListTransformer(params);
+
+ // Create the URL map, which is keyed on slice name.
+ // The value is a list of URLs for each replica in the slice.
+ // The first value in the list is the leader for the slice.
+ final Map<String, List<String>> urlMap = buildUrlMap(col, replicaListTransformer);
+ String routeField =
+ (col.getRouter().getRouteField(col) == null) ? ID : col.getRouter().getRouteField(col);
+ final Map<String, ? extends LBSolrClient.Req> routes =
+ createRoutes(request, routableParams, col, router, urlMap, routeField);
+ if (routes == null) {
+ if (directUpdatesToLeadersOnly && hasInfoToFindLeaders(request, routeField)) {
+ // we have info (documents with ids and/or ids to delete) with
+ // which to find the leaders, but we could not find (all of) them
+ throw new SolrException(
+ SolrException.ErrorCode.SERVICE_UNAVAILABLE,
+ "directUpdatesToLeadersOnly==true but could not find leader(s)");
} else {
- return HttpSolrClient.builder(null).build();
+ // we could not find a leader or routes yet - use unoptimized general path
+ log.warn(
+ "No routing info found for update to collection '{}', broadcasting to all shards.",
+ collection);
+ return null;
}
}
- protected LBSolrClient createOrGetLbClient(HttpSolrClient myClient) {
- return myClient.createLBSolrClient();
- }
+ final NamedList<Throwable> exceptions = new NamedList<>();
+ final NamedList<NamedList<?>> shardResponses =
+ new NamedList<>(routes.size() + 1); // +1 for deleteQuery
- protected ClusterStateProvider createZkClusterStateProvider() {
- ClusterStateProvider stateProvider =
- ClusterStateProvider.newZkClusterStateProvider(zkHosts, zkChroot, canUseZkACLs);
- if (stateProvider instanceof SolrZkClientTimeout.SolrZkClientTimeoutAware timeoutAware) {
- timeoutAware.setZkClientTimeout(zkClientTimeout);
- timeoutAware.setZkConnectTimeout(zkConnectTimeout);
- }
- return stateProvider;
- }
+ long start = System.nanoTime();
- protected ClusterStateProvider createHttpClusterStateProvider(HttpSolrClient httpClient) {
- try {
- return new HttpClusterStateProvider<>(solrUrls, httpClient);
- } catch (Exception e) {
- throw new RuntimeException(
- "Couldn't initialize a HttpClusterStateProvider (is/are the "
- + "Solr server(s), "
- + solrUrls
- + ", down?)",
- e);
+ if (parallelUpdates) {
+ final Map<String, Future<NamedList<?>>> responseFutures =
+ CollectionUtil.newHashMap(routes.size());
+ for (final Map.Entry<String, ? extends LBSolrClient.Req> entry : routes.entrySet()) {
+ final String url = entry.getKey();
+ final LBSolrClient.Req lbRequest = entry.getValue();
+ try {
+ MDC.put("CloudSolrClient.url", url);
+ responseFutures.put(
+ url,
+ threadPool.submit(
+ () -> {
+ return getLbClient().request(lbRequest).getResponse();
+ }));
+ } finally {
+ MDC.remove("CloudSolrClient.url");
+ }
}
- }
- }
- protected static class StateCache extends ConcurrentHashMap<String, ExpiringCachedDocCollection> {
- final AtomicLong puts = new AtomicLong();
- final AtomicLong hits = new AtomicLong();
- final Lock evictLock = new ReentrantLock(true);
- public volatile long timeToLiveMs = 60 * 1000L;
+ for (final Map.Entry<String, Future<NamedList<?>>> entry : responseFutures.entrySet()) {
+ final String url = entry.getKey();
+ final Future<NamedList<?>> responseFuture = entry.getValue();
+ try {
+ shardResponses.add(url, responseFuture.get());
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ } catch (ExecutionException e) {
+ exceptions.add(url, e.getCause());
+ }
+ }
- @Override
- public ExpiringCachedDocCollection get(Object key) {
- ExpiringCachedDocCollection val = super.get(key);
- if (val == null) {
- // a new collection is likely to be added now.
- // check if there are stale items and remove them
- evictStale();
- return null;
+ if (exceptions.size() > 0) {
+ Throwable firstException = exceptions.getVal(0);
+ if (firstException instanceof SolrException e) {
+ throw getRouteException(
+ SolrException.ErrorCode.getErrorCode(e.code()), exceptions, routes);
+ } else {
+ throw getRouteException(SolrException.ErrorCode.SERVER_ERROR, exceptions, routes);
+ }
}
- if (val.isExpired(timeToLiveMs)) {
- super.remove(key);
- return null;
+ } else {
+ for (Map.Entry<String, ? extends LBSolrClient.Req> entry : routes.entrySet()) {
+ String url = entry.getKey();
+ LBSolrClient.Req lbRequest = entry.getValue();
+ try {
+ NamedList<Object> rsp = getLbClient().request(lbRequest).getResponse();
+ shardResponses.add(url, rsp);
+ } catch (Exception e) {
+ if (e instanceof SolrException) {
+ throw (SolrException) e;
+ } else {
+ throw new SolrServerException(e);
+ }
+ }
}
- hits.incrementAndGet();
- return val;
}
- ExpiringCachedDocCollection peek(Object key) {
- return super.get(key);
+ UpdateRequest nonRoutableRequest = null;
+ List<String> deleteQuery = request.getDeleteQuery();
+ if (deleteQuery != null && deleteQuery.size() > 0) {
+ UpdateRequest deleteQueryRequest = new UpdateRequest();
+ deleteQueryRequest.setDeleteQuery(deleteQuery);
+ nonRoutableRequest = deleteQueryRequest;
}
- @Override
- public ExpiringCachedDocCollection put(String key, ExpiringCachedDocCollection value) {
- puts.incrementAndGet();
- return super.put(key, value);
- }
+ Set<String> paramNames = nonRoutableParams.getParameterNames();
- void evictStale() {
- if (!evictLock.tryLock()) return;
+ Set<String> intersection = new HashSet<>(paramNames);
+ intersection.retainAll(NON_ROUTABLE_PARAMS);
+
+ if (nonRoutableRequest != null || intersection.size() > 0) {
+ if (nonRoutableRequest == null) {
+ nonRoutableRequest = new UpdateRequest();
+ }
+ nonRoutableRequest.setParams(nonRoutableParams);
+ nonRoutableRequest.setBasicAuthCredentials(
+ request.getBasicAuthUser(), request.getBasicAuthPassword());
+ final var endpoints =
+ routes.keySet().stream()
+ .map(url -> LBSolrClient.Endpoint.from(url))
+ .collect(Collectors.toList());
+ Collections.shuffle(endpoints, rand);
+ LBSolrClient.Req req = new LBSolrClient.Req(nonRoutableRequest, endpoints);
try {
- for (Entry<String, ExpiringCachedDocCollection> e : entrySet()) {
- if (e.getValue().isExpired(timeToLiveMs)) {
- super.remove(e.getKey());
- }
- }
- } finally {
- evictLock.unlock();
+ LBSolrClient.Rsp rsp = getLbClient().request(req);
+ shardResponses.add(endpoints.get(0).toString(), rsp.getResponse());
+ } catch (Exception e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, endpoints.get(0).toString(), e);
}
}
- }
- protected final StateCache collectionStateCache = new StateCache();
+ long end = System.nanoTime();
- class ExpiringCachedDocCollection {
- final DocCollection cached;
- final long cachedAtNano;
- // This is the time at which the collection is retried and got the same old version
- volatile long retriedAtNano = -1;
- // flag that suggests that this is potentially to be rechecked
- volatile boolean maybeStale = false;
+ @SuppressWarnings({"rawtypes"})
+ RouteResponse rr =
+ condenseResponse(
+ shardResponses, (int) TimeUnit.MILLISECONDS.convert(end - start, TimeUnit.NANOSECONDS));
+ rr.setRouteResponses(shardResponses);
+ rr.setRoutes(routes);
+ return rr;
+ }
- ExpiringCachedDocCollection(DocCollection cached) {
- this.cached = cached;
- this.cachedAtNano = System.nanoTime();
- }
+ protected RouteException getRouteException(
+ SolrException.ErrorCode serverError,
+ NamedList<Throwable> exceptions,
+ Map<String, ? extends LBSolrClient.Req> routes) {
+ return new RouteException(serverError, exceptions, routes);
+ }
- boolean isExpired(long timeToLiveMs) {
- return (System.nanoTime() - cachedAtNano)
- > TimeUnit.NANOSECONDS.convert(timeToLiveMs, TimeUnit.MILLISECONDS);
- }
+ protected Map<String, ? extends LBSolrClient.Req> createRoutes(
+ UpdateRequest updateRequest,
+ ModifiableSolrParams routableParams,
+ DocCollection col,
+ DocRouter router,
+ Map<String, List<String>> urlMap,
+ String routeField) {
+ return urlMap == null
+ ? null
+ : updateRequest.getRoutesToCollection(router, col, urlMap, routableParams, routeField);
+ }
- boolean shouldRetry() {
- if (maybeStale) { // we are not sure if it is stale so check with retry time
- if ((retriedAtNano == -1 || (System.nanoTime() - retriedAtNano) > retryExpiryTimeNano)) {
- return true; // we retried a while back. and we could not get anything new.
- // it's likely that it is not going to be available now also.
+ private Map<String, List<String>> buildUrlMap(
+ DocCollection col, ReplicaListTransformer replicaListTransformer) {
+ Map<String, List<String>> urlMap = new HashMap<>();
+ Collection<Slice> slices = col.getActiveSlices();
+ Set<String> liveNodes = getClusterStateProvider().getLiveNodes();
+ for (Slice slice : slices) {
+ String name = slice.getName();
+ List<Replica> sortedReplicas = new ArrayList<>();
+ Replica leader = slice.getLeader();
+ if (directUpdatesToLeadersOnly && leader == null) {
+ for (Replica replica :
+ slice.getReplicas(
+ replica -> replica.isActive(liveNodes) && replica.getType() == Replica.Type.NRT)) {
+ leader = replica;
+ break;
}
}
- return false;
- }
-
- void setRetriedAt() {
- retriedAtNano = System.nanoTime();
- }
-
- /**
- * Marks this entry as {@code maybeStale} if the provided backoff window has elapsed since the
- * last retry.
- *
- * @return {@code true} if the entry was flagged as maybe stale
- */
- boolean markMaybeStaleIfOutsideBackoff(long retryBackoffNano) {
- if (maybeStale) {
- return true;
+ if (leader == null) {
+ if (directUpdatesToLeadersOnly) {
+ continue;
+ }
+ // take unoptimized general path - we cannot find a leader yet
+ return null;
}
- long lastRetry = retriedAtNano;
- if (lastRetry != -1 && (System.nanoTime() - lastRetry) <= retryBackoffNano) {
- return false;
+
+ if (!directUpdatesToLeadersOnly) {
+ for (Replica replica : slice.getReplicas()) {
+ if (!replica.equals(leader)) {
+ sortedReplicas.add(replica);
+ }
+ }
}
- maybeStale = true;
- return true;
- }
- }
- protected CloudSolrClient(
- boolean updatesToLeaders, boolean parallelUpdates, boolean directUpdatesToLeadersOnly) {
- this(
- updatesToLeaders,
- parallelUpdates,
- directUpdatesToLeadersOnly,
- DEFAULT_STATE_REFRESH_PARALLELISM);
- }
+ // Sort the non-leader replicas according to the request parameters
+ replicaListTransformer.transform(sortedReplicas);
- protected CloudSolrClient(
- boolean updatesToLeaders,
- boolean parallelUpdates,
- boolean directUpdatesToLeadersOnly,
- int stateRefreshThreads) {
- this.updatesToLeaders = updatesToLeaders;
- this.parallelUpdates = parallelUpdates;
- this.directUpdatesToLeadersOnly = directUpdatesToLeadersOnly;
- this.requestRLTGenerator = new RequestReplicaListTransformerGenerator();
- this.stateRefreshParallelism = Math.max(1, stateRefreshThreads);
- this.stateRefreshSemaphore = new Semaphore(this.stateRefreshParallelism);
+ // put the leaderUrl first.
+ sortedReplicas.add(0, leader);
+
+ urlMap.put(
+ name, sortedReplicas.stream().map(Replica::getCoreUrl).collect(Collectors.toList()));
+ }
+ return urlMap;
}
- protected abstract LBSolrClient getLbClient();
+ protected <T extends RouteResponse<?>> T condenseResponse(
+ NamedList<?> response, int timeMillis, Supplier<T> supplier) {
+ T condensed = supplier.get();
+ int status = 0;
+ Integer rf = null;
- public abstract ClusterStateProvider getClusterStateProvider();
+ // TolerantUpdateProcessor
+ List<SimpleOrderedMap<String>> toleratedErrors = null;
+ int maxToleratedErrors = Integer.MAX_VALUE;
- /**
- * @deprecated problematic as a 'get' method, since one implementation will do a remote request
- * each time this is called, potentially return lots of data that isn't even needed.
- */
- @Deprecated
- public ClusterState getClusterState() {
- // The future of "ClusterState" isn't clear. Could make it more of a cache instead of a
- // snapshot, so we un-deprecate. Or we avoid it and maybe make the ClusterStateProvider as that
- // cache. SOLR-17604 is related.
- return getClusterStateProvider().getClusterState();
- }
+ // For "adds", "deletes", "deleteByQuery" etc.
+ Map<String, NamedList<Object>> versions = new HashMap<>();
- /** Is this a communication error? We will retry if so. */
- protected boolean wasCommError(Throwable t) {
- return t instanceof SocketException || t instanceof UnknownHostException;
- }
+ for (int i = 0; i < response.size(); i++) {
+ NamedList<?> shardResponse = (NamedList<?>) response.getVal(i);
+ NamedList<?> header = (NamedList<?>) shardResponse.get("responseHeader");
+ Integer shardStatus = (Integer) header.get("status");
+ int s = shardStatus.intValue();
+ if (s > 0) {
+ status = s;
+ }
+ Object rfObj = header.get(UpdateRequest.REPFACT);
+ if (rfObj != null && rfObj instanceof Integer routeRf) {
+ if (rf == null || routeRf < rf) rf = routeRf;
+ }
- @Override
- public void close() {
- closed = true;
- collectionRefreshes.clear();
- if (!ExecutorUtil.isShutdown(this.threadPool)) {
- ExecutorUtil.shutdownAndAwaitTermination(this.threadPool);
+ @SuppressWarnings("unchecked")
+ List<SimpleOrderedMap<String>> shardTolerantErrors =
+ (List<SimpleOrderedMap<String>>) header.get("errors");
+ if (null != shardTolerantErrors) {
+ Integer shardMaxToleratedErrors = (Integer) header.get("maxErrors");
+ assert null != shardMaxToleratedErrors
+ : "TolerantUpdateProcessor reported errors but not maxErrors";
+ // if we get into some weird state where the nodes disagree about the effective maxErrors,
+ // assume the min value seen to decide if we should fail.
+ maxToleratedErrors =
+ Math.min(
+ maxToleratedErrors,
+ ToleratedUpdateError.getEffectiveMaxErrors(shardMaxToleratedErrors.intValue()));
+
+ if (null == toleratedErrors) {
+ toleratedErrors = new ArrayList<SimpleOrderedMap<String>>(shardTolerantErrors.size());
+ }
+ for (SimpleOrderedMap<String> err : shardTolerantErrors) {
+ toleratedErrors.add(err);
+ }
+ }
+ for (String updateType : Arrays.asList("adds", "deletes", "deleteByQuery")) {
+ Object obj = shardResponse.get(updateType);
+ if (obj instanceof NamedList<?> nl) {
+ NamedList<Object> versionsList =
+ versions.containsKey(updateType) ? versions.get(updateType) : new NamedList<>();
+ versionsList.addAll(nl);
+ versions.put(updateType, versionsList);
+ }
+ }
}
- }
- public ResponseParser getParser() {
- return getLbClient().getParser();
- }
+ NamedList<Object> cheader = new NamedList<>();
+ cheader.add("status", status);
+ cheader.add("QTime", timeMillis);
+ if (rf != null) cheader.add(UpdateRequest.REPFACT, rf);
+ if (null != toleratedErrors) {
+ cheader.add("maxErrors", ToleratedUpdateError.getUserFriendlyMaxErrors(maxToleratedErrors));
+ cheader.add("errors", toleratedErrors);
+ if (maxToleratedErrors < toleratedErrors.size()) {
+ // cumulative errors are too high, we need to throw a client exception w/correct metadata
- public RequestWriter getRequestWriter() {
- return getLbClient().getRequestWriter();
+ // NOTE: it shouldn't be possible for 1 == toleratedErrors.size(), because if that were the
+ // case then at least one shard should have thrown a real error before this, so we don't
+ // worry about having a more "singular" exception msg for that situation
+ StringBuilder msgBuf =
+ new StringBuilder()
+ .append(toleratedErrors.size())
+ .append(" Async failures during distributed update: ");
+
+ NamedList<String> metadata = new NamedList<>();
+ for (SimpleOrderedMap<String> err : toleratedErrors) {
+ ToleratedUpdateError te = ToleratedUpdateError.parseMap(err);
+ metadata.add(te.getMetadataKey(), te.getMetadataValue());
+
+ msgBuf.append("\n").append(te.getMessage());
+ }
+
+ SolrException toThrow =
+ new SolrException(SolrException.ErrorCode.BAD_REQUEST, msgBuf.toString());
+ toThrow.setMetadata(metadata);
+ throw toThrow;
+ }
+ }
+ for (Map.Entry<String, NamedList<Object>> entry : versions.entrySet()) {
+ condensed.add(entry.getKey(), entry.getValue());
+ }
+ condensed.add("responseHeader", cheader);
+ return condensed;
}
- /** Gets whether direct updates are sent in parallel */
- public boolean isParallelUpdates() {
- return parallelUpdates;
+ @SuppressWarnings({"rawtypes"})
+ public RouteResponse condenseResponse(NamedList<?> response, int timeMillis) {
+ return condenseResponse(response, timeMillis, RouteResponse::new);
}
- /**
- * Connect to the zookeeper ensemble. This is an optional method that may be used to force a
- * connection before any other requests are sent.
- *
- * @deprecated Call {@link ClusterStateProvider#getLiveNodes()} instead.
- */
- @Deprecated
- public void connect() {
- getClusterStateProvider().connect();
+ @Override
+ public NamedList<Object> request(SolrRequest<?> request, String collection)
+ throws SolrServerException, IOException {
+ // the collection parameter of the request overrides that of the parameter to this method
+ String requestCollection = request.getCollection();
+ if (requestCollection != null) {
+ collection = requestCollection;
+ } else if (collection == null) {
+ collection = defaultCollection;
+ }
+
+ List<String> inputCollections =
+ collection == null ? List.of() : StrUtils.splitSmart(collection, ",", true);
+ return requestWithRetryOnStaleState(
+ request,
+ 0,
+ inputCollections,
+ /*skipStateVersion*/ false,
+ Map.of(),
+ /*waitedForRefresh*/ false);
}
/**
- * Connect to a cluster. If the cluster is not ready, retry connection up to a given timeout.
- *
- * @param duration the timeout
- * @param timeUnit the units of the timeout
- * @throws TimeoutException if the cluster is not ready after the timeout
- * @throws InterruptedException if the wait is interrupted
+ * As this class doesn't watch external collections on the client side, there's a chance that the
+ * request will fail due to cached stale state, which means the state must be refreshed from ZK
+ * and retried.
*/
- @Deprecated
- public void connect(long duration, TimeUnit timeUnit)
- throws TimeoutException, InterruptedException {
- if (log.isInfoEnabled()) {
- log.info(
- "Waiting for {} {} for cluster at {} to be ready",
- duration,
- timeUnit,
- getClusterStateProvider());
+ protected NamedList<Object> requestWithRetryOnStaleState(
+ SolrRequest<?> request,
+ int retryCount,
+ List<String> inputCollections,
+ boolean skipStateVersion,
+ Map<String, CompletableFuture<DocCollection>> pendingRefreshes,
+ boolean waitedForRefresh)
+ throws SolrServerException, IOException {
+ // build up a _stateVer_ param to pass to the server containing all the
+ // external collection state versions involved in this request, which allows
+ // the server to notify us that our cached state for one or more of the external
+ // collections is stale and needs to be refreshed ... this code has no impact on internal
+ // collections
+ String stateVerParam = null;
+ List<DocCollection> requestedCollections = null;
+ boolean isCollectionRequestOfV2 = false;
+ if (request instanceof V2Request) {
+ isCollectionRequestOfV2 = ((V2Request) request).isPerCollectionRequest();
}
- long timeout = System.nanoTime() + timeUnit.toNanos(duration);
- while (System.nanoTime() < timeout) {
- try {
- connect();
- if (log.isInfoEnabled()) {
- log.info("Cluster at {} ready", getClusterStateProvider());
+ boolean isAdmin =
+ request.getRequestType() == SolrRequestType.ADMIN && !request.requiresCollection();
+ if (!inputCollections.isEmpty()
+ && !isAdmin
+ && !isCollectionRequestOfV2) { // don't do _stateVer_ checking for admin, v2 api requests
+ Set<String> requestedCollectionNames = resolveAliases(inputCollections);
+
+ StringBuilder stateVerParamBuilder = null;
+ for (String requestedCollection : requestedCollectionNames) {
+ // track the version of state we're using on the client side using the _stateVer_ param
+ DocCollection coll = getDocCollection(requestedCollection, null);
+ if (coll == null) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST, "Collection not found: " + requestedCollection);
}
- return;
- } catch (RuntimeException e) {
- // not ready yet, then...
- }
- TimeUnit.MILLISECONDS.sleep(250);
- }
- throw new TimeoutException("Timed out waiting for cluster");
- }
+ int collVer = coll.getZNodeVersion();
+ if (requestedCollections == null)
+ requestedCollections = new ArrayList<>(requestedCollectionNames.size());
+ requestedCollections.add(coll);
- @SuppressWarnings({"unchecked"})
- private NamedList<Object> directUpdate(UpdateRequest request, String collection)
- throws SolrServerException {
- SolrParams params = request.getParams();
- ModifiableSolrParams routableParams = new ModifiableSolrParams();
- ModifiableSolrParams nonRoutableParams = new ModifiableSolrParams();
+ if (stateVerParamBuilder == null) {
+ stateVerParamBuilder = new StringBuilder();
+ } else {
+ stateVerParamBuilder.append(
+ "|"); // hopefully pipe is not an allowed char in a collection name
+ }
- if (params != null) {
- nonRoutableParams.add(params);
- routableParams.add(params);
- for (String param : NON_ROUTABLE_PARAMS) {
- routableParams.remove(param);
+ stateVerParamBuilder.append(coll.getName()).append(":").append(collVer);
}
- } else {
- params = new ModifiableSolrParams();
- }
- if (collection == null) {
- throw new SolrServerException(
- "No collection param specified on request and no default collection has been set.");
+ if (stateVerParamBuilder != null) {
+ stateVerParam = stateVerParamBuilder.toString();
+ }
}
- // Check to see if the collection is an alias. Updates to multi-collection aliases are ok as
- // long as they are routed aliases
- List<String> aliasedCollections = new ArrayList<>(resolveAliases(List.of(collection)));
- if (aliasedCollections.size() == 1 || getClusterStateProvider().isRoutedAlias(collection)) {
- collection = aliasedCollections.get(0); // pick 1st (consistent with HttpSolrCall behavior)
- } else {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "Update request to non-routed multi-collection alias not supported: "
- + collection
- + " -> "
- + aliasedCollections);
- }
-
- DocCollection col = getDocCollection(collection, null);
-
- DocRouter router = col.getRouter();
-
- if (router instanceof ImplicitDocRouter) {
- // short circuit as optimization
- return null;
- }
-
- ReplicaListTransformer replicaListTransformer =
- requestRLTGenerator.getReplicaListTransformer(params);
-
- // Create the URL map, which is keyed on slice name.
- // The value is a list of URLs for each replica in the slice.
- // The first value in the list is the leader for the slice.
- final Map<String, List<String>> urlMap = buildUrlMap(col, replicaListTransformer);
- String routeField =
- (col.getRouter().getRouteField(col) == null) ? ID : col.getRouter().getRouteField(col);
- final Map<String, ? extends LBSolrClient.Req> routes =
- createRoutes(request, routableParams, col, router, urlMap, routeField);
- if (routes == null) {
- if (directUpdatesToLeadersOnly && hasInfoToFindLeaders(request, routeField)) {
- // we have info (documents with ids and/or ids to delete) with
- // which to find the leaders, but we could not find (all of) them
- throw new SolrException(
- SolrException.ErrorCode.SERVICE_UNAVAILABLE,
- "directUpdatesToLeadersOnly==true but could not find leader(s)");
+ if (request.getParams() instanceof ModifiableSolrParams params) {
+ if (!skipStateVersion && stateVerParam != null) {
+ params.set(STATE_VERSION, stateVerParam);
} else {
- // we could not find a leader or routes yet - use unoptimized general path
- log.warn(
- "No routing info found for update to collection '{}', broadcasting to all shards.",
- collection);
- return null;
+ params.remove(STATE_VERSION);
}
- }
-
- final NamedList<Throwable> exceptions = new NamedList<>();
- final NamedList<NamedList<?>> shardResponses =
- new NamedList<>(routes.size() + 1); // +1 for deleteQuery
+ } // else: ??? how to set this ???
- long start = System.nanoTime();
+ NamedList<Object> resp = null;
+ try {
+ resp = sendRequest(request, inputCollections);
+ // to avoid an O(n) operation we always add STATE_VERSION to the last and try to read it from
+ // there
+ Object o = resp == null || resp.size() == 0 ? null : resp.get(STATE_VERSION, resp.size() - 1);
+ if (o != null && o instanceof Map<?, ?> invalidStates) {
+ // remove this because no one else needs this and tests would fail if they are comparing
+ // responses
+ resp.remove(resp.size() - 1);
+ for (Map.Entry<?, ?> e : invalidStates.entrySet()) {
+ getDocCollection((String) e.getKey(), (Integer) e.getValue());
+ }
+ }
+ } catch (Exception exc) {
- if (parallelUpdates) {
- final Map<String, Future<NamedList<?>>> responseFutures =
- CollectionUtil.newHashMap(routes.size());
- for (final Map.Entry<String, ? extends LBSolrClient.Req> entry : routes.entrySet()) {
- final String url = entry.getKey();
- final LBSolrClient.Req lbRequest = entry.getValue();
- try {
- MDC.put("CloudSolrClient.url", url);
- responseFutures.put(
- url,
- threadPool.submit(
- () -> {
- return getLbClient().request(lbRequest).getResponse();
- }));
- } finally {
- MDC.remove("CloudSolrClient.url");
+ Throwable rootCause = SolrException.getRootCause(exc);
+ // don't do retry support for admin requests
+ // or if the request doesn't have a collection specified
+ // or request is v2 api and its method is not GET
+ if (inputCollections.isEmpty()
+ || isAdmin
+ || (request.getApiVersion() == SolrRequest.ApiVersion.V2
+ && request.getMethod() != SolrRequest.METHOD.GET)) {
+ if (exc instanceof SolrServerException) {
+ throw (SolrServerException) exc;
+ } else if (exc instanceof IOException) {
+ throw (IOException) exc;
+ } else if (exc instanceof RuntimeException) {
+ throw (RuntimeException) exc;
+ } else {
+ throw new SolrServerException(rootCause);
}
}
- for (final Map.Entry<String, Future<NamedList<?>>> entry : responseFutures.entrySet()) {
- final String url = entry.getKey();
- final Future<NamedList<?>> responseFuture = entry.getValue();
- try {
- shardResponses.add(url, responseFuture.get());
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new RuntimeException(e);
- } catch (ExecutionException e) {
- exceptions.add(url, e.getCause());
+ int errorCode =
+ (rootCause instanceof SolrException)
+ ? ((SolrException) rootCause).code()
+ : SolrException.ErrorCode.UNKNOWN.code;
+
+ final boolean wasCommError = wasCommError(rootCause);
+
+ if (wasCommError
+ || (exc instanceof RouteException
+ && (errorCode == 503)) // 404 because the core does not exist 503 service unavailable
+ // TODO there are other reasons for 404. We need to change the solr response format from HTML
+ // to structured data to know that
+ ) {
+ // it was a communication error. it is likely that
+ // the node to which the request to be sent is down . So , expire the state
+ // so that the next attempt would fetch the fresh state
+ // just re-read state for all of them, if it has not been retried
+ // in retryExpiryTime time
+ if (requestedCollections != null) {
+ for (DocCollection ext : requestedCollections) {
+ String name = ext.getName();
+ ExpiringCachedDocCollection cacheEntry = collectionStateCache.peek(name);
+ if (cacheEntry != null) {
+ if (wasCommError) {
+ cacheEntry.maybeStale = true;
+ } else {
+ boolean markedStale =
+ cacheEntry.markMaybeStaleIfOutsideBackoff(retryExpiryTimeNano);
+ if (markedStale && cacheEntry.shouldRetry()) {
+ triggerCollectionRefresh(name);
+ }
+ }
+ } else {
+ triggerCollectionRefresh(name);
+ }
+ }
+ }
+ if (retryCount < MAX_STALE_RETRIES) { // if it is a communication error , we must try again
+ // may be, we have a stale version of the collection state,
+ // and we could not get any information from the server
+ // it is probably not worth trying again and again because
+ // the state would not have been updated
+ log.info(
+ "Request to collection {} failed due to ({}) {}, retry={} maxRetries={} commError={} errorCode={} - retrying",
+ inputCollections,
+ errorCode,
+ rootCause,
+ retryCount,
+ MAX_STALE_RETRIES,
+ wasCommError,
+ errorCode);
+ return requestWithRetryOnStaleState(
+ request,
+ retryCount + 1,
+ inputCollections,
+ skipStateVersion,
+ pendingRefreshes,
+ waitedForRefresh);
}
+ } else {
+ log.info("request was not communication error it seems");
}
+ log.info(
+ "Request to collection {} failed due to ({}) {}, retry={} maxRetries={} commError={} errorCode={} ",
+ inputCollections,
+ errorCode,
+ rootCause,
+ retryCount,
+ MAX_STALE_RETRIES,
+ wasCommError,
+ errorCode);
- if (exceptions.size() > 0) {
- Throwable firstException = exceptions.getVal(0);
- if (firstException instanceof SolrException e) {
- throw getRouteException(
- SolrException.ErrorCode.getErrorCode(e.code()), exceptions, routes);
- } else {
- throw getRouteException(SolrException.ErrorCode.SERVER_ERROR, exceptions, routes);
+ boolean stateWasStale = false;
+ if (retryCount < MAX_STALE_RETRIES
+ && requestedCollections != null
+ && !requestedCollections.isEmpty()
+ && (SolrException.ErrorCode.getErrorCode(errorCode)
+ == SolrException.ErrorCode.INVALID_STATE
+ || errorCode == 404)) {
+ // cached state for one or more external collections was stale
+ // re-issue request using updated state
+ stateWasStale = true;
+
+ // just re-read state for all of them, which is a little heavy-handed but hopefully a rare
+ // occurrence
+ for (DocCollection ext : requestedCollections) {
+ collectionStateCache.remove(ext.getName());
}
}
- } else {
- for (Map.Entry<String, ? extends LBSolrClient.Req> entry : routes.entrySet()) {
- String url = entry.getKey();
- LBSolrClient.Req lbRequest = entry.getValue();
- try {
- NamedList<Object> rsp = getLbClient().request(lbRequest).getResponse();
- shardResponses.add(url, rsp);
- } catch (Exception e) {
- if (e instanceof SolrException) {
- throw (SolrException) e;
- } else {
- throw new SolrServerException(e);
+
+ // if we experienced a communication error, it's worth checking the state
+ // with ZK just to make sure the node we're trying to hit is still part of the collection
+ if (retryCount < MAX_STALE_RETRIES
+ && !stateWasStale
+ && requestedCollections != null
+ && !requestedCollections.isEmpty()
+ && wasCommError) {
+ for (DocCollection ext : requestedCollections) {
+ DocCollection latestStateFromZk = getDocCollection(ext.getName(), null);
+ if (latestStateFromZk.getZNodeVersion() != ext.getZNodeVersion()) {
+ // looks like we couldn't reach the server because the state was stale == retry
+ stateWasStale = true;
+ // we just pulled state from ZK, so update the cache so that the retry uses it
+ collectionStateCache.put(
+ ext.getName(), new ExpiringCachedDocCollection(latestStateFromZk));
}
}
}
- }
- UpdateRequest nonRoutableRequest = null;
- List<String> deleteQuery = request.getDeleteQuery();
- if (deleteQuery != null && deleteQuery.size() > 0) {
- UpdateRequest deleteQueryRequest = new UpdateRequest();
- deleteQueryRequest.setDeleteQuery(deleteQuery);
- nonRoutableRequest = deleteQueryRequest;
- }
-
- Set<String> paramNames = nonRoutableParams.getParameterNames();
+ // if the state was stale, then we retry the request once with new state pulled from Zk
+ if (stateWasStale) {
+ log.warn(
+ "Re-trying request to collection(s) {} after stale state error from server.",
+ inputCollections);
- Set<String> intersection = new HashSet<>(paramNames);
- intersection.retainAll(NON_ROUTABLE_PARAMS);
+ Map<String, CompletableFuture<DocCollection>> refreshesToWaitFor = pendingRefreshes;
+ if (!waitedForRefresh && (pendingRefreshes == null || pendingRefreshes.isEmpty())) {
+ refreshesToWaitFor = new HashMap<>();
+ for (DocCollection ext : requestedCollections) {
+ refreshesToWaitFor.put(ext.getName(), triggerCollectionRefresh(ext.getName()));
+ }
+ }
- if (nonRoutableRequest != null || intersection.size() > 0) {
- if (nonRoutableRequest == null) {
- nonRoutableRequest = new UpdateRequest();
- }
- nonRoutableRequest.setParams(nonRoutableParams);
- nonRoutableRequest.setBasicAuthCredentials(
- request.getBasicAuthUser(), request.getBasicAuthPassword());
- final var endpoints =
- routes.keySet().stream()
- .map(url -> LBSolrClient.Endpoint.from(url))
- .collect(Collectors.toList());
- Collections.shuffle(endpoints, rand);
- LBSolrClient.Req req = new LBSolrClient.Req(nonRoutableRequest, endpoints);
- try {
- LBSolrClient.Rsp rsp = getLbClient().request(req);
- shardResponses.add(endpoints.get(0).toString(), rsp.getResponse());
- } catch (Exception e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, endpoints.get(0).toString(), e);
+ // First retry without sending state versions so the server does not immediately reject the
+ // request while we intentionally rely on stale routing (e.g., to allow forwarding to a new
+ // leader) as the background refresh completes.
+ if (!skipStateVersion && !waitedForRefresh) {
+ resp =
+ requestWithRetryOnStaleState(
+ request,
+ retryCount + 1,
+ inputCollections,
+ /*skipStateVersion*/ true,
+ refreshesToWaitFor,
+ waitedForRefresh);
+ } else if (!waitedForRefresh
+ && refreshesToWaitFor != null
+ && !refreshesToWaitFor.isEmpty()) {
+ for (Map.Entry<String, CompletableFuture<DocCollection>> entry :
+ refreshesToWaitFor.entrySet()) {
+ waitForCollectionRefresh(entry.getKey(), entry.getValue());
+ }
+ resp =
+ requestWithRetryOnStaleState(
+ request,
+ retryCount + 1,
+ inputCollections,
+ /*skipStateVersion*/ false,
+ Map.of(),
+ /*waitedForRefresh*/ true);
+ } else {
+ resp =
+ requestWithRetryOnStaleState(
+ request,
+ retryCount + 1,
+ inputCollections,
+ /*skipStateVersion*/ false,
+ Map.of(),
+ /*waitedForRefresh*/ waitedForRefresh);
+ }
+ } else {
+ if (exc instanceof SolrException
+ || exc instanceof SolrServerException
+ || exc instanceof IOException) {
+ throw exc;
+ } else {
+ throw new SolrServerException(rootCause);
+ }
}
- }
- long end = System.nanoTime();
+ if (requestedCollections != null) {
+ requestedCollections.clear(); // done with this
+ }
+ }
- @SuppressWarnings({"rawtypes"})
- RouteResponse rr =
- condenseResponse(
- shardResponses, (int) TimeUnit.MILLISECONDS.convert(end - start, TimeUnit.NANOSECONDS));
- rr.setRouteResponses(shardResponses);
- rr.setRoutes(routes);
- return rr;
+ return resp;
}
- protected RouteException getRouteException(
- SolrException.ErrorCode serverError,
- NamedList<Throwable> exceptions,
- Map<String, ? extends LBSolrClient.Req> routes) {
- return new RouteException(serverError, exceptions, routes);
- }
+ protected NamedList<Object> sendRequest(SolrRequest<?> request, List<String> inputCollections)
+ throws SolrServerException, IOException {
+ boolean sendToLeaders = false;
- protected Map<String, ? extends LBSolrClient.Req> createRoutes(
- UpdateRequest updateRequest,
- ModifiableSolrParams routableParams,
- DocCollection col,
- DocRouter router,
- Map<String, List<String>> urlMap,
- String routeField) {
- return urlMap == null
- ? null
- : updateRequest.getRoutesToCollection(router, col, urlMap, routableParams, routeField);
- }
+ if (request.getRequestType() == SolrRequestType.UPDATE) {
+ sendToLeaders = this.isUpdatesToLeaders();
- private Map<String, List<String>> buildUrlMap(
- DocCollection col, ReplicaListTransformer replicaListTransformer) {
- Map<String, List<String>> urlMap = new HashMap<>();
- Collection<Slice> slices = col.getActiveSlices();
- Set<String> liveNodes = getClusterStateProvider().getLiveNodes();
- for (Slice slice : slices) {
- String name = slice.getName();
- List<Replica> sortedReplicas = new ArrayList<>();
- Replica leader = slice.getLeader();
- if (directUpdatesToLeadersOnly && leader == null) {
- for (Replica replica :
- slice.getReplicas(
- replica -> replica.isActive(liveNodes) && replica.getType() == Replica.Type.NRT)) {
- leader = replica;
- break;
- }
- }
- if (leader == null) {
- if (directUpdatesToLeadersOnly) {
- continue;
- }
- // take unoptimized general path - we cannot find a leader yet
- return null;
- }
+ if (sendToLeaders && request instanceof UpdateRequest updateRequest) {
+ sendToLeaders = sendToLeaders && updateRequest.isSendToLeaders();
- if (!directUpdatesToLeadersOnly) {
- for (Replica replica : slice.getReplicas()) {
- if (!replica.equals(leader)) {
- sortedReplicas.add(replica);
+ // Check if we can do a "directUpdate" ...
+ if (sendToLeaders) {
+ if (inputCollections.size() > 1) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Update request must be sent to a single collection "
+ + "or an alias: "
+ + inputCollections);
+ }
+ String collection =
+ inputCollections.isEmpty()
+ ? null
+ : inputCollections.get(0); // getting first mimics HttpSolrCall
+ NamedList<Object> response = directUpdate(updateRequest, collection);
+ if (response != null) {
+ return response;
}
}
}
+ }
- // Sort the non-leader replicas according to the request parameters
- replicaListTransformer.transform(sortedReplicas);
-
- // put the leaderUrl first.
- sortedReplicas.add(0, leader);
+ SolrParams reqParams = request.getParams();
+ assert reqParams != null;
- urlMap.put(
- name, sortedReplicas.stream().map(Replica::getCoreUrl).collect(Collectors.toList()));
- }
- return urlMap;
- }
+ ReplicaListTransformer replicaListTransformer =
+ requestRLTGenerator.getReplicaListTransformer(reqParams);
- protected <T extends RouteResponse<?>> T condenseResponse(
- NamedList<?> response, int timeMillis, Supplier<T> supplier) {
- T condensed = supplier.get();
- int status = 0;
- Integer rf = null;
+ final ClusterStateProvider provider = getClusterStateProvider();
+ final String urlScheme = provider.getUrlScheme();
+ final Set<String> liveNodes = provider.getLiveNodes();
- // TolerantUpdateProcessor
- List<SimpleOrderedMap<String>> toleratedErrors = null;
- int maxToleratedErrors = Integer.MAX_VALUE;
+ final List<LBSolrClient.Endpoint> requestEndpoints =
+ new ArrayList<>(); // we populate this as follows...
- // For "adds", "deletes", "deleteByQuery" etc.
- Map<String, NamedList<Object>> versions = new HashMap<>();
+ if (request.getApiVersion() == SolrRequest.ApiVersion.V2) {
+ if (!liveNodes.isEmpty()) {
+ List<String> liveNodesList = new ArrayList<>(liveNodes);
+ Collections.shuffle(liveNodesList, rand);
+ final var chosenNodeUrl = Utils.getBaseUrlForNodeName(liveNodesList.get(0), urlScheme);
+ requestEndpoints.add(new LBSolrClient.Endpoint(chosenNodeUrl));
+ }
- for (int i = 0; i < response.size(); i++) {
- NamedList<?> shardResponse = (NamedList<?>) response.getVal(i);
- NamedList<?> header = (NamedList<?>) shardResponse.get("responseHeader");
- Integer shardStatus = (Integer) header.get("status");
- int s = shardStatus.intValue();
- if (s > 0) {
- status = s;
+ } else if (!request.requiresCollection()) {
+ for (String liveNode : liveNodes) {
+ final var nodeBaseUrl = Utils.getBaseUrlForNodeName(liveNode, urlScheme);
+ requestEndpoints.add(new LBSolrClient.Endpoint(nodeBaseUrl));
}
- Object rfObj = header.get(UpdateRequest.REPFACT);
- if (rfObj != null && rfObj instanceof Integer routeRf) {
- if (rf == null || routeRf < rf) rf = routeRf;
+ } else { // API call to a particular collection / core / alias (i.e.
+ // request.requiresCollection() == true)
+ Set<String> collectionNames = resolveAliases(inputCollections);
+ if (collectionNames.isEmpty()) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "No collection param specified on request and no default collection has been set: "
+ + inputCollections);
}
- @SuppressWarnings("unchecked")
- List<SimpleOrderedMap<String>> shardTolerantErrors =
- (List<SimpleOrderedMap<String>>) header.get("errors");
- if (null != shardTolerantErrors) {
- Integer shardMaxToleratedErrors = (Integer) header.get("maxErrors");
- assert null != shardMaxToleratedErrors
- : "TolerantUpdateProcessor reported errors but not maxErrors";
- // if we get into some weird state where the nodes disagree about the effective maxErrors,
- // assume the min value seen to decide if we should fail.
- maxToleratedErrors =
- Math.min(
- maxToleratedErrors,
- ToleratedUpdateError.getEffectiveMaxErrors(shardMaxToleratedErrors.intValue()));
-
- if (null == toleratedErrors) {
- toleratedErrors = new ArrayList<SimpleOrderedMap<String>>(shardTolerantErrors.size());
- }
- for (SimpleOrderedMap<String> err : shardTolerantErrors) {
- toleratedErrors.add(err);
+ List<String> preferredNodes = request.getPreferredNodes();
+ if (preferredNodes != null && !preferredNodes.isEmpty()) {
+ String joinedInputCollections = StrUtils.join(inputCollections, ',');
+ final var endpoints =
+ preferredNodes.stream()
+ .map(nodeName -> Utils.getBaseUrlForNodeName(nodeName, urlScheme))
+ .map(nodeUrl -> new LBSolrClient.Endpoint(nodeUrl, joinedInputCollections))
+ .collect(Collectors.toList());
+ if (!endpoints.isEmpty()) {
+ LBSolrClient.Req req = new LBSolrClient.Req(request, endpoints);
+ LBSolrClient.Rsp rsp = getLbClient().request(req);
+ return rsp.getResponse();
}
}
- for (String updateType : Arrays.asList("adds", "deletes", "deleteByQuery")) {
- Object obj = shardResponse.get(updateType);
- if (obj instanceof NamedList<?> nl) {
- NamedList<Object> versionsList =
- versions.containsKey(updateType) ? versions.get(updateType) : new NamedList<>();
- versionsList.addAll(nl);
- versions.put(updateType, versionsList);
+
+ // TODO: not a big deal because of the caching, but we could avoid looking
+ // at every shard when getting leaders if we tweaked some things
+
+ // Retrieve slices from the cloud state and, for each collection specified, add it to the Map
+ // of slices.
+ Map<String, Slice> slices = new HashMap<>();
+ String shardKeys = reqParams.get(ShardParams._ROUTE_);
+ for (String collectionName : collectionNames) {
+ DocCollection col = getDocCollection(collectionName, null);
+ if (col == null) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST, "Collection not found: " + collectionName);
}
+ Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams, col);
+ ClientUtils.addSlices(slices, collectionName, routeSlices, true);
}
- }
- NamedList<Object> cheader = new NamedList<>();
- cheader.add("status", status);
- cheader.add("QTime", timeMillis);
- if (rf != null) cheader.add(UpdateRequest.REPFACT, rf);
- if (null != toleratedErrors) {
- cheader.add("maxErrors", ToleratedUpdateError.getUserFriendlyMaxErrors(maxToleratedErrors));
- cheader.add("errors", toleratedErrors);
- if (maxToleratedErrors < toleratedErrors.size()) {
- // cumulative errors are too high, we need to throw a client exception w/correct metadata
+ // Gather URLs, grouped by leader or replica
+ List<Replica> sortedReplicas = new ArrayList<>();
+ List<Replica> replicas = new ArrayList<>();
+ for (Slice slice : slices.values()) {
+ Replica leader = slice.getLeader();
+ for (Replica replica : slice.getReplicas()) {
+ String node = replica.getNodeName();
+ if (!liveNodes.contains(node) // Must be a live node to continue
+ || replica.getState()
+ != Replica.State.ACTIVE) { // Must be an ACTIVE replica to continue
+ continue;
+ }
+ if (sendToLeaders && replica.equals(leader)) {
+ sortedReplicas.add(replica); // put leaders here eagerly (if sendToLeader mode)
+ } else {
+ replicas.add(replica); // replicas here
+ }
+ }
+ }
- // NOTE: it shouldn't be possible for 1 == toleratedErrors.size(), because if that were the
- // case then at least one shard should have thrown a real error before this, so we don't
- // worry about having a more "singular" exception msg for that situation
- StringBuilder msgBuf =
- new StringBuilder()
- .append(toleratedErrors.size())
- .append(" Async failures during distributed update: ");
+ // Sort the leader replicas, if any, according to the request preferences (none if
+ // !sendToLeaders)
+ replicaListTransformer.transform(sortedReplicas);
- NamedList<String> metadata = new NamedList<>();
- for (SimpleOrderedMap<String> err : toleratedErrors) {
- ToleratedUpdateError te = ToleratedUpdateError.parseMap(err);
- metadata.add(te.getMetadataKey(), te.getMetadataValue());
+ // Sort the replicas, if any, according to the request preferences and append to our list
+ replicaListTransformer.transform(replicas);
- msgBuf.append("\n").append(te.getMessage());
- }
+ sortedReplicas.addAll(replicas);
- SolrException toThrow =
- new SolrException(SolrException.ErrorCode.BAD_REQUEST, msgBuf.toString());
- toThrow.setMetadata(metadata);
- throw toThrow;
+ String joinedInputCollections = StrUtils.join(inputCollections, ',');
+ Set<String> seenNodes = new HashSet<>();
+ sortedReplicas.forEach(
+ replica -> {
+ if (seenNodes.add(replica.getNodeName())) {
+ if (inputCollections.size() == 1 && collectionNames.size() == 1) {
+ // If we have a single collection name (and not an alias to multiple collection),
+ // send the query directly to a replica of this collection.
+ requestEndpoints.add(
+ new LBSolrClient.Endpoint(replica.getBaseUrl(), replica.getCoreName()));
+ } else {
+ requestEndpoints.add(
+ new LBSolrClient.Endpoint(replica.getBaseUrl(), joinedInputCollections));
+ }
+ }
+ });
+
+ if (requestEndpoints.isEmpty()) {
+ collectionStateCache.keySet().removeAll(collectionNames);
+ throw new SolrException(
+ SolrException.ErrorCode.INVALID_STATE,
+ "Could not find a healthy node to handle the request.");
}
}
- for (Map.Entry<String, NamedList<Object>> entry : versions.entrySet()) {
- condensed.add(entry.getKey(), entry.getValue());
+
+ LBSolrClient.Req req = new LBSolrClient.Req(request, requestEndpoints);
+ LBSolrClient.Rsp rsp = getLbClient().request(req);
+ return rsp.getResponse();
+ }
+
+ /**
+ * Resolves the input collections to their possible aliased collections. Doesn't validate
+ * collection existence.
+ */
+ private Set<String> resolveAliases(List<String> inputCollections) {
+ if (inputCollections.isEmpty()) {
+ return Set.of();
}
- condensed.add("responseHeader", cheader);
- return condensed;
+ LinkedHashSet<String> uniqueNames = new LinkedHashSet<>(); // consistent ordering
+ for (String collectionName : inputCollections) {
+ if (getDocCollection(collectionName, -1) == null) {
+ // perhaps it's an alias
+ uniqueNames.addAll(getClusterStateProvider().resolveAlias(collectionName));
+ } else {
+ uniqueNames.add(collectionName); // it's a collection
+ }
+ }
+ return uniqueNames;
}
- @SuppressWarnings({"rawtypes"})
- public RouteResponse condenseResponse(NamedList<?> response, int timeMillis) {
- return condenseResponse(response, timeMillis, RouteResponse::new);
+ /**
+ * If true, this client has been configured such that it will generally prefer to send {@link
+ * SolrRequestType#UPDATE} requests to a shard leader, if and only if {@link
+ * UpdateRequest#isSendToLeaders} is also true. If false, then this client has been configured to
+ * obey normal routing preferences when dealing with {@link SolrRequestType#UPDATE} requests.
+ *
+ * @see #isDirectUpdatesToLeadersOnly
+ */
+ public boolean isUpdatesToLeaders() {
+ return updatesToLeaders;
}
- @SuppressWarnings({"rawtypes"})
- public static class RouteResponse<T extends LBSolrClient.Req> extends NamedList<Object> {
- private NamedList<NamedList<?>> routeResponses;
- private Map<String, T> routes;
+ /**
+ * If true, this client has been configured such that "direct updates" will <em>only</em> be sent
+ * to the current leader of the corresponding shard, and will not be retried with other replicas.
+ * This method has no effect if {@link #isUpdatesToLeaders()} or {@link
+ * UpdateRequest#isSendToLeaders} returns false.
+ *
+ * <p>A "direct update" is any update that can be sent directly to a single shard, and does not
+ * need to be broadcast to every shard. (Example: document updates or "delete by id" when using
+ * the default router; non-direct updates are things like commits and "delete by query").
+ *
+ * <p>NOTE: If a single {@link UpdateRequest} contains multiple "direct updates" for different
+ * shards, this client may break the request up and merge the responses.
+ *
+ * @return true if direct updates are sent to shard leaders only
+ */
+ public boolean isDirectUpdatesToLeadersOnly() {
+ return directUpdatesToLeadersOnly;
+ }
- public void setRouteResponses(NamedList<NamedList<?>> routeResponses) {
- this.routeResponses = routeResponses;
- }
+ /** Visible for tests so they can assert the configured refresh parallelism. */
+ protected int getStateRefreshParallelism() {
+ return stateRefreshParallelism;
+ }
- public NamedList<NamedList<?>> getRouteResponses() {
- return routeResponses;
+ protected DocCollection getDocCollection(String collection, Integer expectedVersion)
+ throws SolrException {
+ if (expectedVersion == null) {
+ expectedVersion = -1;
+ }
+ if (collection == null) {
+ return null;
}
- public void setRoutes(Map<String, T> routes) {
- this.routes = routes;
+ ExpiringCachedDocCollection cacheEntry = collectionStateCache.peek(collection);
+ if (cacheEntry != null && cacheEntry.isExpired(collectionStateCache.timeToLiveMs)) {
+ collectionStateCache.remove(collection, cacheEntry);
+ cacheEntry = null;
}
- public Map<String, T> getRoutes() {
- return routes;
+ DocCollection cached = cacheEntry == null ? null : cacheEntry.cached;
+
+ if (cacheEntry != null && cacheEntry.shouldRetry()) {
+ triggerCollectionRefresh(collection);
}
- }
- public static class RouteException extends SolrException {
+ if (cached != null && expectedVersion <= cached.getZNodeVersion()) {
+ return cached;
+ }
- private NamedList<Throwable> throwables;
- private Map<String, ? extends LBSolrClient.Req> routes;
+ CompletableFuture<DocCollection> refreshFuture = triggerCollectionRefresh(collection);
+ return waitForCollectionRefresh(collection, refreshFuture);
+ }
- public RouteException(
- ErrorCode errorCode,
- NamedList<Throwable> throwables,
- Map<String, ? extends LBSolrClient.Req> routes) {
- super(errorCode, throwables.getVal(0).getMessage(), throwables.getVal(0));
- this.throwables = throwables;
- this.routes = routes;
+ private CompletableFuture<DocCollection> triggerCollectionRefresh(String collection) {
+ return collectionRefreshes.compute(
+ collection,
+ (key, existingFuture) -> {
+ // A refresh is still in progress; return it.
+ if (existingFuture != null && !existingFuture.isDone()) {
+ return existingFuture;
+ }
+ // No refresh is in-progress, so trigger it.
- // create a merged copy of the metadata from all wrapped exceptions
- NamedList<String> metadata = new NamedList<String>();
- for (int i = 0; i < throwables.size(); i++) {
- Throwable t = throwables.getVal(i);
- if (t instanceof SolrException e) {
- NamedList<String> eMeta = e.getMetadata();
- if (null != eMeta) {
- metadata.addAll(eMeta);
+ if (ExecutorUtil.isShutdown(threadPool)) {
+ assert closed; // see close() for the sequence
+ ExpiringCachedDocCollection cacheEntry = collectionStateCache.peek(key);
+ DocCollection cached = cacheEntry == null ? null : cacheEntry.cached;
+ return CompletableFuture.completedFuture(cached);
+ } else {
+ return CompletableFuture.supplyAsync(
+ () -> {
+ stateRefreshSemaphore.acquireUninterruptibly();
+ try {
+ return loadDocCollection(key);
+ } finally {
+ stateRefreshSemaphore.release();
+ // Remove the entry in case of many collections
+ collectionRefreshes.remove(key);
+ }
+ },
+ threadPool);
}
- }
- }
- if (0 < metadata.size()) {
- this.setMetadata(metadata);
+ });
+ }
+
+ private DocCollection waitForCollectionRefresh(
+ String collection, CompletableFuture<DocCollection> refreshFuture) {
+ try {
+ return refreshFuture.get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Interrupted while refreshing state for collection " + collection,
+ e);
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof SolrException) {
+ throw (SolrException) cause;
}
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Error refreshing state for collection " + collection,
+ cause);
}
+ }
- public NamedList<Throwable> getThrowables() {
- return throwables;
+ private DocCollection loadDocCollection(String collection) {
+ ClusterState.CollectionRef ref = getCollectionRef(collection);
+ if (ref == null) {
+ collectionStateCache.remove(collection);
+ return null;
}
- public Map<String, ? extends LBSolrClient.Req> getRoutes() {
- return this.routes;
+ DocCollection fetchedCol = ref.get();
+ if (fetchedCol == null) {
+ collectionStateCache.remove(collection);
+ return null;
}
- }
- @Override
- public NamedList<Object> request(SolrRequest<?> request, String collection)
- throws SolrServerException, IOException {
- // the collection parameter of the request overrides that of the parameter to this method
- String requestCollection = request.getCollection();
- if (requestCollection != null) {
- collection = requestCollection;
- } else if (collection == null) {
- collection = defaultCollection;
+ ExpiringCachedDocCollection existing = collectionStateCache.peek(collection);
+ if (existing != null && existing.cached.getZNodeVersion() == fetchedCol.getZNodeVersion()) {
+ existing.setRetriedAt();
+ existing.maybeStale = false;
+ return existing.cached;
}
- List<String> inputCollections =
- collection == null ? List.of() : StrUtils.splitSmart(collection, ",", true);
- return requestWithRetryOnStaleState(
- request,
- 0,
- inputCollections,
- /*skipStateVersion*/ false,
- Map.of(),
- /*waitedForRefresh*/ false);
+ collectionStateCache.put(collection, new ExpiringCachedDocCollection(fetchedCol));
+ return fetchedCol;
+ }
+
+ ClusterState.CollectionRef getCollectionRef(String collection) {
+ return getClusterStateProvider().getState(collection);
}
/**
- * As this class doesn't watch external collections on the client side, there's a chance that the
- * request will fail due to cached stale state, which means the state must be refreshed from ZK
- * and retried.
+ * Useful for determining the minimum achieved replication factor across all shards involved in
+ * processing an update request, typically useful for gauging the replication factor of a batch.
*/
- protected NamedList<Object> requestWithRetryOnStaleState(
- SolrRequest<?> request,
- int retryCount,
- List<String> inputCollections,
- boolean skipStateVersion,
- Map<String, CompletableFuture<DocCollection>> pendingRefreshes,
- boolean waitedForRefresh)
- throws SolrServerException, IOException {
- // build up a _stateVer_ param to pass to the server containing all the
- // external collection state versions involved in this request, which allows
- // the server to notify us that our cached state for one or more of the external
- // collections is stale and needs to be refreshed ... this code has no impact on internal
- // collections
- String stateVerParam = null;
- List<DocCollection> requestedCollections = null;
- boolean isCollectionRequestOfV2 = false;
- if (request instanceof V2Request) {
- isCollectionRequestOfV2 = ((V2Request) request).isPerCollectionRequest();
- }
- boolean isAdmin =
- request.getRequestType() == SolrRequestType.ADMIN && !request.requiresCollection();
- if (!inputCollections.isEmpty()
- && !isAdmin
- && !isCollectionRequestOfV2) { // don't do _stateVer_ checking for admin, v2 api requests
- Set<String> requestedCollectionNames = resolveAliases(inputCollections);
-
- StringBuilder stateVerParamBuilder = null;
- for (String requestedCollection : requestedCollectionNames) {
- // track the version of state we're using on the client side using the _stateVer_ param
- DocCollection coll = getDocCollection(requestedCollection, null);
- if (coll == null) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST, "Collection not found: " + requestedCollection);
- }
- int collVer = coll.getZNodeVersion();
- if (requestedCollections == null)
- requestedCollections = new ArrayList<>(requestedCollectionNames.size());
- requestedCollections.add(coll);
-
- if (stateVerParamBuilder == null) {
- stateVerParamBuilder = new StringBuilder();
- } else {
- stateVerParamBuilder.append(
- "|"); // hopefully pipe is not an allowed char in a collection name
- }
-
- stateVerParamBuilder.append(coll.getName()).append(":").append(collVer);
- }
+ public int getMinAchievedReplicationFactor(String collection, NamedList<?> resp) {
+ // it's probably already on the top-level header set by condense
+ NamedList<?> header = (NamedList<?>) resp.get("responseHeader");
+ Integer achRf = (Integer) header.get(UpdateRequest.REPFACT);
+ if (achRf != null) return achRf.intValue();
- if (stateVerParamBuilder != null) {
- stateVerParam = stateVerParamBuilder.toString();
+ // not on the top-level header, walk the shard route tree
+ Map<String, Integer> shardRf = getShardReplicationFactor(collection, resp);
+ for (Integer rf : shardRf.values()) {
+ if (achRf == null || rf < achRf) {
+ achRf = rf;
}
}
+ return (achRf != null) ? achRf.intValue() : -1;
+ }
- if (request.getParams() instanceof ModifiableSolrParams params) {
- if (!skipStateVersion && stateVerParam != null) {
- params.set(STATE_VERSION, stateVerParam);
- } else {
- params.remove(STATE_VERSION);
- }
- } // else: ??? how to set this ???
-
- NamedList<Object> resp = null;
- try {
- resp = sendRequest(request, inputCollections);
- // to avoid an O(n) operation we always add STATE_VERSION to the last and try to read it from
- // there
- Object o = resp == null || resp.size() == 0 ? null : resp.get(STATE_VERSION, resp.size() - 1);
- if (o != null && o instanceof Map<?, ?> invalidStates) {
- // remove this because no one else needs this and tests would fail if they are comparing
- // responses
- resp.remove(resp.size() - 1);
- for (Map.Entry<?, ?> e : invalidStates.entrySet()) {
- getDocCollection((String) e.getKey(), (Integer) e.getValue());
- }
- }
- } catch (Exception exc) {
-
- Throwable rootCause = SolrException.getRootCause(exc);
- // don't do retry support for admin requests
- // or if the request doesn't have a collection specified
- // or request is v2 api and its method is not GET
- if (inputCollections.isEmpty()
- || isAdmin
- || (request.getApiVersion() == SolrRequest.ApiVersion.V2
- && request.getMethod() != SolrRequest.METHOD.GET)) {
- if (exc instanceof SolrServerException) {
- throw (SolrServerException) exc;
- } else if (exc instanceof IOException) {
- throw (IOException) exc;
- } else if (exc instanceof RuntimeException) {
- throw (RuntimeException) exc;
- } else {
- throw new SolrServerException(rootCause);
+ /**
+ * Walks the NamedList response after performing an update request looking for the replication
+ * factor that was achieved in each shard involved in the request. For single doc updates, there
+ * will be only one shard in the return value.
+ */
+ public Map<String, Integer> getShardReplicationFactor(String collection, NamedList<?> resp) {
+ Map<String, Integer> results = new HashMap<>();
+ if (resp instanceof RouteResponse) {
+ NamedList<NamedList<?>> routes = ((RouteResponse<?>) resp).getRouteResponses();
+ DocCollection coll = getDocCollection(collection, null);
+ Map<String, String> leaders = new HashMap<>();
+ for (Slice slice : coll.getActiveSlices()) {
+ Replica leader = slice.getLeader();
+ if (leader != null) {
+ String leaderUrl = leader.getBaseUrl() + "/" + leader.getCoreName();
+ leaders.put(leaderUrl, slice.getName());
+ String altLeaderUrl = leader.getBaseUrl() + "/" + collection;
+ leaders.put(altLeaderUrl, slice.getName());
}
}
- int errorCode =
- (rootCause instanceof SolrException)
- ? ((SolrException) rootCause).code()
- : SolrException.ErrorCode.UNKNOWN.code;
-
- final boolean wasCommError = wasCommError(rootCause);
-
- if (wasCommError
- || (exc instanceof RouteException
- && (errorCode == 503)) // 404 because the core does not exist 503 service unavailable
- // TODO there are other reasons for 404. We need to change the solr response format from HTML
- // to structured data to know that
- ) {
- // it was a communication error. it is likely that
- // the node to which the request to be sent is down . So , expire the state
- // so that the next attempt would fetch the fresh state
- // just re-read state for all of them, if it has not been retried
- // in retryExpiryTime time
- if (requestedCollections != null) {
- for (DocCollection ext : requestedCollections) {
- String name = ext.getName();
- ExpiringCachedDocCollection cacheEntry = collectionStateCache.peek(name);
- if (cacheEntry != null) {
- if (wasCommError) {
- cacheEntry.maybeStale = true;
- } else {
- boolean markedStale =
- cacheEntry.markMaybeStaleIfOutsideBackoff(retryExpiryTimeNano);
- if (markedStale && cacheEntry.shouldRetry()) {
- triggerCollectionRefresh(name);
- }
- }
- } else {
- triggerCollectionRefresh(name);
+ Iterator<Map.Entry<String, NamedList<?>>> routeIter = routes.iterator();
+ while (routeIter.hasNext()) {
+ Map.Entry<String, NamedList<?>> next = routeIter.next();
+ String host = next.getKey();
+ NamedList<?> hostResp = next.getValue();
+ Integer rf =
+ (Integer) ((NamedList<?>) hostResp.get("responseHeader")).get(UpdateRequest.REPFACT);
+ if (rf != null) {
+ String shard = leaders.get(host);
+ if (shard == null) {
+ if (host.endsWith("/")) shard = leaders.get(host.substring(0, host.length() - 1));
+ if (shard == null) {
+ shard = host;
}
}
+ results.put(shard, rf);
}
- if (retryCount < MAX_STALE_RETRIES) { // if it is a communication error , we must try again
- // may be, we have a stale version of the collection state,
- // and we could not get any information from the server
- // it is probably not worth trying again and again because
- // the state would not have been updated
- log.info(
- "Request to collection {} failed due to ({}) {}, retry={} maxRetries={} commError={} errorCode={} - retrying",
- inputCollections,
- errorCode,
- rootCause,
- retryCount,
- MAX_STALE_RETRIES,
- wasCommError,
- errorCode);
- return requestWithRetryOnStaleState(
- request,
- retryCount + 1,
- inputCollections,
- skipStateVersion,
- pendingRefreshes,
- waitedForRefresh);
- }
- } else {
- log.info("request was not communication error it seems");
}
- log.info(
- "Request to collection {} failed due to ({}) {}, retry={} maxRetries={} commError={} errorCode={} ",
- inputCollections,
- errorCode,
- rootCause,
- retryCount,
- MAX_STALE_RETRIES,
- wasCommError,
- errorCode);
-
- boolean stateWasStale = false;
- if (retryCount < MAX_STALE_RETRIES
- && requestedCollections != null
- && !requestedCollections.isEmpty()
- && (SolrException.ErrorCode.getErrorCode(errorCode)
- == SolrException.ErrorCode.INVALID_STATE
- || errorCode == 404)) {
- // cached state for one or more external collections was stale
- // re-issue request using updated state
- stateWasStale = true;
+ }
+ return results;
+ }
- // just re-read state for all of them, which is a little heavy-handed but hopefully a rare
- // occurrence
- for (DocCollection ext : requestedCollections) {
- collectionStateCache.remove(ext.getName());
- }
- }
+ /**
+ * Constructs {@link CloudSolrClient} instances from provided configuration. It will use a Jetty
+ * based {@code HttpClient} if available, or will otherwise use the JDK.
+ */
+ public static class Builder {
- // if we experienced a communication error, it's worth checking the state
- // with ZK just to make sure the node we're trying to hit is still part of the collection
- if (retryCount < MAX_STALE_RETRIES
- && !stateWasStale
- && requestedCollections != null
- && !requestedCollections.isEmpty()
- && wasCommError) {
- for (DocCollection ext : requestedCollections) {
- DocCollection latestStateFromZk = getDocCollection(ext.getName(), null);
- if (latestStateFromZk.getZNodeVersion() != ext.getZNodeVersion()) {
- // looks like we couldn't reach the server because the state was stale == retry
- stateWasStale = true;
- // we just pulled state from ZK, so update the cache so that the retry uses it
- collectionStateCache.put(
- ext.getName(), new ExpiringCachedDocCollection(latestStateFromZk));
- }
- }
- }
-
- // if the state was stale, then we retry the request once with new state pulled from Zk
- if (stateWasStale) {
- log.warn(
- "Re-trying request to collection(s) {} after stale state error from server.",
- inputCollections);
-
- Map<String, CompletableFuture<DocCollection>> refreshesToWaitFor = pendingRefreshes;
- if (!waitedForRefresh && (pendingRefreshes == null || pendingRefreshes.isEmpty())) {
- refreshesToWaitFor = new HashMap<>();
- for (DocCollection ext : requestedCollections) {
- refreshesToWaitFor.put(ext.getName(), triggerCollectionRefresh(ext.getName()));
- }
- }
+ protected Collection<String> zkHosts = new ArrayList<>();
+ protected List<String> solrUrls = new ArrayList<>();
+ protected String zkChroot;
+ protected HttpSolrClient httpClient;
+ protected boolean shardLeadersOnly = true;
+ protected boolean directUpdatesToLeadersOnly = false;
+ protected boolean parallelUpdates = true;
+ protected ClusterStateProvider stateProvider;
+ protected HttpSolrClient.BuilderBase<?, ?> internalClientBuilder;
+ protected RequestWriter requestWriter;
+ protected ResponseParser responseParser;
+ protected long retryExpiryTimeNano =
+ TimeUnit.NANOSECONDS.convert(3, TimeUnit.SECONDS); // 3 seconds or 3 million nanos
- // First retry without sending state versions so the server does not immediately reject the
- // request while we intentionally rely on stale routing (e.g., to allow forwarding to a new
- // leader) as the background refresh completes.
- if (!skipStateVersion && !waitedForRefresh) {
- resp =
- requestWithRetryOnStaleState(
- request,
- retryCount + 1,
- inputCollections,
- /*skipStateVersion*/ true,
- refreshesToWaitFor,
- waitedForRefresh);
- } else if (!waitedForRefresh
- && refreshesToWaitFor != null
- && !refreshesToWaitFor.isEmpty()) {
- for (Map.Entry<String, CompletableFuture<DocCollection>> entry :
- refreshesToWaitFor.entrySet()) {
- waitForCollectionRefresh(entry.getKey(), entry.getValue());
- }
- resp =
- requestWithRetryOnStaleState(
- request,
- retryCount + 1,
- inputCollections,
- /*skipStateVersion*/ false,
- Map.of(),
- /*waitedForRefresh*/ true);
- } else {
- resp =
- requestWithRetryOnStaleState(
- request,
- retryCount + 1,
- inputCollections,
- /*skipStateVersion*/ false,
- Map.of(),
- /*waitedForRefresh*/ waitedForRefresh);
- }
- } else {
- if (exc instanceof SolrException
- || exc instanceof SolrServerException
- || exc instanceof IOException) {
- throw exc;
- } else {
- throw new SolrServerException(rootCause);
- }
- }
+ protected String defaultCollection;
+ protected long timeToLiveSeconds = 60;
+ protected int parallelCacheRefreshesLocks = DEFAULT_STATE_REFRESH_PARALLELISM;
+ protected int zkConnectTimeout = SolrZkClientTimeout.DEFAULT_ZK_CONNECT_TIMEOUT;
+ protected int zkClientTimeout = SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT;
+ protected boolean canUseZkACLs = true;
- if (requestedCollections != null) {
- requestedCollections.clear(); // done with this
- }
+ /**
+ * Provide a series of Solr URLs to be used when configuring {@link CloudSolrClient} instances.
+ * The solr client will use these urls to understand the cluster topology, which solr nodes are
+ * active etc.
+ *
+ * <p>Provided Solr URLs are expected to point to the root Solr path
+ * ("http://hostname:8983/solr"); they should not include any collections, cores, or other path
+ * components.
+ *
+ * <p>Usage example:
+ *
+ * <pre>
+ * final List<String> solrBaseUrls = new ArrayList<String>();
+ * solrBaseUrls.add("http://solr1:8983/solr"); solrBaseUrls.add("http://solr2:8983/solr"); solrBaseUrls.add("http://solr3:8983/solr");
+ * final SolrClient client = new CloudSolrClient.Builder(solrBaseUrls).build();
+ * </pre>
+ */
+ public Builder(List<String> solrUrls) {
+ this.solrUrls = solrUrls;
}
- return resp;
- }
-
- protected NamedList<Object> sendRequest(SolrRequest<?> request, List<String> inputCollections)
- throws SolrServerException, IOException {
- boolean sendToLeaders = false;
-
- if (request.getRequestType() == SolrRequestType.UPDATE) {
- sendToLeaders = this.isUpdatesToLeaders();
-
- if (sendToLeaders && request instanceof UpdateRequest updateRequest) {
- sendToLeaders = sendToLeaders && updateRequest.isSendToLeaders();
-
- // Check if we can do a "directUpdate" ...
- if (sendToLeaders) {
- if (inputCollections.size() > 1) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "Update request must be sent to a single collection "
- + "or an alias: "
- + inputCollections);
- }
- String collection =
- inputCollections.isEmpty()
- ? null
- : inputCollections.get(0); // getting first mimics HttpSolrCall
- NamedList<Object> response = directUpdate(updateRequest, collection);
- if (response != null) {
- return response;
- }
- }
- }
+ /**
+ * Provide a series of ZK hosts which will be used when configuring {@link CloudSolrClient}
+ * instances.
+ *
+ * <p>Usage example when Solr stores data at the ZooKeeper root ('/'):
+ *
+ * <pre>
+ * final List<String> zkServers = new ArrayList<String>();
+ * zkServers.add("zookeeper1:2181"); zkServers.add("zookeeper2:2181"); zkServers.add("zookeeper3:2181");
+ * final SolrClient client = new CloudSolrClient.Builder(zkServers, Optional.empty()).build();
+ * </pre>
+ *
+ * Usage example when Solr data is stored in a ZooKeeper chroot:
+ *
+ * <pre>
+ * final List<String> zkServers = new ArrayList<String>();
+ * zkServers.add("zookeeper1:2181"); zkServers.add("zookeeper2:2181"); zkServers.add("zookeeper3:2181");
+ * final SolrClient client = new CloudSolrClient.Builder(zkServers, Optional.of("/solr")).build();
+ * </pre>
+ *
+ * @param zkHosts a List of at least one ZooKeeper host and port (e.g. "zookeeper1:2181")
+ * @param zkChroot the path to the root ZooKeeper node containing Solr data. Provide {@code
+ * java.util.Optional.empty()} if no ZK chroot is used.
+ * @deprecated Use a connectionString constructor and/or prefer HTTP URLs instead.
+ */
+ @Deprecated(since = "10.1") // sort of 10.0 but accidentally removed
+ public Builder(List<String> zkHosts, Optional<String> zkChroot) {
+ this.zkHosts = zkHosts;
+ if (zkChroot.isPresent()) this.zkChroot = zkChroot.get();
}
- SolrParams reqParams = request.getParams();
- assert reqParams != null;
-
- ReplicaListTransformer replicaListTransformer =
- requestRLTGenerator.getReplicaListTransformer(reqParams);
-
- final ClusterStateProvider provider = getClusterStateProvider();
- final String urlScheme = provider.getUrlScheme();
- final Set<String> liveNodes = provider.getLiveNodes();
-
- final List<LBSolrClient.Endpoint> requestEndpoints =
- new ArrayList<>(); // we populate this as follows...
-
- if (request.getApiVersion() == SolrRequest.ApiVersion.V2) {
- if (!liveNodes.isEmpty()) {
- List<String> liveNodesList = new ArrayList<>(liveNodes);
- Collections.shuffle(liveNodesList, rand);
- final var chosenNodeUrl = Utils.getBaseUrlForNodeName(liveNodesList.get(0), urlScheme);
- requestEndpoints.add(new LBSolrClient.Endpoint(chosenNodeUrl));
- }
-
- } else if (!request.requiresCollection()) {
- for (String liveNode : liveNodes) {
- final var nodeBaseUrl = Utils.getBaseUrlForNodeName(liveNode, urlScheme);
- requestEndpoints.add(new LBSolrClient.Endpoint(nodeBaseUrl));
- }
- } else { // API call to a particular collection / core / alias (i.e.
- // request.requiresCollection() == true)
- Set<String> collectionNames = resolveAliases(inputCollections);
- if (collectionNames.isEmpty()) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "No collection param specified on request and no default collection has been set: "
- + inputCollections);
- }
-
- List<String> preferredNodes = request.getPreferredNodes();
- if (preferredNodes != null && !preferredNodes.isEmpty()) {
- String joinedInputCollections = StrUtils.join(inputCollections, ',');
- final var endpoints =
- preferredNodes.stream()
- .map(nodeName -> Utils.getBaseUrlForNodeName(nodeName, urlScheme))
- .map(nodeUrl -> new LBSolrClient.Endpoint(nodeUrl, joinedInputCollections))
- .collect(Collectors.toList());
- if (!endpoints.isEmpty()) {
- LBSolrClient.Req req = new LBSolrClient.Req(request, endpoints);
- LBSolrClient.Rsp rsp = getLbClient().request(req);
- return rsp.getResponse();
- }
- }
-
- // TODO: not a big deal because of the caching, but we could avoid looking
- // at every shard when getting leaders if we tweaked some things
+ /** for an expert use-case */
+ public Builder(ClusterStateProvider stateProvider) {
+ this.stateProvider = stateProvider;
+ }
- // Retrieve slices from the cloud state and, for each collection specified, add it to the Map
- // of slices.
- Map<String, Slice> slices = new HashMap<>();
- String shardKeys = reqParams.get(ShardParams._ROUTE_);
- for (String collectionName : collectionNames) {
- DocCollection col = getDocCollection(collectionName, null);
- if (col == null) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST, "Collection not found: " + collectionName);
- }
- Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams, col);
- ClientUtils.addSlices(slices, collectionName, routeSlices, true);
- }
+ /**
+ * Creates a client builder based on a connection string of 2 possible formats:
+ *
+ * <ul>
+ * <li>ZooKeeper connection string (optionally with chroot), e.g. {@code
+ * zk1:2181,zk2:2181,zk3:2181/solr}
+ * <li>Comma-separated list of Solr node base URLs (HTTP or HTTPS), e.g. {@code
+ * http://solr1:8983/solr,http://solr2:8983/solr}
+ * </ul>
+ *
+ * @param connectionString a string specifying either ZooKeeper connection string or HTTP(S)
+ * Solr URLs
+ * @throws IllegalArgumentException if string is null, empty, or malformed
+ */
+ public Builder(String connectionString) {
+ this(CloudSolrClientConnection.parse(connectionString));
+ }
- // Gather URLs, grouped by leader or replica
- List<Replica> sortedReplicas = new ArrayList<>();
- List<Replica> replicas = new ArrayList<>();
- for (Slice slice : slices.values()) {
- Replica leader = slice.getLeader();
- for (Replica replica : slice.getReplicas()) {
- String node = replica.getNodeName();
- if (!liveNodes.contains(node) // Must be a live node to continue
- || replica.getState()
- != Replica.State.ACTIVE) { // Must be an ACTIVE replica to continue
- continue;
- }
- if (sendToLeaders && replica.equals(leader)) {
- sortedReplicas.add(replica); // put leaders here eagerly (if sendToLeader mode)
- } else {
- replicas.add(replica); // replicas here
- }
- }
+ /**
+ * Creates a client builder from a {@link CloudSolrClientConnection}.
+ *
+ * @param connection instance of {@link CloudSolrClientConnection}, which can be obtained from
+ * the solr connection string or created via the constructor
+ */
+ public Builder(CloudSolrClientConnection connection) {
+ if (connection.isZookeeper()) {
+ this.zkHosts = connection.quorumItems();
+ this.zkChroot = connection.zkChroot();
+ } else {
+ this.solrUrls = connection.quorumItems();
}
+ }
- // Sort the leader replicas, if any, according to the request preferences (none if
- // !sendToLeaders)
- replicaListTransformer.transform(sortedReplicas);
+ /** Whether to use the default ZK ACLs when building a ZK Client. */
+ public Builder canUseZkACLs(boolean canUseZkACLs) {
+ this.canUseZkACLs = canUseZkACLs;
+ return this;
+ }
- // Sort the replicas, if any, according to the request preferences and append to our list
- replicaListTransformer.transform(replicas);
+ /**
+ * Tells {@link Builder} that created clients should be configured such that {@link
+ * CloudSolrClient#isUpdatesToLeaders} returns <code>true</code>.
+ *
+ * @see #sendUpdatesToAnyReplica
+ * @see CloudSolrClient#isUpdatesToLeaders
+ */
+ public Builder sendUpdatesOnlyToShardLeaders() {
+ shardLeadersOnly = true;
+ return this;
+ }
- sortedReplicas.addAll(replicas);
+ /**
+ * Tells {@link Builder} that created clients should be configured such that {@link
+ * CloudSolrClient#isUpdatesToLeaders} returns <code>false</code>.
+ *
+ * @see #sendUpdatesOnlyToShardLeaders
+ * @see CloudSolrClient#isUpdatesToLeaders
+ */
+ public Builder sendUpdatesToAnyReplica() {
+ shardLeadersOnly = false;
+ return this;
+ }
- String joinedInputCollections = StrUtils.join(inputCollections, ',');
- Set<String> seenNodes = new HashSet<>();
- sortedReplicas.forEach(
- replica -> {
- if (seenNodes.add(replica.getNodeName())) {
- if (inputCollections.size() == 1 && collectionNames.size() == 1) {
- // If we have a single collection name (and not an alias to multiple collection),
- // send the query directly to a replica of this collection.
- requestEndpoints.add(
- new LBSolrClient.Endpoint(replica.getBaseUrl(), replica.getCoreName()));
- } else {
- requestEndpoints.add(
- new LBSolrClient.Endpoint(replica.getBaseUrl(), joinedInputCollections));
- }
- }
- });
+ /**
+ * Tells {@link CloudSolrClient.Builder} that created clients should send direct updates to
+ * shard leaders only.
+ *
+ * <p>UpdateRequests whose leaders cannot be found will "fail fast" on the client side with a
+ * {@link SolrException}
+ *
+ * @see #sendDirectUpdatesToAnyShardReplica
+ * @see CloudSolrClient#isDirectUpdatesToLeadersOnly
+ */
+ public Builder sendDirectUpdatesToShardLeadersOnly() {
+ directUpdatesToLeadersOnly = true;
+ return this;
+ }
- if (requestEndpoints.isEmpty()) {
- collectionStateCache.keySet().removeAll(collectionNames);
- throw new SolrException(
- SolrException.ErrorCode.INVALID_STATE,
- "Could not find a healthy node to handle the request.");
- }
+ /**
+ * Tells {@link CloudSolrClient.Builder} that created clients can send updates to any shard
+ * replica (shard leaders and non-leaders).
+ *
+ * <p>Shard leaders are still preferred, but the created clients will fall back to using other
+ * replicas if a leader cannot be found.
+ *
+ * @see #sendDirectUpdatesToShardLeadersOnly
+ * @see CloudSolrClient#isDirectUpdatesToLeadersOnly
+ */
+ public Builder sendDirectUpdatesToAnyShardReplica() {
+ directUpdatesToLeadersOnly = false;
+ return this;
}
- LBSolrClient.Req req = new LBSolrClient.Req(request, requestEndpoints);
- LBSolrClient.Rsp rsp = getLbClient().request(req);
- return rsp.getResponse();
- }
-
- /**
- * Resolves the input collections to their possible aliased collections. Doesn't validate
- * collection existence.
- */
- private Set<String> resolveAliases(List<String> inputCollections) {
- if (inputCollections.isEmpty()) {
- return Set.of();
- }
- LinkedHashSet<String> uniqueNames = new LinkedHashSet<>(); // consistent ordering
- for (String collectionName : inputCollections) {
- if (getDocCollection(collectionName, -1) == null) {
- // perhaps it's an alias
- uniqueNames.addAll(getClusterStateProvider().resolveAlias(collectionName));
- } else {
- uniqueNames.add(collectionName); // it's a collection
- }
+ /** Provides a {@link RequestWriter} for created clients to use when handing requests. */
+ public Builder withRequestWriter(RequestWriter requestWriter) {
+ this.requestWriter = requestWriter;
+ return this;
}
- return uniqueNames;
- }
- /**
- * If true, this client has been configured such that it will generally prefer to send {@link
- * SolrRequestType#UPDATE} requests to a shard leader, if and only if {@link
- * UpdateRequest#isSendToLeaders} is also true. If false, then this client has been configured to
- * obey normal routing preferences when dealing with {@link SolrRequestType#UPDATE} requests.
- *
- * @see #isDirectUpdatesToLeadersOnly
- */
- public boolean isUpdatesToLeaders() {
- return updatesToLeaders;
- }
+ /** Provides a {@link ResponseParser} for created clients to use when handling requests. */
+ public Builder withResponseParser(ResponseParser responseParser) {
+ this.responseParser = responseParser;
+ return this;
+ }
- /**
- * If true, this client has been configured such that "direct updates" will <em>only</em> be sent
- * to the current leader of the corresponding shard, and will not be retried with other replicas.
- * This method has no effect if {@link #isUpdatesToLeaders()} or {@link
- * UpdateRequest#isSendToLeaders} returns false.
- *
- * <p>A "direct update" is any update that can be sent directly to a single shard, and does not
- * need to be broadcast to every shard. (Example: document updates or "delete by id" when using
- * the default router; non-direct updates are things like commits and "delete by query").
- *
- * <p>NOTE: If a single {@link UpdateRequest} contains multiple "direct updates" for different
- * shards, this client may break the request up and merge the responses.
- *
- * @return true if direct updates are sent to shard leaders only
- */
- public boolean isDirectUpdatesToLeadersOnly() {
- return directUpdatesToLeadersOnly;
- }
+ /**
+ * Tells {@link CloudSolrClient.Builder} whether created clients should send shard updates
+ * serially or in parallel
+ *
+ * <p>When an {@link UpdateRequest} affects multiple shards, {@link CloudSolrClient} splits it
+ * up and sends a request to each affected shard. This setting chooses whether those
+ * sub-requests are sent serially or in parallel.
+ *
+ * <p>If not set, this defaults to 'true' and sends sub-requests in parallel.
+ */
+ public Builder withParallelUpdates(boolean parallelUpdates) {
+ this.parallelUpdates = parallelUpdates;
+ return this;
+ }
- /** Visible for tests so they can assert the configured refresh parallelism. */
- protected int getStateRefreshParallelism() {
- return stateRefreshParallelism;
- }
+ /**
+ * Configures how many collection state refresh operations may run in parallel using a dedicated
+ * thread pool. This controls the maximum number of concurrent ZooKeeper/cluster state lookups.
+ *
+ * <p>Defaults to 5.
+ */
+ public Builder withParallelCacheRefreshes(int parallelCacheRefreshesLocks) {
+ this.parallelCacheRefreshesLocks = parallelCacheRefreshesLocks;
+ return this;
+ }
- protected DocCollection getDocCollection(String collection, Integer expectedVersion)
- throws SolrException {
- if (expectedVersion == null) {
- expectedVersion = -1;
+ /**
+ * This is the time to wait to re-fetch the state after getting the same state version from ZK
+ */
+ public Builder withRetryExpiryTime(long expiryTime, TimeUnit unit) {
+ this.retryExpiryTimeNano = TimeUnit.NANOSECONDS.convert(expiryTime, unit);
+ return this;
}
- if (collection == null) {
- return null;
+
+ /** Sets the default collection for request. */
+ public Builder withDefaultCollection(String defaultCollection) {
+ this.defaultCollection = defaultCollection;
+ return this;
}
- ExpiringCachedDocCollection cacheEntry = collectionStateCache.peek(collection);
- if (cacheEntry != null && cacheEntry.isExpired(collectionStateCache.timeToLiveMs)) {
- collectionStateCache.remove(collection, cacheEntry);
- cacheEntry = null;
+ /**
+ * Sets the cache ttl for DocCollection Objects cached.
+ *
+ * @param timeToLive ttl value
+ */
+ public Builder withCollectionCacheTtl(long timeToLive, TimeUnit unit) {
+ assert timeToLive > 0;
+ this.timeToLiveSeconds = TimeUnit.SECONDS.convert(timeToLive, unit);
+ return this;
}
- DocCollection cached = cacheEntry == null ? null : cacheEntry.cached;
+ /**
+ * Set the internal Solr HTTP client.
+ *
+ * <p>Note: closing the client instance is the responsibility of the caller.
+ *
+ * @return this
+ */
+ public Builder withHttpClient(HttpSolrClient httpSolrClient) {
+ if (this.internalClientBuilder != null) {
+ throw new IllegalStateException(
+ "The builder can't accept an httpClient AND an internalClientBuilder, only one of those can be provided");
+ }
+ this.httpClient = httpSolrClient;
+ return this;
+ }
- if (cacheEntry != null && cacheEntry.shouldRetry()) {
- triggerCollectionRefresh(collection);
+ /**
+ * If provided, the CloudSolrClient will build it's internal client using this builder (instead
+ * of the empty default one). Providing this builder allows users to configure the internal
+ * clients (authentication, timeouts, etc.).
+ *
+ * @param internalClientBuilder the builder to use for creating the internal http client.
+ * @return this
+ */
+ public Builder withHttpClientBuilder(HttpSolrClient.BuilderBase<?, ?> internalClientBuilder) {
+ if (this.httpClient != null) {
+ throw new IllegalStateException(
+ "The builder can't accept an httpClient AND an internalClientBuilder, only one of those can be provided");
+ }
+ this.internalClientBuilder = internalClientBuilder;
+ return this;
}
- if (cached != null && expectedVersion <= cached.getZNodeVersion()) {
- return cached;
+ @Deprecated(since = "9.10")
+ public Builder withInternalClientBuilder(
+ HttpSolrClient.BuilderBase<?, ?> internalClientBuilder) {
+ return withHttpClientBuilder(internalClientBuilder);
}
- CompletableFuture<DocCollection> refreshFuture = triggerCollectionRefresh(collection);
- return waitForCollectionRefresh(collection, refreshFuture);
- }
+ /**
+ * Sets the Zk connection timeout
+ *
+ * @param zkConnectTimeout timeout value
+ * @param unit time unit
+ */
+ public Builder withZkConnectTimeout(int zkConnectTimeout, TimeUnit unit) {
+ this.zkConnectTimeout = Math.toIntExact(unit.toMillis(zkConnectTimeout));
+ return this;
+ }
- private CompletableFuture<DocCollection> triggerCollectionRefresh(String collection) {
- return collectionRefreshes.compute(
- collection,
- (key, existingFuture) -> {
- // A refresh is still in progress; return it.
- if (existingFuture != null && !existingFuture.isDone()) {
- return existingFuture;
- }
- // No refresh is in-progress, so trigger it.
+ /**
+ * Sets the Zk client session timeout
+ *
+ * @param zkClientTimeout timeout value
+ * @param unit time unit
+ */
+ public Builder withZkClientTimeout(int zkClientTimeout, TimeUnit unit) {
+ this.zkClientTimeout = Math.toIntExact(unit.toMillis(zkClientTimeout));
+ return this;
+ }
- if (ExecutorUtil.isShutdown(threadPool)) {
- assert closed; // see close() for the sequence
- ExpiringCachedDocCollection cacheEntry = collectionStateCache.peek(key);
- DocCollection cached = cacheEntry == null ? null : cacheEntry.cached;
- return CompletableFuture.completedFuture(cached);
- } else {
- return CompletableFuture.supplyAsync(
- () -> {
- stateRefreshSemaphore.acquireUninterruptibly();
- try {
- return loadDocCollection(key);
- } finally {
- stateRefreshSemaphore.release();
- // Remove the entry in case of many collections
- collectionRefreshes.remove(key);
- }
- },
- threadPool);
- }
- });
- }
+ /** Create a {@link CloudSolrClient} based on the provided configuration. */
+ public CloudHttp2SolrClient build() {
+ int providedOptions = 0;
+ if (!zkHosts.isEmpty()) providedOptions++;
+ if (!solrUrls.isEmpty()) providedOptions++;
+ if (stateProvider != null) providedOptions++;
- private DocCollection waitForCollectionRefresh(
- String collection, CompletableFuture<DocCollection> refreshFuture) {
- try {
- return refreshFuture.get();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR,
- "Interrupted while refreshing state for collection " + collection,
- e);
- } catch (ExecutionException e) {
- Throwable cause = e.getCause();
- if (cause instanceof SolrException) {
- throw (SolrException) cause;
+ if (providedOptions > 1) {
+ throw new IllegalArgumentException(
+ "Only one of zkHost(s), solrUrl(s), or stateProvider should be specified.");
+ } else if (providedOptions == 0) {
+ throw new IllegalArgumentException(
+ "One of zkHosts, solrUrls, or stateProvider must be specified.");
}
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR,
- "Error refreshing state for collection " + collection,
- cause);
+
+ return new CloudHttp2SolrClient(this);
}
- }
- private DocCollection loadDocCollection(String collection) {
- ClusterState.CollectionRef ref = getCollectionRef(collection);
- if (ref == null) {
- collectionStateCache.remove(collection);
- return null;
+ protected HttpSolrClient createOrGetHttpClient() {
+ if (httpClient != null) {
+ return httpClient;
+ } else if (internalClientBuilder != null) {
+ return internalClientBuilder.build();
+ } else {
+ return HttpSolrClient.builder(null).build();
+ }
}
- DocCollection fetchedCol = ref.get();
- if (fetchedCol == null) {
- collectionStateCache.remove(collection);
- return null;
+ protected LBSolrClient createOrGetLbClient(HttpSolrClient myClient) {
+ return myClient.createLBSolrClient();
}
- ExpiringCachedDocCollection existing = collectionStateCache.peek(collection);
- if (existing != null && existing.cached.getZNodeVersion() == fetchedCol.getZNodeVersion()) {
- existing.setRetriedAt();
- existing.maybeStale = false;
- return existing.cached;
+ protected ClusterStateProvider createZkClusterStateProvider() {
+ ClusterStateProvider stateProvider =
+ ClusterStateProvider.newZkClusterStateProvider(zkHosts, zkChroot, canUseZkACLs);
+ if (stateProvider instanceof SolrZkClientTimeout.SolrZkClientTimeoutAware timeoutAware) {
+ timeoutAware.setZkClientTimeout(zkClientTimeout);
+ timeoutAware.setZkConnectTimeout(zkConnectTimeout);
+ }
+ return stateProvider;
}
- collectionStateCache.put(collection, new ExpiringCachedDocCollection(fetchedCol));
- return fetchedCol;
- }
-
- ClusterState.CollectionRef getCollectionRef(String collection) {
- return getClusterStateProvider().getState(collection);
+ protected ClusterStateProvider createHttpClusterStateProvider(HttpSolrClient httpClient) {
+ try {
+ return new HttpClusterStateProvider<>(solrUrls, httpClient);
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Couldn't initialize a HttpClusterStateProvider (is/are the "
+ + "Solr server(s), "
+ + solrUrls
+ + ", down?)",
+ e);
+ }
+ }
}
- /**
- * Useful for determining the minimum achieved replication factor across all shards involved in
- * processing an update request, typically useful for gauging the replication factor of a batch.
- */
- public int getMinAchievedReplicationFactor(String collection, NamedList<?> resp) {
- // it's probably already on the top-level header set by condense
- NamedList<?> header = (NamedList<?>) resp.get("responseHeader");
- Integer achRf = (Integer) header.get(UpdateRequest.REPFACT);
- if (achRf != null) return achRf.intValue();
+ protected static class StateCache extends ConcurrentHashMap<String, ExpiringCachedDocCollection> {
+ final AtomicLong puts = new AtomicLong();
+ final AtomicLong hits = new AtomicLong();
+ final Lock evictLock = new ReentrantLock(true);
+ public volatile long timeToLiveMs = 60 * 1000L;
- // not on the top-level header, walk the shard route tree
- Map<String, Integer> shardRf = getShardReplicationFactor(collection, resp);
- for (Integer rf : shardRf.values()) {
- if (achRf == null || rf < achRf) {
- achRf = rf;
+ @Override
+ public ExpiringCachedDocCollection get(Object key) {
+ ExpiringCachedDocCollection val = super.get(key);
+ if (val == null) {
+ // a new collection is likely to be added now.
+ // check if there are stale items and remove them
+ evictStale();
+ return null;
+ }
+ if (val.isExpired(timeToLiveMs)) {
+ super.remove(key);
+ return null;
}
+ hits.incrementAndGet();
+ return val;
}
- return (achRf != null) ? achRf.intValue() : -1;
- }
- /**
- * Walks the NamedList response after performing an update request looking for the replication
- * factor that was achieved in each shard involved in the request. For single doc updates, there
- * will be only one shard in the return value.
- */
- public Map<String, Integer> getShardReplicationFactor(String collection, NamedList<?> resp) {
- Map<String, Integer> results = new HashMap<>();
- if (resp instanceof RouteResponse) {
- NamedList<NamedList<?>> routes = ((RouteResponse<?>) resp).getRouteResponses();
- DocCollection coll = getDocCollection(collection, null);
- Map<String, String> leaders = new HashMap<>();
- for (Slice slice : coll.getActiveSlices()) {
- Replica leader = slice.getLeader();
- if (leader != null) {
- String leaderUrl = leader.getBaseUrl() + "/" + leader.getCoreName();
- leaders.put(leaderUrl, slice.getName());
- String altLeaderUrl = leader.getBaseUrl() + "/" + collection;
- leaders.put(altLeaderUrl, slice.getName());
- }
- }
+ ExpiringCachedDocCollection peek(Object key) {
+ return super.get(key);
+ }
- Iterator<Map.Entry<String, NamedList<?>>> routeIter = routes.iterator();
- while (routeIter.hasNext()) {
- Map.Entry<String, NamedList<?>> next = routeIter.next();
- String host = next.getKey();
- NamedList<?> hostResp = next.getValue();
- Integer rf =
- (Integer) ((NamedList<?>) hostResp.get("responseHeader")).get(UpdateRequest.REPFACT);
- if (rf != null) {
- String shard = leaders.get(host);
- if (shard == null) {
- if (host.endsWith("/")) shard = leaders.get(host.substring(0, host.length() - 1));
- if (shard == null) {
- shard = host;
- }
+ @Override
+ public ExpiringCachedDocCollection put(String key, ExpiringCachedDocCollection value) {
+ puts.incrementAndGet();
+ return super.put(key, value);
+ }
+
+ void evictStale() {
+ if (!evictLock.tryLock()) return;
+ try {
+ for (Entry<String, ExpiringCachedDocCollection> e : entrySet()) {
+ if (e.getValue().isExpired(timeToLiveMs)) {
+ super.remove(e.getKey());
}
- results.put(shard, rf);
}
+ } finally {
+ evictLock.unlock();
}
}
- return results;
}
- /**
- * Determines whether an UpdateRequest contains sufficient routing information to identify shard
- * leaders for direct updates when directUpdatesToLeadersOnly is enabled.
- */
- private static boolean hasInfoToFindLeaders(UpdateRequest updateRequest, String idField) {
- final Map<SolrInputDocument, Map<String, Object>> documents = updateRequest.getDocumentsMap();
- final Map<String, Map<String, Object>> deleteById = updateRequest.getDeleteByIdMap();
+ @SuppressWarnings({"rawtypes"})
+ public static class RouteResponse<T extends LBSolrClient.Req> extends NamedList<Object> {
+ private NamedList<NamedList<?>> routeResponses;
+ private Map<String, T> routes;
- final boolean hasNoDocuments = (documents == null || documents.isEmpty());
- final boolean hasNoDeleteById = (deleteById == null || deleteById.isEmpty());
- if (hasNoDocuments && hasNoDeleteById) {
- // no documents and no delete-by-id, so no info to find leader(s)
- return false;
+ public void setRouteResponses(NamedList<NamedList<?>> routeResponses) {
+ this.routeResponses = routeResponses;
}
- if (documents != null) {
- for (final Map.Entry<SolrInputDocument, Map<String, Object>> entry : documents.entrySet()) {
- final SolrInputDocument doc = entry.getKey();
- final Object fieldValue = doc.getFieldValue(idField);
- if (fieldValue == null) {
- // a document with no id field value, so can't find leader for it
- return false;
- }
- }
+ public NamedList<NamedList<?>> getRouteResponses() {
+ return routeResponses;
}
- if (deleteById != null) {
- for (final Map.Entry<String, Map<String, Object>> entry : deleteById.entrySet()) {
- final Map<String, Object> params = entry.getValue();
- if (params == null || params.get(ShardParams._ROUTE_) == null) {
- // deleteById entry lacks explicit route parameter, can't find leader for it
- return false;
+ public void setRoutes(Map<String, T> routes) {
+ this.routes = routes;
+ }
+
+ public Map<String, T> getRoutes() {
+ return routes;
+ }
+ }
+
+ public static class RouteException extends SolrException {
+
+ private NamedList<Throwable> throwables;
+ private Map<String, ? extends LBSolrClient.Req> routes;
+
+ public RouteException(
+ ErrorCode errorCode,
+ NamedList<Throwable> throwables,
+ Map<String, ? extends LBSolrClient.Req> routes) {
+ super(errorCode, throwables.getVal(0).getMessage(), throwables.getVal(0));
+ this.throwables = throwables;
+ this.routes = routes;
+
+ // create a merged copy of the metadata from all wrapped exceptions
+ NamedList<String> metadata = new NamedList<String>();
+ for (int i = 0; i < throwables.size(); i++) {
+ Throwable t = throwables.getVal(i);
+ if (t instanceof SolrException e) {
+ NamedList<String> eMeta = e.getMetadata();
+ if (null != eMeta) {
+ metadata.addAll(eMeta);
+ }
}
}
+ if (0 < metadata.size()) {
+ this.setMetadata(metadata);
+ }
}
- return true;
+ public NamedList<Throwable> getThrowables() {
+ return throwables;
+ }
+
+ public Map<String, ? extends LBSolrClient.Req> getRoutes() {
+ return this.routes;
+ }
}
/** Universal connection string parser logic. */
@@ -1887,4 +1833,55 @@ public abstract class CloudSolrClient extends SolrClient {
return String.join(",", quorumItems) + (isZookeeper && zkChroot != null ? zkChroot : "");
}
}
+
+ class ExpiringCachedDocCollection {
+ final DocCollection cached;
+ final long cachedAtNano;
+ // This is the time at which the collection is retried and got the same old version
+ volatile long retriedAtNano = -1;
+ // flag that suggests that this is potentially to be rechecked
+ volatile boolean maybeStale = false;
+
+ ExpiringCachedDocCollection(DocCollection cached) {
+ this.cached = cached;
+ this.cachedAtNano = System.nanoTime();
+ }
+
+ boolean isExpired(long timeToLiveMs) {
+ return (System.nanoTime() - cachedAtNano)
+ > TimeUnit.NANOSECONDS.convert(timeToLiveMs, TimeUnit.MILLISECONDS);
+ }
+
+ boolean shouldRetry() {
+ if (maybeStale) { // we are not sure if it is stale so check with retry time
+ if ((retriedAtNano == -1 || (System.nanoTime() - retriedAtNano) > retryExpiryTimeNano)) {
+ return true; // we retried a while back. and we could not get anything new.
+ // it's likely that it is not going to be available now also.
+ }
+ }
+ return false;
+ }
+
+ void setRetriedAt() {
+ retriedAtNano = System.nanoTime();
+ }
+
+ /**
+ * Marks this entry as {@code maybeStale} if the provided backoff window has elapsed since the
+ * last retry.
+ *
+ * @return {@code true} if the entry was flagged as maybe stale
+ */
+ boolean markMaybeStaleIfOutsideBackoff(long retryBackoffNano) {
+ if (maybeStale) {
+ return true;
+ }
+ long lastRetry = retriedAtNano;
+ if (lastRetry != -1 && (System.nanoTime() - lastRetry) <= retryBackoffNano) {
+ return false;
+ }
+ maybeStale = true;
+ return true;
+ }
+ }
}
(solr) 04/07: HttpSolrClient: improve error when text/html (#4526)
Posted by ds...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit d51ed7c096706f6b0fbc9b9329eccb06eda866ca
Author: David Smiley <ds...@apache.org>
AuthorDate: Mon Jun 22 22:21:29 2026 -0400
HttpSolrClient: improve error when text/html (#4526)
made error messages more consistent, avoiding complaining about the content-type when it's text/html
Ex: 429 will clearly give that code and not complain about unexpected text/html. Curiously, this happened less for Apache HttpClient.
(cherry picked from commit ad2d914f208b4ae5a1858e0138024eaa6bb96fc8)
---
.../PR#4526-HttpSolrClient-error-message.yml | 9 ++++++++
.../solr/cloud/TestAuthenticationFramework.java | 13 ++++--------
.../solr/security/BasicAuthIntegrationTest.java | 2 +-
.../solr/servlet/TestRequestRateLimiter.java | 17 +++++++++++----
.../solr/client/solrj/impl/HttpSolrClient.java | 24 ++++++++++------------
5 files changed, 38 insertions(+), 27 deletions(-)
diff --git a/changelog/unreleased/PR#4526-HttpSolrClient-error-message.yml b/changelog/unreleased/PR#4526-HttpSolrClient-error-message.yml
new file mode 100644
index 00000000000..ecf4bfc9a6d
--- /dev/null
+++ b/changelog/unreleased/PR#4526-HttpSolrClient-error-message.yml
@@ -0,0 +1,9 @@
+title: >
+ HttpSolrClient: made error messages more consistent, avoiding complaining about the content-type
+ when it's text/html
+type: changed
+authors:
+ - name: David Smiley
+links:
+ - name: PR#4526
+ url: https://github.com/apache/solr/pull/4526
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestAuthenticationFramework.java b/solr/core/src/test/org/apache/solr/cloud/TestAuthenticationFramework.java
index 130e288ee96..18060fcd016 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestAuthenticationFramework.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestAuthenticationFramework.java
@@ -21,7 +21,7 @@ import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import java.lang.invoke.MethodHandles;
import java.util.Map;
-import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.RemoteSolrException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.SolrQuery;
@@ -43,9 +43,6 @@ public class TestAuthenticationFramework extends SolrCloudTestCase {
private static final String configName = "solrCloudCollectionConfig";
private static final String collectionName = "testcollection";
- static String requestUsername = MockAuthenticationPlugin.expectedUsername;
- static String requestPassword = MockAuthenticationPlugin.expectedPassword;
-
@Override
public void setUp() throws Exception {
setupAuthenticationPlugin();
@@ -70,11 +67,9 @@ public class TestAuthenticationFramework extends SolrCloudTestCase {
// Should fail with 401
try {
- SolrServerException e =
- expectThrows(SolrServerException.class, this::collectionCreateSearchDeleteTwice);
- assertTrue(
- "Should've returned a 401 error",
- e.getMessage().contains("401") || e.getMessage().contains("Authentication"));
+ RemoteSolrException e =
+ expectThrows(RemoteSolrException.class, this::collectionCreateSearchDeleteTwice);
+ assertEquals(401, e.code()); // Authentication failed
} finally {
MockAuthenticationPlugin.expectedUsername = null;
MockAuthenticationPlugin.expectedPassword = null;
diff --git a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java
index 166b9f440c6..9684d5447cf 100644
--- a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java
@@ -259,7 +259,7 @@ public class BasicAuthIntegrationTest extends SolrCloudAuthTestCase {
() -> {
new UpdateRequest().deleteByQuery("*:*").process(aNewClient, COLLECTION);
});
- assertTrue(e.getMessage(), e.getMessage().contains("Authentication failed"));
+ assertEquals(401, e.code()); // Authentication failed
} finally {
aNewClient.close();
cluster.stopJettySolrRunner(aNewJetty);
diff --git a/solr/core/src/test/org/apache/solr/servlet/TestRequestRateLimiter.java b/solr/core/src/test/org/apache/solr/servlet/TestRequestRateLimiter.java
index 87ec473b13a..0054cd1ad19 100644
--- a/solr/core/src/test/org/apache/solr/servlet/TestRequestRateLimiter.java
+++ b/solr/core/src/test/org/apache/solr/servlet/TestRequestRateLimiter.java
@@ -38,12 +38,14 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.LongAdder;
-import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.RemoteSolrException;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider;
+import org.apache.solr.client.solrj.jetty.CloudJettySolrClient;
+import org.apache.solr.client.solrj.jetty.HttpJettySolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.SolrQuery;
import org.apache.solr.client.solrj.response.QueryResponse;
@@ -56,7 +58,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
-@LuceneTestCase.AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17810")
public class TestRequestRateLimiter extends SolrCloudTestCase {
private static final String FIRST_COLLECTION = "c1";
private static final String SECOND_COLLECTION = "c2";
@@ -69,7 +70,7 @@ public class TestRequestRateLimiter extends SolrCloudTestCase {
@Test
public void testConcurrentQueries() throws Exception {
- try (CloudSolrClient client = cluster.newSolrClient(FIRST_COLLECTION)) {
+ try (CloudSolrClient client = newHttp1SolrClient(FIRST_COLLECTION)) {
CollectionAdminRequest.createCollection(FIRST_COLLECTION, 1, 1).process(client);
cluster.waitForActiveCollection(FIRST_COLLECTION, 1, 1);
@@ -289,7 +290,7 @@ public class TestRequestRateLimiter extends SolrCloudTestCase {
@Nightly
public void testSlotBorrowing() throws Exception {
- try (CloudSolrClient client = cluster.newSolrClient(SECOND_COLLECTION)) {
+ try (CloudSolrClient client = newHttp1SolrClient(SECOND_COLLECTION)) {
CollectionAdminRequest.createCollection(SECOND_COLLECTION, 1, 1).process(client);
cluster.waitForActiveCollection(SECOND_COLLECTION, 1, 1);
@@ -429,6 +430,14 @@ public class TestRequestRateLimiter extends SolrCloudTestCase {
}
}
+ private CloudSolrClient newHttp1SolrClient(String collection) {
+ return new CloudJettySolrClient.Builder(
+ new ZkClientClusterStateProvider(cluster.getZkServer().getZkAddress()))
+ .withDefaultCollection(collection)
+ .withHttpClientBuilder(new HttpJettySolrClient.Builder().useHttp1_1(true))
+ .build();
+ }
+
private static class MockBuilder extends RateLimitManager.Builder {
private final RequestRateLimiter queryRequestRateLimiter;
private final RequestRateLimiter indexRequestRateLimiter;
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
index 93ae9e332f1..d1e69255bab 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
@@ -23,11 +23,11 @@ import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Constructor;
import java.net.MalformedURLException;
-import java.net.URLDecoder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Collection;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
@@ -188,11 +188,12 @@ public abstract class HttpSolrClient extends SolrClient {
String responseMethod,
final ResponseParser processor,
InputStream is,
- String mimeType,
+ String mimeType, // aka contentType
String encoding,
final boolean isV2Api,
String urlExceptionMessage)
throws SolrServerException {
+ Objects.requireNonNull(processor);
boolean shouldClose = true;
try {
// handle some http level checks before trying to parse the response
@@ -209,7 +210,7 @@ public abstract class HttpSolrClient extends SolrClient {
}
break;
default:
- if (processor == null || mimeType == null) {
+ if (mimeType == null || mimeType.equals("text/html")) {
throw new RemoteSolrException(
urlExceptionMessage,
httpStatus,
@@ -218,6 +219,8 @@ public abstract class HttpSolrClient extends SolrClient {
}
}
+ checkContentType(processor, is, mimeType, encoding, httpStatus, urlExceptionMessage);
+
if (wantStream(processor)) {
// Only case where stream should not be closed
shouldClose = false;
@@ -225,8 +228,6 @@ public abstract class HttpSolrClient extends SolrClient {
return InputStreamResponseParser.createInputStreamNamedList(httpStatus, is);
}
- checkContentType(processor, is, mimeType, encoding, httpStatus, urlExceptionMessage);
-
NamedList<Object> rsp;
try {
rsp = processor.processResponse(is, encoding);
@@ -240,14 +241,11 @@ public abstract class HttpSolrClient extends SolrClient {
}
if (httpStatus != 200 && !isV2Api) {
if (error == null) {
- StringBuilder msg =
- new StringBuilder()
- .append(responseReason)
- .append("\n")
- .append("request: ")
- .append(responseMethod);
- String reason = URLDecoder.decode(msg.toString(), FALLBACK_CHARSET);
- throw new RemoteSolrException(urlExceptionMessage, httpStatus, reason, null);
+ throw new RemoteSolrException(
+ urlExceptionMessage,
+ httpStatus,
+ "non ok status: " + httpStatus + ", message:" + responseReason,
+ null);
} else {
throw new RemoteSolrException(urlExceptionMessage, httpStatus, error);
}
(solr) 05/07: Replace ConcurrentLRUCache and Cache interface with Caffeine (#4516)
Posted by ds...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit adf4048a86c949fcb217246531fd984af02f48c3
Author: David Smiley <ds...@apache.org>
AuthorDate: Tue Jun 23 00:55:04 2026 -0400
Replace ConcurrentLRUCache and Cache interface with Caffeine (#4516)
Replaces a custom utility with our Caffeine dependency. Unlikely to be noticed.
Co-authored-by: google-labs-jules[bot] <16...@users.noreply.github.com>
(cherry picked from commit ec218b20cc11899e41cc0dfe0fd5f6b8e7dfc28f)
---
.../java/org/apache/solr/schema/IndexSchema.java | 10 +-
.../processor/TemplateUpdateProcessorFactory.java | 31 +-
.../org/apache/solr/util/ConcurrentLRUCache.java | 974 ---------------------
.../java/org/apache/solr/common/util/Cache.java | 44 -
.../org/apache/solr/common/util/JavaBinCodec.java | 19 +-
.../apache/solr/common/util/MapBackedCache.java | 60 --
.../org/apache/solr/common/util/ObjectCache.java | 27 +-
.../apache/solr/common/util/TestJavaBinCodec.java | 77 +-
8 files changed, 94 insertions(+), 1148 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
index 05b362f95b6..d90251d8635 100644
--- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
+++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
@@ -18,6 +18,8 @@ package org.apache.solr.schema;
import static java.util.Arrays.asList;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import java.io.IOException;
import java.io.Writer;
import java.lang.invoke.MethodHandles;
@@ -65,7 +67,6 @@ import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.Cache;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.ConfigSetService;
@@ -76,7 +77,6 @@ import org.apache.solr.response.SchemaXmlWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.similarities.SchemaSimilarityFactory;
import org.apache.solr.uninverting.UninvertingReader;
-import org.apache.solr.util.ConcurrentLRUCache;
import org.apache.solr.util.PayloadUtils;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
@@ -141,8 +141,8 @@ public class IndexSchema {
private static final Set<String> FIELDTYPE_KEYS = Set.of("fieldtype", "fieldType");
private static final Set<String> FIELD_KEYS = Set.of("dynamicField", "field");
- protected Cache<String, SchemaField> dynamicFieldCache =
- new ConcurrentLRUCache<>(10000, 8000, 9000, 100, false, false, null);
+ protected final Cache<String, SchemaField> dynamicFieldCache =
+ Caffeine.newBuilder().initialCapacity(100).maximumSize(10000).build();
private Analyzer indexAnalyzer;
private Analyzer queryAnalyzer;
@@ -1386,7 +1386,7 @@ public class IndexSchema {
public SchemaField getFieldOrNull(String fieldName) {
SchemaField f = fields.get(fieldName);
if (f != null) return f;
- f = dynamicFieldCache.get(fieldName);
+ f = dynamicFieldCache.getIfPresent(fieldName);
if (f != null) return f;
for (DynamicField df : dynamicFields) {
diff --git a/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java
index f37e4a4c419..ce10e143afc 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java
@@ -17,17 +17,17 @@
package org.apache.solr.update.processor;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.util.Cache;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
-import org.apache.solr.util.ConcurrentLRUCache;
/**
* Adds new fields to documents based on a template pattern specified via Template.field request
@@ -41,7 +41,7 @@ import org.apache.solr.util.ConcurrentLRUCache;
*/
public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory {
private Cache<String, Resolved> templateCache =
- new ConcurrentLRUCache<>(1000, 800, 900, 10, false, false, null);
+ Caffeine.newBuilder().initialCapacity(10).maximumSize(1000).build();
public static final String NAME = "template";
@Override
@@ -79,17 +79,20 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory
public static Resolved getResolved(
String template, Cache<String, Resolved> cache, Pattern pattern) {
- Resolved r = cache == null ? null : cache.get(template);
- if (r == null) {
- r = new Resolved();
- Matcher m = pattern.matcher(template);
- while (m.find()) {
- String variable = m.group(1);
- r.startIndexes.add(m.start(0));
- r.endOffsets.add(m.end(0));
- r.variables.add(variable);
- }
- if (cache != null) cache.put(template, r);
+ if (cache == null) {
+ return resolve(template, pattern);
+ }
+ return cache.get(template, t -> resolve(t, pattern));
+ }
+
+ private static Resolved resolve(String template, Pattern pattern) {
+ Resolved r = new Resolved();
+ Matcher m = pattern.matcher(template);
+ while (m.find()) {
+ String variable = m.group(1);
+ r.startIndexes.add(m.start(0));
+ r.endOffsets.add(m.end(0));
+ r.variables.add(variable);
}
return r;
}
diff --git a/solr/core/src/java/org/apache/solr/util/ConcurrentLRUCache.java b/solr/core/src/java/org/apache/solr/util/ConcurrentLRUCache.java
deleted file mode 100644
index a32b6158000..00000000000
--- a/solr/core/src/java/org/apache/solr/util/ConcurrentLRUCache.java
+++ /dev/null
@@ -1,974 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.util;
-
-import static org.apache.lucene.util.RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY;
-import static org.apache.lucene.util.RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED;
-
-import java.lang.ref.WeakReference;
-import java.lang.reflect.Array;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.LongAdder;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.Function;
-import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.PriorityQueue;
-import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.solr.common.util.Cache;
-import org.apache.solr.common.util.TimeSource;
-
-/**
- * A LRU cache implementation based upon ConcurrentHashMap and other techniques to reduce contention
- * and synchronization overhead to utilize multiple CPU cores more effectively.
- *
- * <p>Note that the implementation does not follow a true LRU (least-recently-used) eviction
- * strategy. Instead it strives to remove least recently used items but when the initial cleanup
- * does not remove enough items to reach the 'acceptableWaterMark' limit, it can remove more items
- * forcefully regardless of access order.
- *
- * @since solr 1.4
- */
-public class ConcurrentLRUCache<K, V> implements Cache<K, V>, Accountable {
-
- private static final long BASE_RAM_BYTES_USED =
- RamUsageEstimator.shallowSizeOfInstance(ConcurrentLRUCache.class)
- + new Stats().ramBytesUsed()
- + RamUsageEstimator.primitiveSizes.get(long.class)
- + RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class);
-
- private final ConcurrentHashMap<Object, CacheEntry<K, V>> map;
- private int upperWaterMark, lowerWaterMark;
- private final ReentrantLock markAndSweepLock = new ReentrantLock(true);
- private boolean isCleaning = false; // not volatile... piggybacked on other volatile vars
- private boolean newThreadForCleanup;
- private volatile boolean islive = true;
- private final Stats stats = new Stats();
- private int acceptableWaterMark;
- private long oldestEntry = 0; // not volatile, only accessed in the cleaning method
- private final TimeSource timeSource = TimeSource.NANO_TIME;
- private final AtomicLong oldestEntryNs = new AtomicLong(0);
- private long maxIdleTimeNs;
- private final EvictionListener<K, V> evictionListener;
- private CleanupThread cleanupThread;
- private boolean runCleanupThread;
-
- private long ramLowerWatermark, ramUpperWatermark;
- private final LongAdder ramBytes = new LongAdder();
-
- public ConcurrentLRUCache(
- long ramLowerWatermark,
- long ramUpperWatermark,
- boolean runCleanupThread,
- EvictionListener<K, V> evictionListener) {
- this(ramLowerWatermark, ramUpperWatermark, runCleanupThread, evictionListener, -1);
- }
-
- public ConcurrentLRUCache(
- long ramLowerWatermark,
- long ramUpperWatermark,
- boolean runCleanupThread,
- EvictionListener<K, V> evictionListener,
- int maxIdleTimeSec) {
- this.ramLowerWatermark = ramLowerWatermark;
- this.ramUpperWatermark = ramUpperWatermark;
-
- this.evictionListener = evictionListener;
- this.map = new ConcurrentHashMap<>();
- this.newThreadForCleanup = false;
-
- this.acceptableWaterMark = -1;
- this.lowerWaterMark = Integer.MIN_VALUE;
- this.upperWaterMark = Integer.MAX_VALUE;
-
- setMaxIdleTime(maxIdleTimeSec);
- setRunCleanupThread(runCleanupThread);
- }
-
- public ConcurrentLRUCache(
- int upperWaterMark,
- final int lowerWaterMark,
- int acceptableWaterMark,
- int initialSize,
- boolean runCleanupThread,
- boolean runNewThreadForCleanup,
- EvictionListener<K, V> evictionListener) {
- this(
- upperWaterMark,
- lowerWaterMark,
- acceptableWaterMark,
- initialSize,
- runCleanupThread,
- runNewThreadForCleanup,
- evictionListener,
- -1);
- }
-
- public ConcurrentLRUCache(
- int upperWaterMark,
- final int lowerWaterMark,
- int acceptableWaterMark,
- int initialSize,
- boolean runCleanupThread,
- boolean runNewThreadForCleanup,
- EvictionListener<K, V> evictionListener,
- int maxIdleTimeSec) {
- if (upperWaterMark < 1) throw new IllegalArgumentException("upperWaterMark must be > 0");
- if (lowerWaterMark >= upperWaterMark)
- throw new IllegalArgumentException("lowerWaterMark must be < upperWaterMark");
- map = new ConcurrentHashMap<>(initialSize);
- newThreadForCleanup = runNewThreadForCleanup;
- this.upperWaterMark = upperWaterMark;
- this.lowerWaterMark = lowerWaterMark;
- this.acceptableWaterMark = acceptableWaterMark;
- this.evictionListener = evictionListener;
- this.ramLowerWatermark = Long.MIN_VALUE;
- this.ramUpperWatermark = Long.MAX_VALUE;
- setMaxIdleTime(maxIdleTimeSec);
- setRunCleanupThread(runCleanupThread);
- }
-
- public ConcurrentLRUCache(int size, int lowerWaterMark) {
- this(
- size,
- lowerWaterMark,
- (int) Math.floor((lowerWaterMark + size) / 2.0),
- (int) Math.ceil(0.75 * size),
- false,
- false,
- null,
- -1);
- }
-
- public void setAlive(boolean live) {
- islive = live;
- }
-
- public void setUpperWaterMark(int upperWaterMark) {
- if (upperWaterMark < 1) throw new IllegalArgumentException("upperWaterMark must be >= 1");
- this.upperWaterMark = upperWaterMark;
- }
-
- public void setLowerWaterMark(int lowerWaterMark) {
- this.lowerWaterMark = lowerWaterMark;
- }
-
- public void setAcceptableWaterMark(int acceptableWaterMark) {
- this.acceptableWaterMark = acceptableWaterMark;
- }
-
- public void setRamUpperWatermark(long ramUpperWatermark) {
- if (ramUpperWatermark < 1) {
- throw new IllegalArgumentException("ramUpperWaterMark must be >= 1");
- }
- this.ramUpperWatermark = ramUpperWatermark;
- }
-
- public void setRamLowerWatermark(long ramLowerWatermark) {
- if (ramLowerWatermark < 0) {
- throw new IllegalArgumentException("ramLowerWaterMark must be >= 0");
- }
- this.ramLowerWatermark = ramLowerWatermark;
- }
-
- public void setMaxIdleTime(int maxIdleTime) {
- long oldMaxIdleTimeNs = maxIdleTimeNs;
- maxIdleTimeNs =
- maxIdleTime > 0
- ? TimeUnit.NANOSECONDS.convert(maxIdleTime, TimeUnit.SECONDS)
- : Long.MAX_VALUE;
- if (cleanupThread != null && maxIdleTimeNs < oldMaxIdleTimeNs) {
- cleanupThread.wakeThread();
- }
- }
-
- public synchronized void setRunCleanupThread(boolean runCleanupThread) {
- this.runCleanupThread = runCleanupThread;
- if (this.runCleanupThread) {
- if (cleanupThread == null) {
- cleanupThread = new CleanupThread(this);
- cleanupThread.start();
- }
- } else {
- if (cleanupThread != null) {
- cleanupThread.stopThread();
- cleanupThread = null;
- }
- }
- }
-
- @Override
- public V get(K key) {
- CacheEntry<K, V> e = map.get(key);
- if (e == null) {
- if (islive) stats.missCounter.increment();
- return null;
- }
- if (islive) e.lastAccessed = stats.accessCounter.incrementAndGet();
- return e.value;
- }
-
- @Override
- public V remove(K key) {
- CacheEntry<K, V> cacheEntry = map.remove(key);
- if (cacheEntry != null) {
- stats.size.decrement();
- ramBytes.add(-cacheEntry.ramBytesUsed() - HASHTABLE_RAM_BYTES_PER_ENTRY);
- return cacheEntry.value;
- }
- return null;
- }
-
- @Override
- public V computeIfAbsent(K key, Function<? super K, ? extends V> mappingFunction) {
- // prescreen access first
- V val = get(key);
- if (val != null) {
- return val;
- }
- AtomicBoolean newEntry = new AtomicBoolean();
- CacheEntry<K, V> entry =
- map.computeIfAbsent(
- key,
- k -> {
- V value = mappingFunction.apply(key);
- // preserve the semantics of computeIfAbsent
- if (value == null) {
- return null;
- }
- CacheEntry<K, V> e =
- new CacheEntry<>(
- key,
- value,
- timeSource.getEpochTimeNs(),
- stats.accessCounter.incrementAndGet());
- oldestEntryNs.updateAndGet(x -> x > e.createTime || x == 0 ? e.createTime : x);
- stats.size.increment();
- ramBytes.add(
- e.ramBytesUsed() + HASHTABLE_RAM_BYTES_PER_ENTRY); // added key + value + entry
- if (islive) {
- stats.putCounter.increment();
- } else {
- stats.nonLivePutCounter.increment();
- }
- newEntry.set(true);
- return e;
- });
- if (newEntry.get()) {
- maybeMarkAndSweep();
- } else {
- if (islive && entry != null) {
- entry.lastAccessed = stats.accessCounter.incrementAndGet();
- }
- }
- return entry != null ? entry.value : null;
- }
-
- @Override
- public V put(K key, V val) {
- if (val == null) return null;
- CacheEntry<K, V> e =
- new CacheEntry<>(
- key, val, timeSource.getEpochTimeNs(), stats.accessCounter.incrementAndGet());
- return putCacheEntry(e);
- }
-
- /**
- * Visible for testing to create synthetic cache entries.
- *
- * @lucene.internal
- */
- public V putCacheEntry(CacheEntry<K, V> e) {
- // initialize oldestEntryNs
- oldestEntryNs.updateAndGet(x -> x > e.createTime || x == 0 ? e.createTime : x);
- CacheEntry<K, V> oldCacheEntry = map.put(e.key, e);
- if (oldCacheEntry == null) {
- stats.size.increment();
- ramBytes.add(e.ramBytesUsed() + HASHTABLE_RAM_BYTES_PER_ENTRY); // added key + value + entry
- } else {
- ramBytes.add(-oldCacheEntry.ramBytesUsed());
- ramBytes.add(e.ramBytesUsed());
- }
- if (islive) {
- stats.putCounter.increment();
- } else {
- stats.nonLivePutCounter.increment();
- }
- maybeMarkAndSweep();
- return oldCacheEntry == null ? null : oldCacheEntry.value;
- }
-
- private void maybeMarkAndSweep() {
- // Check if we need to clear out old entries from the cache.
- // isCleaning variable is checked instead of markAndSweepLock.isLocked()
- // for performance because every put invocation will check until
- // the size is back to an acceptable level.
- //
- // There is a race between the check and the call to markAndSweep, but
- // it's unimportant because markAndSweep actually acquires the lock or returns if it can't.
- //
- // Thread safety note: isCleaning read is piggybacked (comes after) other volatile reads
- // in this method.
- long idleCutoff = timeSource.getEpochTimeNs() - maxIdleTimeNs;
- int currentSize = stats.size.intValue();
- if ((currentSize > upperWaterMark
- || ramBytes.sum() > ramUpperWatermark
- || oldestEntryNs.get() < idleCutoff)
- && !isCleaning) {
- if (newThreadForCleanup) {
- new Thread(this::markAndSweep, "CacheCleanupThread").start();
- } else if (cleanupThread != null) {
- cleanupThread.wakeThread();
- } else {
- markAndSweep();
- }
- }
- }
-
- /**
- * Removes items from the cache to bring the size down to an acceptable value.
- *
- * <p>Visible for unit testing.
- *
- * @lucene.internal
- */
- public void markAndSweep() {
- // if we want to keep at least 1000 entries, then timestamps of
- // current through current-1000 are guaranteed not to be the oldest (but that does
- // not mean there are 1000 entries in that group... it's actually anywhere between
- // 1 and 1000).
- // Also, if we want to remove 500 entries, then
- // oldestEntry through oldestEntry+500 are guaranteed to be
- // removed (however many there are there).
-
- if (!markAndSweepLock.tryLock()) return;
- try {
- if (maxIdleTimeNs != Long.MAX_VALUE) {
- long idleCutoff = timeSource.getEpochTimeNs() - maxIdleTimeNs;
- if (oldestEntryNs.get() < idleCutoff) {
- markAndSweepByIdleTime();
- }
- }
- if (upperWaterMark < size()) {
- markAndSweepByCacheSize();
- } else if (ramUpperWatermark < ramBytesUsed()) {
- markAndSweepByRamSize();
- } else if (upperWaterMark == Integer.MAX_VALUE && ramUpperWatermark == Long.MAX_VALUE) {
- // should never happen
- throw new AssertionError(
- "ConcurrentLRUCache initialized with neither size limits nor ram limits");
- }
- } finally {
- isCleaning = false; // set before markAndSweep.unlock() for visibility
- markAndSweepLock.unlock();
- }
- }
-
- /*
- Must be called after acquiring markAndSweepLock
- */
- private void markAndSweepByIdleTime() {
- assert markAndSweepLock.isHeldByCurrentThread() : "markAndSweepLock held by another thread";
- Iterator<Map.Entry<Object, CacheEntry<K, V>>> iterator = map.entrySet().iterator();
- long idleCutoff = timeSource.getEpochTimeNs() - maxIdleTimeNs;
- long currentOldestEntry = Long.MAX_VALUE;
- while (iterator.hasNext()) {
- Map.Entry<Object, CacheEntry<K, V>> entry = iterator.next();
- if (entry.getValue().createTime < idleCutoff) {
- iterator.remove();
- stats.evictionIdleCounter.increment();
- postRemoveEntry(entry.getValue());
- } else {
- if (entry.getValue().createTime < currentOldestEntry) {
- currentOldestEntry = entry.getValue().createTime;
- }
- }
- }
- if (currentOldestEntry != Long.MAX_VALUE) {
- oldestEntryNs.set(currentOldestEntry);
- }
- }
-
- /*
- Must be called after acquiring markAndSweepLock
- */
- private void markAndSweepByRamSize() {
- assert markAndSweepLock.isHeldByCurrentThread() : "markAndSweepLock held by another thread";
- List<CacheEntry<K, V>> entriesInAccessOrder = new ArrayList<>(map.size());
- map.forEach(
- (o, kvCacheEntry) -> {
- // important because we want to avoid volatile read during comparisons
- kvCacheEntry.lastAccessedCopy = kvCacheEntry.lastAccessed;
- entriesInAccessOrder.add(kvCacheEntry);
- });
-
- Collections.sort(entriesInAccessOrder); // newer access is smaller, older access is bigger
-
- // iterate in oldest to newest order
- for (int i = entriesInAccessOrder.size() - 1; i >= 0; i--) {
- CacheEntry<K, V> kvCacheEntry = entriesInAccessOrder.get(i);
- evictEntry(kvCacheEntry.key);
- if (ramBytes.sum() <= ramLowerWatermark) {
- break; // we are done!
- }
- }
- }
-
- /*
- * Removes items from the cache to bring the size down
- * to an acceptable value ('acceptableWaterMark').
- * <p>
- * It is done in two stages. In the first stage, least recently used items are evicted.
- * If, after the first stage, the cache size is still greater than 'acceptableSize'
- * config parameter, the second stage takes over.
- * </p>
- * <p>The second stage is more intensive and tries to bring down the cache size
- * to the 'lowerWaterMark' config parameter.</p>
- * Must be called after acquiring markAndSweepLock
- */
- private void markAndSweepByCacheSize() {
- assert markAndSweepLock.isHeldByCurrentThread() : "markAndSweepLock held by another thread";
- long oldestEntry = this.oldestEntry;
- isCleaning = true;
- this.oldestEntry = oldestEntry; // volatile write to make isCleaning visible
-
- long timeCurrent = stats.accessCounter.longValue();
- int sz = stats.size.intValue();
-
- int numRemoved = 0;
- int numKept = 0;
- long newestEntry = timeCurrent;
- long newNewestEntry = -1;
- long newOldestEntry = Long.MAX_VALUE;
-
- int wantToKeep = lowerWaterMark;
- int wantToRemove = sz - lowerWaterMark;
-
- @SuppressWarnings("unchecked")
- CacheEntry<K, V>[] eset = (CacheEntry<K, V>[]) Array.newInstance(CacheEntry.class, sz);
- int eSize = 0;
-
- // System.out.println("newestEntry="+newestEntry + " oldestEntry="+oldestEntry);
- // System.out.println("items removed:" + numRemoved + " numKept=" + numKept + " esetSz="+ eSize
- // + " sz-numRemoved=" + (sz-numRemoved));
-
- for (CacheEntry<K, V> ce : map.values()) {
- // set lastAccessedCopy to avoid more volatile reads
- ce.lastAccessedCopy = ce.lastAccessed;
- long thisEntry = ce.lastAccessedCopy;
-
- // since the wantToKeep group is likely to be bigger than wantToRemove, check it first
- if (thisEntry > newestEntry - wantToKeep) {
- // this entry is guaranteed not to be in the bottom
- // group, so do nothing.
- numKept++;
- newOldestEntry = Math.min(thisEntry, newOldestEntry);
- } else if (thisEntry < oldestEntry + wantToRemove) { // entry in bottom group?
- // this entry is guaranteed to be in the bottom group
- // so immediately remove it from the map.
- evictEntry(ce.key);
- numRemoved++;
- } else {
- // This entry *could* be in the bottom group.
- // Collect these entries to avoid another full pass... this is wasted
- // effort if enough entries are normally removed in this first pass.
- // An alternate impl could make a full second pass.
- if (eSize < eset.length - 1) {
- eset[eSize++] = ce;
- newNewestEntry = Math.max(thisEntry, newNewestEntry);
- newOldestEntry = Math.min(thisEntry, newOldestEntry);
- }
- }
- }
-
- // System.out.println("items removed:" + numRemoved + " numKept=" + numKept + " esetSz="+ eSize
- // + " sz-numRemoved=" + (sz-numRemoved));
- // TODO: allow this to be customized in the constructor?
- int numPasses = 1; // maximum number of linear passes over the data
-
- // if we didn't remove enough entries, then make more passes
- // over the values we collected, with updated min and max values.
- while (sz - numRemoved > acceptableWaterMark && --numPasses >= 0) {
-
- oldestEntry = newOldestEntry == Long.MAX_VALUE ? oldestEntry : newOldestEntry;
- newOldestEntry = Long.MAX_VALUE;
- newestEntry = newNewestEntry;
- newNewestEntry = -1;
- wantToKeep = lowerWaterMark - numKept;
- wantToRemove = sz - lowerWaterMark - numRemoved;
-
- // iterate backward to make it easy to remove items.
- for (int i = eSize - 1; i >= 0; i--) {
- CacheEntry<K, V> ce = eset[i];
- long thisEntry = ce.lastAccessedCopy;
-
- if (thisEntry > newestEntry - wantToKeep) {
- // this entry is guaranteed not to be in the bottom
- // group, so do nothing but remove it from the eset.
- numKept++;
- // remove the entry by moving the last element to its position
- eset[i] = eset[eSize - 1];
- eSize--;
-
- newOldestEntry = Math.min(thisEntry, newOldestEntry);
-
- } else if (thisEntry < oldestEntry + wantToRemove) { // entry in bottom group?
-
- // this entry is guaranteed to be in the bottom group
- // so immediately remove it from the map.
- evictEntry(ce.key);
- numRemoved++;
-
- // remove the entry by moving the last element to its position
- eset[i] = eset[eSize - 1];
- eSize--;
- } else {
- // This entry *could* be in the bottom group, so keep it in the eset,
- // and update the stats.
- newNewestEntry = Math.max(thisEntry, newNewestEntry);
- newOldestEntry = Math.min(thisEntry, newOldestEntry);
- }
- }
- // System.out.println("items removed:" + numRemoved + " numKept=" + numKept + " esetSz="+
- // eSize + " sz-numRemoved=" + (sz-numRemoved));
- }
-
- // if we still didn't remove enough entries, then make another pass while
- // inserting into a priority queue
- if (sz - numRemoved > acceptableWaterMark) {
-
- oldestEntry = newOldestEntry == Long.MAX_VALUE ? oldestEntry : newOldestEntry;
- newOldestEntry = Long.MAX_VALUE;
- newestEntry = newNewestEntry;
- newNewestEntry = -1;
- wantToKeep = lowerWaterMark - numKept;
- wantToRemove = sz - lowerWaterMark - numRemoved;
-
- PQueue<K, V> queue = new PQueue<>(wantToRemove);
-
- for (int i = eSize - 1; i >= 0; i--) {
- CacheEntry<K, V> ce = eset[i];
- long thisEntry = ce.lastAccessedCopy;
-
- if (thisEntry > newestEntry - wantToKeep) {
- // this entry is guaranteed not to be in the bottom
- // group, so do nothing but remove it from the eset.
- numKept++;
- // removal not necessary on last pass.
- // eset[i] = eset[eSize-1];
- // eSize--;
-
- newOldestEntry = Math.min(thisEntry, newOldestEntry);
-
- } else if (thisEntry < oldestEntry + wantToRemove) { // entry in bottom group?
- // this entry is guaranteed to be in the bottom group
- // so immediately remove it.
- evictEntry(ce.key);
- numRemoved++;
-
- // removal not necessary on last pass.
- // eset[i] = eset[eSize-1];
- // eSize--;
- } else {
- // This entry *could* be in the bottom group.
- // add it to the priority queue
-
- // everything in the priority queue will be removed, so keep track of
- // the lowest value that ever comes back out of the queue.
-
- // first reduce the size of the priority queue to account for
- // the number of items we have already removed while executing
- // this loop so far.
- queue.myMaxSize = sz - lowerWaterMark - numRemoved;
- while (queue.size() > queue.myMaxSize && queue.size() > 0) {
- CacheEntry<K, V> otherEntry = queue.pop();
- newOldestEntry = Math.min(otherEntry.lastAccessedCopy, newOldestEntry);
- }
- if (queue.myMaxSize <= 0) break;
-
- CacheEntry<K, V> o = queue.myInsertWithOverflow(ce);
- if (o != null) {
- newOldestEntry = Math.min(o.lastAccessedCopy, newOldestEntry);
- }
- }
- }
-
- // Now delete everything in the priority queue.
- // avoid using pop() since order doesn't matter anymore
- for (CacheEntry<K, V> ce : queue.getValues()) {
- if (ce == null) continue;
- evictEntry(ce.key);
- numRemoved++;
- }
-
- // System.out.println("items removed:" + numRemoved + " numKept=" + numKept + "
- // initialQueueSize="+ wantToRemove + " finalQueueSize=" + queue.size() + " sz-numRemoved=" +
- // (sz-numRemoved));
- }
-
- oldestEntry = newOldestEntry == Long.MAX_VALUE ? oldestEntry : newOldestEntry;
- this.oldestEntry = oldestEntry;
- }
-
- private static class PQueue<K, V> extends PriorityQueue<CacheEntry<K, V>> {
- int myMaxSize;
- final Object[] heap;
-
- PQueue(int maxSz) {
- super(maxSz);
- heap = getHeapArray();
- myMaxSize = maxSz;
- }
-
- @SuppressWarnings("unchecked")
- Iterable<CacheEntry<K, V>> getValues() {
- return (Iterable) Collections.unmodifiableCollection(Arrays.asList(heap));
- }
-
- @Override
- protected boolean lessThan(CacheEntry<K, V> a, CacheEntry<K, V> b) {
- // reverse the parameter order so that the queue keeps the oldest items
- return b.lastAccessedCopy < a.lastAccessedCopy;
- }
-
- // necessary because maxSize is private in base class
- @SuppressWarnings("unchecked")
- public CacheEntry<K, V> myInsertWithOverflow(CacheEntry<K, V> element) {
- if (size() < myMaxSize) {
- add(element);
- return null;
- } else if (size() > 0 && !lessThan(element, (CacheEntry<K, V>) heap[1])) {
- CacheEntry<K, V> ret = (CacheEntry<K, V>) heap[1];
- heap[1] = element;
- updateTop();
- return ret;
- } else {
- return element;
- }
- }
- }
-
- private void evictEntry(K key) {
- CacheEntry<K, V> o = map.remove(key);
- postRemoveEntry(o);
- }
-
- private void postRemoveEntry(CacheEntry<K, V> o) {
- if (o == null) return;
- ramBytes.add(-(o.ramBytesUsed() + HASHTABLE_RAM_BYTES_PER_ENTRY));
- stats.size.decrement();
- stats.evictionCounter.increment();
- if (evictionListener != null) evictionListener.evictedEntry(o.key, o.value);
- }
-
- /**
- * Returns 'n' number of oldest accessed entries present in this cache.
- *
- * <p>This uses a TreeSet to collect the 'n' oldest items ordered by ascending last access time
- * and returns a LinkedHashMap containing 'n' or less than 'n' entries.
- *
- * @param n the number of oldest items needed
- * @return a LinkedHashMap containing 'n' or less than 'n' entries
- */
- public Map<K, V> getOldestAccessedItems(int n) {
- Map<K, V> result = new LinkedHashMap<>();
- if (n <= 0) return result;
- TreeSet<CacheEntry<K, V>> tree = new TreeSet<>();
- markAndSweepLock.lock();
- try {
- for (Map.Entry<Object, CacheEntry<K, V>> entry : map.entrySet()) {
- CacheEntry<K, V> ce = entry.getValue();
- ce.lastAccessedCopy = ce.lastAccessed;
- if (tree.size() < n) {
- tree.add(ce);
- } else {
- if (ce.lastAccessedCopy < tree.first().lastAccessedCopy) {
- tree.remove(tree.first());
- tree.add(ce);
- }
- }
- }
- } finally {
- markAndSweepLock.unlock();
- }
- for (CacheEntry<K, V> e : tree) {
- result.put(e.key, e.value);
- }
- return result;
- }
-
- public Map<K, V> getLatestAccessedItems(int n) {
- Map<K, V> result = new LinkedHashMap<>();
- if (n <= 0) return result;
- TreeSet<CacheEntry<K, V>> tree = new TreeSet<>();
- // we need to grab the lock since we are changing lastAccessedCopy
- markAndSweepLock.lock();
- try {
- for (Map.Entry<Object, CacheEntry<K, V>> entry : map.entrySet()) {
- CacheEntry<K, V> ce = entry.getValue();
- ce.lastAccessedCopy = ce.lastAccessed;
- if (tree.size() < n) {
- tree.add(ce);
- } else {
- if (ce.lastAccessedCopy > tree.last().lastAccessedCopy) {
- tree.remove(tree.last());
- tree.add(ce);
- }
- }
- }
- } finally {
- markAndSweepLock.unlock();
- }
- for (CacheEntry<K, V> e : tree) {
- result.put(e.key, e.value);
- }
- return result;
- }
-
- public int size() {
- return stats.size.intValue();
- }
-
- @Override
- public void clear() {
- map.clear();
- ramBytes.reset();
- }
-
- public Map<Object, CacheEntry<K, V>> getMap() {
- return map;
- }
-
- public static class CacheEntry<K, V> implements Comparable<CacheEntry<K, V>>, Accountable {
- public static long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOf(CacheEntry.class);
-
- final K key;
- final V value;
- final long createTime;
- final long ramBytesUsed; // cache
- volatile long lastAccessed = 0;
- long lastAccessedCopy = 0;
-
- public CacheEntry(K key, V value, long createTime, long lastAccessed) {
- this.key = key;
- this.value = value;
- this.createTime = createTime;
- this.lastAccessed = lastAccessed;
- this.ramBytesUsed =
- BASE_RAM_BYTES_USED
- + RamUsageEstimator.sizeOfObject(key, QUERY_DEFAULT_RAM_BYTES_USED)
- + RamUsageEstimator.sizeOfObject(value, QUERY_DEFAULT_RAM_BYTES_USED);
- }
-
- public void setLastAccessed(long lastAccessed) {
- this.lastAccessed = lastAccessed;
- }
-
- @Override
- public int compareTo(CacheEntry<K, V> that) {
- if (this.lastAccessedCopy == that.lastAccessedCopy) return 0;
- return this.lastAccessedCopy < that.lastAccessedCopy ? 1 : -1;
- }
-
- @Override
- public int hashCode() {
- return value.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- return value.equals(obj);
- }
-
- @Override
- public String toString() {
- return "key: " + key + " value: " + value + " lastAccessed:" + lastAccessed;
- }
-
- @Override
- public long ramBytesUsed() {
- return ramBytesUsed;
- }
-
- @Override
- public Collection<Accountable> getChildResources() {
- return List.of();
- }
- }
-
- private boolean isDestroyed = false;
-
- public void destroy() {
- try {
- if (cleanupThread != null) {
- cleanupThread.stopThread();
- }
- } finally {
- isDestroyed = true;
- }
- }
-
- public Stats getStats() {
- return stats;
- }
-
- public static class Stats implements Accountable {
- private static final long RAM_BYTES_USED =
- // accounts for field refs
- RamUsageEstimator.shallowSizeOfInstance(Stats.class)
- +
- // LongAdder
- 6
- * (RamUsageEstimator.NUM_BYTES_ARRAY_HEADER
- + RamUsageEstimator.primitiveSizes.get(long.class)
- + 2L
- * (RamUsageEstimator.NUM_BYTES_OBJECT_REF
- + RamUsageEstimator.primitiveSizes.get(long.class)))
- +
- // AtomicLong
- RamUsageEstimator.primitiveSizes.get(long.class);
-
- private final AtomicLong accessCounter = new AtomicLong(0);
- private final LongAdder putCounter = new LongAdder();
- private final LongAdder nonLivePutCounter = new LongAdder();
- private final LongAdder missCounter = new LongAdder();
- private final LongAdder size = new LongAdder();
- private LongAdder evictionCounter = new LongAdder();
- private LongAdder evictionIdleCounter = new LongAdder();
-
- public long getCumulativeLookups() {
- return (accessCounter.longValue() - putCounter.longValue() - nonLivePutCounter.longValue())
- + missCounter.longValue();
- }
-
- public long getCumulativeHits() {
- return accessCounter.longValue() - putCounter.longValue() - nonLivePutCounter.longValue();
- }
-
- public long getCumulativePuts() {
- return putCounter.longValue();
- }
-
- public long getCumulativeEvictions() {
- return evictionCounter.longValue();
- }
-
- public long getCumulativeIdleEvictions() {
- return evictionIdleCounter.longValue();
- }
-
- public int getCurrentSize() {
- return size.intValue();
- }
-
- public long getCumulativeNonLivePuts() {
- return nonLivePutCounter.longValue();
- }
-
- public long getCumulativeMisses() {
- return missCounter.longValue();
- }
-
- public void add(Stats other) {
- accessCounter.addAndGet(other.accessCounter.get());
- putCounter.add(other.putCounter.longValue());
- nonLivePutCounter.add(other.nonLivePutCounter.longValue());
- missCounter.add(other.missCounter.longValue());
- evictionCounter.add(other.evictionCounter.longValue());
- long maxSize = Math.max(size.longValue(), other.size.longValue());
- size.reset();
- size.add(maxSize);
- }
-
- @Override
- public long ramBytesUsed() {
- return RAM_BYTES_USED;
- }
- }
-
- public static interface EvictionListener<K, V> {
- public void evictedEntry(K key, V value);
- }
-
- private static class CleanupThread extends Thread {
- private final WeakReference<ConcurrentLRUCache<?, ?>> cache;
-
- private boolean stop = false;
-
- public CleanupThread(ConcurrentLRUCache<?, ?> c) {
- super("CacheCleanupThread");
- cache = new WeakReference<>(c);
- }
-
- @Override
- public void run() {
- while (true) {
- ConcurrentLRUCache<?, ?> c = cache.get();
- if (c == null) break;
- synchronized (this) {
- if (stop) break;
- long waitTimeMs =
- c.maxIdleTimeNs != Long.MAX_VALUE
- ? TimeUnit.MILLISECONDS.convert(c.maxIdleTimeNs, TimeUnit.NANOSECONDS)
- : 0L;
- try {
- this.wait(waitTimeMs);
- } catch (InterruptedException e) {
- }
- }
- if (stop) break;
- c = cache.get();
- if (c == null) break;
- c.markAndSweep();
- }
- }
-
- void wakeThread() {
- synchronized (this) {
- this.notify();
- }
- }
-
- void stopThread() {
- synchronized (this) {
- stop = true;
- this.notify();
- }
- }
- }
-
- @Override
- public long ramBytesUsed() {
- return BASE_RAM_BYTES_USED + ramBytes.sum();
- }
-
- @Override
- public Collection<Accountable> getChildResources() {
- return List.of();
- }
-}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Cache.java b/solr/solrj/src/java/org/apache/solr/common/util/Cache.java
deleted file mode 100644
index 4f589099485..00000000000
--- a/solr/solrj/src/java/org/apache/solr/common/util/Cache.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.common.util;
-
-import java.util.Objects;
-import java.util.function.Function;
-
-public interface Cache<K, V> {
- V put(K key, V val);
-
- V get(K key);
-
- V remove(K key);
-
- void clear();
-
- default V computeIfAbsent(K key, Function<? super K, ? extends V> mappingFunction) {
- Objects.requireNonNull(mappingFunction);
- V v;
- if ((v = get(key)) == null) {
- V newValue;
- if ((newValue = mappingFunction.apply(key)) != null) {
- put(key, newValue);
- return newValue;
- }
- }
-
- return v;
- }
-}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
index 63e6613ae25..f47ff63b980 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
@@ -1450,15 +1450,12 @@ public class JavaBinCodec implements PushWriter {
boolean wantsAllFields();
}
- public static class StringCache {
- private final Cache<StringBytes, String> cache;
-
- public StringCache(Cache<StringBytes, String> cache) {
- this.cache = cache;
- }
-
+ /**
+ * @lucene.internal
+ */
+ public abstract static class StringCache {
public String get(StringBytes b) {
- String result = cache.get(b);
+ String result = getFromCache(b);
if (result == null) {
// make a copy because the buffer received may be changed later by the caller
StringBytes copy =
@@ -1467,10 +1464,14 @@ public class JavaBinCodec implements PushWriter {
CharArr arr = new CharArr();
ByteUtils.UTF8toUTF16(b.bytes, b.offset, b.length, arr);
result = arr.toString();
- cache.put(copy, result);
+ putIntoCache(copy, result);
}
return result;
}
+
+ protected abstract String getFromCache(StringBytes b);
+
+ protected abstract void putIntoCache(StringBytes b, String val);
}
@Override
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/MapBackedCache.java b/solr/solrj/src/java/org/apache/solr/common/util/MapBackedCache.java
deleted file mode 100644
index c10520bb86e..00000000000
--- a/solr/solrj/src/java/org/apache/solr/common/util/MapBackedCache.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.common.util;
-
-import java.util.Map;
-import java.util.function.Function;
-
-// A cache backed by a map
-public class MapBackedCache<K, V> implements Cache<K, V> {
-
- protected final Map<K, V> map;
-
- public MapBackedCache(Map<K, V> map) {
- this.map = map;
- }
-
- public Map<K, V> asMap() {
- return map;
- }
-
- @Override
- public V put(K key, V val) {
- return map.put(key, val);
- }
-
- @Override
- public V get(K key) {
- return map.get(key);
- }
-
- @Override
- public V remove(K key) {
- return map.remove(key);
- }
-
- @Override
- public void clear() {
- map.clear();
- }
-
- @Override
- public V computeIfAbsent(K key, Function<? super K, ? extends V> mappingFunction) {
- return map.computeIfAbsent(key, mappingFunction);
- }
-}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ObjectCache.java b/solr/solrj/src/java/org/apache/solr/common/util/ObjectCache.java
index 096998b76c8..d3dd33a705a 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/ObjectCache.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/ObjectCache.java
@@ -19,18 +19,20 @@ package org.apache.solr.common.util;
import java.io.Closeable;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import org.apache.solr.common.SolrCloseable;
/** Simple object cache with a type-safe accessor. */
-public class ObjectCache extends MapBackedCache<String, Object> implements SolrCloseable {
+public class ObjectCache implements SolrCloseable {
+ protected final ConcurrentMap<String, Object> map;
private final AtomicBoolean isClosed = new AtomicBoolean(false);
public ObjectCache() {
- super(new ConcurrentHashMap<>());
+ this.map = new ConcurrentHashMap<>();
}
private void ensureNotClosed() {
@@ -39,28 +41,24 @@ public class ObjectCache extends MapBackedCache<String, Object> implements SolrC
}
}
- @Override
public Object put(String key, Object val) {
ensureNotClosed();
- return super.put(key, val);
+ return map.put(key, val);
}
- @Override
public Object get(String key) {
ensureNotClosed();
- return super.get(key);
+ return map.get(key);
}
- @Override
public Object remove(String key) {
ensureNotClosed();
- return super.remove(key);
+ return map.remove(key);
}
- @Override
public void clear() {
ensureNotClosed();
- super.clear();
+ map.clear();
}
public <T> T get(String key, Class<T> clazz) {
@@ -72,11 +70,14 @@ public class ObjectCache extends MapBackedCache<String, Object> implements SolrC
}
}
+ public Object computeIfAbsent(String key, Function<String, ?> mappingFunction) {
+ ensureNotClosed();
+ return map.computeIfAbsent(key, mappingFunction);
+ }
+
public <T> T computeIfAbsent(
String key, Class<T> clazz, Function<String, ? extends T> mappingFunction) {
- ensureNotClosed();
- Object o = super.computeIfAbsent(key, mappingFunction);
- return clazz.cast(o);
+ return clazz.cast(computeIfAbsent(key, mappingFunction));
}
@Override
diff --git a/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java b/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java
index 65a89bbc21b..07c4cb64cd3 100644
--- a/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java
+++ b/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java
@@ -32,6 +32,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.EnumFieldValue;
@@ -39,7 +40,6 @@ import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
-import org.apache.solr.util.ConcurrentLRUCache;
import org.apache.solr.util.RTimer;
import org.junit.Test;
import org.noggit.CharArr;
@@ -480,7 +480,19 @@ public class TestJavaBinCodec extends SolrTestCaseJ4 {
assertNotSame(l1.get(1), l2.get(1));
JavaBinCodec.StringCache stringCache =
- new JavaBinCodec.StringCache(new MapBackedCache<>(new HashMap<>()));
+ new JavaBinCodec.StringCache() {
+ Map<StringBytes, String> cache = new HashMap<>();
+
+ @Override
+ protected String getFromCache(StringBytes b) {
+ return cache.get(b);
+ }
+
+ @Override
+ protected void putIntoCache(StringBytes b, String val) {
+ cache.put(b, val);
+ }
+ };
try (JavaBinCodec c1 = new JavaBinCodec(null, stringCache);
JavaBinCodec c2 = new JavaBinCodec(null, stringCache)) {
@@ -525,21 +537,24 @@ public class TestJavaBinCodec extends SolrTestCaseJ4 {
private void testPerf() throws InterruptedException {
final ArrayList<StringBytes> l = new ArrayList<>();
- Cache<StringBytes, String> cache = null;
- /* cache = new ConcurrentLRUCache<JavaBinCodec.StringBytes,String>(10000, 9000, 10000, 1000, false, true, null){
- @Override
- public String put(JavaBinCodec.StringBytes key, String val) {
- l.add(key);
- return super.put(key, val);
- }
- };*/
Runtime.getRuntime().gc();
printMem("before cache init");
- Cache<StringBytes, String> cache1 = new MapBackedCache<>(new HashMap<>());
- final JavaBinCodec.StringCache STRING_CACHE = new JavaBinCodec.StringCache(cache1);
+ final JavaBinCodec.StringCache STRING_CACHE =
+ new JavaBinCodec.StringCache() {
+ Map<StringBytes, String> cache = new HashMap<>();
+
+ @Override
+ protected String getFromCache(StringBytes b) {
+ return cache.get(b);
+ }
+
+ @Override
+ protected void putIntoCache(StringBytes b, String val) {
+ cache.put(b, val);
+ }
+ };
- // STRING_CACHE = new JavaBinCodec.StringCache(cache);
byte[] bytes = new byte[0];
StringBytes stringBytes = new StringBytes(null, 0, 0);
@@ -656,14 +671,26 @@ public class TestJavaBinCodec extends SolrTestCaseJ4 {
int ret = 0;
final RTimer timer = new RTimer();
- ConcurrentLRUCache<StringBytes, String> underlyingCache =
- cacheSz > 0
- ? new ConcurrentLRUCache<>(
- cacheSz, cacheSz - cacheSz / 10, cacheSz, cacheSz / 10, false, true, null)
- : null; // the cache in the first version of the patch was
- // 10000,9000,10000,1000,false,true,null
- final JavaBinCodec.StringCache stringCache =
- underlyingCache == null ? null : new JavaBinCodec.StringCache(underlyingCache);
+ final JavaBinCodec.StringCache stringCache;
+ if (cacheSz > 0) {
+ stringCache =
+ new JavaBinCodec.StringCache() {
+ private final Map<StringBytes, String> cache = new ConcurrentHashMap<>(cacheSz);
+
+ @Override
+ protected String getFromCache(StringBytes b) {
+ return cache.get(b);
+ }
+
+ @Override
+ protected void putIntoCache(StringBytes b, String val) {
+ cache.put(b, val);
+ }
+ };
+ } else {
+ stringCache = null;
+ }
+
if (nThreads <= 0) {
ret += doDecode(buffers, iter, stringCache);
} else {
@@ -680,14 +707,6 @@ public class TestJavaBinCodec extends SolrTestCaseJ4 {
long n = iter * Math.max(1, nThreads);
System.out.println("ret=" + ret + " THROUGHPUT=" + (n * 1000 / timer.getTime()));
- if (underlyingCache != null)
- System.out.println(
- "cache: hits="
- + underlyingCache.getStats().getCumulativeHits()
- + " lookups="
- + underlyingCache.getStats().getCumulativeLookups()
- + " size="
- + underlyingCache.getStats().getCurrentSize());
}
public static int doDecode(byte[][] buffers, long iter, JavaBinCodec.StringCache stringCache)
(solr) 03/07: SOLR-18188: Fix ForceLeaderTest
Posted by ds...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit f271f86c68c545db9bdc4ef7b158abcc8a53cc58
Author: David Smiley <ds...@apache.org>
AuthorDate: Mon Jun 22 22:07:07 2026 -0400
SOLR-18188: Fix ForceLeaderTest
Inaccurately ported CloudSolrClient settings. Also confused zkServer.getZkAddress() & getZkHost()
(cherry picked from commit b3a721bb836d7b72b020e35b29178dd0c3b2b42a)
---
solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java
index d9758361901..ba6efab0c3e 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java
@@ -193,9 +193,12 @@ public class ForceLeaderTest extends HttpPartitionTest {
@Override
protected CloudSolrClient createCloudClient(String defaultCollection) {
- boolean shardLeadersOnly = false; // this test requires we use any replica
- return createNewCloudSolrClient(
- zkServer.getZkHost(), TEST_COLLECTION, shardLeadersOnly, 30_000, 120_000);
+ // Use non-randomized builder; this test requires deterministic routing:
+ // shardLeadersOnly=true (default), directUpdatesToLeadersOnly=false.
+ return new CloudSolrClient.Builder(zkServer.getZkAddress())
+ .withDefaultCollection(defaultCollection)
+ .sendDirectUpdatesToAnyShardReplica()
+ .build();
}
private void putNonLeadersIntoLowerTerm(
@@ -312,7 +315,7 @@ public class ForceLeaderTest extends HttpPartitionTest {
CollectionAdminRequest.forceLeaderElection(collectionName, shard);
try (var cloudClient =
- createNewCloudSolrClient(zkServer.getZkHost(), null, true, 3_000, 60_000)) {
+ createNewCloudSolrClient(zkServer.getZkAddress(), null, true, 3_000, 60_000)) {
cloudClient.request(forceLeader);
}
}