You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@solr.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2026/05/15 16:36:08 UTC
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 210 - Still Failing!
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/210/
No tests ran.
Build Log:
[...truncated 424 lines...]
# ERROR: Server refused connection at: http://localhost:35851/solr/monitors_verbose/update
#
# org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://localhost:35851/solr/monitors_verbose/update
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:506)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
# at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:466)
# at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:442)
# at org.apache.solr.cli.PostTool.commit(PostTool.java:778)
# at org.apache.solr.cli.PostTool.execute(PostTool.java:346)
# at org.apache.solr.cli.PostTool.runImpl(PostTool.java:320)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.net.ConnectException: Connection refused
# at java.base/sun.nio.ch.Net.pollConnect(Native Method)
# at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:682)
# at java.base/sun.nio.ch.SocketChannelImpl.finishTimedConnect(SocketChannelImpl.java:1191)
# at java.base/sun.nio.ch.SocketChannelImpl.blockingConnect(SocketChannelImpl.java:1233)
# at java.base/sun.nio.ch.SocketAdaptor.connect(SocketAdaptor.java:102)
# at org.eclipse.jetty.io.ClientConnector.connect(ClientConnector.java:467)
# at org.eclipse.jetty.io.Transport$Socket.connect(Transport.java:175)
# at org.eclipse.jetty.http2.client.HTTP2Client.connect(HTTP2Client.java:486)
# at org.eclipse.jetty.http2.client.transport.HttpClientTransportOverHTTP2.connect(HttpClientTransportOverHTTP2.java:140)
# at org.eclipse.jetty.http2.client.transport.HttpClientTransportOverHTTP2.connect(HttpClientTransportOverHTTP2.java:128)
# at org.eclipse.jetty.client.HttpClient$1.connect(HttpClient.java:579)
# at org.eclipse.jetty.client.HttpClient$1.succeeded(HttpClient.java:556)
# at org.eclipse.jetty.client.HttpClient$1.succeeded(HttpClient.java:552)
# at org.eclipse.jetty.util.SocketAddressResolver$Async.lambda$resolve$0(SocketAddressResolver.java:186)
# at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
# at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
# at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
# at java.base/java.lang.Thread.run(Thread.java:1583)
# --
#
# Please find the SOLR_HOME snapshot for failed test #12 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/81-12
# Last output:
# Posting files to [base] url http://localhost:35851/solr/monitors_verbose/update using content-type application/xml...
# POSTing file monitor.xml to [base]
# PostTool: WARNING: IOException while reading response: java.net.SocketException: Unexpected end of file from server
# 1 files indexed.
# COMMITting Solr index changes to http://localhost:35851/solr/monitors_verbose/update...
#
# ERROR: Server refused connection at: http://localhost:35851/solr/monitors_verbose/update
#
# org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://localhost:35851/solr/monitors_verbose/update
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:506)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
# at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:466)
# at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:442)
# at org.apache.solr.cli.PostTool.commit(PostTool.java:778)
# at org.apache.solr.cli.PostTool.execute(PostTool.java:346)
# at org.apache.solr.cli.PostTool.runImpl(PostTool.java:320)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.net.ConnectException: Connection refused
# at java.base/sun.nio.ch.Net.pollConnect(Native Method)
# at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:682)
# at java.base/sun.nio.ch.SocketChannelImpl.finishTimedConnect(SocketChannelImpl.java:1191)
# at java.base/sun.nio.ch.SocketChannelImpl.blockingConnect(SocketChannelImpl.java:1233)
# at java.base/sun.nio.ch.SocketAdaptor.connect(SocketAdaptor.java:102)
# at org.eclipse.jetty.io.ClientConnector.connect(ClientConnector.java:467)
# at org.eclipse.jetty.io.Transport$Socket.connect(Transport.java:175)
# at org.eclipse.jetty.http2.client.HTTP2Client.connect(HTTP2Client.java:486)
# at org.eclipse.jetty.http2.client.transport.HttpClientTransportOverHTTP2.connect(HttpClientTransportOverHTTP2.java:140)
# at org.eclipse.jetty.http2.client.transport.HttpClientTransportOverHTTP2.connect(HttpClientTransportOverHTTP2.java:128)
# at org.eclipse.jetty.client.HttpClient$1.connect(HttpClient.java:579)
# at org.eclipse.jetty.client.HttpClient$1.succeeded(HttpClient.java:556)
# at org.eclipse.jetty.client.HttpClient$1.succeeded(HttpClient.java:552)
# at org.eclipse.jetty.util.SocketAddressResolver$Async.lambda$resolve$0(SocketAddressResolver.java:186)
# at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
# at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
# at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
# at java.base/java.lang.Thread.run(Thread.java:1583)
ok 82 post solr log into solr via cli # in 9182 ms
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 17 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 17 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 29 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-05-15T16:22:30.332019868Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-15T16:22:30.332055727Z It should be set to 65000 to avoid operational disruption.
2026-05-15T16:22:30.332059171Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-15T16:22:30.332289820Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-15T16:22:30.332301390Z
2026-05-15T16:22:30.344612183Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-15T16:22:30.355404909Z [0.001s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-15T16:22:30.451832404Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-15T16:22:30.587796270Z WARNING: A command line option has enabled the Security Manager
2026-05-15T16:22:30.587826508Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-15T16:22:32.825865424Z 2026-05-15 16:22:32.820 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-15T16:22:33.432818995Z 2026-05-15 16:22:33.432 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-15T16:22:33.498393783Z 2026-05-15 16:22:33.497 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-15T16:22:33.515433082Z 2026-05-15 16:22:33.514 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-15T16:22:33.515505812Z 2026-05-15 16:22:33.515 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-05-15T16:22:33.515619397Z 2026-05-15 16:22:33.515 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-15T16:22:33.515734575Z 2026-05-15 16:22:33.515 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-15T16:22:33.515259460Z
2026-05-15T16:22:33.519506623Z 2026-05-15 16:22:33.519 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-15T16:22:33.588966659Z 2026-05-15 16:22:33.588 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-15T16:22:33.644303327Z 2026-05-15 16:22:33.643 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-15T16:22:33.644480725Z 2026-05-15 16:22:33.643 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-15T16:22:33.738250761Z 2026-05-15 16:22:33.737 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-15T16:22:35.370673308Z 2026-05-15 16:22:35.369 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-15T16:22:36.326510569Z 2026-05-15 16:22:36.325 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-15T16:22:36.342769143Z 2026-05-15 16:22:36.342 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-05-15T16:22:36.342788116Z 2026-05-15 16:22:36.342 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-05-15T16:22:36.349055630Z 2026-05-15 16:22:36.348 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-05-15T16:22:36.349070472Z 2026-05-15 16:22:36.348 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-05-15T16:22:36.455618719Z 2026-05-15 16:22:36.455 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-05-15T16:22:36.852077130Z 2026-05-15 16:22:36.851 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-05-15T16:22:36.884846121Z 2026-05-15 16:22:36.884 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-15T16:22:36.962866269Z 2026-05-15 16:22:36.962 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-15T16:22:36.993262401Z 2026-05-15 16:22:36.992 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-15T16:22:36.993356233Z 2026-05-15 16:22:36.992 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-15T16:22:37.130452877Z 2026-05-15 16:22:37.129 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-15T16:22:37.170519256Z 2026-05-15 16:22:37.169 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-15T16:22:37.183729619Z 2026-05-15 16:22:37.183 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-05-15T16:22:37.193527714Z 2026-05-15 16:22:37.193 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72059903898681344-solr-node1:8983_solr-n_0000000000) starting
2026-05-15T16:22:37.321246167Z 2026-05-15 16:22:37.320 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-05-15T16:22:37.321288763Z 2026-05-15 16:22:37.320 INFO (OverseerStateUpdate-72059903898681344-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-05-15T16:22:37.385536647Z 2026-05-15 16:22:37.385 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-15T16:22:37.521333263Z 2026-05-15 16:22:37.520 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-15T16:22:38.209333359Z 2026-05-15 16:22:38.208 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-15T16:22:38.543424974Z 2026-05-15 16:22:38.542 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-15T16:22:38.946832537Z 2026-05-15 16:22:38.946 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-15T16:22:39.370410083Z 2026-05-15 16:22:39.369 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-15T16:22:39.506170003Z 2026-05-15 16:22:39.504 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_15.request.log
2026-05-15T16:22:39.510388831Z 2026-05-15 16:22:39.509 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-05-15T16:22:39.513045396Z 2026-05-15 16:22:39.512 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @9158ms
2026-05-15T16:22:40.275660827Z 2026-05-15 16:22:40.274 INFO (qtp985324122-29-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=105
2026-05-15T16:22:41.010523561Z 2026-05-15 16:22:41.010 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=19
2026-05-15T16:22:45.600929194Z 2026-05-15 16:22:45.600 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
2026-05-15T16:22:52.893223793Z 2026-05-15 16:22:52.892 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=44
2026-05-15T16:22:53.037932384Z 2026-05-15 16:22:53.037 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=54
2026-05-15T16:22:53.076630017Z 2026-05-15 16:22:53.076 INFO (qtp985324122-29-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=5
2026-05-15T16:22:53.705372375Z 2026-05-15 16:22:53.704 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=20
2026-05-15T16:22:53.757319061Z 2026-05-15 16:22:53.756 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-05-15T16:22:53.807528657Z 2026-05-15 16:22:53.807 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-05-15T16:22:53.832036203Z 2026-05-15 16:22:53.831 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 173] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-15T16:22:53.957511863Z 2026-05-15 16:22:53.957 INFO (OverseerStateUpdate-72059903898681344-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-15T16:22:53.957534019Z "core":"test-collection_shard3_replica_n1",
2026-05-15T16:22:53.957536735Z "node_name":"solr-node1:8983_solr",
2026-05-15T16:22:53.957538894Z "base_url":"http://solr-node1:8983/solr",
2026-05-15T16:22:53.957540960Z "collection":"test-collection",
2026-05-15T16:22:53.957543834Z "shard":"shard3",
2026-05-15T16:22:53.957545820Z "state":"down",
2026-05-15T16:22:53.957547763Z "type":"NRT",
2026-05-15T16:22:53.957549751Z "operation":"ADDREPLICA",
2026-05-15T16:22:53.957552031Z "waitForFinalState":"false"}
2026-05-15T16:22:53.985968399Z 2026-05-15 16:22:53.984 INFO (OverseerStateUpdate-72059903898681344-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-15T16:22:53.985985569Z "core":"test-collection_shard2_replica_n2",
2026-05-15T16:22:53.985988213Z "node_name":"solr-node3:8985_solr",
2026-05-15T16:22:53.985990416Z "base_url":"http://solr-node3:8985/solr",
2026-05-15T16:22:53.985992723Z "collection":"test-collection",
2026-05-15T16:22:53.985994733Z "shard":"shard2",
2026-05-15T16:22:53.985996631Z "state":"down",
2026-05-15T16:22:53.985998780Z "type":"NRT",
2026-05-15T16:22:53.986000764Z "operation":"ADDREPLICA",
2026-05-15T16:22:53.986002719Z "waitForFinalState":"false"}
2026-05-15T16:22:53.994703900Z 2026-05-15 16:22:53.993 INFO (OverseerStateUpdate-72059903898681344-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-15T16:22:53.994722569Z "core":"test-collection_shard1_replica_n4",
2026-05-15T16:22:53.994735197Z "node_name":"solr-node2:8984_solr",
2026-05-15T16:22:53.994737463Z "base_url":"http://solr-node2:8984/solr",
2026-05-15T16:22:53.994739486Z "collection":"test-collection",
2026-05-15T16:22:53.994741498Z "shard":"shard1",
2026-05-15T16:22:53.994743947Z "state":"down",
2026-05-15T16:22:53.994745845Z "type":"NRT",
2026-05-15T16:22:53.994747740Z "operation":"ADDREPLICA",
2026-05-15T16:22:53.994749756Z "waitForFinalState":"false"}
2026-05-15T16:22:54.107911835Z 2026-05-15 16:22:54.107 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 185] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-15T16:22:54.255534627Z 2026-05-15 16:22:54.255 INFO (qtp985324122-28-solr-node1-8) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-05-15T16:22:54.366823096Z 2026-05-15 16:22:54.366 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-15T16:22:54.504918310Z 2026-05-15 16:22:54.504 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-05-15T16:22:55.658754750Z 2026-05-15 16:22:55.658 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-15T16:22:55.658777902Z 2026-05-15 16:22:55.658 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true
2026-05-15T16:22:55.770700732Z 2026-05-15 16:22:55.769 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/]
2026-05-15T16:22:58.143710376Z 2026-05-15 16:22:58.143 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-15T16:22:58.196652992Z 2026-05-15 16:22:58.195 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-15T16:22:58.203258487Z 2026-05-15 16:22:58.202 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-15T16:22:58.204529589Z 2026-05-15 16:22:58.204 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-15T16:22:58.204608672Z 2026-05-15 16:22:58.204 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-15T16:22:58.307232260Z 2026-05-15 16:22:58.306 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-15T16:22:58.311217027Z 2026-05-15 16:22:58.310 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-15T16:22:58.313999624Z 2026-05-15 16:22:58.313 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-15T16:22:58.322001602Z 2026-05-15 16:22:58.321 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-15T16:22:58.335702328Z 2026-05-15 16:22:58.335 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-05-15T16:22:58.342792888Z 2026-05-15 16:22:58.342 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-15T16:22:58.351152768Z 2026-05-15 16:22:58.350 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-15T16:22:58.351167419Z 2026-05-15 16:22:58.350 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-15T16:22:58.359303493Z 2026-05-15 16:22:58.359 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-15T16:22:58.371447450Z 2026-05-15 16:22:58.370 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-05-15T16:22:58.375740912Z 2026-05-15 16:22:58.375 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-05-15T16:22:58.398782212Z 2026-05-15 16:22:58.398 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-05-15T16:22:58.398799780Z 2026-05-15 16:22:58.398 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-05-15T16:22:58.398803388Z 2026-05-15 16:22:58.398 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/
2026-05-15T16:22:58.400671114Z 2026-05-15 16:22:58.400 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-05-15T16:22:58.401625863Z 2026-05-15 16:22:58.401 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas
2026-05-15T16:22:58.404199682Z 2026-05-15 16:22:58.403 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72059903898681344-core_node3-n_0000000000
2026-05-15T16:22:58.410814074Z 2026-05-15 16:22:58.410 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3
2026-05-15T16:22:58.418915697Z 2026-05-15 16:22:58.418 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-05-15T16:22:58.431216967Z 2026-05-15 16:22:58.430 INFO (qtp985324122-28-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=4177
2026-05-15T16:22:58.447113266Z 2026-05-15 16:22:58.446 INFO (qtp985324122-101-solr-node1-7) [c:test-collection s: r: x: t:solr-node1-7] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
2026-05-15T16:22:58.526561925Z 2026-05-15 16:22:58.526 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 243] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-15T16:22:58.529088541Z 2026-05-15 16:22:58.528 INFO (qtp985324122-101-solr-node1-7) [c:test-collection s: r: x: t:solr-node1-7] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=4803
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1457614,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-05-15T16:22:29.243323028Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-05-15T16:22:41.905028544Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-15T16:22:41.905055235Z It should be set to 65000 to avoid operational disruption.
2026-05-15T16:22:41.905058311Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-15T16:22:41.905405233Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-15T16:22:41.905414380Z
2026-05-15T16:22:41.916773279Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-15T16:22:41.927055447Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-15T16:22:41.987521199Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-15T16:22:42.028451217Z WARNING: A command line option has enabled the Security Manager
2026-05-15T16:22:42.028469962Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-15T16:22:43.431569216Z 2026-05-15 16:22:43.428 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-15T16:22:43.710302531Z 2026-05-15 16:22:43.709 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-15T16:22:43.741491860Z 2026-05-15 16:22:43.741 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-15T16:22:43.750735583Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-15T16:22:43.751084812Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-05-15T16:22:43.751095116Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-15T16:22:43.751097761Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-15T16:22:43.750534382Z
2026-05-15T16:22:43.753704303Z 2026-05-15 16:22:43.753 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-15T16:22:43.796934258Z 2026-05-15 16:22:43.796 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-15T16:22:43.848423068Z 2026-05-15 16:22:43.847 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-15T16:22:44.010063620Z 2026-05-15 16:22:44.009 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-05-15T16:22:44.032428278Z 2026-05-15 16:22:44.032 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-15T16:22:44.032546986Z 2026-05-15 16:22:44.032 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-15T16:22:44.032677206Z 2026-05-15 16:22:44.032 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-15T16:22:44.147731736Z 2026-05-15 16:22:44.147 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-05-15T16:22:44.147994179Z 2026-05-15 16:22:44.147 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-15T16:22:44.148101993Z 2026-05-15 16:22:44.147 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-15T16:22:44.201809785Z 2026-05-15 16:22:44.201 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-15T16:22:45.311567614Z 2026-05-15 16:22:45.310 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-15T16:22:45.467617852Z 2026-05-15 16:22:45.467 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-15T16:22:45.469052756Z 2026-05-15 16:22:45.467 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-05-15T16:22:45.477902421Z 2026-05-15 16:22:45.477 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-15T16:22:45.483193337Z 2026-05-15 16:22:45.482 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-15T16:22:45.486055340Z 2026-05-15 16:22:45.485 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-15T16:22:45.486067556Z 2026-05-15 16:22:45.485 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-15T16:22:45.527769716Z 2026-05-15 16:22:45.527 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-15T16:22:45.537995338Z 2026-05-15 16:22:45.537 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-15T16:22:45.560924181Z 2026-05-15 16:22:45.560 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-15T16:22:45.577611649Z 2026-05-15 16:22:45.577 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-05-15T16:22:45.589533356Z 2026-05-15 16:22:45.588 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-05-15T16:22:45.602136236Z 2026-05-15 16:22:45.601 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
2026-05-15T16:22:45.664474950Z 2026-05-15 16:22:45.663 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-15T16:22:46.068796963Z 2026-05-15 16:22:46.068 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-15T16:22:46.339950000Z 2026-05-15 16:22:46.339 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-15T16:22:46.635498919Z 2026-05-15 16:22:46.635 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-15T16:22:46.792433280Z 2026-05-15 16:22:46.791 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-15T16:22:46.798014409Z 2026-05-15 16:22:46.797 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_15.request.log
2026-05-15T16:22:46.801769735Z 2026-05-15 16:22:46.801 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-05-15T16:22:46.803150758Z 2026-05-15 16:22:46.802 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4877ms
2026-05-15T16:22:47.067886583Z 2026-05-15 16:22:47.066 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=44
2026-05-15T16:22:47.143134589Z 2026-05-15 16:22:47.142 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=13
2026-05-15T16:22:54.254149485Z 2026-05-15 16:22:54.253 INFO (qtp985324122-28-solr-node1-8) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-05-15T16:22:54.325512012Z 2026-05-15 16:22:54.325 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-15T16:22:54.427761192Z 2026-05-15 16:22:54.427 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-05-15T16:22:55.026501743Z 2026-05-15 16:22:55.025 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-15T16:22:55.026519387Z 2026-05-15 16:22:55.026 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-05-15T16:22:55.075155350Z 2026-05-15 16:22:55.074 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-05-15T16:22:55.720372206Z 2026-05-15 16:22:55.719 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-15T16:22:55.749705878Z 2026-05-15 16:22:55.749 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-15T16:22:55.759399830Z 2026-05-15 16:22:55.758 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-15T16:22:55.760531433Z 2026-05-15 16:22:55.759 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-15T16:22:55.760597758Z 2026-05-15 16:22:55.760 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-15T16:22:55.815611644Z 2026-05-15 16:22:55.815 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-15T16:22:55.819783634Z 2026-05-15 16:22:55.819 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-15T16:22:55.822389201Z 2026-05-15 16:22:55.822 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-15T16:22:55.824797847Z 2026-05-15 16:22:55.824 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-15T16:22:55.841841491Z 2026-05-15 16:22:55.841 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4}
2026-05-15T16:22:55.850462069Z 2026-05-15 16:22:55.850 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-15T16:22:55.856821993Z 2026-05-15 16:22:55.856 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-15T16:22:55.856852501Z 2026-05-15 16:22:55.856 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-15T16:22:55.863961753Z 2026-05-15 16:22:55.863 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-15T16:22:55.888285784Z 2026-05-15 16:22:55.887 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-05-15T16:22:55.889578988Z 2026-05-15 16:22:55.889 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1
2026-05-15T16:22:55.914642969Z 2026-05-15 16:22:55.914 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-05-15T16:22:55.914799994Z 2026-05-15 16:22:55.914 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-05-15T16:22:55.914812069Z 2026-05-15 16:22:55.914 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node2:8984/solr/test-collection_shard1_replica_n4/
2026-05-15T16:22:55.916158538Z 2026-05-15 16:22:55.915 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-05-15T16:22:55.916518899Z 2026-05-15 16:22:55.916 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ has no replicas
2026-05-15T16:22:55.917750925Z 2026-05-15 16:22:55.917 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard1/leader after winning as /collections/test-collection/leader_elect/shard1/election/72059903898681348-core_node6-n_0000000000
2026-05-15T16:22:55.937144288Z 2026-05-15 16:22:55.936 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ shard1
2026-05-15T16:22:56.044490181Z 2026-05-15 16:22:56.044 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 218] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-15T16:22:56.048152616Z 2026-05-15 16:22:56.047 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-05-15T16:22:56.058576648Z 2026-05-15 16:22:56.058 INFO (qtp985324122-28-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1806
2026-05-15T16:22:56.161561369Z 2026-05-15 16:22:56.161 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 222] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-15T16:22:58.416031078Z 2026-05-15 16:22:58.415 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 237] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-15T16:22:58.526717514Z 2026-05-15 16:22:58.526 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 243] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node2 ===
{
"Name": "/solr-node2",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1458025,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-05-15T16:22:41.194129254Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node3 ===
2026-05-15T16:22:41.904992740Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-15T16:22:41.905029126Z It should be set to 65000 to avoid operational disruption.
2026-05-15T16:22:41.905032451Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-15T16:22:41.905280439Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-15T16:22:41.905289945Z
2026-05-15T16:22:41.916173928Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-15T16:22:41.928941200Z [0.004s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-15T16:22:41.987585643Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-15T16:22:42.028382506Z WARNING: A command line option has enabled the Security Manager
2026-05-15T16:22:42.028410428Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-15T16:22:43.431146643Z 2026-05-15 16:22:43.428 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-15T16:22:43.716281283Z 2026-05-15 16:22:43.715 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-15T16:22:43.741702523Z 2026-05-15 16:22:43.741 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-15T16:22:43.751200653Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-15T16:22:43.751214402Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8985
2026-05-15T16:22:43.751217022Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-15T16:22:43.751220037Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-15T16:22:43.750600842Z
2026-05-15T16:22:43.753837301Z 2026-05-15 16:22:43.753 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-15T16:22:43.796967800Z 2026-05-15 16:22:43.796 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-15T16:22:43.849355227Z 2026-05-15 16:22:43.848 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-15T16:22:44.010230583Z 2026-05-15 16:22:44.009 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-05-15T16:22:44.031521582Z 2026-05-15 16:22:44.030 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-15T16:22:44.031597535Z 2026-05-15 16:22:44.031 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-15T16:22:44.031774325Z 2026-05-15 16:22:44.031 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-15T16:22:44.147466027Z 2026-05-15 16:22:44.146 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-05-15T16:22:44.148006931Z 2026-05-15 16:22:44.147 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-15T16:22:44.148156319Z 2026-05-15 16:22:44.147 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-15T16:22:44.201781177Z 2026-05-15 16:22:44.201 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-15T16:22:45.231740946Z 2026-05-15 16:22:45.231 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-15T16:22:45.458334600Z 2026-05-15 16:22:45.457 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-15T16:22:45.460592331Z 2026-05-15 16:22:45.457 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-05-15T16:22:45.473610977Z 2026-05-15 16:22:45.473 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-15T16:22:45.480792438Z 2026-05-15 16:22:45.480 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-15T16:22:45.485303569Z 2026-05-15 16:22:45.484 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-15T16:22:45.485367701Z 2026-05-15 16:22:45.485 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-15T16:22:45.527724460Z 2026-05-15 16:22:45.527 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-15T16:22:45.538101513Z 2026-05-15 16:22:45.537 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-15T16:22:45.562225201Z 2026-05-15 16:22:45.561 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-15T16:22:45.577653693Z 2026-05-15 16:22:45.577 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node3:8985_solr as DOWN
2026-05-15T16:22:45.588305608Z 2026-05-15 16:22:45.587 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node3:8985_solr
2026-05-15T16:22:45.601901849Z 2026-05-15 16:22:45.601 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
2026-05-15T16:22:45.664521474Z 2026-05-15 16:22:45.663 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-15T16:22:46.068796015Z 2026-05-15 16:22:46.068 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-15T16:22:46.340104135Z 2026-05-15 16:22:46.339 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-15T16:22:46.635087411Z 2026-05-15 16:22:46.634 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-15T16:22:46.791615765Z 2026-05-15 16:22:46.790 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-15T16:22:46.797985721Z 2026-05-15 16:22:46.797 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_15.request.log
2026-05-15T16:22:46.801766587Z 2026-05-15 16:22:46.801 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8985}
2026-05-15T16:22:46.803125023Z 2026-05-15 16:22:46.802 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4878ms
2026-05-15T16:22:49.612040234Z 2026-05-15 16:22:49.603 INFO (qtp985324122-27-solr-node3-1) [c: s: r: x: t:solr-node3-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=78
2026-05-15T16:22:49.680327943Z 2026-05-15 16:22:49.679 INFO (qtp985324122-56-solr-node3-2) [c: s: r: x: t:solr-node3-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=20
2026-05-15T16:22:54.269223436Z 2026-05-15 16:22:54.267 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node5&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&shard=shard2&wt=javabin
2026-05-15T16:22:54.346633296Z 2026-05-15 16:22:54.346 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-15T16:22:54.426420431Z 2026-05-15 16:22:54.425 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-05-15T16:22:55.035639611Z 2026-05-15 16:22:55.035 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-15T16:22:55.035663652Z 2026-05-15 16:22:55.035 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard2_replica_n2' using configuration from configset techproducts, trusted=true
2026-05-15T16:22:55.074968152Z 2026-05-15 16:22:55.074 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard2_replica_n2], dataDir=[/var/solr/data/test-collection_shard2_replica_n2/data/]
2026-05-15T16:22:55.722256218Z 2026-05-15 16:22:55.721 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-15T16:22:55.749593210Z 2026-05-15 16:22:55.749 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-15T16:22:55.759394531Z 2026-05-15 16:22:55.758 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-15T16:22:55.760493244Z 2026-05-15 16:22:55.759 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-15T16:22:55.760550216Z 2026-05-15 16:22:55.760 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-15T16:22:55.814991171Z 2026-05-15 16:22:55.814 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-15T16:22:55.818155168Z 2026-05-15 16:22:55.817 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-15T16:22:55.820079156Z 2026-05-15 16:22:55.819 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-15T16:22:55.822513890Z 2026-05-15 16:22:55.822 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-15T16:22:55.841798598Z 2026-05-15 16:22:55.841 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01}
2026-05-15T16:22:55.849625774Z 2026-05-15 16:22:55.849 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-15T16:22:55.856881812Z 2026-05-15 16:22:55.856 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-8 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-15T16:22:55.856923791Z 2026-05-15 16:22:55.856 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-8 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-15T16:22:55.863215517Z 2026-05-15 16:22:55.862 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-8 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-15T16:22:55.888899138Z 2026-05-15 16:22:55.888 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard2 to Terms{values={core_node5=0}, version=0} for registerTerm
2026-05-15T16:22:55.890224232Z 2026-05-15 16:22:55.889 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard2
2026-05-15T16:22:55.914776531Z 2026-05-15 16:22:55.914 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-05-15T16:22:55.914783890Z 2026-05-15 16:22:55.914 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-05-15T16:22:55.914786688Z 2026-05-15 16:22:55.914 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node3:8985/solr/test-collection_shard2_replica_n2/
2026-05-15T16:22:55.916627658Z 2026-05-15 16:22:55.916 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-05-15T16:22:55.917459848Z 2026-05-15 16:22:55.917 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ has no replicas
2026-05-15T16:22:55.918495297Z 2026-05-15 16:22:55.918 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard2/leader after winning as /collections/test-collection/leader_elect/shard2/election/72059903898681347-core_node5-n_0000000000
2026-05-15T16:22:55.933367104Z 2026-05-15 16:22:55.932 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ shard2
2026-05-15T16:22:56.044210593Z 2026-05-15 16:22:56.043 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 218] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-15T16:22:56.048066330Z 2026-05-15 16:22:56.047 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-05-15T16:22:56.058347277Z 2026-05-15 16:22:56.058 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=techproducts&newCollection=true&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&collection=test-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1806
2026-05-15T16:22:56.162026545Z 2026-05-15 16:22:56.161 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 222] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-15T16:22:58.415922758Z 2026-05-15 16:22:58.415 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 237] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-15T16:22:58.526597510Z 2026-05-15 16:22:58.526 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 243] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node3 ===
{
"Name": "/solr-node3",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1458155,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-05-15T16:22:41.414999113Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# Docker artifacts saved to: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/docker
not ok 83 Docker SolrCloud rolling upgrade # in 33135 ms
# (in test file test/test_rolling_upgrade.bats, line 118)
# `wait_for 30 1 docker exec solr-node1 solr healthcheck -c test-collection' failed with status 127
# 9.10.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# Digest: sha256:15c8155ceffe1042f37c77bb537aa531804a3510702ff87f2af5fedf523f4203
# Status: Image is up to date for apache/solr-nightly:9.10.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:9.10.0-SNAPSHOT-slim
# 10.0.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# Digest: sha256:7b9bb30f4dc4d340f52255f6cab0a7e3bc0bf4e6c8f5b0d5e0744356ef86b458
# Status: Image is up to date for apache/solr-nightly:10.0.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:10.0.0-SNAPSHOT-slim
# 241765af35e954f0d0c4b276a2481823836dba20b6a8794cc2cb31d24c6c3c4f
# solr-data1
# solr-data2
# solr-data3
# Starting solr-node1 with embedded ZooKeeper
# 7b256613583cc2d5ad747092de8aef692fbbf9929bb95c80141cf9108506f438
# Starting solr-node2 connected to first node's ZooKeeper
# 578e8e5144cb6661b822ec468ce1c04a3734d993d1d08f33a220883567834858
# Started solr-node3 connected to first node's ZooKeeper
# 8d6ce433f0356088d4b73bf98a3f5a28d2cc219a2eabf74e632d0d22271d8851
# Creating a Collection
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# WARN - 2026-05-15 16:22:53.291; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-15 16:22:53.386; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. "/solr"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [http://solr-node1:8983/solr] to [http://solr-node1:8983/].
# Created collection 'test-collection' with 3 shard(s), 1 replica(s) with config-set 'techproducts'
# Checking collection health
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_rolling_upgrade.bats: line 118: wait_for: command not found
# solr-node1
# solr-node2
# solr-node3
# solr-node1
# solr-node2
# solr-node3
# solr-data1
# solr-data2
# solr-data3
# solrcloud-test
ok 84 allowPaths - backup # in 15187 ms
ok 85 snapshot lifecycle # in 4713 ms
ok 86 snapshot list # in 3297 ms
ok 87 snapshot describe # in 3298 ms
ok 88 start solr with ssl # in 15955 ms
ok 89 use different hostname when not checking peer-name # in 24414 ms
ok 90 start solr with ssl and auth # in 14272 ms
ok 91 start solr with client truststore and security manager # in 16089 ms
ok 92 start solr with mTLS needed # in 45604 ms
ok 93 start solr with mTLS wanted # in 34906 ms
ok 94 test keystore reload # in 57897 ms
ok 95 SOLR-11740 check 'solr stop' connection # in 15770 ms
ok 96 stop command for single port # in 18112 ms
ok 97 check stop command doesn't hang # in 213 ms
ok 98 SOLR-16976 solr starts with remote JMX enabled # in 8244 ms
ok 99 deprecated system properties converted to modern properties # in 194561 ms
ok 100 start with custom jetty options # in 8003 ms
ok 101 bootstrapping a configset # in 9158 ms
ok 102 status detects locally running solr # in 9388 ms
ok 103 status with --solr-url from user # in 7918 ms
ok 104 status with --port from user # in 7983 ms
ok 105 multiple connection options are prevented # in 843 ms
ok 106 status with invalid --solr-url from user # in 1222 ms
ok 107 status with --short format # in 195718 ms
ok 108 searching solr via locally executed streaming expression # in 1708 ms
ok 109 searching solr via remotely executed streaming expression # in 1270 ms
ok 110 variable interpolation # in 1664 ms
ok 111 searching solr without credentials fails with error # in 1112 ms
ok 112 --version returns Solr version # in 724 ms
ok 113 version as direct tool call still runs # in 722 ms
ok 114 short help # in 762 ms
ok 115 short help is inferred # in 754 ms
ok 116 long help # in 762 ms
ok 117 running subcommands with zk is prevented # in 702 ms
ok 118 listing out files # in 2156 ms
ok 119 connecting to solr via various solr urls and zk hosts # in 7698 ms
ok 120 copying files around # in 10885 ms
ok 121 upconfig # in 2575 ms
ok 122 SOLR-12429 test upconfig fails with symlink # in 1182 ms
ok 123 downconfig # in 1183 ms
ok 124 bin/solr zk cp gets 'solrhome' from '--solr-home' command line option # in 1432 ms
ok 125 env var ZK_HOST is honored # in 3160 ms
ok 126 Cleanup # in 1841 ms
> Task :solr:packaging:integrationTests FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build.gradle' line: 339
* What went wrong:
Execution failed for task ':solr:packaging:integrationTests'.
> Process 'command '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/.gradle/node/node_modules/bats/bin/bats'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/8.10/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 29m 33s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/vi4h6xumkait4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 251 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/251/
No tests ran.
Build Log:
[...truncated 350 lines...]
# ERROR: KeeperErrorCode = NoNode for /security.json
# --
#
# Last output:
# WARN - 2026-07-02 14:31:15.001; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-07-02 14:31:15.004; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-07-02 14:31:15.472; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-07-02 14:31:15.472; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: KeeperErrorCode = NoNode for /security.json
not ok 9 create, config, api, and delete with basic auth # in 1314 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 44869 is already being used by another process (pid: 1668090)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-07-02 14:31:16.641; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-07-02 14:31:16.644; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
ok 10 post, postlogs and export with basic auth # in 17570 ms
ok 11 package with basic auth # in 12254 ms
ok 12 admin UI static content is excepted from basic auth # in 10586 ms
ok 13 nothing # in 10 ms
ok 14 server does not compress response without Accept-Encoding header # in 144 ms
ok 15 server compresses response when Accept-Encoding: gzip is requested # in 81 ms
ok 16 compressed response can be decompressed and parsed # in 78 ms
ok 17 setting property # in 10995 ms
ok 18 short form of setting property # in 6461 ms
ok 19 connecting to solr via various connection types # in 10503 ms
ok 20 create for non cloud mode # in 8075 ms
ok 21 create for cloud mode # in 10187 ms
ok 22 multiple connection options are prevented # in 869 ms
ok 23 create collection # in 7815 ms
ok 24 create collection using solrUrl # in 5242 ms
ok 25 create collection using legacy solrUrl # in 5361 ms
ok 26 create collection using Zookeeper # in 5508 ms
ok 27 reject d option with invalid config dir # in 2714 ms
ok 28 accept d option with builtin config # in 5322 ms
ok 29 accept d option with explicit path to config # in 5285 ms
ok 30 accept n option as config name # in 5286 ms
ok 31 allow config reuse when n option specifies same config # in 9163 ms
ok 32 create multisharded collections when s provided # in 5321 ms
ok 33 create replicated collections when rf provided # in 6149 ms
ok 34 can delete collections # in 10977 ms
ok 35 can delete collections with solr-url # in 8562 ms
ok 36 can delete collections with solr-connection # in 8062 ms
ok 37 collection delete also deletes zk config # in 7409 ms
ok 38 deletes accompanying zk config with nondefault name # in 7317 ms
ok 39 delete-config option can opt to leave config in zk # in 7150 ms
ok 40 start -e cloud works with --no-prompt # in 2705 ms
ok 41 start -e cloud works with --jvm-opts # in 106 ms
ok 42 Check export command # in 18993 ms
ok 43 export fails on non cloud mode # in 9613 ms
Waiting for Tika Server to be ready on port 44874
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 44 using curl to extract a single pdf file # in 16574 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to extract content from pdf # in 16538 ms
WARNING: Docker not available; bypassing test.
ok 46 using the bin/solr post tool to crawl web site # in 16543 ms
Stopping Tika Server container
ok 47 healthcheck on cloud solr # in 13349 ms
ok 48 healthcheck errors on standalone solr # in 6811 ms
ok 49 solr help flag prints help # in 752 ms
ok 50 solr with no flags prints help # in 740 ms
ok 51 start help flag prints help # in 147 ms
ok 52 start h flag prints help # in 143 ms
ok 53 stop help flag prints help # in 148 ms
ok 54 restart help flag prints help # in 144 ms
ok 55 status help flag prints help # in 788 ms
ok 56 healthcheck help flag prints help # in 753 ms
ok 57 create help flag prints help # in 751 ms
ok 58 delete help flag prints help # in 812 ms
ok 59 zk help flag prints help # in 766 ms
ok 60 auth help flag prints help # in 761 ms
ok 61 assert help flag prints help # in 754 ms
ok 62 post help flag prints help # in 779 ms
ok 63 SQL Module # in 16272 ms
ok 64 icu collation in analysis-extras module # in 12534 ms
ok 65 Check lifecycle of sentiment classification # in 23561 ms
ok 66 Affinity placement plugin using sysprop # in 15839 ms
ok 67 Random placement plugin using ENV # in 15305 ms
ok 68 Check help command # in 2316 ms
ok 69 basic post with a type specified # in 5487 ms
ok 70 basic post with collection instead of url parameter # in 3964 ms
ok 71 basic post with solr-url and collection # in 3962 ms
ok 72 basic post with -s (solr-connection short form) and collection # in 3932 ms
ok 73 basic post WITHOUT a type specified # in 11294 ms
ok 74 crawling a directory as a dry-run # in 763 ms
ok 75 crawling a directory # in 4294 ms
ok 76 skipcommit and optimize and delete # in 3784 ms
ok 77 args mode # in 6848 ms
ok 78 stdin mode # in 3407 ms
ok 79 verbose echo the Solr response # in 3694 ms
ok 80 post solr log into solr via cli # in 9744 ms
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 14 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 14 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 25 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-07-02T14:40:19.606527161Z *** [WARN] *** Your open file limit is currently 1024.
2026-07-02T14:40:19.606572796Z It should be set to 65000 to avoid operational disruption.
2026-07-02T14:40:19.606577499Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-07-02T14:40:19.606830320Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-07-02T14:40:19.606833549Z
2026-07-02T14:40:19.623828557Z Java 17 detected. Enabled workaround for SOLR-16463
2026-07-02T14:40:19.636344550Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-07-02T14:40:19.692223525Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-07-02T14:40:19.739373337Z WARNING: A command line option has enabled the Security Manager
2026-07-02T14:40:19.739403993Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-07-02T14:40:21.923545647Z 2026-07-02 14:40:21.918 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-07-02T14:40:22.459142044Z 2026-07-02 14:40:22.458 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-07-02T14:40:22.511966025Z 2026-07-02 14:40:22.511 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-07-02T14:40:22.525781070Z 2026-07-02 14:40:22.525 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-07-02T14:40:22.525850140Z 2026-07-02 14:40:22.525 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-07-02T14:40:22.526079965Z 2026-07-02 14:40:22.525 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-07-02T14:40:22.526086653Z 2026-07-02 14:40:22.525 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-07-02T14:40:22.525548069Z
2026-07-02T14:40:22.529902728Z 2026-07-02 14:40:22.529 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-07-02T14:40:22.575286985Z 2026-07-02 14:40:22.574 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-07-02T14:40:22.624536554Z 2026-07-02 14:40:22.624 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-07-02T14:40:22.624652769Z 2026-07-02 14:40:22.624 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-07-02T14:40:22.714572874Z 2026-07-02 14:40:22.713 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-07-02T14:40:24.178740445Z 2026-07-02 14:40:24.178 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-07-02T14:40:24.645719265Z 2026-07-02 14:40:24.645 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-07-02T14:40:24.658455148Z 2026-07-02 14:40:24.657 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-07-02T14:40:24.658478216Z 2026-07-02 14:40:24.657 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-07-02T14:40:24.663425254Z 2026-07-02 14:40:24.663 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-07-02T14:40:24.663474199Z 2026-07-02 14:40:24.663 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-07-02T14:40:24.727440674Z 2026-07-02 14:40:24.726 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-07-02T14:40:25.165989019Z 2026-07-02 14:40:25.165 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-07-02T14:40:25.188631696Z 2026-07-02 14:40:25.188 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-07-02T14:40:25.239904046Z 2026-07-02 14:40:25.239 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-07-02T14:40:25.268973442Z 2026-07-02 14:40:25.267 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-07-02T14:40:25.268996752Z 2026-07-02 14:40:25.268 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-07-02T14:40:25.391258457Z 2026-07-02 14:40:25.390 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-07-02T14:40:25.416567818Z 2026-07-02 14:40:25.416 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-07-02T14:40:25.427240372Z 2026-07-02 14:40:25.426 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-07-02T14:40:25.433138397Z 2026-07-02 14:40:25.432 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72059597335363584-solr-node1:8983_solr-n_0000000000) starting
2026-07-02T14:40:25.512875303Z 2026-07-02 14:40:25.512 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-07-02T14:40:25.515794951Z 2026-07-02 14:40:25.515 INFO (OverseerStateUpdate-72059597335363584-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-07-02T14:40:25.531799378Z 2026-07-02 14:40:25.531 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-07-02T14:40:25.625031015Z 2026-07-02 14:40:25.624 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-07-02T14:40:26.257483407Z 2026-07-02 14:40:26.256 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-07-02T14:40:26.533915670Z 2026-07-02 14:40:26.533 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-07-02T14:40:26.992184947Z 2026-07-02 14:40:26.991 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-07-02T14:40:27.603602322Z 2026-07-02 14:40:27.602 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-07-02T14:40:27.612007952Z 2026-07-02 14:40:27.611 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_07_02.request.log
2026-07-02T14:40:27.617091138Z 2026-07-02 14:40:27.616 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-07-02T14:40:27.618631115Z 2026-07-02 14:40:27.618 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @7983ms
2026-07-02T14:40:27.937396488Z 2026-07-02 14:40:27.936 INFO (qtp985324122-28-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=78
2026-07-02T14:40:28.208672071Z 2026-07-02 14:40:28.208 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=21
2026-07-02T14:40:33.073907368Z 2026-07-02 14:40:33.073 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
2026-07-02T14:40:39.231537059Z 2026-07-02 14:40:39.230 INFO (qtp985324122-100-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=34
2026-07-02T14:40:39.267339298Z 2026-07-02 14:40:39.266 INFO (qtp985324122-100-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=11
2026-07-02T14:40:39.303995831Z 2026-07-02 14:40:39.303 INFO (qtp985324122-28-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=4
2026-07-02T14:40:39.723619301Z 2026-07-02 14:40:39.723 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-07-02T14:40:39.763159329Z 2026-07-02 14:40:39.762 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-07-02T14:40:39.785445186Z 2026-07-02 14:40:39.784 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 173] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-07-02T14:40:39.935884771Z 2026-07-02 14:40:39.934 INFO (OverseerStateUpdate-72059597335363584-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-07-02T14:40:39.935909838Z "core":"test-collection_shard3_replica_n1",
2026-07-02T14:40:39.935913351Z "node_name":"solr-node1:8983_solr",
2026-07-02T14:40:39.935916690Z "base_url":"http://solr-node1:8983/solr",
2026-07-02T14:40:39.935919519Z "collection":"test-collection",
2026-07-02T14:40:39.935922345Z "shard":"shard3",
2026-07-02T14:40:39.935924902Z "state":"down",
2026-07-02T14:40:39.935927620Z "type":"NRT",
2026-07-02T14:40:39.935930128Z "operation":"ADDREPLICA",
2026-07-02T14:40:39.935932787Z "waitForFinalState":"false"}
2026-07-02T14:40:39.947205925Z 2026-07-02 14:40:39.946 INFO (OverseerStateUpdate-72059597335363584-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-07-02T14:40:39.947237095Z "core":"test-collection_shard2_replica_n2",
2026-07-02T14:40:39.947241815Z "node_name":"solr-node3:8985_solr",
2026-07-02T14:40:39.947245870Z "base_url":"http://solr-node3:8985/solr",
2026-07-02T14:40:39.947248765Z "collection":"test-collection",
2026-07-02T14:40:39.947251444Z "shard":"shard2",
2026-07-02T14:40:39.947254281Z "state":"down",
2026-07-02T14:40:39.947256996Z "type":"NRT",
2026-07-02T14:40:39.947259516Z "operation":"ADDREPLICA",
2026-07-02T14:40:39.947262022Z "waitForFinalState":"false"}
2026-07-02T14:40:39.962030489Z 2026-07-02 14:40:39.961 INFO (OverseerStateUpdate-72059597335363584-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-07-02T14:40:39.962052407Z "core":"test-collection_shard1_replica_n4",
2026-07-02T14:40:39.962056250Z "node_name":"solr-node2:8984_solr",
2026-07-02T14:40:39.962059210Z "base_url":"http://solr-node2:8984/solr",
2026-07-02T14:40:39.962061850Z "collection":"test-collection",
2026-07-02T14:40:39.962077134Z "shard":"shard1",
2026-07-02T14:40:39.962079437Z "state":"down",
2026-07-02T14:40:39.962081354Z "type":"NRT",
2026-07-02T14:40:39.962083368Z "operation":"ADDREPLICA",
2026-07-02T14:40:39.962085306Z "waitForFinalState":"false"}
2026-07-02T14:40:40.074363039Z 2026-07-02 14:40:40.073 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 185] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-07-02T14:40:40.214284766Z 2026-07-02 14:40:40.213 INFO (qtp985324122-100-solr-node1-6) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-07-02T14:40:40.336736309Z 2026-07-02 14:40:40.336 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-07-02T14:40:40.423804240Z 2026-07-02 14:40:40.423 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-07-02T14:40:41.150804086Z 2026-07-02 14:40:41.150 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-07-02T14:40:41.150957942Z 2026-07-02 14:40:41.150 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true
2026-07-02T14:40:41.206620617Z 2026-07-02 14:40:41.206 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/]
2026-07-02T14:40:42.525124465Z 2026-07-02 14:40:42.524 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-07-02T14:40:42.567053871Z 2026-07-02 14:40:42.566 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-07-02T14:40:42.570622160Z 2026-07-02 14:40:42.570 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-07-02T14:40:42.571922299Z 2026-07-02 14:40:42.571 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-07-02T14:40:42.572027684Z 2026-07-02 14:40:42.571 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-07-02T14:40:42.638759772Z 2026-07-02 14:40:42.638 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-07-02T14:40:42.642074881Z 2026-07-02 14:40:42.641 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-07-02T14:40:42.644548387Z 2026-07-02 14:40:42.644 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-07-02T14:40:42.646297236Z 2026-07-02 14:40:42.646 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-07-02T14:40:42.686270974Z 2026-07-02 14:40:42.685 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-07-02T14:40:42.694682220Z 2026-07-02 14:40:42.694 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-07-02T14:40:42.703452624Z 2026-07-02 14:40:42.703 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-07-02T14:40:42.703474824Z 2026-07-02 14:40:42.703 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-07-02T14:40:42.712820919Z 2026-07-02 14:40:42.712 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-07-02T14:40:42.722506150Z 2026-07-02 14:40:42.721 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-07-02T14:40:42.727160303Z 2026-07-02 14:40:42.726 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-07-02T14:40:42.742306775Z 2026-07-02 14:40:42.741 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-07-02T14:40:42.742328707Z 2026-07-02 14:40:42.741 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-07-02T14:40:42.742382374Z 2026-07-02 14:40:42.742 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/
2026-07-02T14:40:42.743890302Z 2026-07-02 14:40:42.743 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-07-02T14:40:42.744350903Z 2026-07-02 14:40:42.744 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas
2026-07-02T14:40:42.745526000Z 2026-07-02 14:40:42.745 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72059597335363584-core_node3-n_0000000000
2026-07-02T14:40:42.756704381Z 2026-07-02 14:40:42.756 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3
2026-07-02T14:40:42.864440217Z 2026-07-02 14:40:42.863 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 237] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-07-02T14:40:42.867782713Z 2026-07-02 14:40:42.867 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-07-02T14:40:42.877781749Z 2026-07-02 14:40:42.877 INFO (qtp985324122-100-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2664
2026-07-02T14:40:42.975506932Z 2026-07-02 14:40:42.975 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 243] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1697989,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-07-02T14:40:18.042463931Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-07-02T14:40:28.873517158Z *** [WARN] *** Your open file limit is currently 1024.
2026-07-02T14:40:28.873550859Z It should be set to 65000 to avoid operational disruption.
2026-07-02T14:40:28.873565014Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-07-02T14:40:28.873806035Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-07-02T14:40:28.873817912Z
2026-07-02T14:40:28.883294564Z Java 17 detected. Enabled workaround for SOLR-16463
2026-07-02T14:40:28.893826930Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-07-02T14:40:28.954781053Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-07-02T14:40:28.988402216Z WARNING: A command line option has enabled the Security Manager
2026-07-02T14:40:28.988420870Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-07-02T14:40:30.528390223Z 2026-07-02 14:40:30.525 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-07-02T14:40:30.996244436Z 2026-07-02 14:40:30.995 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-07-02T14:40:31.041527575Z 2026-07-02 14:40:31.041 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-07-02T14:40:31.052516299Z 2026-07-02 14:40:31.051 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-07-02T14:40:31.052529653Z 2026-07-02 14:40:31.051 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-07-02T14:40:31.052532425Z 2026-07-02 14:40:31.051 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-07-02T14:40:31.052534989Z 2026-07-02 14:40:31.051 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-07-02T14:40:31.051974839Z
2026-07-02T14:40:31.055707618Z 2026-07-02 14:40:31.055 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-07-02T14:40:31.106433037Z 2026-07-02 14:40:31.105 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-07-02T14:40:31.157075522Z 2026-07-02 14:40:31.156 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-07-02T14:40:31.431156907Z 2026-07-02 14:40:31.430 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-07-02T14:40:31.454653697Z 2026-07-02 14:40:31.454 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-07-02T14:40:31.454734416Z 2026-07-02 14:40:31.454 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-07-02T14:40:31.454887540Z 2026-07-02 14:40:31.454 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-07-02T14:40:31.567865591Z 2026-07-02 14:40:31.567 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-07-02T14:40:31.568143238Z 2026-07-02 14:40:31.567 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-07-02T14:40:31.568247707Z 2026-07-02 14:40:31.567 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-07-02T14:40:31.623879241Z 2026-07-02 14:40:31.623 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-07-02T14:40:32.670773520Z 2026-07-02 14:40:32.670 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-07-02T14:40:32.917103883Z 2026-07-02 14:40:32.916 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-07-02T14:40:32.918937607Z 2026-07-02 14:40:32.916 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-07-02T14:40:32.931284192Z 2026-07-02 14:40:32.930 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-07-02T14:40:32.938593211Z 2026-07-02 14:40:32.938 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-07-02T14:40:32.943340882Z 2026-07-02 14:40:32.942 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-07-02T14:40:32.943357336Z 2026-07-02 14:40:32.942 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-07-02T14:40:32.986631343Z 2026-07-02 14:40:32.986 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-07-02T14:40:32.996822064Z 2026-07-02 14:40:32.996 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-07-02T14:40:33.035650631Z 2026-07-02 14:40:33.035 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-07-02T14:40:33.052647661Z 2026-07-02 14:40:33.052 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-07-02T14:40:33.063711593Z 2026-07-02 14:40:33.063 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-07-02T14:40:33.075390380Z 2026-07-02 14:40:33.074 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
2026-07-02T14:40:33.144925977Z 2026-07-02 14:40:33.144 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-07-02T14:40:33.513093466Z 2026-07-02 14:40:33.512 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-07-02T14:40:33.785251972Z 2026-07-02 14:40:33.784 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-07-02T14:40:34.079897622Z 2026-07-02 14:40:34.079 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-07-02T14:40:34.236641933Z 2026-07-02 14:40:34.235 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-07-02T14:40:34.261383026Z 2026-07-02 14:40:34.260 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_07_02.request.log
2026-07-02T14:40:34.265992866Z 2026-07-02 14:40:34.265 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-07-02T14:40:34.267454138Z 2026-07-02 14:40:34.267 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @5375ms
2026-07-02T14:40:34.518090223Z 2026-07-02 14:40:34.517 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=63
2026-07-02T14:40:34.592513171Z 2026-07-02 14:40:34.592 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=13
2026-07-02T14:40:40.213268726Z 2026-07-02 14:40:40.212 INFO (qtp985324122-59-solr-node1-6) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-07-02T14:40:40.276056868Z 2026-07-02 14:40:40.275 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-07-02T14:40:40.380750327Z 2026-07-02 14:40:40.380 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-07-02T14:40:40.874932471Z 2026-07-02 14:40:40.874 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-07-02T14:40:40.875009977Z 2026-07-02 14:40:40.874 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-07-02T14:40:40.925174604Z 2026-07-02 14:40:40.924 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-07-02T14:40:41.530252064Z 2026-07-02 14:40:41.529 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-07-02T14:40:41.566992782Z 2026-07-02 14:40:41.566 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-07-02T14:40:41.581045691Z 2026-07-02 14:40:41.580 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-07-02T14:40:41.582484694Z 2026-07-02 14:40:41.581 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-07-02T14:40:41.582571129Z 2026-07-02 14:40:41.582 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-07-02T14:40:41.639466313Z 2026-07-02 14:40:41.639 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-07-02T14:40:41.642456580Z 2026-07-02 14:40:41.642 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-07-02T14:40:41.645148732Z 2026-07-02 14:40:41.644 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-07-02T14:40:41.647060708Z 2026-07-02 14:40:41.646 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-07-02T14:40:41.666505561Z 2026-07-02 14:40:41.665 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, accuracy=0.5, maxQueryFrequency=0.01}
2026-07-02T14:40:41.674128927Z 2026-07-02 14:40:41.673 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-07-02T14:40:41.681471660Z 2026-07-02 14:40:41.680 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-07-02T14:40:41.681478974Z 2026-07-02 14:40:41.681 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-07-02T14:40:41.688267505Z 2026-07-02 14:40:41.687 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-07-02T14:40:41.702466414Z 2026-07-02 14:40:41.701 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-07-02T14:40:41.704028873Z 2026-07-02 14:40:41.703 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1
2026-07-02T14:40:41.721438020Z 2026-07-02 14:40:41.720 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-07-02T14:40:41.721444155Z 2026-07-02 14:40:41.721 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-07-02T14:40:41.721502134Z 2026-07-02 14:40:41.721 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node2:8984/solr/test-collection_shard1_replica_n4/
2026-07-02T14:40:41.723525605Z 2026-07-02 14:40:41.723 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-07-02T14:40:41.724497822Z 2026-07-02 14:40:41.724 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ has no replicas
2026-07-02T14:40:41.725848862Z 2026-07-02 14:40:41.725 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard1/leader after winning as /collections/test-collection/leader_elect/shard1/election/72059597335363588-core_node6-n_0000000000
2026-07-02T14:40:41.739387645Z 2026-07-02 14:40:41.738 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ shard1
2026-07-02T14:40:41.850247969Z 2026-07-02 14:40:41.849 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 218] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-07-02T14:40:41.856462041Z 2026-07-02 14:40:41.856 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-07-02T14:40:41.866804119Z 2026-07-02 14:40:41.866 INFO (qtp985324122-59-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1655
2026-07-02T14:40:41.972863700Z 2026-07-02 14:40:41.972 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 222] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-07-02T14:40:42.864794513Z 2026-07-02 14:40:42.864 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 237] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-07-02T14:40:42.975789809Z 2026-07-02 14:40:42.975 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 243] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node2 ===
{
"Name": "/solr-node2",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1698390,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-07-02T14:40:28.467845525Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node3 ===
2026-07-02T14:40:28.881193194Z *** [WARN] *** Your open file limit is currently 1024.
2026-07-02T14:40:28.881221498Z It should be set to 65000 to avoid operational disruption.
2026-07-02T14:40:28.881224568Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-07-02T14:40:28.881550800Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-07-02T14:40:28.881600193Z
2026-07-02T14:40:28.890985463Z Java 17 detected. Enabled workaround for SOLR-16463
2026-07-02T14:40:28.898465185Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-07-02T14:40:28.951650107Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-07-02T14:40:28.988393781Z WARNING: A command line option has enabled the Security Manager
2026-07-02T14:40:28.988416004Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-07-02T14:40:30.528346910Z 2026-07-02 14:40:30.525 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-07-02T14:40:30.996450459Z 2026-07-02 14:40:30.995 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-07-02T14:40:31.041659604Z 2026-07-02 14:40:31.041 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-07-02T14:40:31.052040007Z 2026-07-02 14:40:31.051 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-07-02T14:40:31.052068071Z 2026-07-02 14:40:31.051 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8985
2026-07-02T14:40:31.052232491Z 2026-07-02 14:40:31.051 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-07-02T14:40:31.052351865Z 2026-07-02 14:40:31.051 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-07-02T14:40:31.051794634Z
2026-07-02T14:40:31.055774586Z 2026-07-02 14:40:31.055 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-07-02T14:40:31.106478211Z 2026-07-02 14:40:31.105 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-07-02T14:40:31.157117541Z 2026-07-02 14:40:31.156 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-07-02T14:40:31.431216133Z 2026-07-02 14:40:31.430 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-07-02T14:40:31.455186365Z 2026-07-02 14:40:31.454 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-07-02T14:40:31.455298821Z 2026-07-02 14:40:31.454 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-07-02T14:40:31.455409860Z 2026-07-02 14:40:31.454 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-07-02T14:40:31.567909016Z 2026-07-02 14:40:31.567 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-07-02T14:40:31.568165651Z 2026-07-02 14:40:31.567 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-07-02T14:40:31.568258191Z 2026-07-02 14:40:31.568 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-07-02T14:40:31.623844801Z 2026-07-02 14:40:31.623 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-07-02T14:40:32.697848437Z 2026-07-02 14:40:32.697 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-07-02T14:40:32.915295940Z 2026-07-02 14:40:32.914 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-07-02T14:40:32.917503700Z 2026-07-02 14:40:32.914 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-07-02T14:40:32.930843640Z 2026-07-02 14:40:32.930 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-07-02T14:40:32.937833619Z 2026-07-02 14:40:32.937 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-07-02T14:40:32.942358012Z 2026-07-02 14:40:32.941 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-07-02T14:40:32.942735450Z 2026-07-02 14:40:32.942 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-07-02T14:40:32.986591929Z 2026-07-02 14:40:32.986 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-07-02T14:40:32.996738891Z 2026-07-02 14:40:32.996 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-07-02T14:40:33.035623932Z 2026-07-02 14:40:33.034 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-07-02T14:40:33.051132226Z 2026-07-02 14:40:33.050 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node3:8985_solr as DOWN
2026-07-02T14:40:33.063028215Z 2026-07-02 14:40:33.062 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node3:8985_solr
2026-07-02T14:40:33.074869555Z 2026-07-02 14:40:33.074 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
2026-07-02T14:40:33.144870702Z 2026-07-02 14:40:33.144 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-07-02T14:40:33.512665504Z 2026-07-02 14:40:33.512 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-07-02T14:40:33.786030358Z 2026-07-02 14:40:33.785 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-07-02T14:40:34.079862135Z 2026-07-02 14:40:34.079 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-07-02T14:40:34.236638173Z 2026-07-02 14:40:34.235 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-07-02T14:40:34.261397120Z 2026-07-02 14:40:34.260 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_07_02.request.log
2026-07-02T14:40:34.265963788Z 2026-07-02 14:40:34.265 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8985}
2026-07-02T14:40:34.267476534Z 2026-07-02 14:40:34.267 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @5370ms
2026-07-02T14:40:37.251924595Z 2026-07-02 14:40:37.243 INFO (qtp985324122-27-solr-node3-1) [c: s: r: x: t:solr-node3-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=99
2026-07-02T14:40:37.344062894Z 2026-07-02 14:40:37.343 INFO (qtp985324122-56-solr-node3-2) [c: s: r: x: t:solr-node3-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=20
2026-07-02T14:40:39.677471980Z 2026-07-02 14:40:39.677 INFO (qtp985324122-58-solr-node3-3) [c: s: r: x: t:solr-node3-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=12
2026-07-02T14:40:40.214061264Z 2026-07-02 14:40:40.212 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node5&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&shard=shard2&wt=javabin
2026-07-02T14:40:40.281009350Z 2026-07-02 14:40:40.280 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-07-02T14:40:40.374284331Z 2026-07-02 14:40:40.373 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-07-02T14:40:40.877444444Z 2026-07-02 14:40:40.876 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-07-02T14:40:40.877472738Z 2026-07-02 14:40:40.877 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard2_replica_n2' using configuration from configset techproducts, trusted=true
2026-07-02T14:40:40.925401098Z 2026-07-02 14:40:40.924 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard2_replica_n2], dataDir=[/var/solr/data/test-collection_shard2_replica_n2/data/]
2026-07-02T14:40:41.536689951Z 2026-07-02 14:40:41.535 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-07-02T14:40:41.566829341Z 2026-07-02 14:40:41.566 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-07-02T14:40:41.581008826Z 2026-07-02 14:40:41.580 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-07-02T14:40:41.582448423Z 2026-07-02 14:40:41.581 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-07-02T14:40:41.582574912Z 2026-07-02 14:40:41.582 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-07-02T14:40:41.641692752Z 2026-07-02 14:40:41.641 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-07-02T14:40:41.645153260Z 2026-07-02 14:40:41.644 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-07-02T14:40:41.648061587Z 2026-07-02 14:40:41.647 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-07-02T14:40:41.650192718Z 2026-07-02 14:40:41.649 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-07-02T14:40:41.666473857Z 2026-07-02 14:40:41.666 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-07-02T14:40:41.674371488Z 2026-07-02 14:40:41.673 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-07-02T14:40:41.681440738Z 2026-07-02 14:40:41.681 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-07-02T14:40:41.681684643Z 2026-07-02 14:40:41.681 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-07-02T14:40:41.689183993Z 2026-07-02 14:40:41.688 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-07-02T14:40:41.702531091Z 2026-07-02 14:40:41.701 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard2 to Terms{values={core_node5=0}, version=0} for registerTerm
2026-07-02T14:40:41.704317786Z 2026-07-02 14:40:41.703 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard2
2026-07-02T14:40:41.721402860Z 2026-07-02 14:40:41.720 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-07-02T14:40:41.721711512Z 2026-07-02 14:40:41.721 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-07-02T14:40:41.721755382Z 2026-07-02 14:40:41.721 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node3:8985/solr/test-collection_shard2_replica_n2/
2026-07-02T14:40:41.723499918Z 2026-07-02 14:40:41.723 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-07-02T14:40:41.724549549Z 2026-07-02 14:40:41.724 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ has no replicas
2026-07-02T14:40:41.725942042Z 2026-07-02 14:40:41.725 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard2/leader after winning as /collections/test-collection/leader_elect/shard2/election/72059597335363587-core_node5-n_0000000000
2026-07-02T14:40:41.740774482Z 2026-07-02 14:40:41.740 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ shard2
2026-07-02T14:40:41.850245383Z 2026-07-02 14:40:41.849 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 218] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-07-02T14:40:41.855136849Z 2026-07-02 14:40:41.854 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-07-02T14:40:41.866435220Z 2026-07-02 14:40:41.866 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=techproducts&newCollection=true&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&collection=test-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1654
2026-07-02T14:40:41.973425225Z 2026-07-02 14:40:41.972 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 222] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-07-02T14:40:42.864829959Z 2026-07-02 14:40:42.864 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 237] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-07-02T14:40:42.891961804Z 2026-07-02 14:40:42.891 INFO (qtp985324122-59-solr-node3-4) [c:test-collection s: r: x: t:solr-node3-4] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
2026-07-02T14:40:42.975872729Z 2026-07-02 14:40:42.975 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 243] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-07-02T14:40:42.978816889Z 2026-07-02 14:40:42.978 INFO (qtp985324122-59-solr-node3-4) [c:test-collection s: r: x: t:solr-node3-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=3284
# === Docker inspect for solr-node3 ===
{
"Name": "/solr-node3",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1698520,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-07-02T14:40:28.67011128Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# Docker artifacts saved to: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/docker
not ok 81 Docker SolrCloud rolling upgrade # in 28729 ms
# (in test file test/test_rolling_upgrade.bats, line 118)
# `wait_for 30 1 docker exec solr-node1 solr healthcheck -c test-collection' failed with status 127
# 9.10.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# Digest: sha256:15c8155ceffe1042f37c77bb537aa531804a3510702ff87f2af5fedf523f4203
# Status: Image is up to date for apache/solr-nightly:9.10.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:9.10.0-SNAPSHOT-slim
# 10.0.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# Digest: sha256:fb25f7a7190e3027ade20e97708a8665d41ab0e9ea9285fa242b2c83b0e5f43c
# Status: Image is up to date for apache/solr-nightly:10.0.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:10.0.0-SNAPSHOT-slim
# 65f0f8844d165941c9ddfc17dd49077bc1d625280a6e84478a8808dc70036917
# solr-data1
# solr-data2
# solr-data3
# Starting solr-node1 with embedded ZooKeeper
# ac6dc439c8913c4207abda402fcf40b0552514c8b66f9032b6952414f369b8ae
# Starting solr-node2 connected to first node's ZooKeeper
# 24b8a184b892a12fbcac81a86d1bf3be9113b0511cf7235f67fb0df817e68b6b
# Started solr-node3 connected to first node's ZooKeeper
# ee1ab1fdb8c719d1472210dfc2b7e96983adf269da059f60c063dfc98f496fdd
# Creating a Collection
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# WARN - 2026-07-02 14:40:39.360; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-07-02 14:40:39.427; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. "/solr"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [http://solr-node3:8985/solr] to [http://solr-node3:8985/].
# Created collection 'test-collection' with 3 shard(s), 1 replica(s) with config-set 'techproducts'
# Checking collection health
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_rolling_upgrade.bats: line 118: wait_for: command not found
# solr-node1
# solr-node2
# solr-node3
# solr-node1
# solr-node2
# solr-node3
# solr-data1
# solr-data2
# solr-data3
# solrcloud-test
ok 82 allowPaths - backup # in 16808 ms
ok 83 snapshot lifecycle # in 5123 ms
ok 84 snapshot list # in 3416 ms
ok 85 snapshot list via --solr-connection # in 3375 ms
ok 86 snapshot describe # in 3345 ms
ok 87 snapshot describe via --solr-connection # in 3370 ms
ok 88 start solr with ssl # in 16374 ms
ok 89 use different hostname when not checking peer-name # in 25064 ms
ok 90 start solr with ssl and auth # in 14992 ms
ok 91 start solr with client truststore and security manager # in 16817 ms
ok 92 start solr with mTLS needed # in 46371 ms
ok 93 start solr with mTLS wanted # in 34701 ms
not ok 94 test keystore reload # in 33351 ms
# (in test file test/test_ssl.bats, line 576)
# `wait_for 30 1 solr api --solr-url "https://localhost:${SOLR_PORT}/solr/admin/info/system"' failed with status 127
# Generating 2,048 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 9,999 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Generating 2,048 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 9,999 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Waiting up to 180 seconds to see Solr running on port 44869 [|] [/] [-] [\] [|]
# Started Solr server on port 44869 (pid=1713085). Happy searching!
#
# INFO - 2026-07-02 14:44:09.082; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.keyStorePassword
# INFO - 2026-07-02 14:44:09.085; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.trustStorePassword
# WARN - 2026-07-02 14:44:09.098; org.apache.solr.common.util.EnvUtils; Deprecated system property host has been replaced by solr.host.advertise. Support for the old property will be removed in a future version of Solr.
# INFO - 2026-07-02 14:44:09.374; org.eclipse.jetty.util.ssl.SslContextFactory; x509=X509@294a6b8e(cert1,h=[localhost],a=[/127.0.0.1],w=[]) for Client@4b1d6571[provider=null,keyStore=file:///tmp/bats-run-VauQp6/test/94/ssl/server1.keystore.p12,trustStore=file:///tmp/bats-run-VauQp6/test/94/ssl/server1.keystore.p12]
# Waiting up to 180 seconds to see Solr running on port 44870 [|] [/] [-] [\] [|]
# Started Solr server on port 44870 (pid=1713396). Happy searching!
#
# INFO - 2026-07-02 14:44:15.157; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.keyStorePassword
# INFO - 2026-07-02 14:44:15.160; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.trustStorePassword
# WARN - 2026-07-02 14:44:15.172; org.apache.solr.common.util.EnvUtils; Deprecated system property host has been replaced by solr.host.advertise. Support for the old property will be removed in a future version of Solr.
# INFO - 2026-07-02 14:44:15.452; org.eclipse.jetty.util.ssl.SslContextFactory; x509=X509@294a6b8e(cert1,h=[localhost],a=[/127.0.0.1],w=[]) for Client@4b1d6571[provider=null,keyStore=file:///tmp/bats-run-VauQp6/test/94/ssl/server2.keystore.p12,trustStore=file:///tmp/bats-run-VauQp6/test/94/ssl/server2.keystore.p12]
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_ssl.bats: line 576: wait_for: command not found
# Please find the SOLR_HOME snapshot for failed test #7 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/94-7
# Last output:
# INFO - 2026-07-02 14:44:31.876; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.keyStorePassword
# INFO - 2026-07-02 14:44:31.879; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.trustStorePassword
# WARN - 2026-07-02 14:44:31.892; org.apache.solr.common.util.EnvUtils; Deprecated system property host has been replaced by solr.host.advertise. Support for the old property will be removed in a future version of Solr.
# Neither --solr-connection, --zk-host or --solr-url parameters, nor SOLR_CONNECTION, ZK_HOST env var provided, so assuming solr url is https://localhost:44869.
# WARN - 2026-07-02 14:44:32.843; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-07-02 14:44:32.843; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
ok 95 SOLR-11740 check 'solr stop' connection # in 15991 ms
not ok 96 stop command for single port # in 57579 ms
# (in test file test/test_start_solr.bats, line 48)
# `solr assert --started http://localhost:${SOLR2_PORT} --timeout 5000' failed with status 100
# Waiting up to 180 seconds to see Solr running on port 44869 [|] [/] [-] [\] [|]
# Started Solr server on port 44869 (pid=1715746). Happy searching!
#
# Waiting up to 180 seconds to see Solr running on port 44870 [|] [/] [-] [\] [|]
# Started Solr server on port 44870 (pid=1715880). Happy searching!
#
#
# ERROR: Solr is not running on url http://localhost:44870 after 5 seconds
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/96-2
ok 97 check stop command doesn't hang # in 221 ms
ok 98 SOLR-16976 solr starts with remote JMX enabled # in 8471 ms
ok 99 deprecated system properties converted to modern properties # in 43818 ms
ok 100 start with custom jetty options # in 8212 ms
ok 101 -c flag prints no-op warning and still starts in cloud mode # in 9574 ms
ok 102 bootstrapping a configset # in 9520 ms
ok 103 status detects locally running solr # in 3014 ms
ok 104 status with --solr-url from user # in 1381 ms
ok 105 status with --port from user # in 1365 ms
ok 106 multiple connection options are prevented # in 816 ms
ok 107 status with invalid --solr-url from user # in 1142 ms
ok 108 status with --short format # in 767 ms
ok 109 searching solr via locally executed streaming expression # in 1786 ms
ok 110 searching solr via remotely executed streaming expression by solr-url # in 1572 ms
ok 111 searching solr via remotely executed streaming expression by solr connection # in 1255 ms
ok 112 variable interpolation # in 1684 ms
ok 113 searching solr without credentials fails with error # in 1154 ms
ok 114 --version returns Solr version # in 747 ms
ok 115 version as direct tool call still runs # in 766 ms
ok 116 short help # in 893 ms
ok 117 short help is inferred # in 753 ms
ok 118 long help # in 794 ms
ok 119 running subcommands with zk is prevented # in 773 ms
ok 120 listing out files # in 1168 ms
ok 121 connecting to solr via various solr urls and zk hosts # in 8405 ms
ok 122 copying files around # in 8185 ms
not ok 123 upconfig # in 1478 ms
# (in test file test/test_zk.bats, line 123)
# `wait_for 10 1 bash -c "curl -s 'http://localhost:${SOLR_PORT}/api/configsets' | grep -q techproducts2"' failed with status 127
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_zk.bats: line 123: wait_for: command not found
# Please find the SOLR_HOME snapshot for failed test #8 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/123-8
# Last output:
# WARN - 2026-07-02 14:48:09.702; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-07-02 14:48:09.706; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# Uploading /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server/solr/configsets/sample_techproducts_configs/conf for config techproducts2 to ZooKeeper at localhost:45869
ok 124 SOLR-12429 test upconfig fails with symlink # in 1193 ms
ok 125 downconfig # in 1208 ms
ok 126 bin/solr zk cp gets 'solrhome' from '--solr-home' command line option # in 1461 ms
ok 127 env var ZK_HOST is honored # in 2143 ms
ok 128 Cleanup # in 1917 ms
> Task :solr:packaging:integrationTests FAILED
[Incubating] Problems report is available at: file:///home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/build/reports/problems/problems-report.html
FAILURE: Build failed with an exception.
* Where:
Build file '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build.gradle' line: 354
* What went wrong:
Execution failed for task ':solr:packaging:integrationTests' (registered in build file 'solr/packaging/build.gradle').
> Process 'command '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/.gradle/node/node_modules/bats/bin/bats'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
Deprecated Gradle features were used in this build, making it incompatible with Gradle 10.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/9.6.0/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 22m 40s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/h67zf5wfrehxg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 250 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/250/
No tests ran.
Build Log:
[...truncated 969 lines...]
BUILD FAILED in 29m 30s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/vg4q4fekx2ryy
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 249 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/249/
No tests ran.
Build Log:
[...truncated 360 lines...]
# ERROR: KeeperErrorCode = NoNode for /security.json
# --
#
# Last output:
# WARN - 2026-06-30 12:25:19.784; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-30 12:25:19.789; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-30 12:25:20.278; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-30 12:25:20.278; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: KeeperErrorCode = NoNode for /security.json
not ok 9 create, config, api, and delete with basic auth # in 1715 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 38671 is already being used by another process (pid: 3840819)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-06-30 12:25:21.482; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-30 12:25:21.485; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
ok 10 post, postlogs and export with basic auth # in 18410 ms
ok 11 package with basic auth # in 12774 ms
ok 12 admin UI static content is excepted from basic auth # in 11157 ms
ok 13 nothing # in 10 ms
ok 14 server does not compress response without Accept-Encoding header # in 167 ms
ok 15 server compresses response when Accept-Encoding: gzip is requested # in 81 ms
ok 16 compressed response can be decompressed and parsed # in 82 ms
ok 17 setting property # in 11443 ms
ok 18 short form of setting property # in 6588 ms
ok 19 connecting to solr via various connection types # in 11003 ms
ok 20 create for non cloud mode # in 8074 ms
ok 21 create for cloud mode # in 10348 ms
ok 22 multiple connection options are prevented # in 909 ms
ok 23 create collection # in 7952 ms
ok 24 create collection using solrUrl # in 5369 ms
ok 25 create collection using legacy solrUrl # in 5479 ms
ok 26 create collection using Zookeeper # in 5559 ms
ok 27 reject d option with invalid config dir # in 2810 ms
ok 28 accept d option with builtin config # in 5443 ms
ok 29 accept d option with explicit path to config # in 5394 ms
ok 30 accept n option as config name # in 5374 ms
ok 31 allow config reuse when n option specifies same config # in 9414 ms
ok 32 create multisharded collections when s provided # in 5368 ms
ok 33 create replicated collections when rf provided # in 6246 ms
ok 34 can delete collections # in 11434 ms
ok 35 can delete collections with solr-url # in 8493 ms
ok 36 can delete collections with solr-connection # in 8364 ms
ok 37 collection delete also deletes zk config # in 7642 ms
ok 38 deletes accompanying zk config with nondefault name # in 7526 ms
ok 39 delete-config option can opt to leave config in zk # in 7359 ms
ok 40 start -e cloud works with --jvm-opts # in 21994 ms
ok 41 start -e cloud works with --no-prompt # in 43838 ms
ok 42 Check export command # in 19564 ms
ok 43 export fails on non cloud mode # in 9740 ms
Waiting for Tika Server to be ready on port 38676
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 44 using curl to extract a single pdf file # in 16639 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to extract content from pdf # in 16593 ms
WARNING: Docker not available; bypassing test.
ok 46 using the bin/solr post tool to crawl web site # in 16614 ms
Stopping Tika Server container
ok 47 healthcheck on cloud solr # in 10744 ms
ok 48 healthcheck on remote cloud solr # in 10616 ms
ok 49 healthcheck errors on standalone solr # in 11622 ms
ok 50 solr help flag prints help # in 755 ms
ok 51 solr with no flags prints help # in 776 ms
ok 52 start help flag prints help # in 160 ms
ok 53 start h flag prints help # in 170 ms
ok 54 stop help flag prints help # in 183 ms
ok 55 restart help flag prints help # in 149 ms
ok 56 status help flag prints help # in 902 ms
ok 57 healthcheck help flag prints help # in 779 ms
ok 58 create help flag prints help # in 789 ms
ok 59 delete help flag prints help # in 846 ms
ok 60 zk help flag prints help # in 832 ms
ok 61 auth help flag prints help # in 807 ms
ok 62 assert help flag prints help # in 810 ms
ok 63 post help flag prints help # in 798 ms
ok 64 SQL Module # in 16163 ms
ok 65 icu collation in analysis-extras module # in 13091 ms
ok 66 Check lifecycle of sentiment classification # in 23696 ms
ok 67 lifecycle of package # in 8572 ms
ok 68 deploying and undeploying a collection level package # in 13671 ms
ok 69 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
ok 70 Affinity placement plugin using sysprop # in 16327 ms
ok 71 Random placement plugin using ENV # in 16180 ms
ok 72 Check help command # in 2440 ms
ok 73 basic post with a type specified # in 5521 ms
ok 74 basic post with collection instead of url parameter # in 4076 ms
ok 75 basic post with solr-url and collection # in 4013 ms
ok 76 basic post with -s (solr-connection short form) and collection # in 4052 ms
ok 77 basic post WITHOUT a type specified # in 11873 ms
ok 78 crawling a directory as a dry-run # in 790 ms
ok 79 crawling a directory # in 4499 ms
ok 80 skipcommit and optimize and delete # in 4070 ms
ok 81 args mode # in 7133 ms
ok 82 stdin mode # in 3525 ms
ok 83 verbose echo the Solr response # in 3909 ms
ok 84 post solr log into solr via cli # in 9887 ms
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 15 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 15 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 27 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-06-30T12:35:54.550909185Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-30T12:35:54.550951947Z It should be set to 65000 to avoid operational disruption.
2026-06-30T12:35:54.550955008Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-30T12:35:54.551258270Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-30T12:35:54.551288184Z
2026-06-30T12:35:54.564941461Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-30T12:35:54.575824741Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-30T12:35:54.640864625Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-30T12:35:54.690288316Z WARNING: A command line option has enabled the Security Manager
2026-06-30T12:35:54.690316937Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-30T12:35:57.256469140Z 2026-06-30 12:35:57.250 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-30T12:35:57.804325541Z 2026-06-30 12:35:57.803 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-30T12:35:57.855314415Z 2026-06-30 12:35:57.854 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-30T12:35:57.917530071Z 2026-06-30 12:35:57.916 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-30T12:35:57.917897726Z 2026-06-30 12:35:57.917 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-06-30T12:35:57.917947402Z 2026-06-30 12:35:57.917 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-30T12:35:57.917955776Z 2026-06-30 12:35:57.917 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-30T12:35:57.917291391Z
2026-06-30T12:35:57.921813618Z 2026-06-30 12:35:57.921 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-30T12:35:58.017549573Z 2026-06-30 12:35:58.016 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-30T12:35:58.117251670Z 2026-06-30 12:35:58.116 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-30T12:35:58.117273064Z 2026-06-30 12:35:58.116 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-30T12:35:58.225009411Z 2026-06-30 12:35:58.224 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-30T12:35:59.836348789Z 2026-06-30 12:35:59.835 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-30T12:36:00.343050701Z 2026-06-30 12:36:00.342 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-30T12:36:00.356538268Z 2026-06-30 12:36:00.356 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-06-30T12:36:00.356586392Z 2026-06-30 12:36:00.356 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-06-30T12:36:00.362842261Z 2026-06-30 12:36:00.362 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-06-30T12:36:00.362860204Z 2026-06-30 12:36:00.362 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-06-30T12:36:00.451185734Z 2026-06-30 12:36:00.450 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-06-30T12:36:00.865523017Z 2026-06-30 12:36:00.864 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-06-30T12:36:00.895291722Z 2026-06-30 12:36:00.894 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-30T12:36:00.952192320Z 2026-06-30 12:36:00.951 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-30T12:36:00.981591030Z 2026-06-30 12:36:00.980 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-30T12:36:00.981614125Z 2026-06-30 12:36:00.981 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-30T12:36:01.116294798Z 2026-06-30 12:36:01.115 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-30T12:36:01.144317758Z 2026-06-30 12:36:01.143 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-30T12:36:01.156305071Z 2026-06-30 12:36:01.155 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-06-30T12:36:01.163157173Z 2026-06-30 12:36:01.162 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72211905013022720-solr-node1:8983_solr-n_0000000000) starting
2026-06-30T12:36:01.256719268Z 2026-06-30 12:36:01.256 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-06-30T12:36:01.258241836Z 2026-06-30 12:36:01.257 INFO (OverseerStateUpdate-72211905013022720-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-06-30T12:36:01.269882857Z 2026-06-30 12:36:01.269 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-30T12:36:01.339563200Z 2026-06-30 12:36:01.338 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-30T12:36:02.098931877Z 2026-06-30 12:36:02.098 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-30T12:36:02.448914594Z 2026-06-30 12:36:02.448 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-30T12:36:02.987660217Z 2026-06-30 12:36:02.987 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-30T12:36:03.474744324Z 2026-06-30 12:36:03.473 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-30T12:36:03.537341806Z 2026-06-30 12:36:03.536 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_30.request.log
2026-06-30T12:36:03.544208562Z 2026-06-30 12:36:03.543 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-06-30T12:36:03.546358308Z 2026-06-30 12:36:03.545 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @8971ms
2026-06-30T12:36:04.059633144Z 2026-06-30 12:36:04.059 INFO (qtp985324122-28-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=185
2026-06-30T12:36:04.391558628Z 2026-06-30 12:36:04.391 INFO (qtp985324122-97-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=19
2026-06-30T12:36:09.266627422Z 2026-06-30 12:36:09.266 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-30T12:36:09.873312912Z 2026-06-30 12:36:09.872 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-30T12:36:16.156824845Z 2026-06-30 12:36:16.155 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=41
2026-06-30T12:36:16.250284588Z 2026-06-30 12:36:16.249 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=33
2026-06-30T12:36:16.287984776Z 2026-06-30 12:36:16.287 INFO (qtp985324122-28-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=3
2026-06-30T12:36:16.769002614Z 2026-06-30 12:36:16.768 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-06-30T12:36:16.808289161Z 2026-06-30 12:36:16.807 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-06-30T12:36:16.826741457Z 2026-06-30 12:36:16.826 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 174] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-30T12:36:16.941832654Z 2026-06-30 12:36:16.941 INFO (OverseerStateUpdate-72211905013022720-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-30T12:36:16.941859648Z "core":"test-collection_shard3_replica_n1",
2026-06-30T12:36:16.941863040Z "node_name":"solr-node1:8983_solr",
2026-06-30T12:36:16.941865492Z "base_url":"http://solr-node1:8983/solr",
2026-06-30T12:36:16.941867792Z "collection":"test-collection",
2026-06-30T12:36:16.941870315Z "shard":"shard3",
2026-06-30T12:36:16.941872391Z "state":"down",
2026-06-30T12:36:16.941874441Z "type":"NRT",
2026-06-30T12:36:16.941876437Z "operation":"ADDREPLICA",
2026-06-30T12:36:16.941879022Z "waitForFinalState":"false"}
2026-06-30T12:36:16.955543252Z 2026-06-30 12:36:16.954 INFO (OverseerStateUpdate-72211905013022720-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-30T12:36:16.955571028Z "core":"test-collection_shard2_replica_n2",
2026-06-30T12:36:16.955574050Z "node_name":"solr-node3:8985_solr",
2026-06-30T12:36:16.955577393Z "base_url":"http://solr-node3:8985/solr",
2026-06-30T12:36:16.955579678Z "collection":"test-collection",
2026-06-30T12:36:16.955581761Z "shard":"shard2",
2026-06-30T12:36:16.955583755Z "state":"down",
2026-06-30T12:36:16.955585944Z "type":"NRT",
2026-06-30T12:36:16.955587823Z "operation":"ADDREPLICA",
2026-06-30T12:36:16.955589757Z "waitForFinalState":"false"}
2026-06-30T12:36:16.964320064Z 2026-06-30 12:36:16.963 INFO (OverseerStateUpdate-72211905013022720-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-30T12:36:16.964344747Z "core":"test-collection_shard1_replica_n4",
2026-06-30T12:36:16.964347602Z "node_name":"solr-node2:8984_solr",
2026-06-30T12:36:16.964364080Z "base_url":"http://solr-node2:8984/solr",
2026-06-30T12:36:16.964366345Z "collection":"test-collection",
2026-06-30T12:36:16.964368350Z "shard":"shard1",
2026-06-30T12:36:16.964370283Z "state":"down",
2026-06-30T12:36:16.964372276Z "type":"NRT",
2026-06-30T12:36:16.964374313Z "operation":"ADDREPLICA",
2026-06-30T12:36:16.964376251Z "waitForFinalState":"false"}
2026-06-30T12:36:17.078848605Z 2026-06-30 12:36:17.078 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 186] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-30T12:36:17.224735665Z 2026-06-30 12:36:17.224 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-06-30T12:36:17.385039436Z 2026-06-30 12:36:17.384 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-30T12:36:17.524382511Z 2026-06-30 12:36:17.522 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-30T12:36:18.239618842Z 2026-06-30 12:36:18.238 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-30T12:36:18.239648283Z 2026-06-30 12:36:18.239 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true
2026-06-30T12:36:18.301296188Z 2026-06-30 12:36:18.300 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/]
2026-06-30T12:36:19.768834907Z 2026-06-30 12:36:19.767 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-30T12:36:19.836794016Z 2026-06-30 12:36:19.836 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-30T12:36:19.841868799Z 2026-06-30 12:36:19.840 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-30T12:36:19.842778426Z 2026-06-30 12:36:19.841 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-30T12:36:19.842946327Z 2026-06-30 12:36:19.842 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-30T12:36:19.915598214Z 2026-06-30 12:36:19.915 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-30T12:36:19.922732992Z 2026-06-30 12:36:19.922 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-30T12:36:19.924832029Z 2026-06-30 12:36:19.924 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-30T12:36:19.926637572Z 2026-06-30 12:36:19.926 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-30T12:36:19.950377724Z 2026-06-30 12:36:19.949 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4}
2026-06-30T12:36:19.975689865Z 2026-06-30 12:36:19.975 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-30T12:36:19.993646315Z 2026-06-30 12:36:19.993 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-30T12:36:19.993673597Z 2026-06-30 12:36:19.993 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-30T12:36:20.002596419Z 2026-06-30 12:36:20.002 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-30T12:36:20.012026938Z 2026-06-30 12:36:20.011 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-06-30T12:36:20.016318963Z 2026-06-30 12:36:20.015 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-06-30T12:36:20.033276793Z 2026-06-30 12:36:20.032 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-30T12:36:20.033299248Z 2026-06-30 12:36:20.033 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-30T12:36:20.033355758Z 2026-06-30 12:36:20.033 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/
2026-06-30T12:36:20.034866722Z 2026-06-30 12:36:20.034 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-30T12:36:20.035350675Z 2026-06-30 12:36:20.035 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas
2026-06-30T12:36:20.036542316Z 2026-06-30 12:36:20.036 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72211905013022720-core_node3-n_0000000000
2026-06-30T12:36:20.041638624Z 2026-06-30 12:36:20.041 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3
2026-06-30T12:36:20.148181385Z 2026-06-30 12:36:20.147 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 238] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-30T12:36:20.152110319Z 2026-06-30 12:36:20.151 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-30T12:36:20.159275633Z 2026-06-30 12:36:20.158 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2936
2026-06-30T12:36:20.260022506Z 2026-06-30 12:36:20.259 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 3874439,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-30T12:35:52.85964948Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-06-30T12:36:05.352428393Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-30T12:36:05.352474801Z It should be set to 65000 to avoid operational disruption.
2026-06-30T12:36:05.352480161Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-30T12:36:05.352731650Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-30T12:36:05.352739751Z
2026-06-30T12:36:05.362455677Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-30T12:36:05.372009841Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-30T12:36:05.424723388Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-30T12:36:05.462127653Z WARNING: A command line option has enabled the Security Manager
2026-06-30T12:36:05.462173345Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-30T12:36:07.663109673Z 2026-06-30 12:36:07.659 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-30T12:36:07.965196332Z 2026-06-30 12:36:07.964 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-30T12:36:07.995956641Z 2026-06-30 12:36:07.995 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-30T12:36:08.005144306Z 2026-06-30 12:36:08.004 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-30T12:36:08.005518241Z 2026-06-30 12:36:08.004 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-06-30T12:36:08.005526899Z 2026-06-30 12:36:08.004 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-30T12:36:08.005530641Z 2026-06-30 12:36:08.004 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-30T12:36:08.004959307Z
2026-06-30T12:36:08.009050609Z 2026-06-30 12:36:08.008 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-30T12:36:08.034199901Z 2026-06-30 12:36:08.033 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-30T12:36:08.071472978Z 2026-06-30 12:36:08.070 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-30T12:36:08.357547855Z 2026-06-30 12:36:08.357 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-30T12:36:08.382666565Z 2026-06-30 12:36:08.382 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-30T12:36:08.382700753Z 2026-06-30 12:36:08.382 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-30T12:36:08.382822804Z 2026-06-30 12:36:08.382 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-30T12:36:08.493201133Z 2026-06-30 12:36:08.492 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-30T12:36:08.493350081Z 2026-06-30 12:36:08.493 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-30T12:36:08.493501086Z 2026-06-30 12:36:08.493 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-30T12:36:08.530009957Z 2026-06-30 12:36:08.529 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-30T12:36:09.586780668Z 2026-06-30 12:36:09.586 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-30T12:36:09.743722871Z 2026-06-30 12:36:09.743 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-30T12:36:09.745825393Z 2026-06-30 12:36:09.743 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-30T12:36:09.754710710Z 2026-06-30 12:36:09.754 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-30T12:36:09.760131196Z 2026-06-30 12:36:09.759 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-30T12:36:09.764446794Z 2026-06-30 12:36:09.764 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-30T12:36:09.764641550Z 2026-06-30 12:36:09.764 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-30T12:36:09.800962873Z 2026-06-30 12:36:09.800 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-30T12:36:09.809582523Z 2026-06-30 12:36:09.809 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
2026-06-30T12:36:09.836268950Z 2026-06-30 12:36:09.835 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-30T12:36:09.848651945Z 2026-06-30 12:36:09.848 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-06-30T12:36:09.864199171Z 2026-06-30 12:36:09.863 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-06-30T12:36:09.874498816Z 2026-06-30 12:36:09.874 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-30T12:36:09.928701897Z 2026-06-30 12:36:09.928 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-30T12:36:10.210828671Z 2026-06-30 12:36:10.210 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-30T12:36:10.399466477Z 2026-06-30 12:36:10.398 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-30T12:36:10.667609046Z 2026-06-30 12:36:10.667 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-30T12:36:10.794966884Z 2026-06-30 12:36:10.794 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-30T12:36:10.800969044Z 2026-06-30 12:36:10.800 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_30.request.log
2026-06-30T12:36:10.804461011Z 2026-06-30 12:36:10.803 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-06-30T12:36:10.805258608Z 2026-06-30 12:36:10.805 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @5434ms
2026-06-30T12:36:11.030462368Z 2026-06-30 12:36:11.029 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=58
2026-06-30T12:36:11.160001881Z 2026-06-30 12:36:11.159 INFO (qtp985324122-56-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=17
2026-06-30T12:36:17.224299106Z 2026-06-30 12:36:17.223 INFO (qtp985324122-58-solr-node1-6) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-06-30T12:36:17.291832663Z 2026-06-30 12:36:17.289 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-30T12:36:17.401378993Z 2026-06-30 12:36:17.400 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-30T12:36:17.917359349Z 2026-06-30 12:36:17.916 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-30T12:36:17.917394256Z 2026-06-30 12:36:17.916 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-06-30T12:36:17.949569753Z 2026-06-30 12:36:17.949 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-06-30T12:36:18.729512404Z 2026-06-30 12:36:18.728 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-30T12:36:18.755532434Z 2026-06-30 12:36:18.755 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-30T12:36:18.770900992Z 2026-06-30 12:36:18.770 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-30T12:36:18.772301223Z 2026-06-30 12:36:18.771 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-30T12:36:18.772375185Z 2026-06-30 12:36:18.772 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-30T12:36:18.836414113Z 2026-06-30 12:36:18.835 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-30T12:36:18.840540057Z 2026-06-30 12:36:18.840 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-30T12:36:18.843240280Z 2026-06-30 12:36:18.842 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-30T12:36:18.845993119Z 2026-06-30 12:36:18.845 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-30T12:36:18.867526846Z 2026-06-30 12:36:18.866 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-06-30T12:36:18.877353482Z 2026-06-30 12:36:18.876 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-30T12:36:18.885125923Z 2026-06-30 12:36:18.884 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-30T12:36:18.885148125Z 2026-06-30 12:36:18.884 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-30T12:36:18.906970171Z 2026-06-30 12:36:18.906 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-30T12:36:18.930691200Z 2026-06-30 12:36:18.929 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-06-30T12:36:18.932400239Z 2026-06-30 12:36:18.932 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1
2026-06-30T12:36:18.950225987Z 2026-06-30 12:36:18.949 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-30T12:36:18.950286988Z 2026-06-30 12:36:18.949 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-30T12:36:18.950351077Z 2026-06-30 12:36:18.950 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node2:8984/solr/test-collection_shard1_replica_n4/
2026-06-30T12:36:18.952272822Z 2026-06-30 12:36:18.951 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-30T12:36:18.953258334Z 2026-06-30 12:36:18.953 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ has no replicas
2026-06-30T12:36:18.954926918Z 2026-06-30 12:36:18.954 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard1/leader after winning as /collections/test-collection/leader_elect/shard1/election/72211905013022724-core_node6-n_0000000000
2026-06-30T12:36:18.973141714Z 2026-06-30 12:36:18.972 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ shard1
2026-06-30T12:36:19.083681932Z 2026-06-30 12:36:19.083 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 219] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-30T12:36:19.089548646Z 2026-06-30 12:36:19.089 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-30T12:36:19.103633828Z 2026-06-30 12:36:19.103 INFO (qtp985324122-58-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1881
2026-06-30T12:36:19.206266158Z 2026-06-30 12:36:19.205 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 223] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-30T12:36:20.148495885Z 2026-06-30 12:36:20.147 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 238] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-30T12:36:20.259948258Z 2026-06-30 12:36:20.259 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node2 ===
{
"Name": "/solr-node2",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 3874843,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-30T12:36:04.796102086Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node3 ===
2026-06-30T12:36:05.352697068Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-30T12:36:05.352722043Z It should be set to 65000 to avoid operational disruption.
2026-06-30T12:36:05.352725204Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-30T12:36:05.353083003Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-30T12:36:05.353095323Z
2026-06-30T12:36:05.362672573Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-30T12:36:05.371923394Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-30T12:36:05.424665619Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-30T12:36:05.462181551Z WARNING: A command line option has enabled the Security Manager
2026-06-30T12:36:05.462202669Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-30T12:36:07.018495886Z 2026-06-30 12:36:07.014 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-30T12:36:07.338992303Z 2026-06-30 12:36:07.338 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-30T12:36:07.371255765Z 2026-06-30 12:36:07.370 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-30T12:36:07.381430818Z 2026-06-30 12:36:07.380 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-30T12:36:07.381501108Z 2026-06-30 12:36:07.381 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8985
2026-06-30T12:36:07.381621174Z 2026-06-30 12:36:07.381 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-30T12:36:07.381726936Z 2026-06-30 12:36:07.381 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-30T12:36:07.381199368Z
2026-06-30T12:36:07.384783909Z 2026-06-30 12:36:07.384 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-30T12:36:07.421295016Z 2026-06-30 12:36:07.420 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-30T12:36:07.489356333Z 2026-06-30 12:36:07.488 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-30T12:36:07.663368557Z 2026-06-30 12:36:07.662 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-30T12:36:07.684918498Z 2026-06-30 12:36:07.684 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-30T12:36:07.685024769Z 2026-06-30 12:36:07.684 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-30T12:36:07.685162403Z 2026-06-30 12:36:07.684 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-30T12:36:07.800958684Z 2026-06-30 12:36:07.800 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-30T12:36:07.802034867Z 2026-06-30 12:36:07.800 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-30T12:36:07.802235307Z 2026-06-30 12:36:07.801 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-30T12:36:07.856849195Z 2026-06-30 12:36:07.856 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-30T12:36:08.899447678Z 2026-06-30 12:36:08.898 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-30T12:36:09.123366398Z 2026-06-30 12:36:09.122 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-30T12:36:09.125436362Z 2026-06-30 12:36:09.122 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-30T12:36:09.138662035Z 2026-06-30 12:36:09.138 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-30T12:36:09.146377911Z 2026-06-30 12:36:09.145 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-30T12:36:09.149916037Z 2026-06-30 12:36:09.149 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-30T12:36:09.150133790Z 2026-06-30 12:36:09.149 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-30T12:36:09.189254720Z 2026-06-30 12:36:09.188 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-30T12:36:09.197620347Z 2026-06-30 12:36:09.197 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-30T12:36:09.231503799Z 2026-06-30 12:36:09.230 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-30T12:36:09.243707563Z 2026-06-30 12:36:09.243 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node3:8985_solr as DOWN
2026-06-30T12:36:09.255457775Z 2026-06-30 12:36:09.254 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node3:8985_solr
2026-06-30T12:36:09.267734741Z 2026-06-30 12:36:09.267 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-30T12:36:09.335603339Z 2026-06-30 12:36:09.334 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-30T12:36:09.655132527Z 2026-06-30 12:36:09.654 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-30T12:36:09.873642195Z 2026-06-30 12:36:09.873 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-30T12:36:09.891950947Z 2026-06-30 12:36:09.891 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-30T12:36:10.217638422Z 2026-06-30 12:36:10.217 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-30T12:36:10.368035789Z 2026-06-30 12:36:10.367 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-30T12:36:10.375329305Z 2026-06-30 12:36:10.374 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_30.request.log
2026-06-30T12:36:10.381266526Z 2026-06-30 12:36:10.380 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8985}
2026-06-30T12:36:10.382174004Z 2026-06-30 12:36:10.381 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @5011ms
2026-06-30T12:36:13.860142370Z 2026-06-30 12:36:13.855 INFO (qtp985324122-27-solr-node3-1) [c: s: r: x: t:solr-node3-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=116
2026-06-30T12:36:13.962891801Z 2026-06-30 12:36:13.962 INFO (qtp985324122-56-solr-node3-2) [c: s: r: x: t:solr-node3-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=22
2026-06-30T12:36:16.718824764Z 2026-06-30 12:36:16.718 INFO (qtp985324122-58-solr-node3-3) [c: s: r: x: t:solr-node3-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=10
2026-06-30T12:36:17.224004464Z 2026-06-30 12:36:17.222 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node5&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&shard=shard2&wt=javabin
2026-06-30T12:36:17.301565699Z 2026-06-30 12:36:17.300 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-30T12:36:17.408899788Z 2026-06-30 12:36:17.408 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-30T12:36:17.919108397Z 2026-06-30 12:36:17.918 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-30T12:36:17.919157776Z 2026-06-30 12:36:17.918 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard2_replica_n2' using configuration from configset techproducts, trusted=true
2026-06-30T12:36:17.949576739Z 2026-06-30 12:36:17.949 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard2_replica_n2], dataDir=[/var/solr/data/test-collection_shard2_replica_n2/data/]
2026-06-30T12:36:18.729459412Z 2026-06-30 12:36:18.728 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-30T12:36:18.755590987Z 2026-06-30 12:36:18.755 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-30T12:36:18.770904133Z 2026-06-30 12:36:18.770 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-30T12:36:18.772104995Z 2026-06-30 12:36:18.771 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-30T12:36:18.772172428Z 2026-06-30 12:36:18.771 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-30T12:36:18.836413094Z 2026-06-30 12:36:18.835 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-30T12:36:18.840415249Z 2026-06-30 12:36:18.839 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-30T12:36:18.844296865Z 2026-06-30 12:36:18.843 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-30T12:36:18.855184622Z 2026-06-30 12:36:18.854 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-30T12:36:18.870195349Z 2026-06-30 12:36:18.869 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4}
2026-06-30T12:36:18.879038416Z 2026-06-30 12:36:18.878 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-30T12:36:18.885164852Z 2026-06-30 12:36:18.884 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-30T12:36:18.885244874Z 2026-06-30 12:36:18.884 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-30T12:36:18.907288751Z 2026-06-30 12:36:18.906 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-30T12:36:18.930588718Z 2026-06-30 12:36:18.929 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard2 to Terms{values={core_node5=0}, version=0} for registerTerm
2026-06-30T12:36:18.932238116Z 2026-06-30 12:36:18.931 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard2
2026-06-30T12:36:18.949747160Z 2026-06-30 12:36:18.949 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-30T12:36:18.949776009Z 2026-06-30 12:36:18.949 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-30T12:36:18.949793068Z 2026-06-30 12:36:18.949 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node3:8985/solr/test-collection_shard2_replica_n2/
2026-06-30T12:36:18.951812284Z 2026-06-30 12:36:18.951 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-30T12:36:18.952369594Z 2026-06-30 12:36:18.952 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ has no replicas
2026-06-30T12:36:18.954284507Z 2026-06-30 12:36:18.954 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard2/leader after winning as /collections/test-collection/leader_elect/shard2/election/72211905013022723-core_node5-n_0000000000
2026-06-30T12:36:18.975646137Z 2026-06-30 12:36:18.975 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ shard2
2026-06-30T12:36:19.083723174Z 2026-06-30 12:36:19.083 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 219] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-30T12:36:19.089161179Z 2026-06-30 12:36:19.088 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-30T12:36:19.102369616Z 2026-06-30 12:36:19.101 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=techproducts&newCollection=true&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&collection=test-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1880
2026-06-30T12:36:19.206448907Z 2026-06-30 12:36:19.205 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 223] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-30T12:36:20.148296989Z 2026-06-30 12:36:20.147 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 238] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-30T12:36:20.173438126Z 2026-06-30 12:36:20.172 INFO (qtp985324122-59-solr-node3-4) [c:test-collection s: r: x: t:solr-node3-4] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
2026-06-30T12:36:20.260054617Z 2026-06-30 12:36:20.259 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-30T12:36:20.262618990Z 2026-06-30 12:36:20.262 INFO (qtp985324122-59-solr-node3-4) [c:test-collection s: r: x: t:solr-node3-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=3522
# === Docker inspect for solr-node3 ===
{
"Name": "/solr-node3",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 3874970,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-30T12:36:05.001479307Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# Docker artifacts saved to: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/docker
not ok 85 Docker SolrCloud rolling upgrade # in 31169 ms
# (in test file test/test_rolling_upgrade.bats, line 118)
# `wait_for 30 1 docker exec solr-node1 solr healthcheck -c test-collection' failed with status 127
# 9.10.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# Digest: sha256:15c8155ceffe1042f37c77bb537aa531804a3510702ff87f2af5fedf523f4203
# Status: Image is up to date for apache/solr-nightly:9.10.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:9.10.0-SNAPSHOT-slim
# 10.0.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# Digest: sha256:fb25f7a7190e3027ade20e97708a8665d41ab0e9ea9285fa242b2c83b0e5f43c
# Status: Image is up to date for apache/solr-nightly:10.0.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:10.0.0-SNAPSHOT-slim
# 649b7c42f74ea036f61c7cf952c066c83501dd51fb60557609b30eea9737c0a3
# solr-data1
# solr-data2
# solr-data3
# Starting solr-node1 with embedded ZooKeeper
# a0f85b1c1772d934d438563f26b5e9faa9baa2ae57025c13b15db1a97fd1539f
# Starting solr-node2 connected to first node's ZooKeeper
# 069309e2a2e2dab82a50fc0d975525773d88c7995f809dd0a189ce0c46de2e09
# Started solr-node3 connected to first node's ZooKeeper
# bdaa1dfa287b895579204a77d08082948b265a849c3b5f2cc4d6ad98b5524a64
# Creating a Collection
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# WARN - 2026-06-30 12:36:16.355; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-30 12:36:16.438; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. "/solr"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [http://solr-node3:8985/solr] to [http://solr-node3:8985/].
# Created collection 'test-collection' with 3 shard(s), 1 replica(s) with config-set 'techproducts'
# Checking collection health
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_rolling_upgrade.bats: line 118: wait_for: command not found
# solr-node1
# solr-node2
# solr-node3
# solr-node1
# solr-node2
# solr-node3
# solr-data1
# solr-data2
# solr-data3
# solrcloud-test
ok 86 allowPaths - backup # in 16371 ms
ok 87 snapshot lifecycle # in 5043 ms
ok 88 snapshot list # in 3434 ms
ok 89 snapshot list via --solr-connection # in 3467 ms
ok 90 snapshot describe # in 3506 ms
ok 91 snapshot describe via --solr-connection # in 3619 ms
ok 92 start solr with ssl # in 17316 ms
ok 93 use different hostname when not checking peer-name # in 25267 ms
ok 94 start solr with ssl and auth # in 15169 ms
ok 95 start solr with client truststore and security manager # in 17086 ms
ok 96 start solr with mTLS needed # in 47267 ms
ok 97 start solr with mTLS wanted # in 35975 ms
ok 98 test keystore reload # in 60702 ms
ok 99 SOLR-11740 check 'solr stop' connection # in 16316 ms
ok 100 stop command for single port # in 18749 ms
ok 101 check stop command doesn't hang # in 225 ms
ok 102 SOLR-16976 solr starts with remote JMX enabled # in 8310 ms
ok 103 deprecated system properties converted to modern properties # in 195424 ms
ok 104 start with custom jetty options # in 8138 ms
ok 105 -c flag prints no-op warning and still starts in cloud mode # in 9133 ms
ok 106 bootstrapping a configset # in 9544 ms
ok 107 status detects locally running solr # in 9361 ms
ok 108 status with --solr-url from user # in 8075 ms
ok 109 status with --port from user # in 8034 ms
ok 110 multiple connection options are prevented # in 850 ms
ok 111 status with invalid --solr-url from user # in 1237 ms
ok 112 status with --short format # in 195837 ms
ok 113 searching solr via locally executed streaming expression # in 1741 ms
ok 114 searching solr via remotely executed streaming expression by solr-url # in 1507 ms
ok 115 searching solr via remotely executed streaming expression by solr connection # in 1197 ms
ok 116 variable interpolation # in 1751 ms
ok 117 searching solr without credentials fails with error # in 1164 ms
ok 118 --version returns Solr version # in 731 ms
ok 119 version as direct tool call still runs # in 729 ms
ok 120 short help # in 752 ms
ok 121 short help is inferred # in 763 ms
ok 122 long help # in 768 ms
ok 123 running subcommands with zk is prevented # in 740 ms
ok 124 listing out files # in 2129 ms
ok 125 connecting to solr via various solr urls and zk hosts # in 9191 ms
ok 126 copying files around # in 10834 ms
ok 127 upconfig # in 2566 ms
ok 128 SOLR-12429 test upconfig fails with symlink # in 1154 ms
ok 129 downconfig # in 1191 ms
ok 130 bin/solr zk cp gets 'solrhome' from '--solr-home' command line option # in 1451 ms
ok 131 env var ZK_HOST is honored # in 3173 ms
ok 132 Cleanup # in 1844 ms
> Task :solr:packaging:integrationTests FAILED
[Incubating] Problems report is available at: file:///home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/build/reports/problems/problems-report.html
FAILURE: Build failed with an exception.
* Where:
Build file '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build.gradle' line: 339
* What went wrong:
Execution failed for task ':solr:packaging:integrationTests' (registered in build file 'solr/packaging/build.gradle').
> Process 'command '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/.gradle/node/node_modules/bats/bin/bats'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
Deprecated Gradle features were used in this build, making it incompatible with Gradle 10.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/9.6.0/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 31m 35s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/wtd3utwi3v6hk
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 248 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/248/
No tests ran.
Build Log:
[...truncated 1004 lines...]
BUILD FAILED in 30m 27s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/emaopxwt3jqki
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 247 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/247/
No tests ran.
Build Log:
[...truncated 347 lines...]
# ERROR: KeeperErrorCode = NoNode for /security.json
# --
#
# Last output:
# WARN - 2026-06-28 13:24:53.710; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-28 13:24:53.714; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-28 13:24:54.205; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-28 13:24:54.206; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: KeeperErrorCode = NoNode for /security.json
not ok 9 create, config, api, and delete with basic auth # in 1448 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 46267 is already being used by another process (pid: 2712210)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-06-28 13:24:55.515; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-28 13:24:55.519; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
ok 10 post, postlogs and export with basic auth # in 19521 ms
ok 11 package with basic auth # in 13212 ms
ok 12 admin UI static content is excepted from basic auth # in 11529 ms
ok 13 nothing # in 11 ms
ok 14 server does not compress response without Accept-Encoding header # in 168 ms
ok 15 server compresses response when Accept-Encoding: gzip is requested # in 100 ms
ok 16 compressed response can be decompressed and parsed # in 86 ms
ok 17 setting property # in 13516 ms
ok 18 short form of setting property # in 6994 ms
ok 19 connecting to solr via various connection types # in 11756 ms
ok 20 create for non cloud mode # in 8602 ms
ok 21 create for cloud mode # in 10674 ms
ok 22 multiple connection options are prevented # in 964 ms
ok 23 create collection # in 8990 ms
ok 24 create collection using solrUrl # in 5654 ms
ok 25 create collection using legacy solrUrl # in 5611 ms
ok 26 create collection using Zookeeper # in 5875 ms
ok 27 reject d option with invalid config dir # in 2895 ms
ok 28 accept d option with builtin config # in 5619 ms
ok 29 accept d option with explicit path to config # in 5550 ms
ok 30 accept n option as config name # in 5577 ms
ok 31 allow config reuse when n option specifies same config # in 9770 ms
ok 32 create multisharded collections when s provided # in 5747 ms
ok 33 create replicated collections when rf provided # in 6522 ms
ok 34 can delete collections # in 12124 ms
ok 35 can delete collections with solr-url # in 8993 ms
ok 36 can delete collections with solr-connection # in 8597 ms
ok 37 collection delete also deletes zk config # in 7981 ms
ok 38 deletes accompanying zk config with nondefault name # in 8006 ms
ok 39 delete-config option can opt to leave config in zk # in 7679 ms
ok 40 start -e cloud works with --jvm-opts # in 23296 ms
ok 41 start -e cloud works with --no-prompt # in 44383 ms
ok 42 Check export command # in 20641 ms
ok 43 export fails on non cloud mode # in 9981 ms
Waiting for Tika Server to be ready on port 46272
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 44 using curl to extract a single pdf file # in 16776 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to extract content from pdf # in 16639 ms
WARNING: Docker not available; bypassing test.
ok 46 using the bin/solr post tool to crawl web site # in 16682 ms
Stopping Tika Server container
ok 47 healthcheck on cloud solr # in 11076 ms
ok 48 healthcheck on remote cloud solr # in 11115 ms
ok 49 healthcheck errors on standalone solr # in 12452 ms
ok 50 solr help flag prints help # in 829 ms
ok 51 solr with no flags prints help # in 819 ms
ok 52 start help flag prints help # in 167 ms
ok 53 start h flag prints help # in 163 ms
ok 54 stop help flag prints help # in 158 ms
ok 55 restart help flag prints help # in 161 ms
ok 56 status help flag prints help # in 878 ms
ok 57 healthcheck help flag prints help # in 829 ms
ok 58 create help flag prints help # in 827 ms
ok 59 delete help flag prints help # in 871 ms
ok 60 zk help flag prints help # in 852 ms
ok 61 auth help flag prints help # in 832 ms
ok 62 assert help flag prints help # in 836 ms
ok 63 post help flag prints help # in 834 ms
ok 64 SQL Module # in 17449 ms
ok 65 icu collation in analysis-extras module # in 13779 ms
ok 66 Check lifecycle of sentiment classification # in 24529 ms
ok 67 lifecycle of package # in 9160 ms
ok 68 deploying and undeploying a collection level package # in 14443 ms
ok 69 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
ok 70 Affinity placement plugin using sysprop # in 17266 ms
ok 71 Random placement plugin using ENV # in 17247 ms
ok 72 Check help command # in 2676 ms
ok 73 basic post with a type specified # in 6013 ms
ok 74 basic post with collection instead of url parameter # in 4496 ms
ok 75 basic post with solr-url and collection # in 4409 ms
ok 76 basic post with -s (solr-connection short form) and collection # in 4330 ms
ok 77 basic post WITHOUT a type specified # in 13048 ms
ok 78 crawling a directory as a dry-run # in 853 ms
ok 79 crawling a directory # in 4827 ms
ok 80 skipcommit and optimize and delete # in 4214 ms
ok 81 args mode # in 7612 ms
ok 82 stdin mode # in 3711 ms
ok 83 verbose echo the Solr response # in 4256 ms
ok 84 post solr log into solr via cli # in 10745 ms
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 15 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-06-28T13:36:07.539438184Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-28T13:36:07.539478755Z It should be set to 65000 to avoid operational disruption.
2026-06-28T13:36:07.539481917Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-28T13:36:07.539781475Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-28T13:36:07.539792845Z
2026-06-28T13:36:07.551073984Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-28T13:36:07.559888889Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-28T13:36:07.605340559Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-28T13:36:07.639510463Z WARNING: A command line option has enabled the Security Manager
2026-06-28T13:36:07.639540653Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-28T13:36:08.904684420Z 2026-06-28 13:36:08.901 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-28T13:36:09.194715929Z 2026-06-28 13:36:09.194 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-28T13:36:09.221361743Z 2026-06-28 13:36:09.220 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-28T13:36:09.229580238Z 2026-06-28 13:36:09.228 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-28T13:36:09.229784382Z 2026-06-28 13:36:09.228 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-06-28T13:36:09.230030325Z 2026-06-28 13:36:09.228 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-28T13:36:09.230237650Z 2026-06-28 13:36:09.228 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-28T13:36:09.228914300Z
2026-06-28T13:36:09.232757949Z 2026-06-28 13:36:09.232 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-28T13:36:09.255915768Z 2026-06-28 13:36:09.255 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-28T13:36:09.270574147Z 2026-06-28 13:36:09.269 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-28T13:36:09.270596286Z 2026-06-28 13:36:09.269 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-28T13:36:09.308430012Z 2026-06-28 13:36:09.307 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-28T13:36:10.084324211Z 2026-06-28 13:36:10.083 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-28T13:36:10.344278280Z 2026-06-28 13:36:10.343 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-28T13:36:10.350028653Z 2026-06-28 13:36:10.349 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-06-28T13:36:10.350192270Z 2026-06-28 13:36:10.349 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-06-28T13:36:10.353034217Z 2026-06-28 13:36:10.352 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-06-28T13:36:10.353056998Z 2026-06-28 13:36:10.352 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-06-28T13:36:10.390732252Z 2026-06-28 13:36:10.390 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-06-28T13:36:10.859940802Z 2026-06-28 13:36:10.858 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-06-28T13:36:10.874824248Z 2026-06-28 13:36:10.874 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-28T13:36:10.925971862Z 2026-06-28 13:36:10.925 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-28T13:36:11.020984500Z 2026-06-28 13:36:11.019 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-28T13:36:11.021246302Z 2026-06-28 13:36:11.019 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-28T13:36:11.176089981Z 2026-06-28 13:36:11.175 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-28T13:36:11.206686098Z 2026-06-28 13:36:11.206 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-28T13:36:11.223157566Z 2026-06-28 13:36:11.222 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-06-28T13:36:11.230273536Z 2026-06-28 13:36:11.229 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72200816973905920-solr-node1:8983_solr-n_0000000000) starting
2026-06-28T13:36:11.308505164Z 2026-06-28 13:36:11.307 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-06-28T13:36:11.309469463Z 2026-06-28 13:36:11.309 INFO (OverseerStateUpdate-72200816973905920-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-06-28T13:36:11.321450110Z 2026-06-28 13:36:11.320 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-28T13:36:11.370159651Z 2026-06-28 13:36:11.369 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-28T13:36:11.827056014Z 2026-06-28 13:36:11.826 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-28T13:36:12.013541339Z 2026-06-28 13:36:12.013 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-28T13:36:12.295731583Z 2026-06-28 13:36:12.295 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-28T13:36:12.435948701Z 2026-06-28 13:36:12.435 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-28T13:36:12.441957700Z 2026-06-28 13:36:12.441 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_28.request.log
2026-06-28T13:36:12.446443937Z 2026-06-28 13:36:12.445 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-06-28T13:36:12.487366548Z 2026-06-28 13:36:12.486 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4927ms
2026-06-28T13:36:12.669777164Z 2026-06-28 13:36:12.669 INFO (qtp985324122-27-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=51
2026-06-28T13:36:12.732821299Z 2026-06-28 13:36:12.732 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=16
2026-06-28T13:36:16.124848460Z 2026-06-28 13:36:16.124 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-28T13:36:16.440961283Z 2026-06-28 13:36:16.440 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-28T13:36:20.438515384Z 2026-06-28 13:36:20.438 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=9
2026-06-28T13:36:20.466363717Z 2026-06-28 13:36:20.465 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=9
2026-06-28T13:36:20.495054995Z 2026-06-28 13:36:20.494 INFO (qtp985324122-27-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=2
2026-06-28T13:36:20.736817691Z 2026-06-28 13:36:20.735 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=6
2026-06-28T13:36:20.765417908Z 2026-06-28 13:36:20.765 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-06-28T13:36:20.784593977Z 2026-06-28 13:36:20.784 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-06-28T13:36:20.790491099Z 2026-06-28 13:36:20.790 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 174] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-28T13:36:20.839319425Z 2026-06-28 13:36:20.838 INFO (OverseerStateUpdate-72200816973905920-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-28T13:36:20.839339994Z "core":"test-collection_shard3_replica_n1",
2026-06-28T13:36:20.839343213Z "node_name":"solr-node1:8983_solr",
2026-06-28T13:36:20.839345381Z "base_url":"http://solr-node1:8983/solr",
2026-06-28T13:36:20.839347495Z "collection":"test-collection",
2026-06-28T13:36:20.839349641Z "shard":"shard3",
2026-06-28T13:36:20.839351730Z "state":"down",
2026-06-28T13:36:20.839353747Z "type":"NRT",
2026-06-28T13:36:20.839355830Z "operation":"ADDREPLICA",
2026-06-28T13:36:20.839357789Z "waitForFinalState":"false"}
2026-06-28T13:36:20.847394919Z 2026-06-28 13:36:20.846 INFO (OverseerStateUpdate-72200816973905920-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-28T13:36:20.847420288Z "core":"test-collection_shard2_replica_n2",
2026-06-28T13:36:20.847423202Z "node_name":"solr-node3:8985_solr",
2026-06-28T13:36:20.847425414Z "base_url":"http://solr-node3:8985/solr",
2026-06-28T13:36:20.847427674Z "collection":"test-collection",
2026-06-28T13:36:20.847429731Z "shard":"shard2",
2026-06-28T13:36:20.847431841Z "state":"down",
2026-06-28T13:36:20.847433834Z "type":"NRT",
2026-06-28T13:36:20.847435811Z "operation":"ADDREPLICA",
2026-06-28T13:36:20.847437883Z "waitForFinalState":"false"}
2026-06-28T13:36:20.858456720Z 2026-06-28 13:36:20.857 INFO (OverseerStateUpdate-72200816973905920-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-28T13:36:20.858489456Z "core":"test-collection_shard1_replica_n4",
2026-06-28T13:36:20.858492412Z "node_name":"solr-node2:8984_solr",
2026-06-28T13:36:20.858494544Z "base_url":"http://solr-node2:8984/solr",
2026-06-28T13:36:20.858496525Z "collection":"test-collection",
2026-06-28T13:36:20.858498481Z "shard":"shard1",
2026-06-28T13:36:20.858500393Z "state":"down",
2026-06-28T13:36:20.858502278Z "type":"NRT",
2026-06-28T13:36:20.858504324Z "operation":"ADDREPLICA",
2026-06-28T13:36:20.858506292Z "waitForFinalState":"false"}
2026-06-28T13:36:20.972694362Z 2026-06-28 13:36:20.972 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 186] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-28T13:36:21.062312630Z 2026-06-28 13:36:21.061 INFO (qtp985324122-29-solr-node1-8) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-06-28T13:36:21.165112712Z 2026-06-28 13:36:21.164 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-28T13:36:21.213747945Z 2026-06-28 13:36:21.213 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-06-28T13:36:21.544680830Z 2026-06-28 13:36:21.544 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-28T13:36:21.544703728Z 2026-06-28 13:36:21.544 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true
2026-06-28T13:36:21.579726809Z 2026-06-28 13:36:21.579 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/]
2026-06-28T13:36:22.369822464Z 2026-06-28 13:36:22.368 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-28T13:36:22.407898332Z 2026-06-28 13:36:22.406 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-28T13:36:22.415950172Z 2026-06-28 13:36:22.415 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-28T13:36:22.417363755Z 2026-06-28 13:36:22.416 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-28T13:36:22.417461812Z 2026-06-28 13:36:22.416 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-28T13:36:22.491952769Z 2026-06-28 13:36:22.491 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-28T13:36:22.496737667Z 2026-06-28 13:36:22.496 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-28T13:36:22.503251987Z 2026-06-28 13:36:22.502 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-28T13:36:22.505995348Z 2026-06-28 13:36:22.504 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-28T13:36:22.526179802Z 2026-06-28 13:36:22.525 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4}
2026-06-28T13:36:22.535416601Z 2026-06-28 13:36:22.535 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-28T13:36:22.544875166Z 2026-06-28 13:36:22.544 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-28T13:36:22.545016524Z 2026-06-28 13:36:22.544 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-28T13:36:22.554905614Z 2026-06-28 13:36:22.554 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-28T13:36:22.575857237Z 2026-06-28 13:36:22.575 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-06-28T13:36:22.577704034Z 2026-06-28 13:36:22.577 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-06-28T13:36:22.604258309Z 2026-06-28 13:36:22.603 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-28T13:36:22.604419987Z 2026-06-28 13:36:22.603 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-28T13:36:22.604433060Z 2026-06-28 13:36:22.604 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/
2026-06-28T13:36:22.605968736Z 2026-06-28 13:36:22.605 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-28T13:36:22.606737753Z 2026-06-28 13:36:22.606 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas
2026-06-28T13:36:22.608096595Z 2026-06-28 13:36:22.607 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72200816973905920-core_node3-n_0000000000
2026-06-28T13:36:22.614889002Z 2026-06-28 13:36:22.614 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3
2026-06-28T13:36:22.719895730Z 2026-06-28 13:36:22.719 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 237] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-28T13:36:22.726757929Z 2026-06-28 13:36:22.726 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-28T13:36:22.742397679Z 2026-06-28 13:36:22.741 INFO (qtp985324122-29-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1682
2026-06-28T13:36:22.753125660Z 2026-06-28 13:36:22.752 INFO (qtp985324122-101-solr-node1-7) [c:test-collection s: r: x: t:solr-node1-7] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
2026-06-28T13:36:22.842691929Z 2026-06-28 13:36:22.842 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-28T13:36:22.846769678Z 2026-06-28 13:36:22.846 INFO (qtp985324122-101-solr-node1-7) [c:test-collection s: r: x: t:solr-node1-7] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=2097
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 2746146,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-28T13:36:07.30852805Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-06-28T13:36:13.049647037Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-28T13:36:13.049682417Z It should be set to 65000 to avoid operational disruption.
2026-06-28T13:36:13.049685758Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-28T13:36:13.050013365Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-28T13:36:13.050026301Z
2026-06-28T13:36:13.060746850Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-28T13:36:13.069336663Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-28T13:36:13.113759727Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-28T13:36:13.153561217Z WARNING: A command line option has enabled the Security Manager
2026-06-28T13:36:13.153585885Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-28T13:36:14.319307615Z 2026-06-28 13:36:14.315 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-28T13:36:14.623581639Z 2026-06-28 13:36:14.622 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-28T13:36:14.651783101Z 2026-06-28 13:36:14.650 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-28T13:36:14.659046834Z 2026-06-28 13:36:14.658 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-28T13:36:14.659956032Z 2026-06-28 13:36:14.658 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-06-28T13:36:14.659975380Z 2026-06-28 13:36:14.658 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-28T13:36:14.659978628Z 2026-06-28 13:36:14.658 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-28T13:36:14.658616029Z
2026-06-28T13:36:14.662089049Z 2026-06-28 13:36:14.661 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-28T13:36:14.685760391Z 2026-06-28 13:36:14.685 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-28T13:36:14.712061463Z 2026-06-28 13:36:14.711 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-28T13:36:14.876684153Z 2026-06-28 13:36:14.876 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-28T13:36:14.895970270Z 2026-06-28 13:36:14.895 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-28T13:36:14.896209175Z 2026-06-28 13:36:14.895 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-28T13:36:14.896359566Z 2026-06-28 13:36:14.895 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-28T13:36:15.007452635Z 2026-06-28 13:36:15.006 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-28T13:36:15.007701414Z 2026-06-28 13:36:15.007 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-28T13:36:15.007790378Z 2026-06-28 13:36:15.007 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-28T13:36:15.045543164Z 2026-06-28 13:36:15.044 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-28T13:36:15.816073900Z 2026-06-28 13:36:15.815 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-28T13:36:15.995345100Z 2026-06-28 13:36:15.992 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-28T13:36:15.997548830Z 2026-06-28 13:36:15.993 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-28T13:36:16.004613988Z 2026-06-28 13:36:16.004 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-28T13:36:16.011567941Z 2026-06-28 13:36:16.010 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-28T13:36:16.016634524Z 2026-06-28 13:36:16.016 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-28T13:36:16.016792065Z 2026-06-28 13:36:16.016 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-28T13:36:16.052530109Z 2026-06-28 13:36:16.051 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-28T13:36:16.061997630Z 2026-06-28 13:36:16.061 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-28T13:36:16.084012619Z 2026-06-28 13:36:16.083 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-28T13:36:16.100307838Z 2026-06-28 13:36:16.099 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-06-28T13:36:16.113173602Z 2026-06-28 13:36:16.112 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-06-28T13:36:16.126291146Z 2026-06-28 13:36:16.125 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-28T13:36:16.240136669Z 2026-06-28 13:36:16.238 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-28T13:36:16.441670781Z 2026-06-28 13:36:16.440 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-28T13:36:16.575626680Z 2026-06-28 13:36:16.574 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-28T13:36:16.789717228Z 2026-06-28 13:36:16.788 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-28T13:36:17.094061285Z 2026-06-28 13:36:17.092 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-28T13:36:17.245095966Z 2026-06-28 13:36:17.244 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-28T13:36:17.252371543Z 2026-06-28 13:36:17.251 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_28.request.log
2026-06-28T13:36:17.258240199Z 2026-06-28 13:36:17.256 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-06-28T13:36:17.258964150Z 2026-06-28 13:36:17.258 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4190ms
2026-06-28T13:36:17.432951807Z 2026-06-28 13:36:17.432 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=49
2026-06-28T13:36:17.488244788Z 2026-06-28 13:36:17.487 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=14
2026-06-28T13:36:21.062496471Z 2026-06-28 13:36:21.061 INFO (qtp985324122-59-solr-node1-8) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-06-28T13:36:21.122251551Z 2026-06-28 13:36:21.121 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-28T13:36:21.183600427Z 2026-06-28 13:36:21.182 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-06-28T13:36:21.515407702Z 2026-06-28 13:36:21.514 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-28T13:36:21.515432195Z 2026-06-28 13:36:21.515 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-06-28T13:36:21.535942007Z 2026-06-28 13:36:21.535 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-06-28T13:36:22.039933891Z 2026-06-28 13:36:22.039 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-28T13:36:22.060443104Z 2026-06-28 13:36:22.059 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-28T13:36:22.063950078Z 2026-06-28 13:36:22.063 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-28T13:36:22.065184561Z 2026-06-28 13:36:22.064 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-28T13:36:22.065200780Z 2026-06-28 13:36:22.064 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-28T13:36:22.172578717Z 2026-06-28 13:36:22.172 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-28T13:36:22.176581683Z 2026-06-28 13:36:22.176 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-28T13:36:22.180140002Z 2026-06-28 13:36:22.179 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-28T13:36:22.182398599Z 2026-06-28 13:36:22.182 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-28T13:36:22.195537180Z 2026-06-28 13:36:22.194 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4}
2026-06-28T13:36:22.203650720Z 2026-06-28 13:36:22.203 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-28T13:36:22.210553023Z 2026-06-28 13:36:22.210 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-28T13:36:22.210573893Z 2026-06-28 13:36:22.210 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-28T13:36:22.219492823Z 2026-06-28 13:36:22.218 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-28T13:36:22.235836129Z 2026-06-28 13:36:22.234 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-06-28T13:36:22.237539933Z 2026-06-28 13:36:22.237 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1
2026-06-28T13:36:22.256758044Z 2026-06-28 13:36:22.256 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-28T13:36:22.256780546Z 2026-06-28 13:36:22.256 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-28T13:36:22.256865833Z 2026-06-28 13:36:22.256 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node2:8984/solr/test-collection_shard1_replica_n4/
2026-06-28T13:36:22.258582813Z 2026-06-28 13:36:22.258 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-28T13:36:22.259588512Z 2026-06-28 13:36:22.259 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ has no replicas
2026-06-28T13:36:22.260881137Z 2026-06-28 13:36:22.260 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard1/leader after winning as /collections/test-collection/leader_elect/shard1/election/72200816973905923-core_node6-n_0000000000
2026-06-28T13:36:22.273667169Z 2026-06-28 13:36:22.273 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ shard1
2026-06-28T13:36:22.383088557Z 2026-06-28 13:36:22.382 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 206] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-28T13:36:22.387394557Z 2026-06-28 13:36:22.386 INFO (qtp985324122-59-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-28T13:36:22.399437428Z 2026-06-28 13:36:22.399 INFO (qtp985324122-59-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1338
2026-06-28T13:36:22.501168498Z 2026-06-28 13:36:22.499 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 209] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-28T13:36:22.720208791Z 2026-06-28 13:36:22.719 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 237] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-28T13:36:22.843276129Z 2026-06-28 13:36:22.842 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node2 ===
{
"Name": "/solr-node2",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 2746543,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-28T13:36:12.831103404Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node3 ===
2026-06-28T13:36:13.278493121Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-28T13:36:13.278536525Z It should be set to 65000 to avoid operational disruption.
2026-06-28T13:36:13.278541510Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-28T13:36:13.278852190Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-28T13:36:13.278859308Z
2026-06-28T13:36:13.289146488Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-28T13:36:13.297896227Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-28T13:36:13.344454947Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-28T13:36:13.377732640Z WARNING: A command line option has enabled the Security Manager
2026-06-28T13:36:13.377757390Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-28T13:36:14.582170591Z 2026-06-28 13:36:14.577 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-28T13:36:14.885658502Z 2026-06-28 13:36:14.884 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-28T13:36:14.913490600Z 2026-06-28 13:36:14.912 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-28T13:36:14.921248552Z 2026-06-28 13:36:14.920 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-28T13:36:14.921361806Z 2026-06-28 13:36:14.920 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8985
2026-06-28T13:36:14.921441414Z 2026-06-28 13:36:14.920 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-28T13:36:14.921603661Z 2026-06-28 13:36:14.920 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-28T13:36:14.920851485Z
2026-06-28T13:36:14.924487710Z 2026-06-28 13:36:14.924 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-28T13:36:14.948021256Z 2026-06-28 13:36:14.947 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-28T13:36:14.974295493Z 2026-06-28 13:36:14.973 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-28T13:36:15.135616144Z 2026-06-28 13:36:15.134 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-28T13:36:15.154750189Z 2026-06-28 13:36:15.154 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-28T13:36:15.154949770Z 2026-06-28 13:36:15.154 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-28T13:36:15.155084276Z 2026-06-28 13:36:15.154 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-28T13:36:15.265718384Z 2026-06-28 13:36:15.265 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-28T13:36:15.265884701Z 2026-06-28 13:36:15.265 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-28T13:36:15.266056265Z 2026-06-28 13:36:15.265 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-28T13:36:15.301455292Z 2026-06-28 13:36:15.300 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-28T13:36:16.155873685Z 2026-06-28 13:36:16.155 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-28T13:36:16.308108190Z 2026-06-28 13:36:16.307 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-28T13:36:16.310368337Z 2026-06-28 13:36:16.307 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-28T13:36:16.319399323Z 2026-06-28 13:36:16.318 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-28T13:36:16.325582295Z 2026-06-28 13:36:16.324 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-28T13:36:16.330516687Z 2026-06-28 13:36:16.330 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-28T13:36:16.330624949Z 2026-06-28 13:36:16.330 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-28T13:36:16.371905515Z 2026-06-28 13:36:16.371 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-28T13:36:16.381400028Z 2026-06-28 13:36:16.380 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
2026-06-28T13:36:16.406361136Z 2026-06-28 13:36:16.405 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-28T13:36:16.419765851Z 2026-06-28 13:36:16.419 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node3:8985_solr as DOWN
2026-06-28T13:36:16.431515535Z 2026-06-28 13:36:16.431 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node3:8985_solr
2026-06-28T13:36:16.442230824Z 2026-06-28 13:36:16.441 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-28T13:36:16.487793712Z 2026-06-28 13:36:16.487 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-28T13:36:16.785274279Z 2026-06-28 13:36:16.784 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-28T13:36:16.970625708Z 2026-06-28 13:36:16.970 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-28T13:36:17.267510993Z 2026-06-28 13:36:17.266 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-28T13:36:17.402637813Z 2026-06-28 13:36:17.401 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-28T13:36:17.408544174Z 2026-06-28 13:36:17.408 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_28.request.log
2026-06-28T13:36:17.412405376Z 2026-06-28 13:36:17.411 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8985}
2026-06-28T13:36:17.413312566Z 2026-06-28 13:36:17.412 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4116ms
2026-06-28T13:36:19.180159214Z 2026-06-28 13:36:19.173 INFO (qtp985324122-27-solr-node3-1) [c: s: r: x: t:solr-node3-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=139
2026-06-28T13:36:19.270031145Z 2026-06-28 13:36:19.269 INFO (qtp985324122-56-solr-node3-2) [c: s: r: x: t:solr-node3-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=27
2026-06-28T13:36:21.066457837Z 2026-06-28 13:36:21.065 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node5&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&shard=shard2&wt=javabin
2026-06-28T13:36:21.159950322Z 2026-06-28 13:36:21.159 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-28T13:36:21.272048024Z 2026-06-28 13:36:21.271 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-06-28T13:36:21.693741037Z 2026-06-28 13:36:21.693 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-28T13:36:21.693765997Z 2026-06-28 13:36:21.693 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard2_replica_n2' using configuration from configset techproducts, trusted=true
2026-06-28T13:36:21.717317421Z 2026-06-28 13:36:21.716 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard2_replica_n2], dataDir=[/var/solr/data/test-collection_shard2_replica_n2/data/]
2026-06-28T13:36:22.361817214Z 2026-06-28 13:36:22.361 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-28T13:36:22.386366859Z 2026-06-28 13:36:22.385 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-28T13:36:22.394204747Z 2026-06-28 13:36:22.393 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-28T13:36:22.398251116Z 2026-06-28 13:36:22.394 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-28T13:36:22.398270042Z 2026-06-28 13:36:22.397 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-28T13:36:22.458654858Z 2026-06-28 13:36:22.458 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-28T13:36:22.463082169Z 2026-06-28 13:36:22.462 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-28T13:36:22.465735560Z 2026-06-28 13:36:22.465 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-28T13:36:22.468891962Z 2026-06-28 13:36:22.468 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-28T13:36:22.482661797Z 2026-06-28 13:36:22.482 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01}
2026-06-28T13:36:22.491026568Z 2026-06-28 13:36:22.490 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-28T13:36:22.498112621Z 2026-06-28 13:36:22.497 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-8 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-28T13:36:22.498520008Z 2026-06-28 13:36:22.497 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-8 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-28T13:36:22.508710382Z 2026-06-28 13:36:22.508 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-8 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-28T13:36:22.523020933Z 2026-06-28 13:36:22.522 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard2 to Terms{values={core_node5=0}, version=0} for registerTerm
2026-06-28T13:36:22.525546906Z 2026-06-28 13:36:22.525 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard2
2026-06-28T13:36:22.543050830Z 2026-06-28 13:36:22.542 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-28T13:36:22.543302330Z 2026-06-28 13:36:22.542 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-28T13:36:22.543320478Z 2026-06-28 13:36:22.542 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node3:8985/solr/test-collection_shard2_replica_n2/
2026-06-28T13:36:22.544642679Z 2026-06-28 13:36:22.544 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-28T13:36:22.545644030Z 2026-06-28 13:36:22.545 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ has no replicas
2026-06-28T13:36:22.546983199Z 2026-06-28 13:36:22.546 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard2/leader after winning as /collections/test-collection/leader_elect/shard2/election/72200816973905924-core_node5-n_0000000000
2026-06-28T13:36:22.569596301Z 2026-06-28 13:36:22.569 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ shard2
2026-06-28T13:36:22.720547891Z 2026-06-28 13:36:22.720 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 237] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-28T13:36:22.725691509Z 2026-06-28 13:36:22.725 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-28T13:36:22.741816093Z 2026-06-28 13:36:22.741 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=techproducts&newCollection=true&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&collection=test-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1678
2026-06-28T13:36:22.843282907Z 2026-06-28 13:36:22.842 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node3 ===
{
"Name": "/solr-node3",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 2746729,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-28T13:36:13.040668007Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# Docker artifacts saved to: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/docker
not ok 85 Docker SolrCloud rolling upgrade # in 30277 ms
# (in test file test/test_rolling_upgrade.bats, line 118)
# `wait_for 30 1 docker exec solr-node1 solr healthcheck -c test-collection' failed with status 127
# 9.10.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# af6eca94c810: Pulling fs layer
# cb0efb96dabd: Pulling fs layer
# 3e9d91201f40: Pulling fs layer
# 66b76b382631: Pulling fs layer
# 601f2c23751f: Pulling fs layer
# 0ed719aef393: Pulling fs layer
# 69a09c96e8bf: Pulling fs layer
# c937aee9a870: Pulling fs layer
# 9cdfe5320de8: Pulling fs layer
# 3aa9120e46f5: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# 69a09c96e8bf: Waiting
# 3aa9120e46f5: Waiting
# c937aee9a870: Waiting
# 9cdfe5320de8: Waiting
# 66b76b382631: Waiting
# 4f4fb700ef54: Waiting
# 601f2c23751f: Waiting
# 0ed719aef393: Waiting
# af6eca94c810: Verifying Checksum
# af6eca94c810: Download complete
# cb0efb96dabd: Verifying Checksum
# cb0efb96dabd: Download complete
# 66b76b382631: Download complete
# 601f2c23751f: Verifying Checksum
# 601f2c23751f: Download complete
# 69a09c96e8bf: Verifying Checksum
# 69a09c96e8bf: Download complete
# 3e9d91201f40: Verifying Checksum
# 3e9d91201f40: Download complete
# c937aee9a870: Verifying Checksum
# c937aee9a870: Download complete
# 9cdfe5320de8: Download complete
# 3aa9120e46f5: Download complete
# 4f4fb700ef54: Verifying Checksum
# 4f4fb700ef54: Download complete
# 0ed719aef393: Verifying Checksum
# 0ed719aef393: Download complete
# af6eca94c810: Pull complete
# cb0efb96dabd: Pull complete
# 3e9d91201f40: Pull complete
# 66b76b382631: Pull complete
# 601f2c23751f: Pull complete
# 0ed719aef393: Pull complete
# 69a09c96e8bf: Pull complete
# c937aee9a870: Pull complete
# 9cdfe5320de8: Pull complete
# 3aa9120e46f5: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:15c8155ceffe1042f37c77bb537aa531804a3510702ff87f2af5fedf523f4203
# Status: Downloaded newer image for apache/solr-nightly:9.10.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:9.10.0-SNAPSHOT-slim
# 10.0.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# 40d16f30db40: Pulling fs layer
# 9d5a0b6cb36c: Pulling fs layer
# 49518c28c0b3: Pulling fs layer
# 41162acdcc29: Pulling fs layer
# 2ed0fd558ca3: Pulling fs layer
# 2488415fda1c: Pulling fs layer
# 09511c8fb203: Pulling fs layer
# a0c57fb89c4f: Pulling fs layer
# d0641dcc41d7: Pulling fs layer
# 315722fd9570: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# 41162acdcc29: Waiting
# 2ed0fd558ca3: Waiting
# 2488415fda1c: Waiting
# 09511c8fb203: Waiting
# a0c57fb89c4f: Waiting
# 315722fd9570: Waiting
# 4f4fb700ef54: Waiting
# d0641dcc41d7: Waiting
# 40d16f30db40: Verifying Checksum
# 40d16f30db40: Download complete
# 9d5a0b6cb36c: Verifying Checksum
# 9d5a0b6cb36c: Download complete
# 41162acdcc29: Verifying Checksum
# 41162acdcc29: Download complete
# 2ed0fd558ca3: Download complete
# 09511c8fb203: Verifying Checksum
# 09511c8fb203: Download complete
# a0c57fb89c4f: Download complete
# 49518c28c0b3: Verifying Checksum
# 49518c28c0b3: Download complete
# d0641dcc41d7: Verifying Checksum
# d0641dcc41d7: Download complete
# 315722fd9570: Verifying Checksum
# 315722fd9570: Download complete
# 4f4fb700ef54: Verifying Checksum
# 4f4fb700ef54: Download complete
# 2488415fda1c: Verifying Checksum
# 2488415fda1c: Download complete
# 40d16f30db40: Pull complete
# 9d5a0b6cb36c: Pull complete
# 49518c28c0b3: Pull complete
# 41162acdcc29: Pull complete
# 2ed0fd558ca3: Pull complete
# 2488415fda1c: Pull complete
# 09511c8fb203: Pull complete
# a0c57fb89c4f: Pull complete
# d0641dcc41d7: Pull complete
# 315722fd9570: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:fb25f7a7190e3027ade20e97708a8665d41ab0e9ea9285fa242b2c83b0e5f43c
# Status: Downloaded newer image for apache/solr-nightly:10.0.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:10.0.0-SNAPSHOT-slim
# 4d3c843a4df37b20def9b41ba3626be43b69ada825462be05f6bac1c335f8771
# solr-data1
# solr-data2
# solr-data3
# Starting solr-node1 with embedded ZooKeeper
# 99c55f7ac6834bff851837fe6f8a80e3bc32986561adcd7bf1cc6f047e8d16df
# Starting solr-node2 connected to first node's ZooKeeper
# eb8f91e796d14efedcbf4b1489a667a046a5fc810414d5a02eb0bc016f37d3a9
# Started solr-node3 connected to first node's ZooKeeper
# 80c1902d7f0892d94773df9cf518f00ba4baf1fccd10881571f7f3da3fd78dc2
# Creating a Collection
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# WARN - 2026-06-28 13:36:20.525; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-28 13:36:20.566; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. "/solr"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [http://solr-node1:8983/solr] to [http://solr-node1:8983/].
# Created collection 'test-collection' with 3 shard(s), 1 replica(s) with config-set 'techproducts'
# Checking collection health
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_rolling_upgrade.bats: line 118: wait_for: command not found
# solr-node1
# solr-node2
# solr-node3
# solr-node1
# solr-node2
# solr-node3
# solr-data1
# solr-data2
# solr-data3
# solrcloud-test
ok 86 allowPaths - backup # in 16748 ms
ok 87 snapshot lifecycle # in 5270 ms
ok 88 snapshot list # in 3675 ms
ok 89 snapshot list via --solr-connection # in 3730 ms
ok 90 snapshot describe # in 3705 ms
ok 91 snapshot describe via --solr-connection # in 3663 ms
ok 92 start solr with ssl # in 17674 ms
ok 93 use different hostname when not checking peer-name # in 26935 ms
ok 94 start solr with ssl and auth # in 16039 ms
ok 95 start solr with client truststore and security manager # in 18137 ms
ok 96 start solr with mTLS needed # in 50342 ms
ok 97 start solr with mTLS wanted # in 38325 ms
ok 98 test keystore reload # in 63134 ms
ok 99 SOLR-11740 check 'solr stop' connection # in 16849 ms
ok 100 stop command for single port # in 19359 ms
ok 101 check stop command doesn't hang # in 244 ms
ok 102 SOLR-16976 solr starts with remote JMX enabled # in 8698 ms
not ok 103 deprecated system properties converted to modern properties # in 195672 ms
# (from function `assert_file_contains' in file .gradle/node/node_modules/bats-file/src/file.bash, line 566,
# in test file test/test_start_solr.bats, line 86)
# `assert_file_contains "${SOLR_LOGS_DIR}/solr.log" 'You are passing in deprecated system property disable.config.edit and should upgrade to using solr.api.config.edit.enabled instead.'' failed
# Waiting up to 180 seconds to see Solr running on port 46267 [|] [/] [-] [\] [|]
# Started Solr server on port 46267 (pid=2764542). Happy searching!
#
#
# -- file does not contain regex --
# path : /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/solr-home/logs/solr.log
# --
#
# Please find the SOLR_HOME snapshot for failed test #5 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/103-5
ok 104 start with custom jetty options # in 8896 ms
ok 105 -c flag prints no-op warning and still starts in cloud mode # in 10065 ms
not ok 106 bootstrapping a configset # in 10075 ms
# (from function `config_exists' in file test/bats_helper.bash, line 84,
# in test file test/test_start_solr.bats, line 116)
# `config_exists "techproducts"' failed
# Waiting up to 180 seconds to see Solr running on port 46267 [|] [/] [-] [\] [|]
# Started Solr server on port 46267 (pid=2767279). Happy searching!
#
# WARN - 2026-06-28 13:45:29.289; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-28 13:45:29.293; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# Please find the SOLR_HOME snapshot for failed test #8 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/106-8
ok 107 status detects locally running solr # in 10357 ms
ok 108 status with --solr-url from user # in 8692 ms
ok 109 status with --port from user # in 8711 ms
ok 110 multiple connection options are prevented # in 1102 ms
ok 111 status with invalid --solr-url from user # in 1426 ms
ok 112 status with --short format # in 196655 ms
ok 113 searching solr via locally executed streaming expression # in 1898 ms
ok 114 searching solr via remotely executed streaming expression by solr-url # in 1728 ms
ok 115 searching solr via remotely executed streaming expression by solr connection # in 1349 ms
ok 116 variable interpolation # in 1865 ms
ok 117 searching solr without credentials fails with error # in 1233 ms
ok 118 --version returns Solr version # in 821 ms
ok 119 version as direct tool call still runs # in 819 ms
ok 120 short help # in 918 ms
ok 121 short help is inferred # in 867 ms
ok 122 long help # in 841 ms
ok 123 running subcommands with zk is prevented # in 801 ms
ok 124 listing out files # in 2255 ms
ok 125 connecting to solr via various solr urls and zk hosts # in 10115 ms
ok 126 copying files around # in 11809 ms
ok 127 upconfig # in 2757 ms
ok 128 SOLR-12429 test upconfig fails with symlink # in 1299 ms
ok 129 downconfig # in 1324 ms
ok 130 bin/solr zk cp gets 'solrhome' from '--solr-home' command line option # in 1561 ms
ok 131 env var ZK_HOST is honored # in 3358 ms
ok 132 Cleanup # in 1926 ms
> Task :solr:packaging:integrationTests FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build.gradle' line: 339
* What went wrong:
Execution failed for task ':solr:packaging:integrationTests'.
> Process 'command '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/.gradle/node/node_modules/bats/bin/bats'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/8.10/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 31m 57s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/oeqtasufdylmu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 246 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/246/
No tests ran.
Build Log:
[...truncated 1000 lines...]
BUILD FAILED in 29m 28s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/4tdkvllpljkku
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 245 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/245/
No tests ran.
Build Log:
[...truncated 1005 lines...]
BUILD FAILED in 28m 51s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/3ogxaj5gaxgem
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 244 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/244/
No tests ran.
Build Log:
[...truncated 1004 lines...]
BUILD FAILED in 28m 51s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/vf5vqmhbiq5ne
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 243 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/243/
No tests ran.
Build Log:
[...truncated 347 lines...]
# ERROR: KeeperErrorCode = NoNode for /security.json
# --
#
# Last output:
# WARN - 2026-06-24 12:38:52.647; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-24 12:38:52.651; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-24 12:38:53.146; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-24 12:38:53.146; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: KeeperErrorCode = NoNode for /security.json
not ok 9 create, config, api, and delete with basic auth # in 11029 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 40685 is already being used by another process (pid: 2536087)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-06-24 12:38:54.390; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-24 12:38:54.394; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: KeeperErrorCode = ConnectionLoss for /security.json
ok 10 post, postlogs and export with basic auth # in 18735 ms
ok 11 package with basic auth # in 12859 ms
ok 12 admin UI static content is excepted from basic auth # in 11299 ms
ok 13 nothing # in 13 ms
ok 14 server does not compress response without Accept-Encoding header # in 168 ms
ok 15 server compresses response when Accept-Encoding: gzip is requested # in 82 ms
ok 16 compressed response can be decompressed and parsed # in 84 ms
ok 17 setting property # in 12538 ms
ok 18 short form of setting property # in 6982 ms
ok 19 connecting to solr via various connection types # in 11486 ms
ok 20 create for non cloud mode # in 8491 ms
ok 21 create for cloud mode # in 10481 ms
ok 22 multiple connection options are prevented # in 946 ms
ok 23 create collection # in 8434 ms
ok 24 create collection using solrUrl # in 5750 ms
ok 25 create collection using legacy solrUrl # in 5500 ms
ok 26 create collection using Zookeeper # in 5815 ms
ok 27 reject d option with invalid config dir # in 2847 ms
ok 28 accept d option with builtin config # in 5704 ms
ok 29 accept d option with explicit path to config # in 5568 ms
ok 30 accept n option as config name # in 5429 ms
ok 31 allow config reuse when n option specifies same config # in 9842 ms
ok 32 create multisharded collections when s provided # in 5404 ms
ok 33 create replicated collections when rf provided # in 6388 ms
ok 34 can delete collections # in 12069 ms
ok 35 can delete collections with solr-url # in 8881 ms
ok 36 can delete collections with solr-connection # in 8562 ms
ok 37 collection delete also deletes zk config # in 7936 ms
ok 38 deletes accompanying zk config with nondefault name # in 8037 ms
ok 39 delete-config option can opt to leave config in zk # in 7491 ms
ok 40 start -e cloud works with --jvm-opts # in 23009 ms
ok 41 start -e cloud works with --no-prompt # in 45672 ms
ok 42 Check export command # in 20545 ms
ok 43 export fails on non cloud mode # in 9858 ms
Waiting for Tika Server to be ready on port 40690
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 44 using curl to extract a single pdf file # in 16648 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to extract content from pdf # in 16668 ms
WARNING: Docker not available; bypassing test.
ok 46 using the bin/solr post tool to crawl web site # in 16630 ms
Stopping Tika Server container
ok 47 healthcheck on cloud solr # in 10883 ms
ok 48 healthcheck on remote cloud solr # in 10436 ms
ok 49 healthcheck errors on standalone solr # in 12183 ms
ok 50 solr help flag prints help # in 764 ms
ok 51 solr with no flags prints help # in 779 ms
ok 52 start help flag prints help # in 155 ms
ok 53 start h flag prints help # in 157 ms
ok 54 stop help flag prints help # in 150 ms
ok 55 restart help flag prints help # in 156 ms
ok 56 status help flag prints help # in 826 ms
ok 57 healthcheck help flag prints help # in 781 ms
ok 58 create help flag prints help # in 796 ms
ok 59 delete help flag prints help # in 836 ms
ok 60 zk help flag prints help # in 817 ms
ok 61 auth help flag prints help # in 785 ms
ok 62 assert help flag prints help # in 810 ms
ok 63 post help flag prints help # in 888 ms
ok 64 SQL Module # in 16947 ms
ok 65 icu collation in analysis-extras module # in 13068 ms
ok 66 Check lifecycle of sentiment classification # in 24364 ms
ok 67 lifecycle of package # in 8719 ms
ok 68 deploying and undeploying a collection level package # in 14073 ms
ok 69 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
ok 70 Affinity placement plugin using sysprop # in 16398 ms
ok 71 Random placement plugin using ENV # in 16227 ms
ok 72 Check help command # in 2554 ms
ok 73 basic post with a type specified # in 5908 ms
ok 74 basic post with collection instead of url parameter # in 4098 ms
ok 75 basic post with solr-url and collection # in 4220 ms
ok 76 basic post with -s (solr-connection short form) and collection # in 4212 ms
ok 77 basic post WITHOUT a type specified # in 12100 ms
ok 78 crawling a directory as a dry-run # in 862 ms
ok 79 crawling a directory # in 4775 ms
ok 80 skipcommit and optimize and delete # in 4041 ms
ok 81 args mode # in 7349 ms
ok 82 stdin mode # in 3592 ms
ok 83 verbose echo the Solr response # in 4130 ms
ok 84 post solr log into solr via cli # in 10230 ms
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 15 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-06-24T12:50:02.602157180Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-24T12:50:02.602201591Z It should be set to 65000 to avoid operational disruption.
2026-06-24T12:50:02.602205032Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-24T12:50:02.602570572Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-24T12:50:02.602582884Z
2026-06-24T12:50:02.614223433Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-24T12:50:02.622537514Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-24T12:50:02.668637442Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-24T12:50:02.700003577Z WARNING: A command line option has enabled the Security Manager
2026-06-24T12:50:02.700027459Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-24T12:50:03.994557352Z 2026-06-24 12:50:03.991 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-24T12:50:04.258527055Z 2026-06-24 12:50:04.257 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-24T12:50:04.284037622Z 2026-06-24 12:50:04.283 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-24T12:50:04.292165744Z 2026-06-24 12:50:04.291 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-24T12:50:04.292384628Z 2026-06-24 12:50:04.291 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-06-24T12:50:04.292579679Z 2026-06-24 12:50:04.291 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-24T12:50:04.292708472Z 2026-06-24 12:50:04.291 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-24T12:50:04.291718583Z
2026-06-24T12:50:04.295594185Z 2026-06-24 12:50:04.295 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-24T12:50:04.318459686Z 2026-06-24 12:50:04.317 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-24T12:50:04.332736225Z 2026-06-24 12:50:04.332 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-24T12:50:04.333031939Z 2026-06-24 12:50:04.332 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-24T12:50:04.369761053Z 2026-06-24 12:50:04.369 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-24T12:50:05.114067530Z 2026-06-24 12:50:05.113 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-24T12:50:05.362793281Z 2026-06-24 12:50:05.362 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-24T12:50:05.367836382Z 2026-06-24 12:50:05.367 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-06-24T12:50:05.367854812Z 2026-06-24 12:50:05.367 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-06-24T12:50:05.370408800Z 2026-06-24 12:50:05.369 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-06-24T12:50:05.370435554Z 2026-06-24 12:50:05.370 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-06-24T12:50:05.407363009Z 2026-06-24 12:50:05.406 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-06-24T12:50:05.877319007Z 2026-06-24 12:50:05.876 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-06-24T12:50:05.892176914Z 2026-06-24 12:50:05.891 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-24T12:50:05.932753883Z 2026-06-24 12:50:05.932 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-24T12:50:05.965484429Z 2026-06-24 12:50:05.964 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-24T12:50:05.965596570Z 2026-06-24 12:50:05.964 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-24T12:50:06.091513908Z 2026-06-24 12:50:06.090 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-24T12:50:06.114647834Z 2026-06-24 12:50:06.114 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-24T12:50:06.127648600Z 2026-06-24 12:50:06.127 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-06-24T12:50:06.137838592Z 2026-06-24 12:50:06.137 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72177986523693056-solr-node1:8983_solr-n_0000000000) starting
2026-06-24T12:50:06.202985634Z 2026-06-24 12:50:06.202 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-06-24T12:50:06.206367993Z 2026-06-24 12:50:06.206 INFO (OverseerStateUpdate-72177986523693056-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-06-24T12:50:06.215322401Z 2026-06-24 12:50:06.214 INFO (OverseerStateUpdate-72177986523693056-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-24T12:50:06.257930162Z 2026-06-24 12:50:06.257 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-24T12:50:06.645104577Z 2026-06-24 12:50:06.644 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-24T12:50:06.834663439Z 2026-06-24 12:50:06.834 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-24T12:50:07.120230700Z 2026-06-24 12:50:07.119 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-24T12:50:07.267293009Z 2026-06-24 12:50:07.266 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-24T12:50:07.274523591Z 2026-06-24 12:50:07.274 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_24.request.log
2026-06-24T12:50:07.279416533Z 2026-06-24 12:50:07.278 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-06-24T12:50:07.281969560Z 2026-06-24 12:50:07.281 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4660ms
2026-06-24T12:50:07.522745531Z 2026-06-24 12:50:07.522 INFO (qtp985324122-27-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=48
2026-06-24T12:50:07.575298814Z 2026-06-24 12:50:07.574 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=13
2026-06-24T12:50:10.965209111Z 2026-06-24 12:50:10.964 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-24T12:50:11.045021108Z 2026-06-24 12:50:11.044 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-24T12:50:15.063090849Z 2026-06-24 12:50:15.062 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=8
2026-06-24T12:50:15.091090421Z 2026-06-24 12:50:15.090 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=9
2026-06-24T12:50:15.126712825Z 2026-06-24 12:50:15.126 INFO (qtp985324122-27-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=2
2026-06-24T12:50:15.526632794Z 2026-06-24 12:50:15.525 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=5
2026-06-24T12:50:15.554942374Z 2026-06-24 12:50:15.554 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-06-24T12:50:15.570623955Z 2026-06-24 12:50:15.570 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-06-24T12:50:15.579476770Z 2026-06-24 12:50:15.578 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 173] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-24T12:50:15.626535932Z 2026-06-24 12:50:15.626 INFO (OverseerStateUpdate-72177986523693056-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-24T12:50:15.626558320Z "core":"test-collection_shard3_replica_n1",
2026-06-24T12:50:15.626562159Z "node_name":"solr-node1:8983_solr",
2026-06-24T12:50:15.626565141Z "base_url":"http://solr-node1:8983/solr",
2026-06-24T12:50:15.626568375Z "collection":"test-collection",
2026-06-24T12:50:15.626571163Z "shard":"shard3",
2026-06-24T12:50:15.626573978Z "state":"down",
2026-06-24T12:50:15.626576600Z "type":"NRT",
2026-06-24T12:50:15.626579167Z "operation":"ADDREPLICA",
2026-06-24T12:50:15.626581899Z "waitForFinalState":"false"}
2026-06-24T12:50:15.634206804Z 2026-06-24 12:50:15.633 INFO (OverseerStateUpdate-72177986523693056-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-24T12:50:15.634226874Z "core":"test-collection_shard2_replica_n2",
2026-06-24T12:50:15.634229274Z "node_name":"solr-node3:8985_solr",
2026-06-24T12:50:15.634231766Z "base_url":"http://solr-node3:8985/solr",
2026-06-24T12:50:15.634233900Z "collection":"test-collection",
2026-06-24T12:50:15.634236177Z "shard":"shard2",
2026-06-24T12:50:15.634238153Z "state":"down",
2026-06-24T12:50:15.634240203Z "type":"NRT",
2026-06-24T12:50:15.634242134Z "operation":"ADDREPLICA",
2026-06-24T12:50:15.634245153Z "waitForFinalState":"false"}
2026-06-24T12:50:15.642358693Z 2026-06-24 12:50:15.641 INFO (OverseerStateUpdate-72177986523693056-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-24T12:50:15.642393585Z "core":"test-collection_shard1_replica_n4",
2026-06-24T12:50:15.642396560Z "node_name":"solr-node2:8984_solr",
2026-06-24T12:50:15.642398771Z "base_url":"http://solr-node2:8984/solr",
2026-06-24T12:50:15.642400740Z "collection":"test-collection",
2026-06-24T12:50:15.642402830Z "shard":"shard1",
2026-06-24T12:50:15.642404800Z "state":"down",
2026-06-24T12:50:15.642406707Z "type":"NRT",
2026-06-24T12:50:15.642408703Z "operation":"ADDREPLICA",
2026-06-24T12:50:15.642410686Z "waitForFinalState":"false"}
2026-06-24T12:50:15.756734129Z 2026-06-24 12:50:15.756 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 185] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-24T12:50:15.856106102Z 2026-06-24 12:50:15.855 INFO (qtp985324122-29-solr-node1-8) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-06-24T12:50:15.949661268Z 2026-06-24 12:50:15.948 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-24T12:50:16.026551167Z 2026-06-24 12:50:16.025 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-06-24T12:50:16.476891973Z 2026-06-24 12:50:16.476 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-24T12:50:16.477005046Z 2026-06-24 12:50:16.476 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true
2026-06-24T12:50:16.510762084Z 2026-06-24 12:50:16.510 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/]
2026-06-24T12:50:17.664752441Z 2026-06-24 12:50:17.663 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-24T12:50:17.719872361Z 2026-06-24 12:50:17.718 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-24T12:50:17.731060555Z 2026-06-24 12:50:17.730 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-24T12:50:17.733720499Z 2026-06-24 12:50:17.733 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-24T12:50:17.733794028Z 2026-06-24 12:50:17.733 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-24T12:50:17.811174338Z 2026-06-24 12:50:17.810 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-24T12:50:17.813949430Z 2026-06-24 12:50:17.813 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-24T12:50:17.816148287Z 2026-06-24 12:50:17.815 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-24T12:50:17.818256026Z 2026-06-24 12:50:17.818 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-24T12:50:17.841061898Z 2026-06-24 12:50:17.840 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-06-24T12:50:17.850565040Z 2026-06-24 12:50:17.850 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-24T12:50:17.859209354Z 2026-06-24 12:50:17.858 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-24T12:50:17.859238884Z 2026-06-24 12:50:17.858 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-24T12:50:17.870498690Z 2026-06-24 12:50:17.870 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-24T12:50:17.878953985Z 2026-06-24 12:50:17.878 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-06-24T12:50:17.881280875Z 2026-06-24 12:50:17.880 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-06-24T12:50:17.897475725Z 2026-06-24 12:50:17.897 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-24T12:50:17.897503757Z 2026-06-24 12:50:17.897 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-24T12:50:17.897530049Z 2026-06-24 12:50:17.897 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/
2026-06-24T12:50:17.898891660Z 2026-06-24 12:50:17.898 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-24T12:50:17.899493379Z 2026-06-24 12:50:17.899 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas
2026-06-24T12:50:17.900592550Z 2026-06-24 12:50:17.900 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72177986523693056-core_node3-n_0000000000
2026-06-24T12:50:17.907786530Z 2026-06-24 12:50:17.907 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3
2026-06-24T12:50:18.013775889Z 2026-06-24 12:50:18.013 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 241] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-24T12:50:18.017623113Z 2026-06-24 12:50:18.017 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-24T12:50:18.026562571Z 2026-06-24 12:50:18.026 INFO (qtp985324122-29-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2171
2026-06-24T12:50:18.036848804Z 2026-06-24 12:50:18.036 INFO (qtp985324122-101-solr-node1-7) [c:test-collection s: r: x: t:solr-node1-7] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
2026-06-24T12:50:18.125977513Z 2026-06-24 12:50:18.125 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 247] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-24T12:50:18.128972243Z 2026-06-24 12:50:18.128 INFO (qtp985324122-101-solr-node1-7) [c:test-collection s: r: x: t:solr-node1-7] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=2591
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 2567588,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-24T12:50:02.365514837Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-06-24T12:50:07.884518158Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-24T12:50:07.884544549Z It should be set to 65000 to avoid operational disruption.
2026-06-24T12:50:07.884547685Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-24T12:50:07.884833505Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-24T12:50:07.884847359Z
2026-06-24T12:50:07.895751345Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-24T12:50:07.903884678Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-24T12:50:07.944112086Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-24T12:50:07.983308884Z WARNING: A command line option has enabled the Security Manager
2026-06-24T12:50:07.983328404Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-24T12:50:09.206105833Z 2026-06-24 12:50:09.202 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-24T12:50:09.489623922Z 2026-06-24 12:50:09.488 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-24T12:50:09.516394901Z 2026-06-24 12:50:09.515 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-24T12:50:09.524499084Z 2026-06-24 12:50:09.523 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-24T12:50:09.524580893Z 2026-06-24 12:50:09.524 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-06-24T12:50:09.524893580Z 2026-06-24 12:50:09.524 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-24T12:50:09.525156525Z 2026-06-24 12:50:09.524 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-24T12:50:09.524162200Z
2026-06-24T12:50:09.527662572Z 2026-06-24 12:50:09.527 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-24T12:50:09.555529979Z 2026-06-24 12:50:09.554 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-24T12:50:09.582758259Z 2026-06-24 12:50:09.582 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-24T12:50:09.742517185Z 2026-06-24 12:50:09.741 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-24T12:50:09.766261810Z 2026-06-24 12:50:09.765 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-24T12:50:09.766357831Z 2026-06-24 12:50:09.765 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-24T12:50:09.766462367Z 2026-06-24 12:50:09.765 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-24T12:50:09.880092497Z 2026-06-24 12:50:09.878 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-24T12:50:09.880122653Z 2026-06-24 12:50:09.879 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-24T12:50:09.880311668Z 2026-06-24 12:50:09.879 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-24T12:50:09.922255614Z 2026-06-24 12:50:09.921 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-24T12:50:10.746990585Z 2026-06-24 12:50:10.746 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-24T12:50:10.926574644Z 2026-06-24 12:50:10.925 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-24T12:50:10.928271908Z 2026-06-24 12:50:10.926 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-24T12:50:10.936857132Z 2026-06-24 12:50:10.936 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-24T12:50:10.941923572Z 2026-06-24 12:50:10.941 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-24T12:50:10.945492757Z 2026-06-24 12:50:10.945 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-24T12:50:10.945581986Z 2026-06-24 12:50:10.945 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-24T12:50:10.979699116Z 2026-06-24 12:50:10.979 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-24T12:50:10.988639547Z 2026-06-24 12:50:10.988 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
2026-06-24T12:50:11.010570532Z 2026-06-24 12:50:11.009 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-24T12:50:11.024181702Z 2026-06-24 12:50:11.023 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-06-24T12:50:11.036625746Z 2026-06-24 12:50:11.036 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-06-24T12:50:11.045645708Z 2026-06-24 12:50:11.045 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-24T12:50:11.112231863Z 2026-06-24 12:50:11.111 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-24T12:50:11.417384430Z 2026-06-24 12:50:11.416 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-24T12:50:11.610396033Z 2026-06-24 12:50:11.609 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-24T12:50:11.899617725Z 2026-06-24 12:50:11.899 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-24T12:50:12.050244425Z 2026-06-24 12:50:12.049 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-24T12:50:12.061863363Z 2026-06-24 12:50:12.061 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_24.request.log
2026-06-24T12:50:12.070501110Z 2026-06-24 12:50:12.068 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-06-24T12:50:12.071601184Z 2026-06-24 12:50:12.071 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4168ms
2026-06-24T12:50:12.263769104Z 2026-06-24 12:50:12.263 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=48
2026-06-24T12:50:12.336911990Z 2026-06-24 12:50:12.336 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=17
2026-06-24T12:50:15.856041635Z 2026-06-24 12:50:15.855 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-06-24T12:50:15.919031551Z 2026-06-24 12:50:15.918 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-24T12:50:15.978123348Z 2026-06-24 12:50:15.977 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-06-24T12:50:16.305644093Z 2026-06-24 12:50:16.305 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-24T12:50:16.305673184Z 2026-06-24 12:50:16.305 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-06-24T12:50:16.324276675Z 2026-06-24 12:50:16.323 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-06-24T12:50:16.746145182Z 2026-06-24 12:50:16.745 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-24T12:50:16.766028981Z 2026-06-24 12:50:16.765 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-24T12:50:16.769632655Z 2026-06-24 12:50:16.769 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-24T12:50:16.770950503Z 2026-06-24 12:50:16.770 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-24T12:50:16.771010431Z 2026-06-24 12:50:16.770 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-24T12:50:16.815531238Z 2026-06-24 12:50:16.814 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-24T12:50:16.818708310Z 2026-06-24 12:50:16.818 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-24T12:50:16.821214867Z 2026-06-24 12:50:16.820 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-24T12:50:16.823387151Z 2026-06-24 12:50:16.823 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-24T12:50:16.834673612Z 2026-06-24 12:50:16.834 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4}
2026-06-24T12:50:16.842197339Z 2026-06-24 12:50:16.841 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-24T12:50:16.848360997Z 2026-06-24 12:50:16.847 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-24T12:50:16.848392109Z 2026-06-24 12:50:16.848 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-24T12:50:16.856537937Z 2026-06-24 12:50:16.856 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-24T12:50:16.866514832Z 2026-06-24 12:50:16.865 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-06-24T12:50:16.867858276Z 2026-06-24 12:50:16.867 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1
2026-06-24T12:50:16.883418414Z 2026-06-24 12:50:16.882 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-24T12:50:16.883448291Z 2026-06-24 12:50:16.883 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-24T12:50:16.883545269Z 2026-06-24 12:50:16.883 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node2:8984/solr/test-collection_shard1_replica_n4/
2026-06-24T12:50:16.885408802Z 2026-06-24 12:50:16.885 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-24T12:50:16.886098924Z 2026-06-24 12:50:16.885 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ has no replicas
2026-06-24T12:50:16.887231265Z 2026-06-24 12:50:16.886 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard1/leader after winning as /collections/test-collection/leader_elect/shard1/election/72177986523693060-core_node6-n_0000000000
2026-06-24T12:50:16.895854250Z 2026-06-24 12:50:16.895 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ shard1
2026-06-24T12:50:17.003866540Z 2026-06-24 12:50:17.003 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 205] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-24T12:50:17.008034675Z 2026-06-24 12:50:17.007 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-24T12:50:17.019668861Z 2026-06-24 12:50:17.019 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1165
2026-06-24T12:50:17.119782485Z 2026-06-24 12:50:17.119 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 208] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-24T12:50:17.666532674Z 2026-06-24 12:50:17.666 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 223] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-24T12:50:17.779768515Z 2026-06-24 12:50:17.779 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 226] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-24T12:50:18.014365747Z 2026-06-24 12:50:18.013 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 241] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-24T12:50:18.126318333Z 2026-06-24 12:50:18.125 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 247] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node2 ===
{
"Name": "/solr-node2",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 2567981,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-24T12:50:07.669009526Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node3 ===
2026-06-24T12:50:08.129493825Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-24T12:50:08.129530239Z It should be set to 65000 to avoid operational disruption.
2026-06-24T12:50:08.129533266Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-24T12:50:08.129920574Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-24T12:50:08.129935422Z
2026-06-24T12:50:08.140139593Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-24T12:50:08.147663856Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-24T12:50:08.193686494Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-24T12:50:08.226834353Z WARNING: A command line option has enabled the Security Manager
2026-06-24T12:50:08.226859177Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-24T12:50:09.348768398Z 2026-06-24 12:50:09.345 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-24T12:50:09.652713595Z 2026-06-24 12:50:09.652 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-24T12:50:09.683463283Z 2026-06-24 12:50:09.682 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-24T12:50:09.691785872Z 2026-06-24 12:50:09.690 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-24T12:50:09.692117224Z 2026-06-24 12:50:09.690 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8985
2026-06-24T12:50:09.692534998Z 2026-06-24 12:50:09.691 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-24T12:50:09.692620147Z 2026-06-24 12:50:09.691 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-24T12:50:09.691174886Z
2026-06-24T12:50:09.695331181Z 2026-06-24 12:50:09.694 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-24T12:50:09.720173091Z 2026-06-24 12:50:09.719 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-24T12:50:09.745497671Z 2026-06-24 12:50:09.744 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-24T12:50:09.897645336Z 2026-06-24 12:50:09.897 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-24T12:50:09.919786886Z 2026-06-24 12:50:09.918 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-24T12:50:09.920009513Z 2026-06-24 12:50:09.919 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-24T12:50:09.920267731Z 2026-06-24 12:50:09.919 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-24T12:50:10.031351518Z 2026-06-24 12:50:10.030 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-24T12:50:10.031475886Z 2026-06-24 12:50:10.031 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-24T12:50:10.031616385Z 2026-06-24 12:50:10.031 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-24T12:50:10.070575462Z 2026-06-24 12:50:10.069 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-24T12:50:10.704180541Z 2026-06-24 12:50:10.703 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-24T12:50:10.842468751Z 2026-06-24 12:50:10.841 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-24T12:50:10.844140200Z 2026-06-24 12:50:10.841 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-24T12:50:10.852257493Z 2026-06-24 12:50:10.851 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-24T12:50:10.857071156Z 2026-06-24 12:50:10.856 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-24T12:50:10.861008003Z 2026-06-24 12:50:10.860 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-24T12:50:10.861025728Z 2026-06-24 12:50:10.860 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-24T12:50:10.896481079Z 2026-06-24 12:50:10.895 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-24T12:50:10.905600711Z 2026-06-24 12:50:10.905 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-24T12:50:10.929476394Z 2026-06-24 12:50:10.928 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-24T12:50:10.941191827Z 2026-06-24 12:50:10.940 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node3:8985_solr as DOWN
2026-06-24T12:50:10.955137151Z 2026-06-24 12:50:10.954 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node3:8985_solr
2026-06-24T12:50:10.965551679Z 2026-06-24 12:50:10.965 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-24T12:50:11.007764507Z 2026-06-24 12:50:11.007 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-24T12:50:11.045170104Z 2026-06-24 12:50:11.044 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-24T12:50:11.283626494Z 2026-06-24 12:50:11.282 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-24T12:50:11.449686022Z 2026-06-24 12:50:11.449 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-24T12:50:11.707764133Z 2026-06-24 12:50:11.707 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-24T12:50:11.836143653Z 2026-06-24 12:50:11.835 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-24T12:50:11.841603214Z 2026-06-24 12:50:11.841 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_24.request.log
2026-06-24T12:50:11.845324051Z 2026-06-24 12:50:11.844 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8985}
2026-06-24T12:50:11.846189873Z 2026-06-24 12:50:11.845 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @3699ms
2026-06-24T12:50:13.826053778Z 2026-06-24 12:50:13.824 INFO (qtp985324122-27-solr-node3-1) [c: s: r: x: t:solr-node3-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=62
2026-06-24T12:50:13.928421768Z 2026-06-24 12:50:13.927 INFO (qtp985324122-56-solr-node3-2) [c: s: r: x: t:solr-node3-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=55
2026-06-24T12:50:15.857850029Z 2026-06-24 12:50:15.857 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node5&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&shard=shard2&wt=javabin
2026-06-24T12:50:15.933038898Z 2026-06-24 12:50:15.932 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-24T12:50:16.132689128Z 2026-06-24 12:50:16.132 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-06-24T12:50:16.530869779Z 2026-06-24 12:50:16.530 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-24T12:50:16.530967373Z 2026-06-24 12:50:16.530 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard2_replica_n2' using configuration from configset techproducts, trusted=true
2026-06-24T12:50:16.555352367Z 2026-06-24 12:50:16.554 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard2_replica_n2], dataDir=[/var/solr/data/test-collection_shard2_replica_n2/data/]
2026-06-24T12:50:17.289486480Z 2026-06-24 12:50:17.288 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-24T12:50:17.315404762Z 2026-06-24 12:50:17.314 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-24T12:50:17.318968434Z 2026-06-24 12:50:17.318 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-24T12:50:17.320038391Z 2026-06-24 12:50:17.319 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-24T12:50:17.320053502Z 2026-06-24 12:50:17.319 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-24T12:50:17.477903914Z 2026-06-24 12:50:17.477 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-24T12:50:17.480915587Z 2026-06-24 12:50:17.480 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-24T12:50:17.484227447Z 2026-06-24 12:50:17.483 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-24T12:50:17.488192156Z 2026-06-24 12:50:17.487 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-24T12:50:17.500253582Z 2026-06-24 12:50:17.499 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-06-24T12:50:17.507566309Z 2026-06-24 12:50:17.507 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-24T12:50:17.514381472Z 2026-06-24 12:50:17.513 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-8 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-24T12:50:17.514440245Z 2026-06-24 12:50:17.514 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-8 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-24T12:50:17.521184375Z 2026-06-24 12:50:17.520 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-8 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-24T12:50:17.533419775Z 2026-06-24 12:50:17.532 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard2 to Terms{values={core_node5=0}, version=0} for registerTerm
2026-06-24T12:50:17.534828168Z 2026-06-24 12:50:17.534 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard2
2026-06-24T12:50:17.548903080Z 2026-06-24 12:50:17.548 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-24T12:50:17.548970347Z 2026-06-24 12:50:17.548 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-24T12:50:17.548975379Z 2026-06-24 12:50:17.548 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node3:8985/solr/test-collection_shard2_replica_n2/
2026-06-24T12:50:17.550264191Z 2026-06-24 12:50:17.550 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-24T12:50:17.550833014Z 2026-06-24 12:50:17.550 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ has no replicas
2026-06-24T12:50:17.552059662Z 2026-06-24 12:50:17.551 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard2/leader after winning as /collections/test-collection/leader_elect/shard2/election/72177986523693059-core_node5-n_0000000000
2026-06-24T12:50:17.560002639Z 2026-06-24 12:50:17.559 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ shard2
2026-06-24T12:50:17.666697050Z 2026-06-24 12:50:17.666 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 223] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-24T12:50:17.670996055Z 2026-06-24 12:50:17.670 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-24T12:50:17.679533338Z 2026-06-24 12:50:17.679 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=techproducts&newCollection=true&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&collection=test-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1823
2026-06-24T12:50:17.780397097Z 2026-06-24 12:50:17.779 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 226] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-24T12:50:18.013935281Z 2026-06-24 12:50:18.013 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 241] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-24T12:50:18.126305518Z 2026-06-24 12:50:18.125 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 247] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node3 ===
{
"Name": "/solr-node3",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 2568160,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-24T12:50:07.880351272Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# Docker artifacts saved to: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/docker
not ok 85 Docker SolrCloud rolling upgrade # in 30307 ms
# (in test file test/test_rolling_upgrade.bats, line 118)
# `wait_for 30 1 docker exec solr-node1 solr healthcheck -c test-collection' failed with status 127
# 9.10.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# af6eca94c810: Pulling fs layer
# cb0efb96dabd: Pulling fs layer
# 3e9d91201f40: Pulling fs layer
# 66b76b382631: Pulling fs layer
# 601f2c23751f: Pulling fs layer
# 0ed719aef393: Pulling fs layer
# 69a09c96e8bf: Pulling fs layer
# c937aee9a870: Pulling fs layer
# 9cdfe5320de8: Pulling fs layer
# 3aa9120e46f5: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# 66b76b382631: Waiting
# 601f2c23751f: Waiting
# 0ed719aef393: Waiting
# 69a09c96e8bf: Waiting
# c937aee9a870: Waiting
# 9cdfe5320de8: Waiting
# 3aa9120e46f5: Waiting
# 4f4fb700ef54: Waiting
# cb0efb96dabd: Verifying Checksum
# cb0efb96dabd: Download complete
# 66b76b382631: Download complete
# af6eca94c810: Verifying Checksum
# af6eca94c810: Download complete
# 601f2c23751f: Verifying Checksum
# 601f2c23751f: Download complete
# 3e9d91201f40: Verifying Checksum
# 3e9d91201f40: Download complete
# 69a09c96e8bf: Verifying Checksum
# 69a09c96e8bf: Download complete
# c937aee9a870: Verifying Checksum
# c937aee9a870: Download complete
# 9cdfe5320de8: Verifying Checksum
# 9cdfe5320de8: Download complete
# 3aa9120e46f5: Verifying Checksum
# 3aa9120e46f5: Download complete
# 4f4fb700ef54: Download complete
# 0ed719aef393: Download complete
# af6eca94c810: Pull complete
# cb0efb96dabd: Pull complete
# 3e9d91201f40: Pull complete
# 66b76b382631: Pull complete
# 601f2c23751f: Pull complete
# 0ed719aef393: Pull complete
# 69a09c96e8bf: Pull complete
# c937aee9a870: Pull complete
# 9cdfe5320de8: Pull complete
# 3aa9120e46f5: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:15c8155ceffe1042f37c77bb537aa531804a3510702ff87f2af5fedf523f4203
# Status: Downloaded newer image for apache/solr-nightly:9.10.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:9.10.0-SNAPSHOT-slim
# 10.0.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# 40d16f30db40: Pulling fs layer
# 9d5a0b6cb36c: Pulling fs layer
# 49518c28c0b3: Pulling fs layer
# 41162acdcc29: Pulling fs layer
# 2ed0fd558ca3: Pulling fs layer
# 2488415fda1c: Pulling fs layer
# 09511c8fb203: Pulling fs layer
# a0c57fb89c4f: Pulling fs layer
# d0641dcc41d7: Pulling fs layer
# 315722fd9570: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# 41162acdcc29: Waiting
# 2ed0fd558ca3: Waiting
# 2488415fda1c: Waiting
# 315722fd9570: Waiting
# 4f4fb700ef54: Waiting
# 09511c8fb203: Waiting
# d0641dcc41d7: Waiting
# a0c57fb89c4f: Waiting
# 9d5a0b6cb36c: Verifying Checksum
# 9d5a0b6cb36c: Download complete
# 41162acdcc29: Verifying Checksum
# 41162acdcc29: Download complete
# 2ed0fd558ca3: Verifying Checksum
# 2ed0fd558ca3: Download complete
# 40d16f30db40: Verifying Checksum
# 40d16f30db40: Download complete
# 09511c8fb203: Verifying Checksum
# 09511c8fb203: Download complete
# 49518c28c0b3: Verifying Checksum
# 49518c28c0b3: Download complete
# a0c57fb89c4f: Verifying Checksum
# a0c57fb89c4f: Download complete
# d0641dcc41d7: Verifying Checksum
# d0641dcc41d7: Download complete
# 315722fd9570: Verifying Checksum
# 315722fd9570: Download complete
# 4f4fb700ef54: Verifying Checksum
# 4f4fb700ef54: Download complete
# 2488415fda1c: Verifying Checksum
# 2488415fda1c: Download complete
# 40d16f30db40: Pull complete
# 9d5a0b6cb36c: Pull complete
# 49518c28c0b3: Pull complete
# 41162acdcc29: Pull complete
# 2ed0fd558ca3: Pull complete
# 2488415fda1c: Pull complete
# 09511c8fb203: Pull complete
# a0c57fb89c4f: Pull complete
# d0641dcc41d7: Pull complete
# 315722fd9570: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:fb25f7a7190e3027ade20e97708a8665d41ab0e9ea9285fa242b2c83b0e5f43c
# Status: Downloaded newer image for apache/solr-nightly:10.0.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:10.0.0-SNAPSHOT-slim
# d0cca0692e3d313d6251a5fb6992d83eee62e45a97f5f4c5b2d2d51bee849740
# solr-data1
# solr-data2
# solr-data3
# Starting solr-node1 with embedded ZooKeeper
# 8b2b0c2f23482326f6352bd96a17858d2ee429f6313a8dec5fdaeaa8df9e933c
# Starting solr-node2 connected to first node's ZooKeeper
# 30e0ee109ba7cc8fbd3c7156b6bc80dc2e36a84a95afc0838d2dbc53526acc4c
# Started solr-node3 connected to first node's ZooKeeper
# a5f9fdebcff4c2ced8675cb9310de7f7f4d586ddafc2e7b73d704062e6022d9f
# Creating a Collection
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# WARN - 2026-06-24 12:50:15.157; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-24 12:50:15.189; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. "/solr"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [http://solr-node1:8983/solr] to [http://solr-node1:8983/].
# Created collection 'test-collection' with 3 shard(s), 1 replica(s) with config-set 'techproducts'
# Checking collection health
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_rolling_upgrade.bats: line 118: wait_for: command not found
# solr-node1
# solr-node2
# solr-node3
# solr-node1
# solr-node2
# solr-node3
# solr-data1
# solr-data2
# solr-data3
# solrcloud-test
ok 86 allowPaths - backup # in 16420 ms
ok 87 snapshot lifecycle # in 5272 ms
ok 88 snapshot list # in 3513 ms
ok 89 snapshot list via --solr-connection # in 3674 ms
ok 90 snapshot describe # in 3573 ms
ok 91 snapshot describe via --solr-connection # in 3666 ms
ok 92 start solr with ssl # in 17626 ms
ok 93 use different hostname when not checking peer-name # in 26492 ms
ok 94 start solr with ssl and auth # in 15116 ms
ok 95 start solr with client truststore and security manager # in 17882 ms
ok 96 start solr with mTLS needed # in 49057 ms
ok 97 start solr with mTLS wanted # in 36457 ms
ok 98 test keystore reload # in 61694 ms
ok 99 SOLR-11740 check 'solr stop' connection # in 16921 ms
ok 100 stop command for single port # in 19227 ms
ok 101 check stop command doesn't hang # in 227 ms
ok 102 SOLR-16976 solr starts with remote JMX enabled # in 8887 ms
not ok 103 deprecated system properties converted to modern properties # in 195478 ms
# (from function `assert_file_contains' in file .gradle/node/node_modules/bats-file/src/file.bash, line 566,
# in test file test/test_start_solr.bats, line 86)
# `assert_file_contains "${SOLR_LOGS_DIR}/solr.log" 'You are passing in deprecated system property disable.config.edit and should upgrade to using solr.api.config.edit.enabled instead.'' failed
# Waiting up to 180 seconds to see Solr running on port 40685 [|] [/] [-] [\] [|]
# Started Solr server on port 40685 (pid=2585689). Happy searching!
#
#
# -- file does not contain regex --
# path : /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/solr-home/logs/solr.log
# --
#
# Please find the SOLR_HOME snapshot for failed test #5 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/103-5
ok 104 start with custom jetty options # in 8537 ms
ok 105 -c flag prints no-op warning and still starts in cloud mode # in 10112 ms
not ok 106 bootstrapping a configset # in 9868 ms
# (from function `config_exists' in file test/bats_helper.bash, line 84,
# in test file test/test_start_solr.bats, line 116)
# `config_exists "techproducts"' failed
# Waiting up to 180 seconds to see Solr running on port 40685 [|] [/] [-] [\] [|]
# Started Solr server on port 40685 (pid=2588401). Happy searching!
#
# WARN - 2026-06-24 12:59:16.847; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-24 12:59:16.851; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# Please find the SOLR_HOME snapshot for failed test #8 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/106-8
ok 107 status detects locally running solr # in 10381 ms
ok 108 status with --solr-url from user # in 8643 ms
ok 109 status with --port from user # in 8480 ms
ok 110 multiple connection options are prevented # in 949 ms
ok 111 status with invalid --solr-url from user # in 1376 ms
ok 112 status with --short format # in 196535 ms
ok 113 searching solr via locally executed streaming expression # in 1928 ms
ok 114 searching solr via remotely executed streaming expression by solr-url # in 1669 ms
ok 115 searching solr via remotely executed streaming expression by solr connection # in 1268 ms
ok 116 variable interpolation # in 1811 ms
ok 117 searching solr without credentials fails with error # in 1238 ms
ok 118 --version returns Solr version # in 786 ms
ok 119 version as direct tool call still runs # in 782 ms
ok 120 short help # in 931 ms
ok 121 short help is inferred # in 811 ms
ok 122 long help # in 912 ms
ok 123 running subcommands with zk is prevented # in 834 ms
ok 124 listing out files # in 2215 ms
ok 125 connecting to solr via various solr urls and zk hosts # in 10151 ms
ok 126 copying files around # in 11822 ms
ok 127 upconfig # in 2790 ms
ok 128 SOLR-12429 test upconfig fails with symlink # in 1435 ms
ok 129 downconfig # in 1434 ms
ok 130 bin/solr zk cp gets 'solrhome' from '--solr-home' command line option # in 1569 ms
ok 131 env var ZK_HOST is honored # in 3453 ms
ok 132 Cleanup # in 1922 ms
> Task :solr:packaging:integrationTests FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build.gradle' line: 339
* What went wrong:
Execution failed for task ':solr:packaging:integrationTests'.
> Process 'command '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/.gradle/node/node_modules/bats/bin/bats'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/8.10/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 31m 39s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/n22mbsns353xy
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 242 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/242/
No tests ran.
Build Log:
[...truncated 902 lines...]
BUILD FAILED in 30m 56s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/y7yc6fpigol3o
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 241 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/241/
No tests ran.
Build Log:
[...truncated 903 lines...]
BUILD FAILED in 29m
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/fi2zajm4onmxg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 240 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/240/
No tests ran.
Build Log:
[...truncated 347 lines...]
# ERROR: KeeperErrorCode = NoNode for /security.json
# --
#
# Last output:
# WARN - 2026-06-21 12:04:28.517; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-21 12:04:28.520; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-21 12:04:28.966; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-21 12:04:28.966; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: KeeperErrorCode = NoNode for /security.json
not ok 9 create, config, api, and delete with basic auth # in 16261 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 44801 is already being used by another process (pid: 3253843)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-06-21 12:04:30.081; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-21 12:04:30.084; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
ok 10 post, postlogs and export with basic auth # in 17243 ms
ok 11 package with basic auth # in 11696 ms
ok 12 admin UI static content is excepted from basic auth # in 10143 ms
ok 13 nothing # in 9 ms
ok 14 server does not compress response without Accept-Encoding header # in 137 ms
ok 15 server compresses response when Accept-Encoding: gzip is requested # in 76 ms
ok 16 compressed response can be decompressed and parsed # in 77 ms
ok 17 setting property # in 10377 ms
ok 18 short form of setting property # in 6235 ms
ok 19 connecting to solr via various connection types # in 10040 ms
ok 20 create for non cloud mode # in 7551 ms
ok 21 create for cloud mode # in 9600 ms
ok 22 multiple connection options are prevented # in 792 ms
ok 23 create collection # in 7154 ms
ok 24 create collection using solrUrl # in 4932 ms
ok 25 create collection using legacy solrUrl # in 4928 ms
ok 26 create collection using Zookeeper # in 5092 ms
ok 27 reject d option with invalid config dir # in 2684 ms
ok 28 accept d option with builtin config # in 4904 ms
ok 29 accept d option with explicit path to config # in 5077 ms
ok 30 accept n option as config name # in 4927 ms
ok 31 allow config reuse when n option specifies same config # in 8728 ms
ok 32 create multisharded collections when s provided # in 4889 ms
ok 33 create replicated collections when rf provided # in 5819 ms
ok 34 can delete collections # in 10019 ms
ok 35 can delete collections with solr-url # in 7850 ms
ok 36 can delete collections with solr-connection # in 7733 ms
ok 37 collection delete also deletes zk config # in 7070 ms
ok 38 deletes accompanying zk config with nondefault name # in 7088 ms
ok 39 delete-config option can opt to leave config in zk # in 6780 ms
ok 40 start -e cloud works with --jvm-opts # in 20931 ms
ok 41 start -e cloud works with --no-prompt # in 46186 ms
ok 42 Check export command # in 18071 ms
ok 43 export fails on non cloud mode # in 9039 ms
Waiting for Tika Server to be ready on port 44806
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 44 using curl to extract a single pdf file # in 16541 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to extract content from pdf # in 16504 ms
WARNING: Docker not available; bypassing test.
ok 46 using the bin/solr post tool to crawl web site # in 16500 ms
Stopping Tika Server container
ok 47 healthcheck on cloud solr # in 9861 ms
ok 48 healthcheck on remote cloud solr # in 9393 ms
ok 49 healthcheck errors on standalone solr # in 10625 ms
ok 50 solr help flag prints help # in 695 ms
ok 51 solr with no flags prints help # in 695 ms
ok 52 start help flag prints help # in 137 ms
ok 53 start h flag prints help # in 142 ms
ok 54 stop help flag prints help # in 138 ms
ok 55 restart help flag prints help # in 138 ms
ok 56 status help flag prints help # in 737 ms
ok 57 healthcheck help flag prints help # in 702 ms
ok 58 create help flag prints help # in 718 ms
ok 59 delete help flag prints help # in 715 ms
ok 60 zk help flag prints help # in 720 ms
ok 61 auth help flag prints help # in 725 ms
ok 62 assert help flag prints help # in 706 ms
ok 63 post help flag prints help # in 728 ms
ok 64 SQL Module # in 14980 ms
ok 65 icu collation in analysis-extras module # in 11859 ms
ok 66 Check lifecycle of sentiment classification # in 25992 ms
ok 67 lifecycle of package # in 8143 ms
ok 68 deploying and undeploying a collection level package # in 12641 ms
ok 69 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
ok 70 Affinity placement plugin using sysprop # in 14916 ms
ok 71 Random placement plugin using ENV # in 14783 ms
ok 72 Check help command # in 2210 ms
ok 73 basic post with a type specified # in 5072 ms
ok 74 basic post with collection instead of url parameter # in 3830 ms
ok 75 basic post with solr-url and collection # in 3733 ms
ok 76 basic post with -s (solr-connection short form) and collection # in 3691 ms
ok 77 basic post WITHOUT a type specified # in 10756 ms
ok 78 crawling a directory as a dry-run # in 787 ms
ok 79 crawling a directory # in 4235 ms
ok 80 skipcommit and optimize and delete # in 3652 ms
ok 81 args mode # in 6591 ms
ok 82 stdin mode # in 3189 ms
ok 83 verbose echo the Solr response # in 3656 ms
ok 84 post solr log into solr via cli # in 8907 ms
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 9 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 9 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 14 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-06-21T12:14:52.467128068Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-21T12:14:52.467189998Z It should be set to 65000 to avoid operational disruption.
2026-06-21T12:14:52.467193736Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-21T12:14:52.467435365Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-21T12:14:52.467446772Z
2026-06-21T12:14:52.476537828Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-21T12:14:52.484133871Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-21T12:14:52.526901975Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-21T12:14:52.558083148Z WARNING: A command line option has enabled the Security Manager
2026-06-21T12:14:52.558121883Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-21T12:14:53.597934808Z 2026-06-21 12:14:53.594 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-21T12:14:53.855872460Z 2026-06-21 12:14:53.855 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-21T12:14:53.879642099Z 2026-06-21 12:14:53.879 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-21T12:14:53.886572878Z 2026-06-21 12:14:53.886 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-21T12:14:53.886638739Z 2026-06-21 12:14:53.886 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-06-21T12:14:53.886758010Z 2026-06-21 12:14:53.886 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-21T12:14:53.886895159Z 2026-06-21 12:14:53.886 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-21T12:14:53.886311774Z
2026-06-21T12:14:53.889308019Z 2026-06-21 12:14:53.888 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-21T12:14:53.910599279Z 2026-06-21 12:14:53.910 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-21T12:14:53.923256629Z 2026-06-21 12:14:53.922 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-21T12:14:53.923318491Z 2026-06-21 12:14:53.922 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-21T12:14:53.956077726Z 2026-06-21 12:14:53.955 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-21T12:14:54.608742136Z 2026-06-21 12:14:54.608 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-21T12:14:54.824738791Z 2026-06-21 12:14:54.824 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-21T12:14:54.829015336Z 2026-06-21 12:14:54.828 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-06-21T12:14:54.829028165Z 2026-06-21 12:14:54.828 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-06-21T12:14:54.831120146Z 2026-06-21 12:14:54.830 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-06-21T12:14:54.831228217Z 2026-06-21 12:14:54.830 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-06-21T12:14:54.864850947Z 2026-06-21 12:14:54.864 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-06-21T12:14:55.334618416Z 2026-06-21 12:14:55.333 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-06-21T12:14:55.347431449Z 2026-06-21 12:14:55.346 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-21T12:14:55.384403353Z 2026-06-21 12:14:55.383 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-21T12:14:55.427368831Z 2026-06-21 12:14:55.424 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-21T12:14:55.427446378Z 2026-06-21 12:14:55.425 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-21T12:14:55.616838271Z 2026-06-21 12:14:55.615 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-21T12:14:55.638846679Z 2026-06-21 12:14:55.638 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-21T12:14:55.654143816Z 2026-06-21 12:14:55.653 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-06-21T12:14:55.660357329Z 2026-06-21 12:14:55.660 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72160861276012544-solr-node1:8983_solr-n_0000000000) starting
2026-06-21T12:14:55.721844232Z 2026-06-21 12:14:55.721 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-06-21T12:14:55.723191329Z 2026-06-21 12:14:55.722 INFO (OverseerStateUpdate-72160861276012544-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-06-21T12:14:55.731628919Z 2026-06-21 12:14:55.731 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-21T12:14:55.777598909Z 2026-06-21 12:14:55.777 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-21T12:14:56.204558278Z 2026-06-21 12:14:56.203 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-21T12:14:56.397436814Z 2026-06-21 12:14:56.396 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-21T12:14:56.648350489Z 2026-06-21 12:14:56.647 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-21T12:14:56.773178877Z 2026-06-21 12:14:56.772 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-21T12:14:56.778651866Z 2026-06-21 12:14:56.778 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_21.request.log
2026-06-21T12:14:56.782068109Z 2026-06-21 12:14:56.781 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-06-21T12:14:56.783692508Z 2026-06-21 12:14:56.783 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4300ms
2026-06-21T12:14:56.968020104Z 2026-06-21 12:14:56.967 INFO (qtp985324122-27-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=41
2026-06-21T12:14:57.030357820Z 2026-06-21 12:14:57.029 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=14
2026-06-21T12:15:00.174668403Z 2026-06-21 12:15:00.174 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-21T12:15:00.203057669Z 2026-06-21 12:15:00.202 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-21T12:15:04.074894989Z 2026-06-21 12:15:04.074 INFO (qtp985324122-100-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=8
2026-06-21T12:15:04.099059028Z 2026-06-21 12:15:04.098 INFO (qtp985324122-100-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=7
2026-06-21T12:15:04.124150726Z 2026-06-21 12:15:04.123 INFO (qtp985324122-101-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=1
2026-06-21T12:15:04.610488646Z 2026-06-21 12:15:04.609 INFO (qtp985324122-100-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=4
2026-06-21T12:15:04.637143278Z 2026-06-21 12:15:04.634 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-06-21T12:15:04.651767540Z 2026-06-21 12:15:04.651 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-06-21T12:15:04.656301794Z 2026-06-21 12:15:04.655 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 173] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-21T12:15:04.691364791Z 2026-06-21 12:15:04.690 INFO (OverseerStateUpdate-72160861276012544-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-21T12:15:04.691381507Z "core":"test-collection_shard2_replica_n1",
2026-06-21T12:15:04.691384222Z "node_name":"solr-node3:8985_solr",
2026-06-21T12:15:04.691386494Z "base_url":"http://solr-node3:8985/solr",
2026-06-21T12:15:04.691388764Z "collection":"test-collection",
2026-06-21T12:15:04.691390809Z "shard":"shard2",
2026-06-21T12:15:04.691392765Z "state":"down",
2026-06-21T12:15:04.691394717Z "type":"NRT",
2026-06-21T12:15:04.691396600Z "operation":"ADDREPLICA",
2026-06-21T12:15:04.691399050Z "waitForFinalState":"false"}
2026-06-21T12:15:04.697257900Z 2026-06-21 12:15:04.696 INFO (OverseerStateUpdate-72160861276012544-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-21T12:15:04.697266334Z "core":"test-collection_shard3_replica_n2",
2026-06-21T12:15:04.697268717Z "node_name":"solr-node1:8983_solr",
2026-06-21T12:15:04.697270663Z "base_url":"http://solr-node1:8983/solr",
2026-06-21T12:15:04.697272740Z "collection":"test-collection",
2026-06-21T12:15:04.697274818Z "shard":"shard3",
2026-06-21T12:15:04.697276773Z "state":"down",
2026-06-21T12:15:04.697278658Z "type":"NRT",
2026-06-21T12:15:04.697280634Z "operation":"ADDREPLICA",
2026-06-21T12:15:04.697282669Z "waitForFinalState":"false"}
2026-06-21T12:15:04.703114610Z 2026-06-21 12:15:04.702 INFO (OverseerStateUpdate-72160861276012544-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-21T12:15:04.703139957Z "core":"test-collection_shard1_replica_n4",
2026-06-21T12:15:04.703143562Z "node_name":"solr-node2:8984_solr",
2026-06-21T12:15:04.703146527Z "base_url":"http://solr-node2:8984/solr",
2026-06-21T12:15:04.703149313Z "collection":"test-collection",
2026-06-21T12:15:04.703151980Z "shard":"shard1",
2026-06-21T12:15:04.703154757Z "state":"down",
2026-06-21T12:15:04.703157582Z "type":"NRT",
2026-06-21T12:15:04.703161054Z "operation":"ADDREPLICA",
2026-06-21T12:15:04.703163787Z "waitForFinalState":"false"}
2026-06-21T12:15:04.813036112Z 2026-06-21 12:15:04.812 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 185] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-21T12:15:04.871533162Z 2026-06-21 12:15:04.871 INFO (qtp985324122-29-solr-node1-8) [c: s: r: x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node5&name=test-collection_shard3_replica_n2&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-06-21T12:15:04.924169860Z 2026-06-21 12:15:04.923 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-21T12:15:04.976384647Z 2026-06-21 12:15:04.975 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-06-21T12:15:05.252660591Z 2026-06-21 12:15:05.249 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-21T12:15:05.253280385Z 2026-06-21 12:15:05.249 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n2' using configuration from configset techproducts, trusted=true
2026-06-21T12:15:05.279271733Z 2026-06-21 12:15:05.278 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n2], dataDir=[/var/solr/data/test-collection_shard3_replica_n2/data/]
2026-06-21T12:15:05.947188950Z 2026-06-21 12:15:05.944 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-21T12:15:06.000548924Z 2026-06-21 12:15:05.999 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-21T12:15:06.004704402Z 2026-06-21 12:15:06.004 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-21T12:15:06.007331862Z 2026-06-21 12:15:06.006 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-21T12:15:06.007720571Z 2026-06-21 12:15:06.007 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-21T12:15:06.075316181Z 2026-06-21 12:15:06.074 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-21T12:15:06.077602004Z 2026-06-21 12:15:06.077 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-21T12:15:06.079933330Z 2026-06-21 12:15:06.079 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-21T12:15:06.081859304Z 2026-06-21 12:15:06.081 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-21T12:15:06.098539733Z 2026-06-21 12:15:06.098 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4}
2026-06-21T12:15:06.107196012Z 2026-06-21 12:15:06.106 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-21T12:15:06.113622555Z 2026-06-21 12:15:06.113 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n2 solr-node1-8 core_node5 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-21T12:15:06.113644451Z 2026-06-21 12:15:06.113 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n2 solr-node1-8 core_node5 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-21T12:15:06.122980394Z 2026-06-21 12:15:06.122 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n2 solr-node1-8 core_node5 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-21T12:15:06.128658176Z 2026-06-21 12:15:06.128 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node5=0}, version=0} for registerTerm
2026-06-21T12:15:06.130288325Z 2026-06-21 12:15:06.129 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-06-21T12:15:06.146058441Z 2026-06-21 12:15:06.145 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-21T12:15:06.146144691Z 2026-06-21 12:15:06.145 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-21T12:15:06.146250319Z 2026-06-21 12:15:06.145 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n2/
2026-06-21T12:15:06.147367431Z 2026-06-21 12:15:06.147 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-21T12:15:06.147910755Z 2026-06-21 12:15:06.147 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n2/ has no replicas
2026-06-21T12:15:06.149535191Z 2026-06-21 12:15:06.149 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72160861276012544-core_node5-n_0000000000
2026-06-21T12:15:06.157246621Z 2026-06-21 12:15:06.156 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n2/ shard3
2026-06-21T12:15:06.321887379Z 2026-06-21 12:15:06.321 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 236] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-21T12:15:06.326321074Z 2026-06-21 12:15:06.325 INFO (qtp985324122-29-solr-node1-8) [c:test-collection s:shard3 r:core_node5 x:test-collection_shard3_replica_n2 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-21T12:15:06.336180183Z 2026-06-21 12:15:06.335 INFO (qtp985324122-29-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n2&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1465
2026-06-21T12:15:06.350362604Z 2026-06-21 12:15:06.350 INFO (qtp985324122-27-solr-node1-7) [c:test-collection s: r: x: t:solr-node1-7] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
2026-06-21T12:15:06.436832288Z 2026-06-21 12:15:06.436 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 243] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-21T12:15:06.439193505Z 2026-06-21 12:15:06.438 INFO (qtp985324122-27-solr-node1-7) [c:test-collection s: r: x: t:solr-node1-7] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=1819
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 3285844,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-21T12:14:52.25531223Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-06-21T12:14:57.353203172Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-21T12:14:57.353239324Z It should be set to 65000 to avoid operational disruption.
2026-06-21T12:14:57.353243340Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-21T12:14:57.353742386Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-21T12:14:57.353751581Z
2026-06-21T12:14:57.365333402Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-21T12:14:57.373971950Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-21T12:14:57.419774543Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-21T12:14:57.459734406Z WARNING: A command line option has enabled the Security Manager
2026-06-21T12:14:57.459753624Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-21T12:14:58.575764869Z 2026-06-21 12:14:58.572 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-21T12:14:58.841441519Z 2026-06-21 12:14:58.840 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-21T12:14:58.867026472Z 2026-06-21 12:14:58.866 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-21T12:14:58.874144295Z 2026-06-21 12:14:58.873 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-21T12:14:58.874236982Z 2026-06-21 12:14:58.873 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-06-21T12:14:58.874334582Z 2026-06-21 12:14:58.873 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-21T12:14:58.874433810Z 2026-06-21 12:14:58.873 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-21T12:14:58.873953314Z
2026-06-21T12:14:58.877032937Z 2026-06-21 12:14:58.876 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-21T12:14:58.899107601Z 2026-06-21 12:14:58.898 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-21T12:14:58.921856941Z 2026-06-21 12:14:58.921 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-21T12:14:59.058791005Z 2026-06-21 12:14:59.058 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-21T12:14:59.075983499Z 2026-06-21 12:14:59.075 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-21T12:14:59.076070281Z 2026-06-21 12:14:59.075 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-21T12:14:59.076176644Z 2026-06-21 12:14:59.075 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-21T12:14:59.185915611Z 2026-06-21 12:14:59.185 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-21T12:14:59.185980625Z 2026-06-21 12:14:59.185 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-21T12:14:59.186094019Z 2026-06-21 12:14:59.185 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-21T12:14:59.217870211Z 2026-06-21 12:14:59.217 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-21T12:14:59.900497465Z 2026-06-21 12:14:59.899 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-21T12:15:00.052972126Z 2026-06-21 12:15:00.052 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-21T12:15:00.054443105Z 2026-06-21 12:15:00.052 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-21T12:15:00.063711987Z 2026-06-21 12:15:00.063 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-21T12:15:00.069481108Z 2026-06-21 12:15:00.069 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-21T12:15:00.072483265Z 2026-06-21 12:15:00.072 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-21T12:15:00.072496908Z 2026-06-21 12:15:00.072 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-21T12:15:00.107128297Z 2026-06-21 12:15:00.106 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-21T12:15:00.115846587Z 2026-06-21 12:15:00.115 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-21T12:15:00.141718407Z 2026-06-21 12:15:00.141 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-21T12:15:00.152420956Z 2026-06-21 12:15:00.152 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-06-21T12:15:00.165285007Z 2026-06-21 12:15:00.164 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-06-21T12:15:00.175690473Z 2026-06-21 12:15:00.175 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-21T12:15:00.203739143Z 2026-06-21 12:15:00.202 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-21T12:15:00.219676952Z 2026-06-21 12:15:00.219 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-21T12:15:00.522380695Z 2026-06-21 12:15:00.521 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-21T12:15:00.706138585Z 2026-06-21 12:15:00.705 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-21T12:15:00.978477849Z 2026-06-21 12:15:00.977 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-21T12:15:01.118124027Z 2026-06-21 12:15:01.117 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-21T12:15:01.123787744Z 2026-06-21 12:15:01.123 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_21.request.log
2026-06-21T12:15:01.127879717Z 2026-06-21 12:15:01.127 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-06-21T12:15:01.128755040Z 2026-06-21 12:15:01.128 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @3756ms
2026-06-21T12:15:01.285062905Z 2026-06-21 12:15:01.284 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=43
2026-06-21T12:15:01.361205801Z 2026-06-21 12:15:01.358 INFO (qtp985324122-56-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=16
2026-06-21T12:15:04.871816262Z 2026-06-21 12:15:04.871 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-06-21T12:15:04.941515468Z 2026-06-21 12:15:04.940 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-21T12:15:05.018405483Z 2026-06-21 12:15:05.017 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-06-21T12:15:05.282115249Z 2026-06-21 12:15:05.281 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-21T12:15:05.282133541Z 2026-06-21 12:15:05.281 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-06-21T12:15:05.300064210Z 2026-06-21 12:15:05.299 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-06-21T12:15:05.711129515Z 2026-06-21 12:15:05.710 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-21T12:15:05.731088557Z 2026-06-21 12:15:05.730 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-21T12:15:05.734944236Z 2026-06-21 12:15:05.734 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-21T12:15:05.736061659Z 2026-06-21 12:15:05.735 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-21T12:15:05.736082046Z 2026-06-21 12:15:05.735 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-21T12:15:05.842314006Z 2026-06-21 12:15:05.841 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-21T12:15:05.845192354Z 2026-06-21 12:15:05.844 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-21T12:15:05.847129625Z 2026-06-21 12:15:05.846 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-21T12:15:05.849093678Z 2026-06-21 12:15:05.848 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-21T12:15:05.860089671Z 2026-06-21 12:15:05.859 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-06-21T12:15:05.867482793Z 2026-06-21 12:15:05.867 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-21T12:15:05.872985169Z 2026-06-21 12:15:05.872 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-21T12:15:05.873089265Z 2026-06-21 12:15:05.872 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-21T12:15:05.879453232Z 2026-06-21 12:15:05.879 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-21T12:15:05.892295805Z 2026-06-21 12:15:05.891 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-06-21T12:15:05.893755422Z 2026-06-21 12:15:05.893 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1
2026-06-21T12:15:05.907920158Z 2026-06-21 12:15:05.907 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-21T12:15:05.907940871Z 2026-06-21 12:15:05.907 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-21T12:15:05.907997397Z 2026-06-21 12:15:05.907 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node2:8984/solr/test-collection_shard1_replica_n4/
2026-06-21T12:15:05.909165201Z 2026-06-21 12:15:05.908 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-21T12:15:05.909872741Z 2026-06-21 12:15:05.909 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ has no replicas
2026-06-21T12:15:05.910857809Z 2026-06-21 12:15:05.910 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard1/leader after winning as /collections/test-collection/leader_elect/shard1/election/72160861276012547-core_node6-n_0000000000
2026-06-21T12:15:05.919856805Z 2026-06-21 12:15:05.919 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ shard1
2026-06-21T12:15:06.030574387Z 2026-06-21 12:15:06.030 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 205] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-21T12:15:06.035093781Z 2026-06-21 12:15:06.034 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-21T12:15:06.044762509Z 2026-06-21 12:15:06.044 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1174
2026-06-21T12:15:06.143102550Z 2026-06-21 12:15:06.142 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 219] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-21T12:15:06.321174986Z 2026-06-21 12:15:06.320 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 236] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-21T12:15:06.437055297Z 2026-06-21 12:15:06.436 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 243] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node2 ===
{
"Name": "/solr-node2",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 3286243,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-21T12:14:57.164387683Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node3 ===
2026-06-21T12:14:57.546418945Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-21T12:14:57.546460796Z It should be set to 65000 to avoid operational disruption.
2026-06-21T12:14:57.546463822Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-21T12:14:57.546710228Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-21T12:14:57.546716924Z
2026-06-21T12:14:57.556546266Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-21T12:14:57.564209985Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-21T12:14:57.606787455Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-21T12:14:57.636923125Z WARNING: A command line option has enabled the Security Manager
2026-06-21T12:14:57.636947229Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-21T12:14:58.639048029Z 2026-06-21 12:14:58.635 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-21T12:14:58.902818962Z 2026-06-21 12:14:58.902 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-21T12:14:58.927533320Z 2026-06-21 12:14:58.927 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-21T12:14:58.934445592Z 2026-06-21 12:14:58.933 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-21T12:14:58.934520205Z 2026-06-21 12:14:58.934 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8985
2026-06-21T12:14:58.934620445Z 2026-06-21 12:14:58.934 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-21T12:14:58.934765328Z 2026-06-21 12:14:58.934 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-21T12:14:58.934142400Z
2026-06-21T12:14:58.937481332Z 2026-06-21 12:14:58.937 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-21T12:14:58.959172784Z 2026-06-21 12:14:58.958 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-21T12:14:58.981929833Z 2026-06-21 12:14:58.981 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-21T12:14:59.118901024Z 2026-06-21 12:14:59.118 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-21T12:14:59.135865147Z 2026-06-21 12:14:59.135 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-21T12:14:59.135990091Z 2026-06-21 12:14:59.135 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-21T12:14:59.136123894Z 2026-06-21 12:14:59.135 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-21T12:14:59.244288005Z 2026-06-21 12:14:59.243 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-21T12:14:59.244502389Z 2026-06-21 12:14:59.244 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-21T12:14:59.244512876Z 2026-06-21 12:14:59.244 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-21T12:14:59.276792540Z 2026-06-21 12:14:59.276 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-21T12:14:59.954590087Z 2026-06-21 12:14:59.954 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-21T12:15:00.097475828Z 2026-06-21 12:15:00.095 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-21T12:15:00.100599388Z 2026-06-21 12:15:00.095 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-21T12:15:00.108180014Z 2026-06-21 12:15:00.107 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-21T12:15:00.113031944Z 2026-06-21 12:15:00.112 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-21T12:15:00.116096115Z 2026-06-21 12:15:00.115 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-21T12:15:00.116108756Z 2026-06-21 12:15:00.115 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-21T12:15:00.145477934Z 2026-06-21 12:15:00.144 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-21T12:15:00.153543671Z 2026-06-21 12:15:00.153 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-21T12:15:00.173576494Z 2026-06-21 12:15:00.173 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-21T12:15:00.175436927Z 2026-06-21 12:15:00.175 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-21T12:15:00.183952203Z 2026-06-21 12:15:00.183 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node3:8985_solr as DOWN
2026-06-21T12:15:00.193853545Z 2026-06-21 12:15:00.193 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node3:8985_solr
2026-06-21T12:15:00.204383190Z 2026-06-21 12:15:00.203 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-21T12:15:00.251782885Z 2026-06-21 12:15:00.251 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-21T12:15:00.515770885Z 2026-06-21 12:15:00.515 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-21T12:15:00.683501596Z 2026-06-21 12:15:00.682 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-21T12:15:00.929458536Z 2026-06-21 12:15:00.929 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-21T12:15:01.048293619Z 2026-06-21 12:15:01.047 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-21T12:15:01.053976405Z 2026-06-21 12:15:01.053 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_21.request.log
2026-06-21T12:15:01.057363309Z 2026-06-21 12:15:01.056 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8985}
2026-06-21T12:15:01.058159068Z 2026-06-21 12:15:01.057 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @3495ms
2026-06-21T12:15:02.947584687Z 2026-06-21 12:15:02.946 INFO (qtp985324122-27-solr-node3-1) [c: s: r: x: t:solr-node3-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=191
2026-06-21T12:15:03.011170632Z 2026-06-21 12:15:03.010 INFO (qtp985324122-56-solr-node3-2) [c: s: r: x: t:solr-node3-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=13
2026-06-21T12:15:04.876497183Z 2026-06-21 12:15:04.875 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard2_replica_n1&action=CREATE&numShards=3&shard=shard2&wt=javabin
2026-06-21T12:15:04.937141512Z 2026-06-21 12:15:04.936 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-21T12:15:04.993786775Z 2026-06-21 12:15:04.993 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-06-21T12:15:05.261524930Z 2026-06-21 12:15:05.260 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-21T12:15:05.261542848Z 2026-06-21 12:15:05.261 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard2_replica_n1' using configuration from configset techproducts, trusted=true
2026-06-21T12:15:05.279862871Z 2026-06-21 12:15:05.279 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard2_replica_n1], dataDir=[/var/solr/data/test-collection_shard2_replica_n1/data/]
2026-06-21T12:15:05.921613714Z 2026-06-21 12:15:05.921 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-21T12:15:05.954347330Z 2026-06-21 12:15:05.953 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-21T12:15:05.968023714Z 2026-06-21 12:15:05.967 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-21T12:15:05.971055459Z 2026-06-21 12:15:05.970 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-21T12:15:05.971323041Z 2026-06-21 12:15:05.970 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-21T12:15:06.062111640Z 2026-06-21 12:15:06.061 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-21T12:15:06.068308886Z 2026-06-21 12:15:06.067 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-21T12:15:06.073209056Z 2026-06-21 12:15:06.072 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-21T12:15:06.077876755Z 2026-06-21 12:15:06.077 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-21T12:15:06.120573110Z 2026-06-21 12:15:06.119 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01}
2026-06-21T12:15:06.156200533Z 2026-06-21 12:15:06.155 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-21T12:15:06.167872676Z 2026-06-21 12:15:06.167 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n1 solr-node1-8 core_node3 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-21T12:15:06.167890332Z 2026-06-21 12:15:06.167 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n1 solr-node1-8 core_node3 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-21T12:15:06.175584491Z 2026-06-21 12:15:06.175 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n1 solr-node1-8 core_node3 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-21T12:15:06.185109755Z 2026-06-21 12:15:06.184 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard2 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-06-21T12:15:06.186625278Z 2026-06-21 12:15:06.186 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard2
2026-06-21T12:15:06.201557538Z 2026-06-21 12:15:06.201 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-21T12:15:06.201704305Z 2026-06-21 12:15:06.201 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-21T12:15:06.201717980Z 2026-06-21 12:15:06.201 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node3:8985/solr/test-collection_shard2_replica_n1/
2026-06-21T12:15:06.203369219Z 2026-06-21 12:15:06.203 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-21T12:15:06.203993896Z 2026-06-21 12:15:06.203 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node3:8985/solr/test-collection_shard2_replica_n1/ has no replicas
2026-06-21T12:15:06.206316279Z 2026-06-21 12:15:06.206 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard2/leader after winning as /collections/test-collection/leader_elect/shard2/election/72160861276012548-core_node3-n_0000000000
2026-06-21T12:15:06.215129698Z 2026-06-21 12:15:06.214 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node3:8985/solr/test-collection_shard2_replica_n1/ shard2
2026-06-21T12:15:06.321253049Z 2026-06-21 12:15:06.320 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 236] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-21T12:15:06.326979336Z 2026-06-21 12:15:06.326 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-21T12:15:06.336621507Z 2026-06-21 12:15:06.336 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard2_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1461
2026-06-21T12:15:06.437206024Z 2026-06-21 12:15:06.436 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 243] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node3 ===
{
"Name": "/solr-node3",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 3286412,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-21T12:14:57.341748162Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# Docker artifacts saved to: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/docker
not ok 85 Docker SolrCloud rolling upgrade # in 28456 ms
# (in test file test/test_rolling_upgrade.bats, line 118)
# `wait_for 30 1 docker exec solr-node1 solr healthcheck -c test-collection' failed with status 127
# 9.10.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# af6eca94c810: Pulling fs layer
# cb0efb96dabd: Pulling fs layer
# 3e9d91201f40: Pulling fs layer
# 66b76b382631: Pulling fs layer
# 66b76b382631: Waiting
# 601f2c23751f: Pulling fs layer
# 601f2c23751f: Waiting
# 0ed719aef393: Pulling fs layer
# 0ed719aef393: Waiting
# 69a09c96e8bf: Pulling fs layer
# c937aee9a870: Pulling fs layer
# 9cdfe5320de8: Pulling fs layer
# 3aa9120e46f5: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# 69a09c96e8bf: Waiting
# c937aee9a870: Waiting
# 9cdfe5320de8: Waiting
# 4f4fb700ef54: Waiting
# 3aa9120e46f5: Waiting
# cb0efb96dabd: Verifying Checksum
# cb0efb96dabd: Download complete
# af6eca94c810: Verifying Checksum
# af6eca94c810: Download complete
# 66b76b382631: Download complete
# 601f2c23751f: Verifying Checksum
# 601f2c23751f: Download complete
# 69a09c96e8bf: Verifying Checksum
# 69a09c96e8bf: Download complete
# c937aee9a870: Verifying Checksum
# c937aee9a870: Download complete
# 3e9d91201f40: Verifying Checksum
# 3e9d91201f40: Download complete
# 9cdfe5320de8: Verifying Checksum
# 9cdfe5320de8: Download complete
# 3aa9120e46f5: Download complete
# 4f4fb700ef54: Verifying Checksum
# 4f4fb700ef54: Download complete
# 0ed719aef393: Verifying Checksum
# 0ed719aef393: Download complete
# af6eca94c810: Pull complete
# cb0efb96dabd: Pull complete
# 3e9d91201f40: Pull complete
# 66b76b382631: Pull complete
# 601f2c23751f: Pull complete
# 0ed719aef393: Pull complete
# 69a09c96e8bf: Pull complete
# c937aee9a870: Pull complete
# 9cdfe5320de8: Pull complete
# 3aa9120e46f5: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:15c8155ceffe1042f37c77bb537aa531804a3510702ff87f2af5fedf523f4203
# Status: Downloaded newer image for apache/solr-nightly:9.10.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:9.10.0-SNAPSHOT-slim
# 10.0.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# 40d16f30db40: Pulling fs layer
# 9d5a0b6cb36c: Pulling fs layer
# 49518c28c0b3: Pulling fs layer
# 41162acdcc29: Pulling fs layer
# 2ed0fd558ca3: Pulling fs layer
# 2488415fda1c: Pulling fs layer
# 09511c8fb203: Pulling fs layer
# a0c57fb89c4f: Pulling fs layer
# d0641dcc41d7: Pulling fs layer
# 315722fd9570: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# a0c57fb89c4f: Waiting
# 2ed0fd558ca3: Waiting
# 2488415fda1c: Waiting
# 09511c8fb203: Waiting
# d0641dcc41d7: Waiting
# 4f4fb700ef54: Waiting
# 315722fd9570: Waiting
# 41162acdcc29: Waiting
# 40d16f30db40: Verifying Checksum
# 40d16f30db40: Download complete
# 9d5a0b6cb36c: Verifying Checksum
# 9d5a0b6cb36c: Download complete
# 41162acdcc29: Verifying Checksum
# 41162acdcc29: Download complete
# 2ed0fd558ca3: Verifying Checksum
# 2ed0fd558ca3: Download complete
# 09511c8fb203: Verifying Checksum
# 09511c8fb203: Download complete
# 49518c28c0b3: Verifying Checksum
# 49518c28c0b3: Download complete
# a0c57fb89c4f: Verifying Checksum
# a0c57fb89c4f: Download complete
# d0641dcc41d7: Download complete
# 315722fd9570: Download complete
# 4f4fb700ef54: Download complete
# 2488415fda1c: Verifying Checksum
# 2488415fda1c: Download complete
# 40d16f30db40: Pull complete
# 9d5a0b6cb36c: Pull complete
# 49518c28c0b3: Pull complete
# 41162acdcc29: Pull complete
# 2ed0fd558ca3: Pull complete
# 2488415fda1c: Pull complete
# 09511c8fb203: Pull complete
# a0c57fb89c4f: Pull complete
# d0641dcc41d7: Pull complete
# 315722fd9570: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:fb25f7a7190e3027ade20e97708a8665d41ab0e9ea9285fa242b2c83b0e5f43c
# Status: Downloaded newer image for apache/solr-nightly:10.0.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:10.0.0-SNAPSHOT-slim
# 33b7cbc3d312472ced44d510a7477591dae6b7d9818e60e724a9b09df5dc2cf1
# solr-data1
# solr-data2
# solr-data3
# Starting solr-node1 with embedded ZooKeeper
# 14fec5eb9f516f6341d0d85d816603768c87e7405d4c038c28aafd0668c64131
# Starting solr-node2 connected to first node's ZooKeeper
# eaf9d1d61dbf4094883e7914c12937526587281a4c8e2a7f8d4c8448a9004cf8
# Started solr-node3 connected to first node's ZooKeeper
# 113060fb372336344de57b66d8706f7e5312635e6001b326ef62fb754e966432
# Creating a Collection
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# WARN - 2026-06-21 12:15:04.153; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-21 12:15:04.183; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. "/solr"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [http://solr-node1:8983/solr] to [http://solr-node1:8983/].
# Created collection 'test-collection' with 3 shard(s), 1 replica(s) with config-set 'techproducts'
# Checking collection health
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_rolling_upgrade.bats: line 118: wait_for: command not found
# solr-node1
# solr-node2
# solr-node3
# solr-node1
# solr-node2
# solr-node3
# solr-data1
# solr-data2
# solr-data3
# solrcloud-test
ok 86 allowPaths - backup # in 14802 ms
ok 87 snapshot lifecycle # in 4559 ms
ok 88 snapshot list # in 3133 ms
ok 89 snapshot list via --solr-connection # in 3139 ms
ok 90 snapshot describe # in 3214 ms
ok 91 snapshot describe via --solr-connection # in 3169 ms
ok 92 start solr with ssl # in 15803 ms
ok 93 use different hostname when not checking peer-name # in 23566 ms
ok 94 start solr with ssl and auth # in 14101 ms
ok 95 start solr with client truststore and security manager # in 15819 ms
ok 96 start solr with mTLS needed # in 44476 ms
ok 97 start solr with mTLS wanted # in 36841 ms
ok 98 test keystore reload # in 56790 ms
ok 99 SOLR-11740 check 'solr stop' connection # in 15472 ms
ok 100 stop command for single port # in 17763 ms
ok 101 check stop command doesn't hang # in 205 ms
ok 102 SOLR-16976 solr starts with remote JMX enabled # in 8020 ms
not ok 103 deprecated system properties converted to modern properties # in 195359 ms
# (from function `assert_file_contains' in file .gradle/node/node_modules/bats-file/src/file.bash, line 566,
# in test file test/test_start_solr.bats, line 86)
# `assert_file_contains "${SOLR_LOGS_DIR}/solr.log" 'You are passing in deprecated system property disable.config.edit and should upgrade to using solr.api.config.edit.enabled instead.'' failed
# Waiting up to 180 seconds to see Solr running on port 44801 [|] [/] [-] [\] [|]
# Started Solr server on port 44801 (pid=3303944). Happy searching!
#
#
# -- file does not contain regex --
# path : /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/solr-home/logs/solr.log
# --
#
# Please find the SOLR_HOME snapshot for failed test #5 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/103-5
ok 104 start with custom jetty options # in 7881 ms
ok 105 -c flag prints no-op warning and still starts in cloud mode # in 8917 ms
not ok 106 bootstrapping a configset # in 8782 ms
# (from function `config_exists' in file test/bats_helper.bash, line 84,
# in test file test/test_start_solr.bats, line 116)
# `config_exists "techproducts"' failed
# Waiting up to 180 seconds to see Solr running on port 44801 [|] [/] [-] [\] [|]
# Started Solr server on port 44801 (pid=3306660). Happy searching!
#
# WARN - 2026-06-21 12:23:36.242; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-21 12:23:36.246; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# Please find the SOLR_HOME snapshot for failed test #8 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/106-8
ok 107 status detects locally running solr # in 9168 ms
ok 108 status with --solr-url from user # in 7745 ms
ok 109 status with --port from user # in 7716 ms
ok 110 multiple connection options are prevented # in 809 ms
ok 111 status with invalid --solr-url from user # in 1174 ms
ok 112 status with --short format # in 195594 ms
ok 113 searching solr via locally executed streaming expression # in 1663 ms
ok 114 searching solr via remotely executed streaming expression by solr-url # in 1236 ms
ok 115 searching solr via remotely executed streaming expression by solr connection # in 1116 ms
ok 116 variable interpolation # in 1644 ms
ok 117 searching solr without credentials fails with error # in 1077 ms
ok 118 --version returns Solr version # in 709 ms
ok 119 version as direct tool call still runs # in 730 ms
ok 120 short help # in 743 ms
ok 121 short help is inferred # in 722 ms
ok 122 long help # in 758 ms
ok 123 running subcommands with zk is prevented # in 698 ms
ok 124 listing out files # in 2099 ms
ok 125 connecting to solr via various solr urls and zk hosts # in 9045 ms
ok 126 copying files around # in 10733 ms
ok 127 upconfig # in 2500 ms
ok 128 SOLR-12429 test upconfig fails with symlink # in 1163 ms
ok 129 downconfig # in 1141 ms
ok 130 bin/solr zk cp gets 'solrhome' from '--solr-home' command line option # in 1418 ms
ok 131 env var ZK_HOST is honored # in 3112 ms
ok 132 Cleanup # in 1853 ms
> Task :solr:packaging:integrationTests FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build.gradle' line: 339
* What went wrong:
Execution failed for task ':solr:packaging:integrationTests'.
> Process 'command '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/.gradle/node/node_modules/bats/bin/bats'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/8.10/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 29m 23s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/xmftflqp6munc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 239 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/239/
No tests ran.
Build Log:
[...truncated 990 lines...]
BUILD FAILED in 29m 18s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/pkotaqd5udltm
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 238 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/238/
No tests ran.
Build Log:
[...truncated 347 lines...]
# ERROR: KeeperErrorCode = NoNode for /security.json
# --
#
# Last output:
# WARN - 2026-06-19 12:05:02.024; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-19 12:05:02.027; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-19 12:05:02.503; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-19 12:05:02.504; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: KeeperErrorCode = NoNode for /security.json
not ok 9 create, config, api, and delete with basic auth # in 16906 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 46853 is already being used by another process (pid: 2925526)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-06-19 12:05:03.668; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-19 12:05:03.672; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
ok 10 post, postlogs and export with basic auth # in 17251 ms
ok 11 package with basic auth # in 12031 ms
ok 12 admin UI static content is excepted from basic auth # in 10401 ms
ok 13 nothing # in 11 ms
ok 14 server does not compress response without Accept-Encoding header # in 175 ms
ok 15 server compresses response when Accept-Encoding: gzip is requested # in 77 ms
ok 16 compressed response can be decompressed and parsed # in 85 ms
ok 17 setting property # in 10848 ms
ok 18 short form of setting property # in 6324 ms
ok 19 connecting to solr via various connection types # in 10613 ms
ok 20 create for non cloud mode # in 7996 ms
ok 21 create for cloud mode # in 9625 ms
ok 22 multiple connection options are prevented # in 860 ms
ok 23 create collection # in 7587 ms
ok 24 create collection using solrUrl # in 5147 ms
ok 25 create collection using legacy solrUrl # in 5177 ms
ok 26 create collection using Zookeeper # in 5354 ms
ok 27 reject d option with invalid config dir # in 2624 ms
ok 28 accept d option with builtin config # in 5184 ms
ok 29 accept d option with explicit path to config # in 5136 ms
ok 30 accept n option as config name # in 5256 ms
ok 31 allow config reuse when n option specifies same config # in 9079 ms
ok 32 create multisharded collections when s provided # in 5195 ms
ok 33 create replicated collections when rf provided # in 6159 ms
ok 34 can delete collections # in 10654 ms
ok 35 can delete collections with solr-url # in 8289 ms
ok 36 can delete collections with solr-connection # in 7888 ms
ok 37 collection delete also deletes zk config # in 7384 ms
ok 38 deletes accompanying zk config with nondefault name # in 7317 ms
ok 39 delete-config option can opt to leave config in zk # in 7109 ms
ok 40 start -e cloud works with --jvm-opts # in 21554 ms
ok 41 start -e cloud works with --no-prompt # in 43307 ms
ok 42 Check export command # in 18900 ms
ok 43 export fails on non cloud mode # in 9208 ms
Waiting for Tika Server to be ready on port 46858
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 44 using curl to extract a single pdf file # in 16588 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to extract content from pdf # in 16609 ms
WARNING: Docker not available; bypassing test.
ok 46 using the bin/solr post tool to crawl web site # in 16551 ms
Stopping Tika Server container
ok 47 healthcheck on cloud solr # in 10146 ms
ok 48 healthcheck on remote cloud solr # in 9620 ms
ok 49 healthcheck errors on standalone solr # in 11077 ms
ok 50 solr help flag prints help # in 708 ms
ok 51 solr with no flags prints help # in 718 ms
ok 52 start help flag prints help # in 150 ms
ok 53 start h flag prints help # in 141 ms
ok 54 stop help flag prints help # in 139 ms
ok 55 restart help flag prints help # in 144 ms
ok 56 status help flag prints help # in 770 ms
ok 57 healthcheck help flag prints help # in 749 ms
ok 58 create help flag prints help # in 731 ms
ok 59 delete help flag prints help # in 760 ms
ok 60 zk help flag prints help # in 759 ms
ok 61 auth help flag prints help # in 759 ms
ok 62 assert help flag prints help # in 755 ms
ok 63 post help flag prints help # in 745 ms
ok 64 SQL Module # in 15325 ms
ok 65 icu collation in analysis-extras module # in 12254 ms
ok 66 Check lifecycle of sentiment classification # in 22933 ms
ok 67 lifecycle of package # in 8325 ms
ok 68 deploying and undeploying a collection level package # in 13354 ms
ok 69 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
ok 70 Affinity placement plugin using sysprop # in 15083 ms
ok 71 Random placement plugin using ENV # in 15283 ms
ok 72 Check help command # in 2324 ms
ok 73 basic post with a type specified # in 5335 ms
ok 74 basic post with collection instead of url parameter # in 3925 ms
ok 75 basic post with solr-url and collection # in 3790 ms
ok 76 basic post with -s (solr-connection short form) and collection # in 3791 ms
ok 77 basic post WITHOUT a type specified # in 11267 ms
ok 78 crawling a directory as a dry-run # in 797 ms
ok 79 crawling a directory # in 4173 ms
ok 80 skipcommit and optimize and delete # in 3683 ms
ok 81 args mode # in 6884 ms
ok 82 stdin mode # in 3389 ms
ok 83 verbose echo the Solr response # in 3696 ms
ok 84 post solr log into solr via cli # in 9528 ms
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 9 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 9 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 14 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-06-19T12:15:26.743722495Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-19T12:15:26.743760157Z It should be set to 65000 to avoid operational disruption.
2026-06-19T12:15:26.743763122Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-19T12:15:26.743991452Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-19T12:15:26.743997563Z
2026-06-19T12:15:26.753330474Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-19T12:15:26.761011266Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-19T12:15:26.803817243Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-19T12:15:26.835358723Z WARNING: A command line option has enabled the Security Manager
2026-06-19T12:15:26.835383647Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-19T12:15:27.894007473Z 2026-06-19 12:15:27.890 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-19T12:15:28.155443491Z 2026-06-19 12:15:28.154 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-19T12:15:28.180698181Z 2026-06-19 12:15:28.180 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-19T12:15:28.187590970Z 2026-06-19 12:15:28.187 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-19T12:15:28.187688032Z 2026-06-19 12:15:28.187 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-06-19T12:15:28.187795637Z 2026-06-19 12:15:28.187 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-19T12:15:28.187940785Z 2026-06-19 12:15:28.187 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-19T12:15:28.187389137Z
2026-06-19T12:15:28.190505277Z 2026-06-19 12:15:28.190 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-19T12:15:28.211608910Z 2026-06-19 12:15:28.211 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-19T12:15:28.224698832Z 2026-06-19 12:15:28.224 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-19T12:15:28.224812999Z 2026-06-19 12:15:28.224 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-19T12:15:28.259203382Z 2026-06-19 12:15:28.258 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-19T12:15:29.147327693Z 2026-06-19 12:15:29.146 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-19T12:15:29.390178742Z 2026-06-19 12:15:29.389 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-19T12:15:29.395460198Z 2026-06-19 12:15:29.395 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-06-19T12:15:29.395586729Z 2026-06-19 12:15:29.395 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-06-19T12:15:29.398255373Z 2026-06-19 12:15:29.397 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-06-19T12:15:29.398288977Z 2026-06-19 12:15:29.397 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-06-19T12:15:29.441327461Z 2026-06-19 12:15:29.440 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-06-19T12:15:29.902085449Z 2026-06-19 12:15:29.901 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-06-19T12:15:29.914628780Z 2026-06-19 12:15:29.914 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-19T12:15:29.955697297Z 2026-06-19 12:15:29.954 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-19T12:15:29.977832625Z 2026-06-19 12:15:29.977 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-19T12:15:29.978070236Z 2026-06-19 12:15:29.977 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-19T12:15:30.109826044Z 2026-06-19 12:15:30.108 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-19T12:15:30.131102855Z 2026-06-19 12:15:30.130 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-19T12:15:30.142111782Z 2026-06-19 12:15:30.141 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-06-19T12:15:30.153119225Z 2026-06-19 12:15:30.152 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72149538921578496-solr-node1:8983_solr-n_0000000000) starting
2026-06-19T12:15:30.217969461Z 2026-06-19 12:15:30.217 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-06-19T12:15:30.218362663Z 2026-06-19 12:15:30.218 INFO (OverseerStateUpdate-72149538921578496-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-06-19T12:15:30.230458819Z 2026-06-19 12:15:30.230 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-19T12:15:30.273200141Z 2026-06-19 12:15:30.272 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-19T12:15:30.691133618Z 2026-06-19 12:15:30.690 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-19T12:15:30.870035056Z 2026-06-19 12:15:30.869 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-19T12:15:31.233311698Z 2026-06-19 12:15:31.232 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-19T12:15:31.364859894Z 2026-06-19 12:15:31.364 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-19T12:15:31.370623029Z 2026-06-19 12:15:31.370 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_19.request.log
2026-06-19T12:15:31.376890213Z 2026-06-19 12:15:31.376 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-06-19T12:15:31.377822809Z 2026-06-19 12:15:31.377 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4618ms
2026-06-19T12:15:31.555074736Z 2026-06-19 12:15:31.554 INFO (qtp985324122-27-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=43
2026-06-19T12:15:31.679981805Z 2026-06-19 12:15:31.679 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=12
2026-06-19T12:15:35.037593999Z 2026-06-19 12:15:35.037 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-19T12:15:35.155521005Z 2026-06-19 12:15:35.155 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-19T12:15:39.000305530Z 2026-06-19 12:15:38.999 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=8
2026-06-19T12:15:39.026244856Z 2026-06-19 12:15:39.025 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=9
2026-06-19T12:15:39.053701020Z 2026-06-19 12:15:39.053 INFO (qtp985324122-27-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=2
2026-06-19T12:15:39.328044972Z 2026-06-19 12:15:39.327 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-06-19T12:15:39.345202488Z 2026-06-19 12:15:39.344 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-06-19T12:15:39.352383047Z 2026-06-19 12:15:39.351 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 174] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-19T12:15:39.388505104Z 2026-06-19 12:15:39.387 INFO (OverseerStateUpdate-72149538921578496-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-19T12:15:39.388524951Z "core":"test-collection_shard3_replica_n1",
2026-06-19T12:15:39.388528264Z "node_name":"solr-node1:8983_solr",
2026-06-19T12:15:39.388531110Z "base_url":"http://solr-node1:8983/solr",
2026-06-19T12:15:39.388533738Z "collection":"test-collection",
2026-06-19T12:15:39.388536463Z "shard":"shard3",
2026-06-19T12:15:39.388539039Z "state":"down",
2026-06-19T12:15:39.388541858Z "type":"NRT",
2026-06-19T12:15:39.388544342Z "operation":"ADDREPLICA",
2026-06-19T12:15:39.388546904Z "waitForFinalState":"false"}
2026-06-19T12:15:39.394077396Z 2026-06-19 12:15:39.393 INFO (OverseerStateUpdate-72149538921578496-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-19T12:15:39.394097082Z "core":"test-collection_shard2_replica_n2",
2026-06-19T12:15:39.394111677Z "node_name":"solr-node3:8985_solr",
2026-06-19T12:15:39.394115911Z "base_url":"http://solr-node3:8985/solr",
2026-06-19T12:15:39.394119906Z "collection":"test-collection",
2026-06-19T12:15:39.394123546Z "shard":"shard2",
2026-06-19T12:15:39.394126752Z "state":"down",
2026-06-19T12:15:39.394130711Z "type":"NRT",
2026-06-19T12:15:39.394134486Z "operation":"ADDREPLICA",
2026-06-19T12:15:39.394138198Z "waitForFinalState":"false"}
2026-06-19T12:15:39.402234539Z 2026-06-19 12:15:39.401 INFO (OverseerStateUpdate-72149538921578496-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-19T12:15:39.402249727Z "core":"test-collection_shard1_replica_n4",
2026-06-19T12:15:39.402253125Z "node_name":"solr-node2:8984_solr",
2026-06-19T12:15:39.402267721Z "base_url":"http://solr-node2:8984/solr",
2026-06-19T12:15:39.402269930Z "collection":"test-collection",
2026-06-19T12:15:39.402272112Z "shard":"shard1",
2026-06-19T12:15:39.402274017Z "state":"down",
2026-06-19T12:15:39.402275893Z "type":"NRT",
2026-06-19T12:15:39.402277780Z "operation":"ADDREPLICA",
2026-06-19T12:15:39.402279902Z "waitForFinalState":"false"}
2026-06-19T12:15:39.511473160Z 2026-06-19 12:15:39.510 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 186] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-19T12:15:39.572735090Z 2026-06-19 12:15:39.572 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-06-19T12:15:39.612822221Z 2026-06-19 12:15:39.611 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-19T12:15:39.707552747Z 2026-06-19 12:15:39.707 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-19T12:15:40.010868921Z 2026-06-19 12:15:40.010 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-19T12:15:40.010936691Z 2026-06-19 12:15:40.010 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true
2026-06-19T12:15:40.029191950Z 2026-06-19 12:15:40.028 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/]
2026-06-19T12:15:40.662595468Z 2026-06-19 12:15:40.659 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-19T12:15:40.697381370Z 2026-06-19 12:15:40.696 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-19T12:15:40.705286812Z 2026-06-19 12:15:40.704 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-19T12:15:40.706989167Z 2026-06-19 12:15:40.706 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-19T12:15:40.707282592Z 2026-06-19 12:15:40.706 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-19T12:15:40.764218673Z 2026-06-19 12:15:40.763 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-19T12:15:40.768079436Z 2026-06-19 12:15:40.767 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-19T12:15:40.769843015Z 2026-06-19 12:15:40.769 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-19T12:15:40.772040318Z 2026-06-19 12:15:40.771 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-19T12:15:40.787000227Z 2026-06-19 12:15:40.786 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01}
2026-06-19T12:15:40.794846014Z 2026-06-19 12:15:40.794 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-19T12:15:40.802085721Z 2026-06-19 12:15:40.801 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-19T12:15:40.802104355Z 2026-06-19 12:15:40.801 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-19T12:15:40.809307246Z 2026-06-19 12:15:40.808 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-19T12:15:40.814351438Z 2026-06-19 12:15:40.813 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-06-19T12:15:40.815671718Z 2026-06-19 12:15:40.815 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-06-19T12:15:40.830508306Z 2026-06-19 12:15:40.830 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-19T12:15:40.830532291Z 2026-06-19 12:15:40.830 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-19T12:15:40.830536290Z 2026-06-19 12:15:40.830 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/
2026-06-19T12:15:40.831827926Z 2026-06-19 12:15:40.831 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-19T12:15:40.832333076Z 2026-06-19 12:15:40.832 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas
2026-06-19T12:15:40.833864887Z 2026-06-19 12:15:40.833 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72149538921578496-core_node3-n_0000000000
2026-06-19T12:15:40.840331768Z 2026-06-19 12:15:40.840 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3
2026-06-19T12:15:40.945106453Z 2026-06-19 12:15:40.944 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 235] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-19T12:15:40.949847767Z 2026-06-19 12:15:40.949 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-19T12:15:40.958258435Z 2026-06-19 12:15:40.957 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1386
2026-06-19T12:15:41.065441636Z 2026-06-19 12:15:41.064 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 242] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 2956765,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-19T12:15:26.447953346Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-06-19T12:15:32.016002528Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-19T12:15:32.016031307Z It should be set to 65000 to avoid operational disruption.
2026-06-19T12:15:32.016034301Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-19T12:15:32.016320505Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-19T12:15:32.016333427Z
2026-06-19T12:15:32.025659628Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-19T12:15:32.033182173Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-19T12:15:32.077887142Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-19T12:15:32.115591567Z WARNING: A command line option has enabled the Security Manager
2026-06-19T12:15:32.115609664Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-19T12:15:33.341795994Z 2026-06-19 12:15:33.338 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-19T12:15:33.613610941Z 2026-06-19 12:15:33.613 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-19T12:15:33.640091810Z 2026-06-19 12:15:33.639 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-19T12:15:33.647896224Z 2026-06-19 12:15:33.647 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-19T12:15:33.647919191Z 2026-06-19 12:15:33.647 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-06-19T12:15:33.648006067Z 2026-06-19 12:15:33.647 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-19T12:15:33.648135699Z 2026-06-19 12:15:33.647 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-19T12:15:33.647539544Z
2026-06-19T12:15:33.651091103Z 2026-06-19 12:15:33.650 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-19T12:15:33.674830068Z 2026-06-19 12:15:33.674 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-19T12:15:33.702118045Z 2026-06-19 12:15:33.701 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-19T12:15:33.848100555Z 2026-06-19 12:15:33.847 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-19T12:15:33.867839602Z 2026-06-19 12:15:33.867 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-19T12:15:33.867955903Z 2026-06-19 12:15:33.867 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-19T12:15:33.868199594Z 2026-06-19 12:15:33.867 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-19T12:15:33.978198713Z 2026-06-19 12:15:33.977 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-19T12:15:33.978471393Z 2026-06-19 12:15:33.978 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-19T12:15:33.978573181Z 2026-06-19 12:15:33.978 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-19T12:15:34.012631152Z 2026-06-19 12:15:34.012 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-19T12:15:34.874338191Z 2026-06-19 12:15:34.873 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-19T12:15:35.013229214Z 2026-06-19 12:15:35.012 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-19T12:15:35.014730409Z 2026-06-19 12:15:35.012 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-19T12:15:35.026518249Z 2026-06-19 12:15:35.026 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-19T12:15:35.032751167Z 2026-06-19 12:15:35.032 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-19T12:15:35.055532904Z 2026-06-19 12:15:35.055 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-19T12:15:35.055761551Z 2026-06-19 12:15:35.055 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-19T12:15:35.094845062Z 2026-06-19 12:15:35.094 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-19T12:15:35.106439251Z 2026-06-19 12:15:35.105 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
2026-06-19T12:15:35.126443035Z 2026-06-19 12:15:35.126 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-19T12:15:35.137820822Z 2026-06-19 12:15:35.137 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-06-19T12:15:35.148143698Z 2026-06-19 12:15:35.147 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-06-19T12:15:35.156564374Z 2026-06-19 12:15:35.156 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-19T12:15:35.204244228Z 2026-06-19 12:15:35.203 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-19T12:15:35.542245188Z 2026-06-19 12:15:35.541 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-19T12:15:35.718078019Z 2026-06-19 12:15:35.717 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-19T12:15:35.986305748Z 2026-06-19 12:15:35.985 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-19T12:15:36.118515293Z 2026-06-19 12:15:36.117 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-19T12:15:36.123759209Z 2026-06-19 12:15:36.123 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_19.request.log
2026-06-19T12:15:36.128334865Z 2026-06-19 12:15:36.127 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-06-19T12:15:36.129585437Z 2026-06-19 12:15:36.129 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4097ms
2026-06-19T12:15:36.282290245Z 2026-06-19 12:15:36.281 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=45
2026-06-19T12:15:36.346796572Z 2026-06-19 12:15:36.346 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=17
2026-06-19T12:15:39.300251969Z 2026-06-19 12:15:39.299 INFO (qtp985324122-58-solr-node2-3) [c: s: r: x: t:solr-node2-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=4
2026-06-19T12:15:39.574439392Z 2026-06-19 12:15:39.573 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-06-19T12:15:39.621633648Z 2026-06-19 12:15:39.621 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-19T12:15:39.679900712Z 2026-06-19 12:15:39.679 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-19T12:15:39.982588970Z 2026-06-19 12:15:39.982 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-19T12:15:39.982623008Z 2026-06-19 12:15:39.982 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-06-19T12:15:40.001839469Z 2026-06-19 12:15:40.001 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-06-19T12:15:40.433829615Z 2026-06-19 12:15:40.433 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-19T12:15:40.453579178Z 2026-06-19 12:15:40.453 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-19T12:15:40.457198776Z 2026-06-19 12:15:40.456 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-19T12:15:40.458248859Z 2026-06-19 12:15:40.457 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-19T12:15:40.458345520Z 2026-06-19 12:15:40.458 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-19T12:15:40.502412376Z 2026-06-19 12:15:40.501 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-19T12:15:40.505608029Z 2026-06-19 12:15:40.505 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-19T12:15:40.507530209Z 2026-06-19 12:15:40.507 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-19T12:15:40.509257072Z 2026-06-19 12:15:40.509 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-19T12:15:40.521025085Z 2026-06-19 12:15:40.520 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-06-19T12:15:40.528980325Z 2026-06-19 12:15:40.528 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-19T12:15:40.534285207Z 2026-06-19 12:15:40.533 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-19T12:15:40.534357586Z 2026-06-19 12:15:40.534 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-19T12:15:40.540710870Z 2026-06-19 12:15:40.540 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-19T12:15:40.552203611Z 2026-06-19 12:15:40.551 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-06-19T12:15:40.554055061Z 2026-06-19 12:15:40.553 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1
2026-06-19T12:15:40.567476782Z 2026-06-19 12:15:40.566 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-19T12:15:40.567532433Z 2026-06-19 12:15:40.566 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-19T12:15:40.567626115Z 2026-06-19 12:15:40.567 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node2:8984/solr/test-collection_shard1_replica_n4/
2026-06-19T12:15:40.568573336Z 2026-06-19 12:15:40.568 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-19T12:15:40.569586507Z 2026-06-19 12:15:40.569 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ has no replicas
2026-06-19T12:15:40.570626155Z 2026-06-19 12:15:40.570 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard1/leader after winning as /collections/test-collection/leader_elect/shard1/election/72149538921578500-core_node6-n_0000000000
2026-06-19T12:15:40.612895781Z 2026-06-19 12:15:40.612 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ shard1
2026-06-19T12:15:40.723637520Z 2026-06-19 12:15:40.723 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 206] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-19T12:15:40.727133132Z 2026-06-19 12:15:40.726 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-19T12:15:40.734864620Z 2026-06-19 12:15:40.734 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1161
2026-06-19T12:15:40.945290131Z 2026-06-19 12:15:40.944 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 235] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-19T12:15:40.974558570Z 2026-06-19 12:15:40.974 INFO (qtp985324122-59-solr-node2-4) [c:test-collection s: r: x: t:solr-node2-4] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
2026-06-19T12:15:41.065661327Z 2026-06-19 12:15:41.065 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 242] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-19T12:15:41.068904835Z 2026-06-19 12:15:41.068 INFO (qtp985324122-59-solr-node2-4) [c:test-collection s: r: x: t:solr-node2-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=1757
# === Docker inspect for solr-node2 ===
{
"Name": "/solr-node2",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 2957172,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-19T12:15:31.787261034Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node3 ===
2026-06-19T12:15:32.215736495Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-19T12:15:32.215779461Z It should be set to 65000 to avoid operational disruption.
2026-06-19T12:15:32.215783282Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-19T12:15:32.216031928Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-19T12:15:32.216047841Z
2026-06-19T12:15:32.225272005Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-19T12:15:32.233332874Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-19T12:15:32.278533085Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-19T12:15:32.310926950Z WARNING: A command line option has enabled the Security Manager
2026-06-19T12:15:32.310950147Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-19T12:15:33.361814010Z 2026-06-19 12:15:33.358 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-19T12:15:33.630235058Z 2026-06-19 12:15:33.629 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-19T12:15:33.655709425Z 2026-06-19 12:15:33.655 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-19T12:15:33.662996316Z 2026-06-19 12:15:33.662 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-19T12:15:33.663055902Z 2026-06-19 12:15:33.662 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8985
2026-06-19T12:15:33.663193252Z 2026-06-19 12:15:33.662 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-19T12:15:33.663311742Z 2026-06-19 12:15:33.662 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-19T12:15:33.662770248Z
2026-06-19T12:15:33.666066239Z 2026-06-19 12:15:33.665 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-19T12:15:33.688298073Z 2026-06-19 12:15:33.687 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-19T12:15:33.712342352Z 2026-06-19 12:15:33.711 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-19T12:15:33.852181318Z 2026-06-19 12:15:33.851 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-19T12:15:33.871201846Z 2026-06-19 12:15:33.870 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-19T12:15:33.871426994Z 2026-06-19 12:15:33.870 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-19T12:15:33.871437190Z 2026-06-19 12:15:33.871 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-19T12:15:33.981072804Z 2026-06-19 12:15:33.980 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-19T12:15:33.981341722Z 2026-06-19 12:15:33.981 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-19T12:15:33.981435458Z 2026-06-19 12:15:33.981 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-19T12:15:34.015282491Z 2026-06-19 12:15:34.014 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-19T12:15:34.798074090Z 2026-06-19 12:15:34.797 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-19T12:15:34.930977460Z 2026-06-19 12:15:34.930 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-19T12:15:34.932358282Z 2026-06-19 12:15:34.930 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-19T12:15:34.940889908Z 2026-06-19 12:15:34.940 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-19T12:15:34.945524217Z 2026-06-19 12:15:34.945 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-19T12:15:34.948946418Z 2026-06-19 12:15:34.948 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-19T12:15:34.949026659Z 2026-06-19 12:15:34.948 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-19T12:15:34.978036698Z 2026-06-19 12:15:34.977 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-19T12:15:34.985673495Z 2026-06-19 12:15:34.985 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-19T12:15:35.006078542Z 2026-06-19 12:15:35.005 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-19T12:15:35.017766090Z 2026-06-19 12:15:35.017 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node3:8985_solr as DOWN
2026-06-19T12:15:35.027705974Z 2026-06-19 12:15:35.027 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node3:8985_solr
2026-06-19T12:15:35.039012447Z 2026-06-19 12:15:35.038 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-19T12:15:35.088938540Z 2026-06-19 12:15:35.088 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-19T12:15:35.156345965Z 2026-06-19 12:15:35.155 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-19T12:15:35.347497970Z 2026-06-19 12:15:35.346 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-19T12:15:35.516934636Z 2026-06-19 12:15:35.516 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-19T12:15:35.774353712Z 2026-06-19 12:15:35.773 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-19T12:15:35.895229507Z 2026-06-19 12:15:35.894 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-19T12:15:35.900630559Z 2026-06-19 12:15:35.900 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_19.request.log
2026-06-19T12:15:35.903878705Z 2026-06-19 12:15:35.903 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8985}
2026-06-19T12:15:35.904708822Z 2026-06-19 12:15:35.904 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @3672ms
2026-06-19T12:15:37.884196439Z 2026-06-19 12:15:37.877 INFO (qtp985324122-27-solr-node3-1) [c: s: r: x: t:solr-node3-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=73
2026-06-19T12:15:37.944151746Z 2026-06-19 12:15:37.943 INFO (qtp985324122-56-solr-node3-2) [c: s: r: x: t:solr-node3-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=17
2026-06-19T12:15:39.575045342Z 2026-06-19 12:15:39.574 INFO (qtp985324122-58-solr-node1-6) [c: s: r: x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node5&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&shard=shard2&wt=javabin
2026-06-19T12:15:39.640517194Z 2026-06-19 12:15:39.640 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-19T12:15:39.719336768Z 2026-06-19 12:15:39.718 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-19T12:15:40.135916469Z 2026-06-19 12:15:40.135 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-19T12:15:40.135948994Z 2026-06-19 12:15:40.135 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard2_replica_n2' using configuration from configset techproducts, trusted=true
2026-06-19T12:15:40.154422598Z 2026-06-19 12:15:40.153 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard2_replica_n2], dataDir=[/var/solr/data/test-collection_shard2_replica_n2/data/]
2026-06-19T12:15:40.614749806Z 2026-06-19 12:15:40.614 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-19T12:15:40.636384526Z 2026-06-19 12:15:40.635 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-19T12:15:40.642864994Z 2026-06-19 12:15:40.642 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-19T12:15:40.643856395Z 2026-06-19 12:15:40.643 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-19T12:15:40.643892404Z 2026-06-19 12:15:40.643 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-19T12:15:40.689866236Z 2026-06-19 12:15:40.689 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-19T12:15:40.692677855Z 2026-06-19 12:15:40.692 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-19T12:15:40.694542655Z 2026-06-19 12:15:40.694 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-19T12:15:40.696480109Z 2026-06-19 12:15:40.696 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-19T12:15:40.709762657Z 2026-06-19 12:15:40.709 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01}
2026-06-19T12:15:40.717347867Z 2026-06-19 12:15:40.716 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-19T12:15:40.728651719Z 2026-06-19 12:15:40.728 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-19T12:15:40.728815515Z 2026-06-19 12:15:40.728 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-19T12:15:40.735658164Z 2026-06-19 12:15:40.735 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-19T12:15:40.746292196Z 2026-06-19 12:15:40.745 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard2 to Terms{values={core_node5=0}, version=0} for registerTerm
2026-06-19T12:15:40.747672123Z 2026-06-19 12:15:40.747 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard2
2026-06-19T12:15:40.762562549Z 2026-06-19 12:15:40.762 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-19T12:15:40.762581940Z 2026-06-19 12:15:40.762 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-19T12:15:40.762627161Z 2026-06-19 12:15:40.762 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node3:8985/solr/test-collection_shard2_replica_n2/
2026-06-19T12:15:40.763965125Z 2026-06-19 12:15:40.763 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-19T12:15:40.764489646Z 2026-06-19 12:15:40.764 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ has no replicas
2026-06-19T12:15:40.765574972Z 2026-06-19 12:15:40.765 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard2/leader after winning as /collections/test-collection/leader_elect/shard2/election/72149538921578499-core_node5-n_0000000000
2026-06-19T12:15:40.772266025Z 2026-06-19 12:15:40.771 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ shard2
2026-06-19T12:15:40.945602429Z 2026-06-19 12:15:40.944 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 235] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-19T12:15:40.950314049Z 2026-06-19 12:15:40.950 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-19T12:15:40.958754848Z 2026-06-19 12:15:40.958 INFO (qtp985324122-58-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=techproducts&newCollection=true&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&collection=test-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1385
2026-06-19T12:15:41.066376652Z 2026-06-19 12:15:41.065 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 242] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node3 ===
{
"Name": "/solr-node3",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 2957345,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-19T12:15:31.997705702Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# Docker artifacts saved to: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/docker
not ok 85 Docker SolrCloud rolling upgrade # in 19088 ms
# (in test file test/test_rolling_upgrade.bats, line 118)
# `wait_for 30 1 docker exec solr-node1 solr healthcheck -c test-collection' failed with status 127
# 9.10.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# Digest: sha256:15c8155ceffe1042f37c77bb537aa531804a3510702ff87f2af5fedf523f4203
# Status: Image is up to date for apache/solr-nightly:9.10.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:9.10.0-SNAPSHOT-slim
# 10.0.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# Digest: sha256:fb25f7a7190e3027ade20e97708a8665d41ab0e9ea9285fa242b2c83b0e5f43c
# Status: Image is up to date for apache/solr-nightly:10.0.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:10.0.0-SNAPSHOT-slim
# 7fa88bd647f97d2a92ec9d966e240e3e15dcb70dd0cafbe8120d81742e64c852
# solr-data1
# solr-data2
# solr-data3
# Starting solr-node1 with embedded ZooKeeper
# 5c7bb26e5a73ead60d09ac4b9e622b658c0b81e379c41f647dd69f5d15506f9b
# Starting solr-node2 connected to first node's ZooKeeper
# dbbf48a7db0903fccdd14ef7431cf990c6d90b01040983f2e21c287b81f36909
# Started solr-node3 connected to first node's ZooKeeper
# 3468deb0bdc045016029b0633b79aa1c0eead83051bd3a6ba6bfb036e9307582
# Creating a Collection
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# WARN - 2026-06-19 12:15:39.083; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-19 12:15:39.157; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. "/solr"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [http://solr-node2:8984/solr] to [http://solr-node2:8984/].
# Created collection 'test-collection' with 3 shard(s), 1 replica(s) with config-set 'techproducts'
# Checking collection health
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_rolling_upgrade.bats: line 118: wait_for: command not found
# solr-node1
# solr-node2
# solr-node3
# solr-node1
# solr-node2
# solr-node3
# solr-data1
# solr-data2
# solr-data3
# solrcloud-test
ok 86 allowPaths - backup # in 15296 ms
ok 87 snapshot lifecycle # in 4728 ms
ok 88 snapshot list # in 3301 ms
ok 89 snapshot list via --solr-connection # in 3378 ms
ok 90 snapshot describe # in 3268 ms
ok 91 snapshot describe via --solr-connection # in 3273 ms
ok 92 start solr with ssl # in 16019 ms
ok 93 use different hostname when not checking peer-name # in 24653 ms
ok 94 start solr with ssl and auth # in 14590 ms
ok 95 start solr with client truststore and security manager # in 16482 ms
ok 96 start solr with mTLS needed # in 46096 ms
ok 97 start solr with mTLS wanted # in 35267 ms
ok 98 test keystore reload # in 58818 ms
ok 99 SOLR-11740 check 'solr stop' connection # in 15909 ms
ok 100 stop command for single port # in 18140 ms
ok 101 check stop command doesn't hang # in 225 ms
ok 102 SOLR-16976 solr starts with remote JMX enabled # in 8289 ms
not ok 103 deprecated system properties converted to modern properties # in 195057 ms
# (from function `assert_file_contains' in file .gradle/node/node_modules/bats-file/src/file.bash, line 566,
# in test file test/test_start_solr.bats, line 86)
# `assert_file_contains "${SOLR_LOGS_DIR}/solr.log" 'You are passing in deprecated system property disable.config.edit and should upgrade to using solr.api.config.edit.enabled instead.'' failed
# Waiting up to 180 seconds to see Solr running on port 46853 [|] [/] [-] [\] [|]
# Started Solr server on port 46853 (pid=2974885). Happy searching!
#
#
# -- file does not contain regex --
# path : /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/solr-home/logs/solr.log
# --
#
# Please find the SOLR_HOME snapshot for failed test #5 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/103-5
ok 104 start with custom jetty options # in 8251 ms
ok 105 -c flag prints no-op warning and still starts in cloud mode # in 9210 ms
not ok 106 bootstrapping a configset # in 9205 ms
# (from function `config_exists' in file test/bats_helper.bash, line 84,
# in test file test/test_start_solr.bats, line 116)
# `config_exists "techproducts"' failed
# Waiting up to 180 seconds to see Solr running on port 46853 [|] [/] [-] [\] [|]
# Started Solr server on port 46853 (pid=2977604). Happy searching!
#
# WARN - 2026-06-19 12:24:19.031; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-19 12:24:19.035; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# Please find the SOLR_HOME snapshot for failed test #8 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/106-8
ok 107 status detects locally running solr # in 9564 ms
ok 108 status with --solr-url from user # in 8034 ms
ok 109 status with --port from user # in 7883 ms
ok 110 multiple connection options are prevented # in 858 ms
ok 111 status with invalid --solr-url from user # in 1291 ms
ok 112 status with --short format # in 195871 ms
ok 113 searching solr via locally executed streaming expression # in 1757 ms
ok 114 searching solr via remotely executed streaming expression by solr-url # in 1254 ms
ok 115 searching solr via remotely executed streaming expression by solr connection # in 1148 ms
ok 116 variable interpolation # in 1676 ms
ok 117 searching solr without credentials fails with error # in 1141 ms
ok 118 --version returns Solr version # in 739 ms
ok 119 version as direct tool call still runs # in 748 ms
ok 120 short help # in 765 ms
ok 121 short help is inferred # in 759 ms
ok 122 long help # in 797 ms
ok 123 running subcommands with zk is prevented # in 748 ms
ok 124 listing out files # in 2138 ms
ok 125 connecting to solr via various solr urls and zk hosts # in 9329 ms
ok 126 copying files around # in 10956 ms
ok 127 upconfig # in 2521 ms
ok 128 SOLR-12429 test upconfig fails with symlink # in 1212 ms
ok 129 downconfig # in 1168 ms
ok 130 bin/solr zk cp gets 'solrhome' from '--solr-home' command line option # in 1431 ms
ok 131 env var ZK_HOST is honored # in 3278 ms
ok 132 Cleanup # in 1850 ms
> Task :solr:packaging:integrationTests FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build.gradle' line: 339
* What went wrong:
Execution failed for task ':solr:packaging:integrationTests'.
> Process 'command '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/.gradle/node/node_modules/bats/bin/bats'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/8.10/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 29m 50s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/4cmkf6tmkei2c
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 237 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/237/
No tests ran.
Build Log:
[...truncated 903 lines...]
BUILD FAILED in 29m 50s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/cknmqe7jfh6vg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 236 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/236/
No tests ran.
Build Log:
[...truncated 984 lines...]
BUILD FAILED in 30m 30s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/zgcxifpkyqhsc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 235 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/235/
No tests ran.
Build Log:
[...truncated 347 lines...]
# ERROR: KeeperErrorCode = NoNode for /security.json
# --
#
# Last output:
# WARN - 2026-06-16 11:44:20.150; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-16 11:44:20.154; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-16 11:44:20.612; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-16 11:44:20.612; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: KeeperErrorCode = NoNode for /security.json
not ok 9 create, config, api, and delete with basic auth # in 1466 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 40331 is already being used by another process (pid: 619304)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-06-16 11:44:21.943; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-16 11:44:21.948; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
ok 10 post, postlogs and export with basic auth # in 17555 ms
ok 11 package with basic auth # in 11893 ms
ok 12 admin UI static content is excepted from basic auth # in 10858 ms
ok 13 nothing # in 10 ms
ok 14 server does not compress response without Accept-Encoding header # in 141 ms
ok 15 server compresses response when Accept-Encoding: gzip is requested # in 82 ms
ok 16 compressed response can be decompressed and parsed # in 83 ms
ok 17 setting property # in 11068 ms
ok 18 short form of setting property # in 6295 ms
ok 19 connecting to solr via various connection types # in 10458 ms
ok 20 create for non cloud mode # in 7915 ms
ok 21 create for cloud mode # in 9774 ms
ok 22 multiple connection options are prevented # in 837 ms
ok 23 create collection # in 7789 ms
ok 24 create collection using solrUrl # in 5211 ms
ok 25 create collection using legacy solrUrl # in 5232 ms
ok 26 create collection using Zookeeper # in 5681 ms
ok 27 reject d option with invalid config dir # in 2682 ms
ok 28 accept d option with builtin config # in 5285 ms
ok 29 accept d option with explicit path to config # in 5248 ms
ok 30 accept n option as config name # in 5202 ms
ok 31 allow config reuse when n option specifies same config # in 9197 ms
ok 32 create multisharded collections when s provided # in 5178 ms
ok 33 create replicated collections when rf provided # in 6094 ms
ok 34 can delete collections # in 10519 ms
ok 35 can delete collections with solr-url # in 8082 ms
ok 36 can delete collections with solr-connection # in 7708 ms
ok 37 collection delete also deletes zk config # in 7220 ms
ok 38 deletes accompanying zk config with nondefault name # in 7040 ms
ok 39 delete-config option can opt to leave config in zk # in 6941 ms
ok 40 start -e cloud works with --jvm-opts # in 23379 ms
ok 41 start -e cloud works with --no-prompt # in 45733 ms
ok 42 Check export command # in 18497 ms
ok 43 export fails on non cloud mode # in 9208 ms
Waiting for Tika Server to be ready on port 40336
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 44 using curl to extract a single pdf file # in 16593 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to extract content from pdf # in 16539 ms
WARNING: Docker not available; bypassing test.
ok 46 using the bin/solr post tool to crawl web site # in 16568 ms
Stopping Tika Server container
ok 47 healthcheck on cloud solr # in 10072 ms
ok 48 healthcheck on remote cloud solr # in 9757 ms
ok 49 healthcheck errors on standalone solr # in 11244 ms
ok 50 solr help flag prints help # in 730 ms
ok 51 solr with no flags prints help # in 739 ms
ok 52 start help flag prints help # in 149 ms
ok 53 start h flag prints help # in 148 ms
ok 54 stop help flag prints help # in 139 ms
ok 55 restart help flag prints help # in 146 ms
ok 56 status help flag prints help # in 756 ms
ok 57 healthcheck help flag prints help # in 739 ms
ok 58 create help flag prints help # in 750 ms
ok 59 delete help flag prints help # in 736 ms
ok 60 zk help flag prints help # in 743 ms
ok 61 auth help flag prints help # in 734 ms
ok 62 assert help flag prints help # in 738 ms
ok 63 post help flag prints help # in 739 ms
ok 64 SQL Module # in 15655 ms
ok 65 icu collation in analysis-extras module # in 12330 ms
ok 66 Check lifecycle of sentiment classification # in 23961 ms
ok 67 lifecycle of package # in 8315 ms
ok 68 deploying and undeploying a collection level package # in 12929 ms
ok 69 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
ok 70 Affinity placement plugin using sysprop # in 15071 ms
ok 71 Random placement plugin using ENV # in 15053 ms
ok 72 Check help command # in 2273 ms
ok 73 basic post with a type specified # in 5216 ms
ok 74 basic post with collection instead of url parameter # in 3911 ms
ok 75 basic post with solr-url and collection # in 3787 ms
ok 76 basic post with -s (solr-connection short form) and collection # in 3904 ms
ok 77 basic post WITHOUT a type specified # in 11135 ms
ok 78 crawling a directory as a dry-run # in 781 ms
ok 79 crawling a directory # in 4531 ms
ok 80 skipcommit and optimize and delete # in 3836 ms
ok 81 args mode # in 6807 ms
ok 82 stdin mode # in 3377 ms
ok 83 verbose echo the Solr response # in 3781 ms
ok 84 post solr log into solr via cli # in 9204 ms
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 9 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 9 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 14 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-06-16T11:54:40.621378890Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-16T11:54:40.621423473Z It should be set to 65000 to avoid operational disruption.
2026-06-16T11:54:40.621427100Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-16T11:54:40.621675176Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-16T11:54:40.621687786Z
2026-06-16T11:54:40.632058187Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-16T11:54:40.640170218Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-16T11:54:40.687942683Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-16T11:54:40.725610077Z WARNING: A command line option has enabled the Security Manager
2026-06-16T11:54:40.725641013Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-16T11:54:41.963960545Z 2026-06-16 11:54:41.960 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-16T11:54:42.231060521Z 2026-06-16 11:54:42.230 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-16T11:54:42.254953438Z 2026-06-16 11:54:42.254 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-16T11:54:42.261627478Z 2026-06-16 11:54:42.261 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-16T11:54:42.261690244Z 2026-06-16 11:54:42.261 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-06-16T11:54:42.261856421Z 2026-06-16 11:54:42.261 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-16T11:54:42.261977121Z 2026-06-16 11:54:42.261 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-16T11:54:42.261450481Z
2026-06-16T11:54:42.264460707Z 2026-06-16 11:54:42.264 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-16T11:54:42.284563835Z 2026-06-16 11:54:42.284 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-16T11:54:42.297095480Z 2026-06-16 11:54:42.296 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-16T11:54:42.297191935Z 2026-06-16 11:54:42.296 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-16T11:54:42.329697993Z 2026-06-16 11:54:42.329 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-16T11:54:43.087614904Z 2026-06-16 11:54:43.086 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-16T11:54:43.310633972Z 2026-06-16 11:54:43.310 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-16T11:54:43.315275684Z 2026-06-16 11:54:43.314 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-06-16T11:54:43.315292826Z 2026-06-16 11:54:43.314 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-06-16T11:54:43.317480107Z 2026-06-16 11:54:43.317 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-06-16T11:54:43.317556796Z 2026-06-16 11:54:43.317 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-06-16T11:54:43.354173711Z 2026-06-16 11:54:43.353 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-06-16T11:54:43.822320496Z 2026-06-16 11:54:43.820 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-06-16T11:54:43.835718491Z 2026-06-16 11:54:43.835 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-16T11:54:43.885886438Z 2026-06-16 11:54:43.885 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-16T11:54:43.957103319Z 2026-06-16 11:54:43.956 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-16T11:54:43.957176521Z 2026-06-16 11:54:43.956 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-16T11:54:44.071654636Z 2026-06-16 11:54:44.070 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-16T11:54:44.099706464Z 2026-06-16 11:54:44.099 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-16T11:54:44.113478022Z 2026-06-16 11:54:44.112 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-06-16T11:54:44.119714862Z 2026-06-16 11:54:44.119 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72132540123054080-solr-node1:8983_solr-n_0000000000) starting
2026-06-16T11:54:44.187415011Z 2026-06-16 11:54:44.186 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-06-16T11:54:44.189677818Z 2026-06-16 11:54:44.189 INFO (OverseerStateUpdate-72132540123054080-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-06-16T11:54:44.200563287Z 2026-06-16 11:54:44.200 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-16T11:54:44.242336720Z 2026-06-16 11:54:44.241 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-16T11:54:44.702648423Z 2026-06-16 11:54:44.699 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-16T11:54:44.885136623Z 2026-06-16 11:54:44.884 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-16T11:54:45.163015850Z 2026-06-16 11:54:45.162 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-16T11:54:45.373669370Z 2026-06-16 11:54:45.372 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-16T11:54:45.379715686Z 2026-06-16 11:54:45.379 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_16.request.log
2026-06-16T11:54:45.384465518Z 2026-06-16 11:54:45.384 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-06-16T11:54:45.386196459Z 2026-06-16 11:54:45.385 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4747ms
2026-06-16T11:54:45.554309747Z 2026-06-16 11:54:45.553 INFO (qtp985324122-27-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=45
2026-06-16T11:54:45.607494925Z 2026-06-16 11:54:45.607 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=14
2026-06-16T11:54:48.895276840Z 2026-06-16 11:54:48.894 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-16T11:54:49.214044784Z 2026-06-16 11:54:49.213 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-16T11:54:52.964752624Z 2026-06-16 11:54:52.963 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=11
2026-06-16T11:54:52.995908562Z 2026-06-16 11:54:52.995 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=9
2026-06-16T11:54:53.020852279Z 2026-06-16 11:54:53.020 INFO (qtp985324122-27-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=3
2026-06-16T11:54:53.279564325Z 2026-06-16 11:54:53.279 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-06-16T11:54:53.301591675Z 2026-06-16 11:54:53.300 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-06-16T11:54:53.310061719Z 2026-06-16 11:54:53.309 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 174] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-16T11:54:53.353189096Z 2026-06-16 11:54:53.352 INFO (OverseerStateUpdate-72132540123054080-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-16T11:54:53.353215178Z "core":"test-collection_shard3_replica_n1",
2026-06-16T11:54:53.353218011Z "node_name":"solr-node1:8983_solr",
2026-06-16T11:54:53.353220424Z "base_url":"http://solr-node1:8983/solr",
2026-06-16T11:54:53.353222587Z "collection":"test-collection",
2026-06-16T11:54:53.353224941Z "shard":"shard3",
2026-06-16T11:54:53.353226965Z "state":"down",
2026-06-16T11:54:53.353228943Z "type":"NRT",
2026-06-16T11:54:53.353230807Z "operation":"ADDREPLICA",
2026-06-16T11:54:53.353232730Z "waitForFinalState":"false"}
2026-06-16T11:54:53.360160529Z 2026-06-16 11:54:53.359 INFO (OverseerStateUpdate-72132540123054080-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-16T11:54:53.360180607Z "core":"test-collection_shard2_replica_n2",
2026-06-16T11:54:53.360183360Z "node_name":"solr-node3:8985_solr",
2026-06-16T11:54:53.360185776Z "base_url":"http://solr-node3:8985/solr",
2026-06-16T11:54:53.360187873Z "collection":"test-collection",
2026-06-16T11:54:53.360190402Z "shard":"shard2",
2026-06-16T11:54:53.360192557Z "state":"down",
2026-06-16T11:54:53.360194565Z "type":"NRT",
2026-06-16T11:54:53.360196590Z "operation":"ADDREPLICA",
2026-06-16T11:54:53.360198934Z "waitForFinalState":"false"}
2026-06-16T11:54:53.368832972Z 2026-06-16 11:54:53.368 INFO (OverseerStateUpdate-72132540123054080-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-16T11:54:53.368848689Z "core":"test-collection_shard1_replica_n4",
2026-06-16T11:54:53.368851557Z "node_name":"solr-node2:8984_solr",
2026-06-16T11:54:53.368868316Z "base_url":"http://solr-node2:8984/solr",
2026-06-16T11:54:53.368870926Z "collection":"test-collection",
2026-06-16T11:54:53.368872938Z "shard":"shard1",
2026-06-16T11:54:53.368874847Z "state":"down",
2026-06-16T11:54:53.368876874Z "type":"NRT",
2026-06-16T11:54:53.368878750Z "operation":"ADDREPLICA",
2026-06-16T11:54:53.368880787Z "waitForFinalState":"false"}
2026-06-16T11:54:53.477200332Z 2026-06-16 11:54:53.476 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 187] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-16T11:54:53.550350058Z 2026-06-16 11:54:53.548 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-06-16T11:54:53.623834267Z 2026-06-16 11:54:53.623 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-16T11:54:53.691981184Z 2026-06-16 11:54:53.691 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-16T11:54:54.001102816Z 2026-06-16 11:54:54.000 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-16T11:54:54.001208446Z 2026-06-16 11:54:54.000 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true
2026-06-16T11:54:54.032937525Z 2026-06-16 11:54:54.032 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/]
2026-06-16T11:54:54.758032770Z 2026-06-16 11:54:54.756 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-16T11:54:54.785760658Z 2026-06-16 11:54:54.784 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-16T11:54:54.790503967Z 2026-06-16 11:54:54.789 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-16T11:54:54.791662788Z 2026-06-16 11:54:54.791 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-16T11:54:54.791769255Z 2026-06-16 11:54:54.791 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-16T11:54:54.864557723Z 2026-06-16 11:54:54.863 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-16T11:54:54.868673383Z 2026-06-16 11:54:54.868 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-16T11:54:54.870924099Z 2026-06-16 11:54:54.870 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-16T11:54:54.873462803Z 2026-06-16 11:54:54.873 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-16T11:54:54.901194213Z 2026-06-16 11:54:54.900 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-06-16T11:54:54.910681529Z 2026-06-16 11:54:54.910 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-16T11:54:54.917811550Z 2026-06-16 11:54:54.917 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-16T11:54:54.917833769Z 2026-06-16 11:54:54.917 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-16T11:54:54.928462847Z 2026-06-16 11:54:54.927 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-16T11:54:54.936137292Z 2026-06-16 11:54:54.935 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-06-16T11:54:54.937602182Z 2026-06-16 11:54:54.937 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-06-16T11:54:54.950370906Z 2026-06-16 11:54:54.949 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-16T11:54:54.950394131Z 2026-06-16 11:54:54.950 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-16T11:54:54.950456571Z 2026-06-16 11:54:54.950 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/
2026-06-16T11:54:54.951838604Z 2026-06-16 11:54:54.951 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-16T11:54:54.952362159Z 2026-06-16 11:54:54.952 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas
2026-06-16T11:54:54.953653749Z 2026-06-16 11:54:54.953 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72132540123054080-core_node3-n_0000000000
2026-06-16T11:54:54.960197990Z 2026-06-16 11:54:54.959 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3
2026-06-16T11:54:55.066504028Z 2026-06-16 11:54:55.065 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 238] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-16T11:54:55.071499264Z 2026-06-16 11:54:55.071 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-16T11:54:55.081361020Z 2026-06-16 11:54:55.080 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1533
2026-06-16T11:54:55.183127649Z 2026-06-16 11:54:55.182 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 245] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 653092,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-16T11:54:40.387906515Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-06-16T11:54:45.941133183Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-16T11:54:45.941172792Z It should be set to 65000 to avoid operational disruption.
2026-06-16T11:54:45.941175915Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-16T11:54:45.941493895Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-16T11:54:45.941504620Z
2026-06-16T11:54:45.951312456Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-16T11:54:45.959397675Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-16T11:54:46.008553498Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-16T11:54:46.046953707Z WARNING: A command line option has enabled the Security Manager
2026-06-16T11:54:46.046976871Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-16T11:54:47.172585183Z 2026-06-16 11:54:47.168 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-16T11:54:47.445390608Z 2026-06-16 11:54:47.444 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-16T11:54:47.472449123Z 2026-06-16 11:54:47.471 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-16T11:54:47.479733874Z 2026-06-16 11:54:47.479 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-16T11:54:47.479823479Z 2026-06-16 11:54:47.479 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-06-16T11:54:47.479937426Z 2026-06-16 11:54:47.479 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-16T11:54:47.480049684Z 2026-06-16 11:54:47.479 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-16T11:54:47.479426328Z
2026-06-16T11:54:47.482687222Z 2026-06-16 11:54:47.482 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-16T11:54:47.504475741Z 2026-06-16 11:54:47.503 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-16T11:54:47.528831534Z 2026-06-16 11:54:47.528 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-16T11:54:47.674832659Z 2026-06-16 11:54:47.674 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-16T11:54:47.693969272Z 2026-06-16 11:54:47.693 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-16T11:54:47.694030942Z 2026-06-16 11:54:47.693 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-16T11:54:47.694183869Z 2026-06-16 11:54:47.693 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-16T11:54:47.804490660Z 2026-06-16 11:54:47.803 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-16T11:54:47.804632238Z 2026-06-16 11:54:47.804 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-16T11:54:47.804715804Z 2026-06-16 11:54:47.804 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-16T11:54:47.837284894Z 2026-06-16 11:54:47.836 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-16T11:54:48.590794010Z 2026-06-16 11:54:48.590 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-16T11:54:48.770974567Z 2026-06-16 11:54:48.769 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-16T11:54:48.772535618Z 2026-06-16 11:54:48.770 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-16T11:54:48.785080754Z 2026-06-16 11:54:48.783 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-16T11:54:48.791920150Z 2026-06-16 11:54:48.791 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-16T11:54:48.798089784Z 2026-06-16 11:54:48.797 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-16T11:54:48.798175153Z 2026-06-16 11:54:48.797 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-16T11:54:48.832166602Z 2026-06-16 11:54:48.831 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-16T11:54:48.839518521Z 2026-06-16 11:54:48.839 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-16T11:54:48.862048563Z 2026-06-16 11:54:48.861 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-16T11:54:48.874169623Z 2026-06-16 11:54:48.873 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-06-16T11:54:48.883890535Z 2026-06-16 11:54:48.883 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-06-16T11:54:48.896610917Z 2026-06-16 11:54:48.896 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-16T11:54:48.977925860Z 2026-06-16 11:54:48.977 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-16T11:54:49.215484018Z 2026-06-16 11:54:49.214 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-16T11:54:49.334548976Z 2026-06-16 11:54:49.333 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-16T11:54:49.507208335Z 2026-06-16 11:54:49.506 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-16T11:54:49.795806385Z 2026-06-16 11:54:49.794 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-16T11:54:49.928294947Z 2026-06-16 11:54:49.927 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-16T11:54:49.933931366Z 2026-06-16 11:54:49.933 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_16.request.log
2026-06-16T11:54:49.938434325Z 2026-06-16 11:54:49.938 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-06-16T11:54:49.940356927Z 2026-06-16 11:54:49.940 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @3982ms
2026-06-16T11:54:50.129086155Z 2026-06-16 11:54:50.128 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=56
2026-06-16T11:54:50.200369996Z 2026-06-16 11:54:50.199 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=15
2026-06-16T11:54:53.552151933Z 2026-06-16 11:54:53.551 INFO (qtp985324122-58-solr-node1-6) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-06-16T11:54:53.603788211Z 2026-06-16 11:54:53.603 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-16T11:54:53.702910884Z 2026-06-16 11:54:53.702 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-16T11:54:54.001163529Z 2026-06-16 11:54:54.000 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-16T11:54:54.001198296Z 2026-06-16 11:54:54.000 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-06-16T11:54:54.022763559Z 2026-06-16 11:54:54.022 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-06-16T11:54:54.486784407Z 2026-06-16 11:54:54.486 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-16T11:54:54.510134053Z 2026-06-16 11:54:54.509 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-16T11:54:54.514283272Z 2026-06-16 11:54:54.513 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-16T11:54:54.515464586Z 2026-06-16 11:54:54.514 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-16T11:54:54.515746846Z 2026-06-16 11:54:54.515 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-16T11:54:54.562735432Z 2026-06-16 11:54:54.562 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-16T11:54:54.566082598Z 2026-06-16 11:54:54.565 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-16T11:54:54.568255753Z 2026-06-16 11:54:54.567 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-16T11:54:54.571952238Z 2026-06-16 11:54:54.571 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-16T11:54:54.584623899Z 2026-06-16 11:54:54.584 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01}
2026-06-16T11:54:54.592910626Z 2026-06-16 11:54:54.592 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-16T11:54:54.600524405Z 2026-06-16 11:54:54.599 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-16T11:54:54.600608963Z 2026-06-16 11:54:54.600 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-16T11:54:54.609020772Z 2026-06-16 11:54:54.608 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-16T11:54:54.624004647Z 2026-06-16 11:54:54.623 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-06-16T11:54:54.625874563Z 2026-06-16 11:54:54.625 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1
2026-06-16T11:54:54.639154525Z 2026-06-16 11:54:54.638 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-16T11:54:54.639175058Z 2026-06-16 11:54:54.638 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-16T11:54:54.639178364Z 2026-06-16 11:54:54.638 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node2:8984/solr/test-collection_shard1_replica_n4/
2026-06-16T11:54:54.640950704Z 2026-06-16 11:54:54.640 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-16T11:54:54.641539937Z 2026-06-16 11:54:54.641 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ has no replicas
2026-06-16T11:54:54.642759049Z 2026-06-16 11:54:54.642 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard1/leader after winning as /collections/test-collection/leader_elect/shard1/election/72132540123054083-core_node6-n_0000000000
2026-06-16T11:54:54.653735360Z 2026-06-16 11:54:54.653 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ shard1
2026-06-16T11:54:54.761293192Z 2026-06-16 11:54:54.760 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 207] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-16T11:54:54.765370685Z 2026-06-16 11:54:54.764 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-16T11:54:54.774460915Z 2026-06-16 11:54:54.773 INFO (qtp985324122-58-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1224
2026-06-16T11:54:54.873487991Z 2026-06-16 11:54:54.872 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 221] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-16T11:54:55.066567699Z 2026-06-16 11:54:55.066 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 238] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-16T11:54:55.183325377Z 2026-06-16 11:54:55.182 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 245] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node2 ===
{
"Name": "/solr-node2",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 653492,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-16T11:54:45.719291079Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node3 ===
2026-06-16T11:54:46.162092286Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-16T11:54:46.162149312Z It should be set to 65000 to avoid operational disruption.
2026-06-16T11:54:46.162154838Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-16T11:54:46.162419857Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-16T11:54:46.162431830Z
2026-06-16T11:54:46.172108159Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-16T11:54:46.181096481Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-16T11:54:46.237015110Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-16T11:54:46.273873956Z WARNING: A command line option has enabled the Security Manager
2026-06-16T11:54:46.273909223Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-16T11:54:47.532057262Z 2026-06-16 11:54:47.528 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-16T11:54:47.816308844Z 2026-06-16 11:54:47.815 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-16T11:54:47.842109153Z 2026-06-16 11:54:47.841 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-16T11:54:47.849704484Z 2026-06-16 11:54:47.849 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-16T11:54:47.849829965Z 2026-06-16 11:54:47.849 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8985
2026-06-16T11:54:47.850006211Z 2026-06-16 11:54:47.849 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-16T11:54:47.850160990Z 2026-06-16 11:54:47.849 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-16T11:54:47.849502272Z
2026-06-16T11:54:47.852730116Z 2026-06-16 11:54:47.852 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-16T11:54:47.875957658Z 2026-06-16 11:54:47.875 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-16T11:54:47.901859569Z 2026-06-16 11:54:47.901 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-16T11:54:48.041862871Z 2026-06-16 11:54:48.041 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-16T11:54:48.058722741Z 2026-06-16 11:54:48.058 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-16T11:54:48.058791721Z 2026-06-16 11:54:48.058 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-16T11:54:48.058925259Z 2026-06-16 11:54:48.058 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-16T11:54:48.168056749Z 2026-06-16 11:54:48.167 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-16T11:54:48.168144415Z 2026-06-16 11:54:48.167 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-16T11:54:48.168258432Z 2026-06-16 11:54:48.168 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-16T11:54:48.201198348Z 2026-06-16 11:54:48.200 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-16T11:54:48.966380821Z 2026-06-16 11:54:48.965 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-16T11:54:49.106635939Z 2026-06-16 11:54:49.106 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-16T11:54:49.107934861Z 2026-06-16 11:54:49.106 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-16T11:54:49.116325006Z 2026-06-16 11:54:49.115 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-16T11:54:49.121075725Z 2026-06-16 11:54:49.120 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-16T11:54:49.124623647Z 2026-06-16 11:54:49.124 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-16T11:54:49.124702981Z 2026-06-16 11:54:49.124 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-16T11:54:49.158252821Z 2026-06-16 11:54:49.157 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-16T11:54:49.166427447Z 2026-06-16 11:54:49.166 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
2026-06-16T11:54:49.185467953Z 2026-06-16 11:54:49.184 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-16T11:54:49.196754558Z 2026-06-16 11:54:49.196 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node3:8985_solr as DOWN
2026-06-16T11:54:49.206516085Z 2026-06-16 11:54:49.206 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node3:8985_solr
2026-06-16T11:54:49.215003537Z 2026-06-16 11:54:49.214 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-16T11:54:49.262700790Z 2026-06-16 11:54:49.262 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-16T11:54:49.512283805Z 2026-06-16 11:54:49.511 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-16T11:54:49.675340761Z 2026-06-16 11:54:49.674 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-16T11:54:49.927166612Z 2026-06-16 11:54:49.926 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-16T11:54:50.050877497Z 2026-06-16 11:54:50.050 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-16T11:54:50.056193501Z 2026-06-16 11:54:50.055 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_16.request.log
2026-06-16T11:54:50.059815166Z 2026-06-16 11:54:50.059 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8985}
2026-06-16T11:54:50.060618791Z 2026-06-16 11:54:50.060 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @3880ms
2026-06-16T11:54:51.736399536Z 2026-06-16 11:54:51.726 INFO (qtp985324122-27-solr-node3-1) [c: s: r: x: t:solr-node3-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=79
2026-06-16T11:54:51.803077922Z 2026-06-16 11:54:51.802 INFO (qtp985324122-56-solr-node3-2) [c: s: r: x: t:solr-node3-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=18
2026-06-16T11:54:53.245365195Z 2026-06-16 11:54:53.244 INFO (qtp985324122-58-solr-node3-3) [c: s: r: x: t:solr-node3-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=9
2026-06-16T11:54:53.551496368Z 2026-06-16 11:54:53.550 INFO (qtp985324122-30-solr-node1-6) [c: s: r: x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node5&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&shard=shard2&wt=javabin
2026-06-16T11:54:53.632915930Z 2026-06-16 11:54:53.632 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-16T11:54:53.738538034Z 2026-06-16 11:54:53.737 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-16T11:54:54.124772044Z 2026-06-16 11:54:54.124 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-16T11:54:54.124816158Z 2026-06-16 11:54:54.124 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard2_replica_n2' using configuration from configset techproducts, trusted=true
2026-06-16T11:54:54.147836957Z 2026-06-16 11:54:54.147 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard2_replica_n2], dataDir=[/var/solr/data/test-collection_shard2_replica_n2/data/]
2026-06-16T11:54:54.655631704Z 2026-06-16 11:54:54.655 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-16T11:54:54.733853986Z 2026-06-16 11:54:54.733 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-16T11:54:54.737439100Z 2026-06-16 11:54:54.737 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-16T11:54:54.738620481Z 2026-06-16 11:54:54.737 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-16T11:54:54.738634456Z 2026-06-16 11:54:54.738 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-16T11:54:54.785908337Z 2026-06-16 11:54:54.785 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-16T11:54:54.789387681Z 2026-06-16 11:54:54.789 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-16T11:54:54.791419526Z 2026-06-16 11:54:54.791 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-16T11:54:54.793324829Z 2026-06-16 11:54:54.793 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-16T11:54:54.806595506Z 2026-06-16 11:54:54.806 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01}
2026-06-16T11:54:54.815125262Z 2026-06-16 11:54:54.814 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-16T11:54:54.821715020Z 2026-06-16 11:54:54.821 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-16T11:54:54.821804322Z 2026-06-16 11:54:54.821 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-16T11:54:54.828663312Z 2026-06-16 11:54:54.828 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-16T11:54:54.855077119Z 2026-06-16 11:54:54.854 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard2 to Terms{values={core_node5=0}, version=0} for registerTerm
2026-06-16T11:54:54.856747131Z 2026-06-16 11:54:54.856 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard2
2026-06-16T11:54:54.870591298Z 2026-06-16 11:54:54.870 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-16T11:54:54.870665436Z 2026-06-16 11:54:54.870 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-16T11:54:54.870670233Z 2026-06-16 11:54:54.870 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node3:8985/solr/test-collection_shard2_replica_n2/
2026-06-16T11:54:54.874232012Z 2026-06-16 11:54:54.873 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-16T11:54:54.874883419Z 2026-06-16 11:54:54.874 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ has no replicas
2026-06-16T11:54:54.876323948Z 2026-06-16 11:54:54.876 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard2/leader after winning as /collections/test-collection/leader_elect/shard2/election/72132540123054084-core_node5-n_0000000000
2026-06-16T11:54:54.886181253Z 2026-06-16 11:54:54.885 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ shard2
2026-06-16T11:54:55.066713650Z 2026-06-16 11:54:55.066 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 238] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-16T11:54:55.072029312Z 2026-06-16 11:54:55.071 INFO (qtp985324122-30-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-16T11:54:55.083015696Z 2026-06-16 11:54:55.082 INFO (qtp985324122-30-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=techproducts&newCollection=true&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&collection=test-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1533
2026-06-16T11:54:55.094532324Z 2026-06-16 11:54:55.094 INFO (qtp985324122-27-solr-node3-4) [c:test-collection s: r: x: t:solr-node3-4] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
2026-06-16T11:54:55.183745799Z 2026-06-16 11:54:55.183 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 245] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-16T11:54:55.187518607Z 2026-06-16 11:54:55.187 INFO (qtp985324122-27-solr-node3-4) [c:test-collection s: r: x: t:solr-node3-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=1928
# === Docker inspect for solr-node3 ===
{
"Name": "/solr-node3",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 653674,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-16T11:54:45.92901938Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# Docker artifacts saved to: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/docker
not ok 85 Docker SolrCloud rolling upgrade # in 25644 ms
# (in test file test/test_rolling_upgrade.bats, line 118)
# `wait_for 30 1 docker exec solr-node1 solr healthcheck -c test-collection' failed with status 127
# 9.10.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# af6eca94c810: Pulling fs layer
# cb0efb96dabd: Pulling fs layer
# 3e9d91201f40: Pulling fs layer
# 66b76b382631: Pulling fs layer
# 601f2c23751f: Pulling fs layer
# 0ed719aef393: Pulling fs layer
# 69a09c96e8bf: Pulling fs layer
# c937aee9a870: Pulling fs layer
# 9cdfe5320de8: Pulling fs layer
# 3aa9120e46f5: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# 601f2c23751f: Waiting
# 0ed719aef393: Waiting
# 69a09c96e8bf: Waiting
# c937aee9a870: Waiting
# 9cdfe5320de8: Waiting
# 66b76b382631: Waiting
# 3aa9120e46f5: Waiting
# 4f4fb700ef54: Waiting
# af6eca94c810: Verifying Checksum
# af6eca94c810: Download complete
# cb0efb96dabd: Download complete
# 66b76b382631: Download complete
# 601f2c23751f: Verifying Checksum
# 601f2c23751f: Download complete
# 69a09c96e8bf: Download complete
# 3e9d91201f40: Verifying Checksum
# 3e9d91201f40: Download complete
# c937aee9a870: Download complete
# 9cdfe5320de8: Download complete
# 4f4fb700ef54: Verifying Checksum
# 4f4fb700ef54: Download complete
# 3aa9120e46f5: Verifying Checksum
# 0ed719aef393: Verifying Checksum
# 0ed719aef393: Download complete
# af6eca94c810: Pull complete
# cb0efb96dabd: Pull complete
# 3e9d91201f40: Pull complete
# 66b76b382631: Pull complete
# 601f2c23751f: Pull complete
# 0ed719aef393: Pull complete
# 69a09c96e8bf: Pull complete
# c937aee9a870: Pull complete
# 9cdfe5320de8: Pull complete
# 3aa9120e46f5: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:15c8155ceffe1042f37c77bb537aa531804a3510702ff87f2af5fedf523f4203
# Status: Downloaded newer image for apache/solr-nightly:9.10.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:9.10.0-SNAPSHOT-slim
# 10.0.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# 40d16f30db40: Already exists
# 9d5a0b6cb36c: Already exists
# 49518c28c0b3: Already exists
# 41162acdcc29: Already exists
# 2ed0fd558ca3: Already exists
# 2488415fda1c: Pulling fs layer
# 09511c8fb203: Pulling fs layer
# a0c57fb89c4f: Pulling fs layer
# d0641dcc41d7: Pulling fs layer
# 315722fd9570: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# d0641dcc41d7: Waiting
# 315722fd9570: Waiting
# 4f4fb700ef54: Waiting
# 09511c8fb203: Verifying Checksum
# 09511c8fb203: Download complete
# a0c57fb89c4f: Download complete
# d0641dcc41d7: Verifying Checksum
# d0641dcc41d7: Download complete
# 315722fd9570: Verifying Checksum
# 315722fd9570: Download complete
# 4f4fb700ef54: Verifying Checksum
# 4f4fb700ef54: Download complete
# 2488415fda1c: Verifying Checksum
# 2488415fda1c: Download complete
# 2488415fda1c: Pull complete
# 09511c8fb203: Pull complete
# a0c57fb89c4f: Pull complete
# d0641dcc41d7: Pull complete
# 315722fd9570: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:fb25f7a7190e3027ade20e97708a8665d41ab0e9ea9285fa242b2c83b0e5f43c
# Status: Downloaded newer image for apache/solr-nightly:10.0.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:10.0.0-SNAPSHOT-slim
# 3bb5dc8347b91ee7b001cd47c2807e4b5a7fc9d99e5251e4d1f5a8f9c9dbcc06
# solr-data1
# solr-data2
# solr-data3
# Starting solr-node1 with embedded ZooKeeper
# 02ec503a477a0f1d51aafee998f314445eb9e2315d5a217a69fd194476f60678
# Starting solr-node2 connected to first node's ZooKeeper
# b8cb7d11a901b37efc1b8b9f69e5e90aa4a53e46d3725648a72af837b6b75c55
# Started solr-node3 connected to first node's ZooKeeper
# bc2b0c27ef6004a5b0c1303aec16afc0bd7d56615c570f4f6767ffa10d0040b1
# Creating a Collection
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# WARN - 2026-06-16 11:54:53.063; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-16 11:54:53.097; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. "/solr"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [http://solr-node3:8985/solr] to [http://solr-node3:8985/].
# Created collection 'test-collection' with 3 shard(s), 1 replica(s) with config-set 'techproducts'
# Checking collection health
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_rolling_upgrade.bats: line 118: wait_for: command not found
# solr-node1
# solr-node2
# solr-node3
# solr-node1
# solr-node2
# solr-node3
# solr-data1
# solr-data2
# solr-data3
# solrcloud-test
ok 86 allowPaths - backup # in 16504 ms
ok 87 snapshot lifecycle # in 5234 ms
ok 88 snapshot list # in 3653 ms
ok 89 snapshot list via --solr-connection # in 3502 ms
ok 90 snapshot describe # in 3343 ms
ok 91 snapshot describe via --solr-connection # in 3304 ms
ok 92 start solr with ssl # in 16377 ms
ok 93 use different hostname when not checking peer-name # in 24782 ms
ok 94 start solr with ssl and auth # in 14681 ms
ok 95 start solr with client truststore and security manager # in 16432 ms
ok 96 start solr with mTLS needed # in 46414 ms
ok 97 start solr with mTLS wanted # in 34299 ms
ok 98 test keystore reload # in 58602 ms
ok 99 SOLR-11740 check 'solr stop' connection # in 15987 ms
ok 100 stop command for single port # in 18004 ms
ok 101 check stop command doesn't hang # in 219 ms
ok 102 SOLR-16976 solr starts with remote JMX enabled # in 8216 ms
not ok 103 deprecated system properties converted to modern properties # in 195225 ms
# (from function `assert_file_contains' in file .gradle/node/node_modules/bats-file/src/file.bash, line 566,
# in test file test/test_start_solr.bats, line 86)
# `assert_file_contains "${SOLR_LOGS_DIR}/solr.log" 'You are passing in deprecated system property disable.config.edit and should upgrade to using solr.api.config.edit.enabled instead.'' failed
# Waiting up to 180 seconds to see Solr running on port 40331 [|] [/] [-] [\] [|]
# Started Solr server on port 40331 (pid=671180). Happy searching!
#
#
# -- file does not contain regex --
# path : /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/solr-home/logs/solr.log
# --
#
# Please find the SOLR_HOME snapshot for failed test #5 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/103-5
ok 104 start with custom jetty options # in 8166 ms
ok 105 -c flag prints no-op warning and still starts in cloud mode # in 9193 ms
not ok 106 bootstrapping a configset # in 9259 ms
# (from function `config_exists' in file test/bats_helper.bash, line 84,
# in test file test/test_start_solr.bats, line 116)
# `config_exists "techproducts"' failed
# Waiting up to 180 seconds to see Solr running on port 40331 [|] [/] [-] [\] [|]
# Started Solr server on port 40331 (pid=673907). Happy searching!
#
# WARN - 2026-06-16 12:03:35.557; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-16 12:03:35.562; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# Please find the SOLR_HOME snapshot for failed test #8 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/106-8
ok 107 status detects locally running solr # in 9626 ms
ok 108 status with --solr-url from user # in 7934 ms
ok 109 status with --port from user # in 7924 ms
ok 110 multiple connection options are prevented # in 843 ms
ok 111 status with invalid --solr-url from user # in 1301 ms
ok 112 status with --short format # in 195935 ms
ok 113 searching solr via locally executed streaming expression # in 1737 ms
ok 114 searching solr via remotely executed streaming expression by solr-url # in 1276 ms
ok 115 searching solr via remotely executed streaming expression by solr connection # in 1108 ms
ok 116 variable interpolation # in 1717 ms
ok 117 searching solr without credentials fails with error # in 1134 ms
ok 118 --version returns Solr version # in 748 ms
ok 119 version as direct tool call still runs # in 748 ms
ok 120 short help # in 803 ms
ok 121 short help is inferred # in 741 ms
ok 122 long help # in 763 ms
ok 123 running subcommands with zk is prevented # in 716 ms
ok 124 listing out files # in 2112 ms
ok 125 connecting to solr via various solr urls and zk hosts # in 9356 ms
ok 126 copying files around # in 10804 ms
ok 127 upconfig # in 2641 ms
ok 128 SOLR-12429 test upconfig fails with symlink # in 1089 ms
ok 129 downconfig # in 1143 ms
ok 130 bin/solr zk cp gets 'solrhome' from '--solr-home' command line option # in 1399 ms
ok 131 env var ZK_HOST is honored # in 3191 ms
ok 132 Cleanup # in 1938 ms
> Task :solr:packaging:integrationTests FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build.gradle' line: 339
* What went wrong:
Execution failed for task ':solr:packaging:integrationTests'.
> Process 'command '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/.gradle/node/node_modules/bats/bin/bats'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/8.10/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 29m 46s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/k35tb4obip2zw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 234 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/234/
No tests ran.
Build Log:
[...truncated 347 lines...]
# ERROR: KeeperErrorCode = NoNode for /security.json
# --
#
# Last output:
# WARN - 2026-06-15 12:09:00.069; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-15 12:09:00.074; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-15 12:09:00.628; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-15 12:09:00.628; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: KeeperErrorCode = NoNode for /security.json
not ok 9 create, config, api, and delete with basic auth # in 1531 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 38375 is already being used by another process (pid: 3853610)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-06-15 12:09:02.010; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-15 12:09:02.015; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
not ok 10 post, postlogs and export with basic auth # in 16772 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 38375 is already being used by another process (pid: 3853610)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/10-2
# Last output:
# WARN - 2026-06-15 12:09:03.424; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-15 12:09:03.428; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
ok 11 package with basic auth # in 12940 ms
ok 12 admin UI static content is excepted from basic auth # in 11110 ms
ok 13 nothing # in 11 ms
ok 14 server does not compress response without Accept-Encoding header # in 162 ms
ok 15 server compresses response when Accept-Encoding: gzip is requested # in 87 ms
ok 16 compressed response can be decompressed and parsed # in 88 ms
ok 17 setting property # in 12222 ms
ok 18 short form of setting property # in 6812 ms
ok 19 connecting to solr via various connection types # in 11554 ms
ok 20 create for non cloud mode # in 8406 ms
ok 21 create for cloud mode # in 10595 ms
ok 22 multiple connection options are prevented # in 929 ms
ok 23 create collection # in 8438 ms
ok 24 create collection using solrUrl # in 5542 ms
ok 25 create collection using legacy solrUrl # in 5432 ms
ok 26 create collection using Zookeeper # in 5603 ms
ok 27 reject d option with invalid config dir # in 2810 ms
ok 28 accept d option with builtin config # in 5518 ms
ok 29 accept d option with explicit path to config # in 5550 ms
ok 30 accept n option as config name # in 5441 ms
ok 31 allow config reuse when n option specifies same config # in 9415 ms
ok 32 create multisharded collections when s provided # in 5639 ms
ok 33 create replicated collections when rf provided # in 6379 ms
ok 34 can delete collections # in 11638 ms
ok 35 can delete collections with solr-url # in 8837 ms
ok 36 can delete collections with solr-connection # in 8403 ms
ok 37 collection delete also deletes zk config # in 7774 ms
ok 38 deletes accompanying zk config with nondefault name # in 7879 ms
ok 39 delete-config option can opt to leave config in zk # in 7904 ms
ok 40 start -e cloud works with --jvm-opts # in 22881 ms
ok 41 start -e cloud works with --no-prompt # in 45563 ms
ok 42 Check export command # in 20315 ms
ok 43 export fails on non cloud mode # in 9943 ms
Waiting for Tika Server to be ready on port 38380
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 44 using curl to extract a single pdf file # in 16674 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to extract content from pdf # in 16631 ms
WARNING: Docker not available; bypassing test.
ok 46 using the bin/solr post tool to crawl web site # in 16632 ms
Stopping Tika Server container
ok 47 healthcheck on cloud solr # in 10874 ms
ok 48 healthcheck on remote cloud solr # in 10668 ms
ok 49 healthcheck errors on standalone solr # in 11962 ms
ok 50 solr help flag prints help # in 813 ms
ok 51 solr with no flags prints help # in 806 ms
ok 52 start help flag prints help # in 165 ms
ok 53 start h flag prints help # in 155 ms
ok 54 stop help flag prints help # in 148 ms
ok 55 restart help flag prints help # in 164 ms
ok 56 status help flag prints help # in 837 ms
ok 57 healthcheck help flag prints help # in 810 ms
ok 58 create help flag prints help # in 822 ms
ok 59 delete help flag prints help # in 824 ms
ok 60 zk help flag prints help # in 804 ms
ok 61 auth help flag prints help # in 811 ms
ok 62 assert help flag prints help # in 843 ms
ok 63 post help flag prints help # in 822 ms
ok 64 SQL Module # in 16710 ms
ok 65 icu collation in analysis-extras module # in 13018 ms
ok 66 Check lifecycle of sentiment classification # in 24791 ms
ok 67 lifecycle of package # in 8662 ms
ok 68 deploying and undeploying a collection level package # in 13931 ms
ok 69 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
ok 70 Affinity placement plugin using sysprop # in 16857 ms
ok 71 Random placement plugin using ENV # in 16647 ms
ok 72 Check help command # in 2579 ms
ok 73 basic post with a type specified # in 5671 ms
ok 74 basic post with collection instead of url parameter # in 4241 ms
ok 75 basic post with solr-url and collection # in 4193 ms
ok 76 basic post with -s (solr-connection short form) and collection # in 4279 ms
ok 77 basic post WITHOUT a type specified # in 12612 ms
ok 78 crawling a directory as a dry-run # in 884 ms
ok 79 crawling a directory # in 4653 ms
ok 80 skipcommit and optimize and delete # in 4185 ms
ok 81 args mode # in 7480 ms
ok 82 stdin mode # in 3688 ms
ok 83 verbose echo the Solr response # in 4038 ms
ok 84 post solr log into solr via cli # in 10597 ms
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 11 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 11 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 17 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-06-15T12:19:56.279323243Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-15T12:19:56.279377896Z It should be set to 65000 to avoid operational disruption.
2026-06-15T12:19:56.279381714Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-15T12:19:56.279700350Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-15T12:19:56.279712693Z
2026-06-15T12:19:56.289390493Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-15T12:19:56.297462986Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-15T12:19:56.344562244Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-15T12:19:56.378271454Z WARNING: A command line option has enabled the Security Manager
2026-06-15T12:19:56.378359210Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-15T12:19:57.637988777Z 2026-06-15 12:19:57.634 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-15T12:19:57.952791352Z 2026-06-15 12:19:57.952 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-15T12:19:57.978318851Z 2026-06-15 12:19:57.977 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-15T12:19:57.985425905Z 2026-06-15 12:19:57.984 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-15T12:19:57.985507968Z 2026-06-15 12:19:57.985 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-06-15T12:19:57.985588619Z 2026-06-15 12:19:57.985 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-15T12:19:57.985705495Z 2026-06-15 12:19:57.985 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-15T12:19:57.985206321Z
2026-06-15T12:19:57.988154693Z 2026-06-15 12:19:57.987 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-15T12:19:58.010625766Z 2026-06-15 12:19:58.010 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-15T12:19:58.025094285Z 2026-06-15 12:19:58.024 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-15T12:19:58.025166162Z 2026-06-15 12:19:58.024 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-15T12:19:58.062712112Z 2026-06-15 12:19:58.062 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-15T12:19:58.803348248Z 2026-06-15 12:19:58.802 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-15T12:19:59.050904403Z 2026-06-15 12:19:59.050 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-15T12:19:59.056168458Z 2026-06-15 12:19:59.055 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-06-15T12:19:59.056288311Z 2026-06-15 12:19:59.055 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-06-15T12:19:59.058825592Z 2026-06-15 12:19:59.058 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-06-15T12:19:59.058946848Z 2026-06-15 12:19:59.058 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-06-15T12:19:59.096642185Z 2026-06-15 12:19:59.096 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-06-15T12:19:59.567723528Z 2026-06-15 12:19:59.565 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-06-15T12:19:59.588025592Z 2026-06-15 12:19:59.587 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-15T12:19:59.631673474Z 2026-06-15 12:19:59.631 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-15T12:19:59.746847653Z 2026-06-15 12:19:59.679 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-15T12:19:59.746868347Z 2026-06-15 12:19:59.746 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-15T12:19:59.918486387Z 2026-06-15 12:19:59.917 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-15T12:19:59.943072405Z 2026-06-15 12:19:59.942 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-15T12:19:59.960330624Z 2026-06-15 12:19:59.959 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-06-15T12:19:59.967133991Z 2026-06-15 12:19:59.966 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72126907354775552-solr-node1:8983_solr-n_0000000000) starting
2026-06-15T12:20:00.038552407Z 2026-06-15 12:20:00.037 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-06-15T12:20:00.038667570Z 2026-06-15 12:20:00.037 INFO (OverseerStateUpdate-72126907354775552-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-06-15T12:20:00.053049675Z 2026-06-15 12:20:00.051 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-15T12:20:00.104187028Z 2026-06-15 12:20:00.103 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-15T12:20:00.592869314Z 2026-06-15 12:20:00.590 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-15T12:20:00.775069364Z 2026-06-15 12:20:00.774 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-15T12:20:01.054238273Z 2026-06-15 12:20:01.053 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-15T12:20:01.191613250Z 2026-06-15 12:20:01.190 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-15T12:20:01.198523730Z 2026-06-15 12:20:01.198 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_15.request.log
2026-06-15T12:20:01.293758576Z 2026-06-15 12:20:01.292 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-06-15T12:20:01.294760216Z 2026-06-15 12:20:01.294 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4998ms
2026-06-15T12:20:01.496114573Z 2026-06-15 12:20:01.494 INFO (qtp985324122-28-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=50
2026-06-15T12:20:01.563465522Z 2026-06-15 12:20:01.562 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=15
2026-06-15T12:20:05.226343833Z 2026-06-15 12:20:05.225 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-15T12:20:05.654088393Z 2026-06-15 12:20:05.653 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-15T12:20:10.179311543Z 2026-06-15 12:20:10.178 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=8
2026-06-15T12:20:10.215159544Z 2026-06-15 12:20:10.214 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=9
2026-06-15T12:20:10.250394337Z 2026-06-15 12:20:10.249 INFO (qtp985324122-28-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=2
2026-06-15T12:20:10.582419938Z 2026-06-15 12:20:10.581 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-06-15T12:20:10.602728590Z 2026-06-15 12:20:10.602 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-06-15T12:20:10.609544273Z 2026-06-15 12:20:10.609 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 174] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-15T12:20:10.651728356Z 2026-06-15 12:20:10.651 INFO (OverseerStateUpdate-72126907354775552-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-15T12:20:10.651751390Z "core":"test-collection_shard2_replica_n1",
2026-06-15T12:20:10.651754904Z "node_name":"solr-node3:8985_solr",
2026-06-15T12:20:10.651757663Z "base_url":"http://solr-node3:8985/solr",
2026-06-15T12:20:10.651760365Z "collection":"test-collection",
2026-06-15T12:20:10.651763026Z "shard":"shard2",
2026-06-15T12:20:10.651765710Z "state":"down",
2026-06-15T12:20:10.651768237Z "type":"NRT",
2026-06-15T12:20:10.651770750Z "operation":"ADDREPLICA",
2026-06-15T12:20:10.651773358Z "waitForFinalState":"false"}
2026-06-15T12:20:10.661054397Z 2026-06-15 12:20:10.660 INFO (OverseerStateUpdate-72126907354775552-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-15T12:20:10.661074023Z "core":"test-collection_shard1_replica_n2",
2026-06-15T12:20:10.661077119Z "node_name":"solr-node2:8984_solr",
2026-06-15T12:20:10.661079788Z "base_url":"http://solr-node2:8984/solr",
2026-06-15T12:20:10.661082466Z "collection":"test-collection",
2026-06-15T12:20:10.661085099Z "shard":"shard1",
2026-06-15T12:20:10.661087758Z "state":"down",
2026-06-15T12:20:10.661090362Z "type":"NRT",
2026-06-15T12:20:10.661092962Z "operation":"ADDREPLICA",
2026-06-15T12:20:10.661106645Z "waitForFinalState":"false"}
2026-06-15T12:20:10.668169237Z 2026-06-15 12:20:10.667 INFO (OverseerStateUpdate-72126907354775552-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-15T12:20:10.668186173Z "core":"test-collection_shard3_replica_n4",
2026-06-15T12:20:10.668189138Z "node_name":"solr-node1:8983_solr",
2026-06-15T12:20:10.668207941Z "base_url":"http://solr-node1:8983/solr",
2026-06-15T12:20:10.668210025Z "collection":"test-collection",
2026-06-15T12:20:10.668212002Z "shard":"shard3",
2026-06-15T12:20:10.668213988Z "state":"down",
2026-06-15T12:20:10.668215908Z "type":"NRT",
2026-06-15T12:20:10.668217908Z "operation":"ADDREPLICA",
2026-06-15T12:20:10.668219839Z "waitForFinalState":"false"}
2026-06-15T12:20:10.779533124Z 2026-06-15 12:20:10.778 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 186] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-15T12:20:10.888149643Z 2026-06-15 12:20:10.887 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard3_replica_n4&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-06-15T12:20:10.934678248Z 2026-06-15 12:20:10.934 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-15T12:20:11.014641782Z 2026-06-15 12:20:11.014 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-15T12:20:11.390222008Z 2026-06-15 12:20:11.379 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-15T12:20:11.390245206Z 2026-06-15 12:20:11.388 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n4' using configuration from configset techproducts, trusted=true
2026-06-15T12:20:11.422137314Z 2026-06-15 12:20:11.421 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n4], dataDir=[/var/solr/data/test-collection_shard3_replica_n4/data/]
2026-06-15T12:20:12.316381395Z 2026-06-15 12:20:12.314 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-15T12:20:12.347273118Z 2026-06-15 12:20:12.346 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-15T12:20:12.353541688Z 2026-06-15 12:20:12.352 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-15T12:20:12.354112581Z 2026-06-15 12:20:12.353 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-15T12:20:12.354210072Z 2026-06-15 12:20:12.353 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-15T12:20:12.423323755Z 2026-06-15 12:20:12.422 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-15T12:20:12.426397680Z 2026-06-15 12:20:12.426 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-15T12:20:12.429141592Z 2026-06-15 12:20:12.428 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-15T12:20:12.431154990Z 2026-06-15 12:20:12.430 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-15T12:20:12.468290765Z 2026-06-15 12:20:12.467 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4}
2026-06-15T12:20:12.478449173Z 2026-06-15 12:20:12.477 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-15T12:20:12.489109203Z 2026-06-15 12:20:12.488 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n4 solr-node1-6 core_node6 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-15T12:20:12.489143640Z 2026-06-15 12:20:12.488 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n4 solr-node1-6 core_node6 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-15T12:20:12.497571040Z 2026-06-15 12:20:12.496 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n4 solr-node1-6 core_node6 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-15T12:20:12.509202335Z 2026-06-15 12:20:12.508 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-06-15T12:20:12.511125575Z 2026-06-15 12:20:12.510 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-06-15T12:20:12.527444421Z 2026-06-15 12:20:12.526 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-15T12:20:12.527598852Z 2026-06-15 12:20:12.527 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-15T12:20:12.527675544Z 2026-06-15 12:20:12.527 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n4/
2026-06-15T12:20:12.528977536Z 2026-06-15 12:20:12.528 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-15T12:20:12.529530560Z 2026-06-15 12:20:12.529 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n4/ has no replicas
2026-06-15T12:20:12.530864727Z 2026-06-15 12:20:12.530 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72126907354775552-core_node6-n_0000000000
2026-06-15T12:20:12.539283154Z 2026-06-15 12:20:12.538 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n4/ shard3
2026-06-15T12:20:12.722034846Z 2026-06-15 12:20:12.721 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 237] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-15T12:20:12.726038325Z 2026-06-15 12:20:12.725 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-15T12:20:12.743204777Z 2026-06-15 12:20:12.742 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1857
2026-06-15T12:20:12.844902040Z 2026-06-15 12:20:12.844 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 3884471,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-15T12:19:56.001290926Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-06-15T12:20:01.912588605Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-15T12:20:01.912614861Z It should be set to 65000 to avoid operational disruption.
2026-06-15T12:20:01.912617778Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-15T12:20:01.913065904Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-15T12:20:01.913083195Z
2026-06-15T12:20:01.925689892Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-15T12:20:01.935818083Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-15T12:20:01.982041515Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-15T12:20:02.019215860Z WARNING: A command line option has enabled the Security Manager
2026-06-15T12:20:02.019240049Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-15T12:20:03.268103813Z 2026-06-15 12:20:03.264 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-15T12:20:03.549871390Z 2026-06-15 12:20:03.549 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-15T12:20:03.577794759Z 2026-06-15 12:20:03.577 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-15T12:20:03.585737679Z 2026-06-15 12:20:03.584 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-15T12:20:03.585764086Z 2026-06-15 12:20:03.585 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-06-15T12:20:03.586156018Z 2026-06-15 12:20:03.585 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-15T12:20:03.586173666Z 2026-06-15 12:20:03.585 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-15T12:20:03.585281498Z
2026-06-15T12:20:03.588918030Z 2026-06-15 12:20:03.588 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-15T12:20:03.612551971Z 2026-06-15 12:20:03.611 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-15T12:20:03.637813725Z 2026-06-15 12:20:03.637 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-15T12:20:03.784320369Z 2026-06-15 12:20:03.783 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-15T12:20:03.806457866Z 2026-06-15 12:20:03.805 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-15T12:20:03.806599606Z 2026-06-15 12:20:03.806 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-15T12:20:03.806724885Z 2026-06-15 12:20:03.806 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-15T12:20:03.918495784Z 2026-06-15 12:20:03.917 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-15T12:20:03.918536333Z 2026-06-15 12:20:03.918 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-15T12:20:03.918612760Z 2026-06-15 12:20:03.918 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-15T12:20:03.953040882Z 2026-06-15 12:20:03.952 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-15T12:20:04.790955192Z 2026-06-15 12:20:04.790 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-15T12:20:04.961975755Z 2026-06-15 12:20:04.961 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-15T12:20:04.963371839Z 2026-06-15 12:20:04.961 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-15T12:20:04.976975198Z 2026-06-15 12:20:04.975 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-15T12:20:04.992851673Z 2026-06-15 12:20:04.992 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-15T12:20:04.998839177Z 2026-06-15 12:20:04.998 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-15T12:20:04.998952011Z 2026-06-15 12:20:04.998 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-15T12:20:05.144347133Z 2026-06-15 12:20:05.142 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-15T12:20:05.157456308Z 2026-06-15 12:20:05.156 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-15T12:20:05.182789163Z 2026-06-15 12:20:05.182 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-15T12:20:05.197445506Z 2026-06-15 12:20:05.196 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-06-15T12:20:05.211293016Z 2026-06-15 12:20:05.210 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-06-15T12:20:05.228043133Z 2026-06-15 12:20:05.227 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-15T12:20:05.285399139Z 2026-06-15 12:20:05.284 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-15T12:20:05.593401239Z 2026-06-15 12:20:05.590 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-15T12:20:05.654000828Z 2026-06-15 12:20:05.653 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-15T12:20:05.777243480Z 2026-06-15 12:20:05.776 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-15T12:20:06.082083582Z 2026-06-15 12:20:06.081 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-15T12:20:06.218038138Z 2026-06-15 12:20:06.217 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-15T12:20:06.223843484Z 2026-06-15 12:20:06.223 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_15.request.log
2026-06-15T12:20:06.227559443Z 2026-06-15 12:20:06.226 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-06-15T12:20:06.230756890Z 2026-06-15 12:20:06.230 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4296ms
2026-06-15T12:20:06.414607772Z 2026-06-15 12:20:06.413 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=54
2026-06-15T12:20:06.486908097Z 2026-06-15 12:20:06.485 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=16
2026-06-15T12:20:10.549230537Z 2026-06-15 12:20:10.548 INFO (qtp985324122-58-solr-node2-3) [c: s: r: x: t:solr-node2-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=6
2026-06-15T12:20:10.886079392Z 2026-06-15 12:20:10.885 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node5&name=test-collection_shard1_replica_n2&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-06-15T12:20:10.962926015Z 2026-06-15 12:20:10.962 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-15T12:20:11.166119833Z 2026-06-15 12:20:11.165 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-15T12:20:11.442154778Z 2026-06-15 12:20:11.441 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-15T12:20:11.442308852Z 2026-06-15 12:20:11.441 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n2' using configuration from configset techproducts, trusted=true
2026-06-15T12:20:11.465049330Z 2026-06-15 12:20:11.464 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n2], dataDir=[/var/solr/data/test-collection_shard1_replica_n2/data/]
2026-06-15T12:20:11.949871105Z 2026-06-15 12:20:11.948 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-15T12:20:11.973121152Z 2026-06-15 12:20:11.972 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-15T12:20:11.977426746Z 2026-06-15 12:20:11.976 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-15T12:20:11.978716065Z 2026-06-15 12:20:11.978 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-15T12:20:11.978788602Z 2026-06-15 12:20:11.978 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-15T12:20:12.092882122Z 2026-06-15 12:20:12.092 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-15T12:20:12.099229004Z 2026-06-15 12:20:12.098 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-15T12:20:12.103443203Z 2026-06-15 12:20:12.102 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-15T12:20:12.107986840Z 2026-06-15 12:20:12.107 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-15T12:20:12.122356642Z 2026-06-15 12:20:12.121 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4}
2026-06-15T12:20:12.131991487Z 2026-06-15 12:20:12.131 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-15T12:20:12.139399698Z 2026-06-15 12:20:12.138 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n2 solr-node1-6 core_node5 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-15T12:20:12.139427523Z 2026-06-15 12:20:12.138 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n2 solr-node1-6 core_node5 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-15T12:20:12.146988555Z 2026-06-15 12:20:12.146 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n2 solr-node1-6 core_node5 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-15T12:20:12.169191056Z 2026-06-15 12:20:12.168 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node5=0}, version=0} for registerTerm
2026-06-15T12:20:12.170718784Z 2026-06-15 12:20:12.170 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1
2026-06-15T12:20:12.190182490Z 2026-06-15 12:20:12.189 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-15T12:20:12.190438609Z 2026-06-15 12:20:12.189 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-15T12:20:12.190449390Z 2026-06-15 12:20:12.189 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node2:8984/solr/test-collection_shard1_replica_n2/
2026-06-15T12:20:12.191757513Z 2026-06-15 12:20:12.191 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-15T12:20:12.192482081Z 2026-06-15 12:20:12.192 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node2:8984/solr/test-collection_shard1_replica_n2/ has no replicas
2026-06-15T12:20:12.193725061Z 2026-06-15 12:20:12.193 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard1/leader after winning as /collections/test-collection/leader_elect/shard1/election/72126907354775555-core_node5-n_0000000000
2026-06-15T12:20:12.206064304Z 2026-06-15 12:20:12.205 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node2:8984/solr/test-collection_shard1_replica_n2/ shard1
2026-06-15T12:20:12.319909040Z 2026-06-15 12:20:12.319 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 206] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-15T12:20:12.324114657Z 2026-06-15 12:20:12.323 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard1 r:core_node5 x:test-collection_shard1_replica_n2 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-15T12:20:12.335457923Z 2026-06-15 12:20:12.334 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=techproducts&newCollection=true&name=test-collection_shard1_replica_n2&action=CREATE&numShards=3&collection=test-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1450
2026-06-15T12:20:12.436658990Z 2026-06-15 12:20:12.436 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 209] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-15T12:20:12.722071528Z 2026-06-15 12:20:12.721 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 237] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-15T12:20:12.757880960Z 2026-06-15 12:20:12.757 INFO (qtp985324122-59-solr-node2-4) [c:test-collection s: r: x: t:solr-node2-4] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
2026-06-15T12:20:12.844944812Z 2026-06-15 12:20:12.844 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-15T12:20:12.848610736Z 2026-06-15 12:20:12.848 INFO (qtp985324122-59-solr-node2-4) [c:test-collection s: r: x: t:solr-node2-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=2285
# === Docker inspect for solr-node2 ===
{
"Name": "/solr-node2",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 3884889,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-15T12:20:01.67958601Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node3 ===
2026-06-15T12:20:02.162433317Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-15T12:20:02.162480468Z It should be set to 65000 to avoid operational disruption.
2026-06-15T12:20:02.162484367Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-15T12:20:02.162785725Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-15T12:20:02.162818489Z
2026-06-15T12:20:02.173609211Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-15T12:20:02.183731167Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-15T12:20:02.235644627Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-15T12:20:02.273483984Z WARNING: A command line option has enabled the Security Manager
2026-06-15T12:20:02.273519470Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-15T12:20:03.643687222Z 2026-06-15 12:20:03.640 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-15T12:20:03.940879528Z 2026-06-15 12:20:03.940 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-15T12:20:03.972269780Z 2026-06-15 12:20:03.971 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-15T12:20:03.979771373Z 2026-06-15 12:20:03.979 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-15T12:20:03.980035651Z 2026-06-15 12:20:03.979 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8985
2026-06-15T12:20:03.980242787Z 2026-06-15 12:20:03.979 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-15T12:20:03.980352955Z 2026-06-15 12:20:03.979 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-15T12:20:03.979481474Z
2026-06-15T12:20:03.982990654Z 2026-06-15 12:20:03.982 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-15T12:20:04.004834608Z 2026-06-15 12:20:04.004 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-15T12:20:04.031464382Z 2026-06-15 12:20:04.030 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-15T12:20:04.208315443Z 2026-06-15 12:20:04.207 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-15T12:20:04.228161948Z 2026-06-15 12:20:04.227 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-15T12:20:04.228238131Z 2026-06-15 12:20:04.227 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-15T12:20:04.228436072Z 2026-06-15 12:20:04.227 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-15T12:20:04.339227104Z 2026-06-15 12:20:04.338 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-15T12:20:04.339423802Z 2026-06-15 12:20:04.339 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-15T12:20:04.339477474Z 2026-06-15 12:20:04.339 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-15T12:20:04.375334354Z 2026-06-15 12:20:04.374 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-15T12:20:05.376760224Z 2026-06-15 12:20:05.376 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-15T12:20:05.529010502Z 2026-06-15 12:20:05.528 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-15T12:20:05.530225499Z 2026-06-15 12:20:05.528 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-15T12:20:05.539713251Z 2026-06-15 12:20:05.539 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-15T12:20:05.545142626Z 2026-06-15 12:20:05.544 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-15T12:20:05.549601280Z 2026-06-15 12:20:05.549 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-15T12:20:05.549618585Z 2026-06-15 12:20:05.549 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-15T12:20:05.589306425Z 2026-06-15 12:20:05.588 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-15T12:20:05.598849577Z 2026-06-15 12:20:05.598 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
2026-06-15T12:20:05.621523336Z 2026-06-15 12:20:05.620 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-15T12:20:05.633542500Z 2026-06-15 12:20:05.633 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node3:8985_solr as DOWN
2026-06-15T12:20:05.644835169Z 2026-06-15 12:20:05.644 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node3:8985_solr
2026-06-15T12:20:05.654965355Z 2026-06-15 12:20:05.654 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-15T12:20:05.701413497Z 2026-06-15 12:20:05.700 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-15T12:20:05.973114278Z 2026-06-15 12:20:05.972 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-15T12:20:06.159508467Z 2026-06-15 12:20:06.159 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-15T12:20:06.448323830Z 2026-06-15 12:20:06.447 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-15T12:20:06.601006626Z 2026-06-15 12:20:06.600 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-15T12:20:06.608937991Z 2026-06-15 12:20:06.608 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_15.request.log
2026-06-15T12:20:06.613485139Z 2026-06-15 12:20:06.612 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8985}
2026-06-15T12:20:06.614729424Z 2026-06-15 12:20:06.614 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4432ms
2026-06-15T12:20:08.956967149Z 2026-06-15 12:20:08.933 INFO (qtp985324122-27-solr-node3-1) [c: s: r: x: t:solr-node3-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=121
2026-06-15T12:20:09.031195769Z 2026-06-15 12:20:09.030 INFO (qtp985324122-56-solr-node3-2) [c: s: r: x: t:solr-node3-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=26
2026-06-15T12:20:10.892082755Z 2026-06-15 12:20:10.890 INFO (qtp985324122-58-solr-node1-6) [c: s: r: x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard2_replica_n1&action=CREATE&numShards=3&shard=shard2&wt=javabin
2026-06-15T12:20:11.001592934Z 2026-06-15 12:20:11.001 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-15T12:20:11.146949978Z 2026-06-15 12:20:11.146 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-15T12:20:11.578512956Z 2026-06-15 12:20:11.577 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-15T12:20:11.578626380Z 2026-06-15 12:20:11.577 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard2_replica_n1' using configuration from configset techproducts, trusted=true
2026-06-15T12:20:11.607301587Z 2026-06-15 12:20:11.606 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard2_replica_n1], dataDir=[/var/solr/data/test-collection_shard2_replica_n1/data/]
2026-06-15T12:20:12.399624271Z 2026-06-15 12:20:12.399 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-15T12:20:12.457459633Z 2026-06-15 12:20:12.456 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-15T12:20:12.467259633Z 2026-06-15 12:20:12.466 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-15T12:20:12.468519867Z 2026-06-15 12:20:12.467 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-15T12:20:12.468536149Z 2026-06-15 12:20:12.468 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-15T12:20:12.518841125Z 2026-06-15 12:20:12.518 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-15T12:20:12.522969334Z 2026-06-15 12:20:12.522 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-15T12:20:12.525547117Z 2026-06-15 12:20:12.525 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-15T12:20:12.527276711Z 2026-06-15 12:20:12.526 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-15T12:20:12.540328232Z 2026-06-15 12:20:12.539 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4}
2026-06-15T12:20:12.549554380Z 2026-06-15 12:20:12.549 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-15T12:20:12.557209537Z 2026-06-15 12:20:12.556 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n1 solr-node1-6 core_node3 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-15T12:20:12.557281853Z 2026-06-15 12:20:12.556 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n1 solr-node1-6 core_node3 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-15T12:20:12.567450191Z 2026-06-15 12:20:12.566 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n1 solr-node1-6 core_node3 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-15T12:20:12.575900348Z 2026-06-15 12:20:12.575 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard2 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-06-15T12:20:12.577610415Z 2026-06-15 12:20:12.577 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard2
2026-06-15T12:20:12.601365113Z 2026-06-15 12:20:12.600 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-15T12:20:12.601413482Z 2026-06-15 12:20:12.601 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-15T12:20:12.601484964Z 2026-06-15 12:20:12.601 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node3:8985/solr/test-collection_shard2_replica_n1/
2026-06-15T12:20:12.603166442Z 2026-06-15 12:20:12.602 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-15T12:20:12.603782846Z 2026-06-15 12:20:12.603 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node3:8985/solr/test-collection_shard2_replica_n1/ has no replicas
2026-06-15T12:20:12.605093061Z 2026-06-15 12:20:12.604 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard2/leader after winning as /collections/test-collection/leader_elect/shard2/election/72126907354775556-core_node3-n_0000000000
2026-06-15T12:20:12.613337884Z 2026-06-15 12:20:12.612 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node3:8985/solr/test-collection_shard2_replica_n1/ shard2
2026-06-15T12:20:12.722359149Z 2026-06-15 12:20:12.721 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 237] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-15T12:20:12.726856489Z 2026-06-15 12:20:12.726 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard2 r:core_node3 x:test-collection_shard2_replica_n1 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-15T12:20:12.745487422Z 2026-06-15 12:20:12.745 INFO (qtp985324122-58-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard2_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1858
2026-06-15T12:20:12.845393921Z 2026-06-15 12:20:12.844 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node3 ===
{
"Name": "/solr-node3",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 3885072,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-15T12:20:01.927742403Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# Docker artifacts saved to: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/docker
not ok 85 Docker SolrCloud rolling upgrade # in 29591 ms
# (in test file test/test_rolling_upgrade.bats, line 118)
# `wait_for 30 1 docker exec solr-node1 solr healthcheck -c test-collection' failed with status 127
# 9.10.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# af6eca94c810: Pulling fs layer
# cb0efb96dabd: Pulling fs layer
# 3e9d91201f40: Pulling fs layer
# 66b76b382631: Pulling fs layer
# 601f2c23751f: Pulling fs layer
# 0ed719aef393: Pulling fs layer
# 69a09c96e8bf: Pulling fs layer
# c937aee9a870: Pulling fs layer
# 9cdfe5320de8: Pulling fs layer
# 66b76b382631: Waiting
# 3aa9120e46f5: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# 69a09c96e8bf: Waiting
# 601f2c23751f: Waiting
# 9cdfe5320de8: Waiting
# 0ed719aef393: Waiting
# 3aa9120e46f5: Waiting
# 4f4fb700ef54: Waiting
# c937aee9a870: Waiting
# af6eca94c810: Verifying Checksum
# af6eca94c810: Download complete
# cb0efb96dabd: Verifying Checksum
# cb0efb96dabd: Download complete
# 66b76b382631: Download complete
# 601f2c23751f: Download complete
# 3e9d91201f40: Verifying Checksum
# 3e9d91201f40: Download complete
# 69a09c96e8bf: Verifying Checksum
# 69a09c96e8bf: Download complete
# c937aee9a870: Verifying Checksum
# c937aee9a870: Download complete
# 9cdfe5320de8: Download complete
# 3aa9120e46f5: Verifying Checksum
# 3aa9120e46f5: Download complete
# 4f4fb700ef54: Verifying Checksum
# 4f4fb700ef54: Download complete
# 0ed719aef393: Verifying Checksum
# 0ed719aef393: Download complete
# af6eca94c810: Pull complete
# cb0efb96dabd: Pull complete
# 3e9d91201f40: Pull complete
# 66b76b382631: Pull complete
# 601f2c23751f: Pull complete
# 0ed719aef393: Pull complete
# 69a09c96e8bf: Pull complete
# c937aee9a870: Pull complete
# 9cdfe5320de8: Pull complete
# 3aa9120e46f5: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:15c8155ceffe1042f37c77bb537aa531804a3510702ff87f2af5fedf523f4203
# Status: Downloaded newer image for apache/solr-nightly:9.10.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:9.10.0-SNAPSHOT-slim
# 10.0.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# 40d16f30db40: Already exists
# 9d5a0b6cb36c: Already exists
# 49518c28c0b3: Already exists
# 41162acdcc29: Already exists
# 2ed0fd558ca3: Already exists
# 2488415fda1c: Pulling fs layer
# 09511c8fb203: Pulling fs layer
# a0c57fb89c4f: Pulling fs layer
# d0641dcc41d7: Pulling fs layer
# 315722fd9570: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# d0641dcc41d7: Waiting
# 315722fd9570: Waiting
# 4f4fb700ef54: Waiting
# 09511c8fb203: Verifying Checksum
# 09511c8fb203: Download complete
# d0641dcc41d7: Download complete
# a0c57fb89c4f: Verifying Checksum
# a0c57fb89c4f: Download complete
# 4f4fb700ef54: Verifying Checksum
# 4f4fb700ef54: Download complete
# 315722fd9570: Verifying Checksum
# 315722fd9570: Download complete
# 2488415fda1c: Verifying Checksum
# 2488415fda1c: Download complete
# 2488415fda1c: Pull complete
# 09511c8fb203: Pull complete
# a0c57fb89c4f: Pull complete
# d0641dcc41d7: Pull complete
# 315722fd9570: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:fb25f7a7190e3027ade20e97708a8665d41ab0e9ea9285fa242b2c83b0e5f43c
# Status: Downloaded newer image for apache/solr-nightly:10.0.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:10.0.0-SNAPSHOT-slim
# 83b762bb24919b5fda4309a133871323ffb7b79bbe4c936bd229fbdeecd54500
# solr-data1
# solr-data2
# solr-data3
# Starting solr-node1 with embedded ZooKeeper
# b8e0578d26612256047baed44118e14d7d65609a2c73a3c5a111c0a65007441b
# Starting solr-node2 connected to first node's ZooKeeper
# 57f78c6961ed2a879ffdf191e46c9bb800da931c66d29a2bf7c9ba97836b2cb2
# Started solr-node3 connected to first node's ZooKeeper
# 84714353c1f25cf70f0f6ebd22fcd531a837a4905caeb87342abbd3ae9d1fa06
# Creating a Collection
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# WARN - 2026-06-15 12:20:10.284; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-15 12:20:10.319; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. "/solr"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [http://solr-node2:8984/solr] to [http://solr-node2:8984/].
# Created collection 'test-collection' with 3 shard(s), 1 replica(s) with config-set 'techproducts'
# Checking collection health
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_rolling_upgrade.bats: line 118: wait_for: command not found
# solr-node1
# solr-node2
# solr-node3
# solr-node1
# solr-node2
# solr-node3
# solr-data1
# solr-data2
# solr-data3
# solrcloud-test
ok 86 allowPaths - backup # in 16596 ms
ok 87 snapshot lifecycle # in 5102 ms
ok 88 snapshot list # in 3578 ms
ok 89 snapshot list via --solr-connection # in 3645 ms
ok 90 snapshot describe # in 3649 ms
ok 91 snapshot describe via --solr-connection # in 3656 ms
ok 92 start solr with ssl # in 17687 ms
ok 93 use different hostname when not checking peer-name # in 26383 ms
ok 94 start solr with ssl and auth # in 15374 ms
ok 95 start solr with client truststore and security manager # in 18422 ms
ok 96 start solr with mTLS needed # in 48026 ms
ok 97 start solr with mTLS wanted # in 37224 ms
ok 98 test keystore reload # in 62377 ms
ok 99 SOLR-11740 check 'solr stop' connection # in 16595 ms
ok 100 stop command for single port # in 19058 ms
ok 101 check stop command doesn't hang # in 249 ms
ok 102 SOLR-16976 solr starts with remote JMX enabled # in 8622 ms
not ok 103 deprecated system properties converted to modern properties # in 195627 ms
# (from function `assert_file_contains' in file .gradle/node/node_modules/bats-file/src/file.bash, line 566,
# in test file test/test_start_solr.bats, line 86)
# `assert_file_contains "${SOLR_LOGS_DIR}/solr.log" 'You are passing in deprecated system property disable.config.edit and should upgrade to using solr.api.config.edit.enabled instead.'' failed
# Waiting up to 180 seconds to see Solr running on port 38375 [|] [/] [-] [\] [|]
# Started Solr server on port 38375 (pid=3902658). Happy searching!
#
#
# -- file does not contain regex --
# path : /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/solr-home/logs/solr.log
# --
#
# Please find the SOLR_HOME snapshot for failed test #5 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/103-5
ok 104 start with custom jetty options # in 8732 ms
ok 105 -c flag prints no-op warning and still starts in cloud mode # in 10001 ms
not ok 106 bootstrapping a configset # in 9620 ms
# (from function `config_exists' in file test/bats_helper.bash, line 84,
# in test file test/test_start_solr.bats, line 116)
# `config_exists "techproducts"' failed
# Waiting up to 180 seconds to see Solr running on port 38375 [|] [/] [-] [\] [|]
# Started Solr server on port 38375 (pid=3905353). Happy searching!
#
# WARN - 2026-06-15 12:29:12.279; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-15 12:29:12.283; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# Please find the SOLR_HOME snapshot for failed test #8 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/106-8
ok 107 status detects locally running solr # in 10399 ms
ok 108 status with --solr-url from user # in 8571 ms
ok 109 status with --port from user # in 8437 ms
ok 110 multiple connection options are prevented # in 923 ms
ok 111 status with invalid --solr-url from user # in 1295 ms
ok 112 status with --short format # in 196474 ms
ok 113 searching solr via locally executed streaming expression # in 1889 ms
ok 114 searching solr via remotely executed streaming expression by solr-url # in 1432 ms
ok 115 searching solr via remotely executed streaming expression by solr connection # in 1254 ms
ok 116 variable interpolation # in 1778 ms
ok 117 searching solr without credentials fails with error # in 1221 ms
ok 118 --version returns Solr version # in 789 ms
ok 119 version as direct tool call still runs # in 776 ms
ok 120 short help # in 962 ms
ok 121 short help is inferred # in 813 ms
ok 122 long help # in 860 ms
ok 123 running subcommands with zk is prevented # in 809 ms
ok 124 listing out files # in 2194 ms
ok 125 connecting to solr via various solr urls and zk hosts # in 9954 ms
ok 126 copying files around # in 11541 ms
ok 127 upconfig # in 2638 ms
ok 128 SOLR-12429 test upconfig fails with symlink # in 1265 ms
ok 129 downconfig # in 1332 ms
ok 130 bin/solr zk cp gets 'solrhome' from '--solr-home' command line option # in 1533 ms
ok 131 env var ZK_HOST is honored # in 3434 ms
ok 132 Cleanup # in 1909 ms
> Task :solr:packaging:integrationTests FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build.gradle' line: 339
* What went wrong:
Execution failed for task ':solr:packaging:integrationTests'.
> Process 'command '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/.gradle/node/node_modules/bats/bin/bats'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/8.10/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 31m 15s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/p45d7hflfqyea
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 233 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/233/
No tests ran.
Build Log:
[...truncated 904 lines...]
BUILD FAILED in 29m 42s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/x3hkl6mowsn56
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 232 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/232/
No tests ran.
Build Log:
[...truncated 347 lines...]
# ERROR: KeeperErrorCode = NoNode for /security.json
# --
#
# Last output:
# WARN - 2026-06-13 12:12:38.767; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-13 12:12:38.771; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-13 12:12:39.220; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-13 12:12:39.220; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: KeeperErrorCode = NoNode for /security.json
not ok 9 create, config, api, and delete with basic auth # in 18020 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 38917 is already being used by another process (pid: 2551112)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-06-13 12:12:40.406; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-13 12:12:40.410; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
ok 10 post, postlogs and export with basic auth # in 17983 ms
ok 11 package with basic auth # in 12285 ms
ok 12 admin UI static content is excepted from basic auth # in 10532 ms
ok 13 nothing # in 10 ms
ok 14 server does not compress response without Accept-Encoding header # in 145 ms
ok 15 server compresses response when Accept-Encoding: gzip is requested # in 76 ms
ok 16 compressed response can be decompressed and parsed # in 79 ms
ok 17 setting property # in 11410 ms
ok 18 short form of setting property # in 6545 ms
ok 19 connecting to solr via various connection types # in 10496 ms
ok 20 create for non cloud mode # in 7805 ms
ok 21 create for cloud mode # in 9694 ms
ok 22 multiple connection options are prevented # in 826 ms
ok 23 create collection # in 7692 ms
ok 24 create collection using solrUrl # in 5192 ms
ok 25 create collection using legacy solrUrl # in 5258 ms
ok 26 create collection using Zookeeper # in 5542 ms
ok 27 reject d option with invalid config dir # in 2661 ms
ok 28 accept d option with builtin config # in 5113 ms
ok 29 accept d option with explicit path to config # in 5203 ms
ok 30 accept n option as config name # in 5182 ms
ok 31 allow config reuse when n option specifies same config # in 8892 ms
ok 32 create multisharded collections when s provided # in 5099 ms
ok 33 create replicated collections when rf provided # in 6085 ms
ok 34 can delete collections # in 10903 ms
ok 35 can delete collections with solr-url # in 8387 ms
ok 36 can delete collections with solr-connection # in 7936 ms
ok 37 collection delete also deletes zk config # in 7271 ms
ok 38 deletes accompanying zk config with nondefault name # in 7304 ms
ok 39 delete-config option can opt to leave config in zk # in 7029 ms
not ok 40 start -e cloud works with --jvm-opts # in 222984 ms
# (in test file test/test_example.bats, line 32)
# `solr start -e cloud --no-prompt --jvm-opts "-Dcustom.prop=helloworld"' failed
#
# Welcome to the SolrCloud example!
#
# Starting up 2 Solr nodes for your example SolrCloud cluster.
#
# Creating Solr home directory /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr
# Cloning /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1 into
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node2
#
# Starting up Solr on port 38917 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 38917 --solr-home "build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server" --jvm-opts "-Dcustom.prop=helloworld"
#
# Waiting up to 180 seconds to see Solr running on port 38917 [|] [/] [-] [\] [|]
# Started Solr server on port 38917 (pid=2565418). Happy searching!
#
#
# Starting up Solr on port 38918 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 38918 --solr-home "build/solr-10.1.0-SNAPSHOT/example/cloud/node2/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server" -z 127.0.0.1:39917 --jvm-opts "-Dcustom.prop=helloworld"
#
# Waiting up to 180 seconds to see Solr running on port 38918 [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] [|] [/] [-] [\] Still not seeing Solr listening on 38918 after 180 seconds!
# 2026-06-13 12:17:00.976 INFO (main) [] o.e.j.s.Server jetty-12.0.34; built: 2026-04-01T05:45:03.823Z; git: 57e7adb250592bd15f573e7820be9b64632c6637; jvm 21.0.4+7-LTS
#
# ERROR: Did not see Solr at http://localhost:38918/solr come online within 30 seconds!
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/40-1
ok 41 start -e cloud works with --no-prompt # in 20933 ms
ok 42 Check export command # in 18971 ms
ok 43 export fails on non cloud mode # in 9676 ms
Waiting for Tika Server to be ready on port 38922
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 44 using curl to extract a single pdf file # in 16574 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to extract content from pdf # in 16538 ms
WARNING: Docker not available; bypassing test.
ok 46 using the bin/solr post tool to crawl web site # in 16607 ms
Stopping Tika Server container
ok 47 healthcheck on cloud solr # in 10193 ms
ok 48 healthcheck on remote cloud solr # in 9745 ms
ok 49 healthcheck errors on standalone solr # in 11089 ms
ok 50 solr help flag prints help # in 722 ms
ok 51 solr with no flags prints help # in 711 ms
ok 52 start help flag prints help # in 141 ms
ok 53 start h flag prints help # in 139 ms
ok 54 stop help flag prints help # in 135 ms
ok 55 restart help flag prints help # in 143 ms
ok 56 status help flag prints help # in 767 ms
ok 57 healthcheck help flag prints help # in 808 ms
ok 58 create help flag prints help # in 751 ms
ok 59 delete help flag prints help # in 792 ms
ok 60 zk help flag prints help # in 771 ms
ok 61 auth help flag prints help # in 728 ms
ok 62 assert help flag prints help # in 741 ms
ok 63 post help flag prints help # in 758 ms
ok 64 SQL Module # in 16222 ms
ok 65 icu collation in analysis-extras module # in 12558 ms
ok 66 Check lifecycle of sentiment classification # in 22774 ms
ok 67 lifecycle of package # in 8903 ms
ok 68 deploying and undeploying a collection level package # in 13494 ms
ok 69 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
ok 70 Affinity placement plugin using sysprop # in 15214 ms
ok 71 Random placement plugin using ENV # in 15576 ms
ok 72 Check help command # in 2284 ms
ok 73 basic post with a type specified # in 5661 ms
ok 74 basic post with collection instead of url parameter # in 4078 ms
ok 75 basic post with solr-url and collection # in 3902 ms
ok 76 basic post with -s (solr-connection short form) and collection # in 3883 ms
ok 77 basic post WITHOUT a type specified # in 11467 ms
ok 78 crawling a directory as a dry-run # in 774 ms
ok 79 crawling a directory # in 4180 ms
ok 80 skipcommit and optimize and delete # in 3735 ms
ok 81 args mode # in 6753 ms
ok 82 stdin mode # in 3532 ms
ok 83 verbose echo the Solr response # in 3828 ms
ok 84 post solr log into solr via cli # in 9513 ms
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 11 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 17 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-06-13T12:26:15.812111800Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-13T12:26:15.812158633Z It should be set to 65000 to avoid operational disruption.
2026-06-13T12:26:15.812163635Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-13T12:26:15.812446326Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-13T12:26:15.812466044Z
2026-06-13T12:26:15.822239326Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-13T12:26:15.829864968Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-13T12:26:15.878473229Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-13T12:26:15.909319018Z WARNING: A command line option has enabled the Security Manager
2026-06-13T12:26:15.909350961Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-13T12:26:17.113130016Z 2026-06-13 12:26:17.109 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-13T12:26:17.378699794Z 2026-06-13 12:26:17.378 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-13T12:26:17.404040293Z 2026-06-13 12:26:17.403 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-13T12:26:17.411393076Z 2026-06-13 12:26:17.410 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-13T12:26:17.411422536Z 2026-06-13 12:26:17.411 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-06-13T12:26:17.411608505Z 2026-06-13 12:26:17.411 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-13T12:26:17.411745356Z 2026-06-13 12:26:17.411 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-13T12:26:17.411161157Z
2026-06-13T12:26:17.414493270Z 2026-06-13 12:26:17.414 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-13T12:26:17.437205336Z 2026-06-13 12:26:17.436 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-13T12:26:17.451796751Z 2026-06-13 12:26:17.451 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-13T12:26:17.451829311Z 2026-06-13 12:26:17.451 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-13T12:26:17.489839676Z 2026-06-13 12:26:17.489 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-13T12:26:18.169651239Z 2026-06-13 12:26:18.169 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-13T12:26:18.404483391Z 2026-06-13 12:26:18.403 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-13T12:26:18.409788599Z 2026-06-13 12:26:18.409 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-06-13T12:26:18.409839285Z 2026-06-13 12:26:18.409 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-06-13T12:26:18.412444193Z 2026-06-13 12:26:18.412 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-06-13T12:26:18.412517098Z 2026-06-13 12:26:18.412 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-06-13T12:26:18.448122099Z 2026-06-13 12:26:18.447 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-06-13T12:26:18.914770493Z 2026-06-13 12:26:18.914 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-06-13T12:26:18.928912269Z 2026-06-13 12:26:18.928 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-13T12:26:19.065106389Z 2026-06-13 12:26:19.064 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-13T12:26:19.091188339Z 2026-06-13 12:26:19.090 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-13T12:26:19.091210261Z 2026-06-13 12:26:19.090 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-13T12:26:19.216653853Z 2026-06-13 12:26:19.216 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-13T12:26:19.238923024Z 2026-06-13 12:26:19.238 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-13T12:26:19.250599445Z 2026-06-13 12:26:19.250 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-06-13T12:26:19.257083584Z 2026-06-13 12:26:19.256 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72115607593287680-solr-node1:8983_solr-n_0000000000) starting
2026-06-13T12:26:19.410887243Z 2026-06-13 12:26:19.410 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-06-13T12:26:19.411740596Z 2026-06-13 12:26:19.411 INFO (OverseerStateUpdate-72115607593287680-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-06-13T12:26:19.421500909Z 2026-06-13 12:26:19.421 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-13T12:26:19.588153269Z 2026-06-13 12:26:19.587 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-13T12:26:20.052548266Z 2026-06-13 12:26:20.050 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-13T12:26:20.243930959Z 2026-06-13 12:26:20.243 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-13T12:26:20.536938051Z 2026-06-13 12:26:20.536 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-13T12:26:20.693490588Z 2026-06-13 12:26:20.692 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-13T12:26:20.700999432Z 2026-06-13 12:26:20.700 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_13.request.log
2026-06-13T12:26:20.777713866Z 2026-06-13 12:26:20.776 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-06-13T12:26:20.778789648Z 2026-06-13 12:26:20.778 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4950ms
2026-06-13T12:26:21.135694375Z 2026-06-13 12:26:21.134 INFO (qtp985324122-27-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=46
2026-06-13T12:26:21.415489011Z 2026-06-13 12:26:21.414 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=12
2026-06-13T12:26:24.635339439Z 2026-06-13 12:26:24.634 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-13T12:26:24.946427915Z 2026-06-13 12:26:24.945 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-13T12:26:28.809414639Z 2026-06-13 12:26:28.808 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=9
2026-06-13T12:26:28.838529541Z 2026-06-13 12:26:28.838 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=8
2026-06-13T12:26:28.878279519Z 2026-06-13 12:26:28.877 INFO (qtp985324122-27-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=2
2026-06-13T12:26:29.517300487Z 2026-06-13 12:26:29.516 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-06-13T12:26:29.543668422Z 2026-06-13 12:26:29.543 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-06-13T12:26:29.549615429Z 2026-06-13 12:26:29.549 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 174] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-13T12:26:29.595652029Z 2026-06-13 12:26:29.594 INFO (OverseerStateUpdate-72115607593287680-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-13T12:26:29.595675980Z "core":"test-collection_shard3_replica_n1",
2026-06-13T12:26:29.595679808Z "node_name":"solr-node1:8983_solr",
2026-06-13T12:26:29.595682663Z "base_url":"http://solr-node1:8983/solr",
2026-06-13T12:26:29.595685424Z "collection":"test-collection",
2026-06-13T12:26:29.595688096Z "shard":"shard3",
2026-06-13T12:26:29.595690777Z "state":"down",
2026-06-13T12:26:29.595693342Z "type":"NRT",
2026-06-13T12:26:29.595695890Z "operation":"ADDREPLICA",
2026-06-13T12:26:29.595698507Z "waitForFinalState":"false"}
2026-06-13T12:26:29.603611797Z 2026-06-13 12:26:29.602 INFO (OverseerStateUpdate-72115607593287680-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-13T12:26:29.603637906Z "core":"test-collection_shard2_replica_n2",
2026-06-13T12:26:29.603642676Z "node_name":"solr-node3:8985_solr",
2026-06-13T12:26:29.603646812Z "base_url":"http://solr-node3:8985/solr",
2026-06-13T12:26:29.603650914Z "collection":"test-collection",
2026-06-13T12:26:29.603654899Z "shard":"shard2",
2026-06-13T12:26:29.603658959Z "state":"down",
2026-06-13T12:26:29.603663051Z "type":"NRT",
2026-06-13T12:26:29.603666959Z "operation":"ADDREPLICA",
2026-06-13T12:26:29.603670972Z "waitForFinalState":"false"}
2026-06-13T12:26:29.611897926Z 2026-06-13 12:26:29.611 INFO (OverseerStateUpdate-72115607593287680-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-06-13T12:26:29.611922495Z "core":"test-collection_shard1_replica_n4",
2026-06-13T12:26:29.611926126Z "node_name":"solr-node2:8984_solr",
2026-06-13T12:26:29.611943426Z "base_url":"http://solr-node2:8984/solr",
2026-06-13T12:26:29.611945744Z "collection":"test-collection",
2026-06-13T12:26:29.611947796Z "shard":"shard1",
2026-06-13T12:26:29.611949846Z "state":"down",
2026-06-13T12:26:29.611951780Z "type":"NRT",
2026-06-13T12:26:29.611953811Z "operation":"ADDREPLICA",
2026-06-13T12:26:29.611955826Z "waitForFinalState":"false"}
2026-06-13T12:26:29.722535842Z 2026-06-13 12:26:29.721 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 186] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-13T12:26:29.803905767Z 2026-06-13 12:26:29.802 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-06-13T12:26:29.873439070Z 2026-06-13 12:26:29.872 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-13T12:26:29.950363169Z 2026-06-13 12:26:29.949 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-13T12:26:30.989032981Z 2026-06-13 12:26:30.988 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-13T12:26:30.989116529Z 2026-06-13 12:26:30.988 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true
2026-06-13T12:26:31.018760386Z 2026-06-13 12:26:31.018 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/]
2026-06-13T12:26:31.799522025Z 2026-06-13 12:26:31.798 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-13T12:26:31.838954549Z 2026-06-13 12:26:31.838 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-13T12:26:31.846116100Z 2026-06-13 12:26:31.845 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-13T12:26:31.847101766Z 2026-06-13 12:26:31.846 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-13T12:26:31.847934274Z 2026-06-13 12:26:31.846 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-13T12:26:31.947116503Z 2026-06-13 12:26:31.946 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-13T12:26:31.951772874Z 2026-06-13 12:26:31.951 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-13T12:26:31.955927126Z 2026-06-13 12:26:31.955 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-13T12:26:31.958320316Z 2026-06-13 12:26:31.957 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-13T12:26:31.979738616Z 2026-06-13 12:26:31.979 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01}
2026-06-13T12:26:31.988679970Z 2026-06-13 12:26:31.988 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-13T12:26:31.999000915Z 2026-06-13 12:26:31.998 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-13T12:26:31.999067980Z 2026-06-13 12:26:31.998 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-13T12:26:32.007005166Z 2026-06-13 12:26:32.006 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-13T12:26:32.019011079Z 2026-06-13 12:26:32.018 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-06-13T12:26:32.023349248Z 2026-06-13 12:26:32.022 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-06-13T12:26:32.038824136Z 2026-06-13 12:26:32.038 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-13T12:26:32.038849409Z 2026-06-13 12:26:32.038 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-13T12:26:32.038864137Z 2026-06-13 12:26:32.038 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/
2026-06-13T12:26:32.040652638Z 2026-06-13 12:26:32.040 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-13T12:26:32.041447611Z 2026-06-13 12:26:32.041 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas
2026-06-13T12:26:32.043463416Z 2026-06-13 12:26:32.043 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72115607593287680-core_node3-n_0000000000
2026-06-13T12:26:32.050731006Z 2026-06-13 12:26:32.050 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3
2026-06-13T12:26:32.156831423Z 2026-06-13 12:26:32.156 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 238] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-13T12:26:32.160082200Z 2026-06-13 12:26:32.159 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-13T12:26:32.184212873Z 2026-06-13 12:26:32.183 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2382
2026-06-13T12:26:32.283687621Z 2026-06-13 12:26:32.283 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 2583488,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-13T12:26:15.593210577Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-06-13T12:26:21.723760899Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-13T12:26:21.723781999Z It should be set to 65000 to avoid operational disruption.
2026-06-13T12:26:21.723784882Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-13T12:26:21.724131595Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-13T12:26:21.724147513Z
2026-06-13T12:26:21.733357628Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-13T12:26:21.741442665Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-13T12:26:21.782426913Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-13T12:26:21.817724382Z WARNING: A command line option has enabled the Security Manager
2026-06-13T12:26:21.817747419Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-13T12:26:22.950920254Z 2026-06-13 12:26:22.947 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-13T12:26:23.214228859Z 2026-06-13 12:26:23.213 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-13T12:26:23.239916627Z 2026-06-13 12:26:23.239 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-13T12:26:23.247618363Z 2026-06-13 12:26:23.247 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-13T12:26:23.247741300Z 2026-06-13 12:26:23.247 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-06-13T12:26:23.247874949Z 2026-06-13 12:26:23.247 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-13T12:26:23.247995946Z 2026-06-13 12:26:23.247 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-13T12:26:23.247334625Z
2026-06-13T12:26:23.250720247Z 2026-06-13 12:26:23.250 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-13T12:26:23.273226061Z 2026-06-13 12:26:23.272 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-13T12:26:23.296222660Z 2026-06-13 12:26:23.295 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-13T12:26:23.434848186Z 2026-06-13 12:26:23.434 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-13T12:26:23.454067461Z 2026-06-13 12:26:23.453 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-13T12:26:23.454257793Z 2026-06-13 12:26:23.453 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-13T12:26:23.454375313Z 2026-06-13 12:26:23.453 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-13T12:26:23.565774709Z 2026-06-13 12:26:23.565 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-13T12:26:23.565970330Z 2026-06-13 12:26:23.565 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-13T12:26:23.566138444Z 2026-06-13 12:26:23.565 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-13T12:26:23.602310927Z 2026-06-13 12:26:23.601 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-13T12:26:24.326165072Z 2026-06-13 12:26:24.325 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-13T12:26:24.462425855Z 2026-06-13 12:26:24.461 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-13T12:26:24.463859543Z 2026-06-13 12:26:24.462 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-13T12:26:24.488158339Z 2026-06-13 12:26:24.487 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-13T12:26:24.496246564Z 2026-06-13 12:26:24.495 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-13T12:26:24.511139815Z 2026-06-13 12:26:24.510 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-13T12:26:24.511247972Z 2026-06-13 12:26:24.510 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-13T12:26:24.557234899Z 2026-06-13 12:26:24.556 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-13T12:26:24.566268763Z 2026-06-13 12:26:24.565 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-06-13T12:26:24.590026071Z 2026-06-13 12:26:24.589 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-13T12:26:24.605021654Z 2026-06-13 12:26:24.604 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-06-13T12:26:24.622612019Z 2026-06-13 12:26:24.622 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-06-13T12:26:24.638093394Z 2026-06-13 12:26:24.637 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-06-13T12:26:24.683226732Z 2026-06-13 12:26:24.682 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-13T12:26:24.946777565Z 2026-06-13 12:26:24.946 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-13T12:26:24.964228457Z 2026-06-13 12:26:24.963 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-13T12:26:25.152481995Z 2026-06-13 12:26:25.151 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-13T12:26:25.430121259Z 2026-06-13 12:26:25.429 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-13T12:26:25.578053182Z 2026-06-13 12:26:25.577 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-13T12:26:25.584587657Z 2026-06-13 12:26:25.584 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_13.request.log
2026-06-13T12:26:25.588303580Z 2026-06-13 12:26:25.587 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-06-13T12:26:25.589748679Z 2026-06-13 12:26:25.589 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @3849ms
2026-06-13T12:26:25.797965176Z 2026-06-13 12:26:25.797 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=73
2026-06-13T12:26:25.857627768Z 2026-06-13 12:26:25.857 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=15
2026-06-13T12:26:29.799064319Z 2026-06-13 12:26:29.798 INFO (qtp985324122-59-solr-node1-6) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-06-13T12:26:29.856768664Z 2026-06-13 12:26:29.856 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-13T12:26:29.918207149Z 2026-06-13 12:26:29.917 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-13T12:26:30.738686444Z 2026-06-13 12:26:30.737 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-13T12:26:30.738709600Z 2026-06-13 12:26:30.738 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-06-13T12:26:30.757266713Z 2026-06-13 12:26:30.756 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-06-13T12:26:31.301494387Z 2026-06-13 12:26:31.300 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-13T12:26:31.322221596Z 2026-06-13 12:26:31.321 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-13T12:26:31.326099508Z 2026-06-13 12:26:31.325 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-13T12:26:31.327582023Z 2026-06-13 12:26:31.326 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-13T12:26:31.327620937Z 2026-06-13 12:26:31.327 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-13T12:26:31.373495160Z 2026-06-13 12:26:31.373 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-13T12:26:31.377025848Z 2026-06-13 12:26:31.376 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-13T12:26:31.379512595Z 2026-06-13 12:26:31.379 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-13T12:26:31.381435501Z 2026-06-13 12:26:31.381 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-13T12:26:31.397864069Z 2026-06-13 12:26:31.397 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, accuracy=0.5, maxQueryFrequency=0.01}
2026-06-13T12:26:31.407967158Z 2026-06-13 12:26:31.407 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-13T12:26:31.414550414Z 2026-06-13 12:26:31.414 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-13T12:26:31.414575002Z 2026-06-13 12:26:31.414 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-13T12:26:31.425155526Z 2026-06-13 12:26:31.424 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-13T12:26:31.433443814Z 2026-06-13 12:26:31.432 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-06-13T12:26:31.435411720Z 2026-06-13 12:26:31.434 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1
2026-06-13T12:26:31.454340146Z 2026-06-13 12:26:31.453 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-13T12:26:31.454370274Z 2026-06-13 12:26:31.454 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-13T12:26:31.454376136Z 2026-06-13 12:26:31.454 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node2:8984/solr/test-collection_shard1_replica_n4/
2026-06-13T12:26:31.456353872Z 2026-06-13 12:26:31.455 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-13T12:26:31.459874458Z 2026-06-13 12:26:31.459 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ has no replicas
2026-06-13T12:26:31.461429579Z 2026-06-13 12:26:31.461 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard1/leader after winning as /collections/test-collection/leader_elect/shard1/election/72115607593287683-core_node6-n_0000000000
2026-06-13T12:26:31.560955818Z 2026-06-13 12:26:31.560 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ shard1
2026-06-13T12:26:31.763236446Z 2026-06-13 12:26:31.762 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 219] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-13T12:26:31.767081585Z 2026-06-13 12:26:31.766 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-13T12:26:31.776745951Z 2026-06-13 12:26:31.776 INFO (qtp985324122-59-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1979
2026-06-13T12:26:31.879712130Z 2026-06-13 12:26:31.879 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 223] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-13T12:26:32.157006972Z 2026-06-13 12:26:32.156 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 238] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-13T12:26:32.283684919Z 2026-06-13 12:26:32.283 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node2 ===
{
"Name": "/solr-node2",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 2583883,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-13T12:26:21.50631586Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node3 ===
2026-06-13T12:26:21.954026113Z *** [WARN] *** Your open file limit is currently 1024.
2026-06-13T12:26:21.954061448Z It should be set to 65000 to avoid operational disruption.
2026-06-13T12:26:21.954064461Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-06-13T12:26:21.954347202Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-06-13T12:26:21.954352142Z
2026-06-13T12:26:21.964884524Z Java 17 detected. Enabled workaround for SOLR-16463
2026-06-13T12:26:21.972698500Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-06-13T12:26:22.019786705Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-06-13T12:26:22.052792407Z WARNING: A command line option has enabled the Security Manager
2026-06-13T12:26:22.052833613Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-06-13T12:26:23.087005764Z 2026-06-13 12:26:23.083 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-06-13T12:26:23.383290909Z 2026-06-13 12:26:23.382 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-06-13T12:26:23.409011311Z 2026-06-13 12:26:23.408 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-06-13T12:26:23.416293946Z 2026-06-13 12:26:23.415 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-06-13T12:26:23.416343533Z 2026-06-13 12:26:23.415 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8985
2026-06-13T12:26:23.416451062Z 2026-06-13 12:26:23.415 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-06-13T12:26:23.416588884Z 2026-06-13 12:26:23.416 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-06-13T12:26:23.416049544Z
2026-06-13T12:26:23.419281310Z 2026-06-13 12:26:23.418 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-06-13T12:26:23.444047420Z 2026-06-13 12:26:23.443 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-06-13T12:26:23.472541613Z 2026-06-13 12:26:23.471 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-13T12:26:23.623283779Z 2026-06-13 12:26:23.622 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-06-13T12:26:23.645968706Z 2026-06-13 12:26:23.645 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-13T12:26:23.646016337Z 2026-06-13 12:26:23.645 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-13T12:26:23.646138250Z 2026-06-13 12:26:23.645 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-06-13T12:26:23.756793399Z 2026-06-13 12:26:23.756 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-06-13T12:26:23.757052023Z 2026-06-13 12:26:23.756 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-06-13T12:26:23.757144874Z 2026-06-13 12:26:23.756 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-06-13T12:26:23.792789535Z 2026-06-13 12:26:23.792 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-06-13T12:26:24.634040199Z 2026-06-13 12:26:24.633 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-06-13T12:26:24.799735625Z 2026-06-13 12:26:24.799 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-06-13T12:26:24.801162187Z 2026-06-13 12:26:24.799 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-06-13T12:26:24.812986972Z 2026-06-13 12:26:24.812 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-13T12:26:24.819283347Z 2026-06-13 12:26:24.818 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-06-13T12:26:24.822475386Z 2026-06-13 12:26:24.822 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-06-13T12:26:24.822762340Z 2026-06-13 12:26:24.822 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-06-13T12:26:24.858532440Z 2026-06-13 12:26:24.857 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-06-13T12:26:24.870770033Z 2026-06-13 12:26:24.870 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
2026-06-13T12:26:24.899254972Z 2026-06-13 12:26:24.898 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-06-13T12:26:24.919681667Z 2026-06-13 12:26:24.919 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node3:8985_solr as DOWN
2026-06-13T12:26:24.933772874Z 2026-06-13 12:26:24.933 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node3:8985_solr
2026-06-13T12:26:24.947990889Z 2026-06-13 12:26:24.947 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-06-13T12:26:25.001020377Z 2026-06-13 12:26:25.000 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-06-13T12:26:25.284173790Z 2026-06-13 12:26:25.283 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-06-13T12:26:25.453077181Z 2026-06-13 12:26:25.452 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-06-13T12:26:25.714340037Z 2026-06-13 12:26:25.713 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-13T12:26:25.844362706Z 2026-06-13 12:26:25.843 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-06-13T12:26:25.849731210Z 2026-06-13 12:26:25.849 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_06_13.request.log
2026-06-13T12:26:25.853103965Z 2026-06-13 12:26:25.852 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8985}
2026-06-13T12:26:25.853889686Z 2026-06-13 12:26:25.853 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @3882ms
2026-06-13T12:26:27.555058978Z 2026-06-13 12:26:27.548 INFO (qtp985324122-27-solr-node3-1) [c: s: r: x: t:solr-node3-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=117
2026-06-13T12:26:27.633101351Z 2026-06-13 12:26:27.632 INFO (qtp985324122-56-solr-node3-2) [c: s: r: x: t:solr-node3-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=24
2026-06-13T12:26:29.477233061Z 2026-06-13 12:26:29.476 INFO (qtp985324122-58-solr-node3-3) [c: s: r: x: t:solr-node3-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=10
2026-06-13T12:26:29.801271128Z 2026-06-13 12:26:29.799 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node5&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&shard=shard2&wt=javabin
2026-06-13T12:26:29.876389237Z 2026-06-13 12:26:29.875 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-06-13T12:26:29.980994171Z 2026-06-13 12:26:29.980 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-06-13T12:26:30.892786325Z 2026-06-13 12:26:30.892 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-06-13T12:26:30.892885803Z 2026-06-13 12:26:30.892 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard2_replica_n2' using configuration from configset techproducts, trusted=true
2026-06-13T12:26:30.916444721Z 2026-06-13 12:26:30.915 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard2_replica_n2], dataDir=[/var/solr/data/test-collection_shard2_replica_n2/data/]
2026-06-13T12:26:31.431369827Z 2026-06-13 12:26:31.430 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-06-13T12:26:31.457383817Z 2026-06-13 12:26:31.456 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-06-13T12:26:31.467068655Z 2026-06-13 12:26:31.466 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-06-13T12:26:31.468428190Z 2026-06-13 12:26:31.467 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-06-13T12:26:31.468651274Z 2026-06-13 12:26:31.468 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-06-13T12:26:31.558776494Z 2026-06-13 12:26:31.558 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-06-13T12:26:31.562694380Z 2026-06-13 12:26:31.562 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-06-13T12:26:31.565540116Z 2026-06-13 12:26:31.565 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-06-13T12:26:31.568258495Z 2026-06-13 12:26:31.567 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-06-13T12:26:31.583845122Z 2026-06-13 12:26:31.583 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-06-13T12:26:31.591416615Z 2026-06-13 12:26:31.590 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-06-13T12:26:31.598015381Z 2026-06-13 12:26:31.597 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-06-13T12:26:31.598044620Z 2026-06-13 12:26:31.597 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-06-13T12:26:31.604517935Z 2026-06-13 12:26:31.604 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-06-13T12:26:31.618380680Z 2026-06-13 12:26:31.617 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard2 to Terms{values={core_node5=0}, version=0} for registerTerm
2026-06-13T12:26:31.620419001Z 2026-06-13 12:26:31.620 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard2
2026-06-13T12:26:31.638303606Z 2026-06-13 12:26:31.637 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-06-13T12:26:31.638536972Z 2026-06-13 12:26:31.638 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-06-13T12:26:31.638622637Z 2026-06-13 12:26:31.638 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node3:8985/solr/test-collection_shard2_replica_n2/
2026-06-13T12:26:31.641007187Z 2026-06-13 12:26:31.640 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-06-13T12:26:31.641968154Z 2026-06-13 12:26:31.641 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ has no replicas
2026-06-13T12:26:31.643843347Z 2026-06-13 12:26:31.643 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard2/leader after winning as /collections/test-collection/leader_elect/shard2/election/72115607593287684-core_node5-n_0000000000
2026-06-13T12:26:31.656593183Z 2026-06-13 12:26:31.656 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ shard2
2026-06-13T12:26:31.763348498Z 2026-06-13 12:26:31.762 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 219] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-13T12:26:31.767950534Z 2026-06-13 12:26:31.767 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-06-13T12:26:31.776809863Z 2026-06-13 12:26:31.776 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=techproducts&newCollection=true&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&collection=test-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1977
2026-06-13T12:26:31.879552862Z 2026-06-13 12:26:31.879 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 223] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-13T12:26:32.157273650Z 2026-06-13 12:26:32.156 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 238] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-13T12:26:32.226398236Z 2026-06-13 12:26:32.225 INFO (qtp985324122-59-solr-node3-4) [c:test-collection s: r: x: t:solr-node3-4] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
2026-06-13T12:26:32.283694010Z 2026-06-13 12:26:32.283 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-06-13T12:26:32.287653089Z 2026-06-13 12:26:32.287 INFO (qtp985324122-59-solr-node3-4) [c:test-collection s: r: x: t:solr-node3-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=2794
# === Docker inspect for solr-node3 ===
{
"Name": "/solr-node3",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 2584064,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-06-13T12:26:21.727617904Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# Docker artifacts saved to: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/docker
not ok 85 Docker SolrCloud rolling upgrade # in 28065 ms
# (in test file test/test_rolling_upgrade.bats, line 118)
# `wait_for 30 1 docker exec solr-node1 solr healthcheck -c test-collection' failed with status 127
# 9.10.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# af6eca94c810: Pulling fs layer
# cb0efb96dabd: Pulling fs layer
# 3e9d91201f40: Pulling fs layer
# 66b76b382631: Pulling fs layer
# 601f2c23751f: Pulling fs layer
# 0ed719aef393: Pulling fs layer
# 69a09c96e8bf: Pulling fs layer
# c937aee9a870: Pulling fs layer
# 9cdfe5320de8: Pulling fs layer
# 3aa9120e46f5: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# 66b76b382631: Waiting
# c937aee9a870: Waiting
# 9cdfe5320de8: Waiting
# 69a09c96e8bf: Waiting
# 3aa9120e46f5: Waiting
# 0ed719aef393: Waiting
# 4f4fb700ef54: Waiting
# 601f2c23751f: Waiting
# cb0efb96dabd: Download complete
# af6eca94c810: Verifying Checksum
# af6eca94c810: Download complete
# 3e9d91201f40: Verifying Checksum
# 3e9d91201f40: Download complete
# 66b76b382631: Download complete
# 601f2c23751f: Verifying Checksum
# 601f2c23751f: Download complete
# 69a09c96e8bf: Verifying Checksum
# 69a09c96e8bf: Download complete
# c937aee9a870: Verifying Checksum
# c937aee9a870: Download complete
# 9cdfe5320de8: Verifying Checksum
# 9cdfe5320de8: Download complete
# 3aa9120e46f5: Verifying Checksum
# 3aa9120e46f5: Download complete
# 4f4fb700ef54: Download complete
# 0ed719aef393: Verifying Checksum
# 0ed719aef393: Download complete
# af6eca94c810: Pull complete
# cb0efb96dabd: Pull complete
# 3e9d91201f40: Pull complete
# 66b76b382631: Pull complete
# 601f2c23751f: Pull complete
# 0ed719aef393: Pull complete
# 69a09c96e8bf: Pull complete
# c937aee9a870: Pull complete
# 9cdfe5320de8: Pull complete
# 3aa9120e46f5: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:15c8155ceffe1042f37c77bb537aa531804a3510702ff87f2af5fedf523f4203
# Status: Downloaded newer image for apache/solr-nightly:9.10.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:9.10.0-SNAPSHOT-slim
# 10.0.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# 40d16f30db40: Already exists
# 9d5a0b6cb36c: Already exists
# 49518c28c0b3: Already exists
# 41162acdcc29: Already exists
# 2ed0fd558ca3: Already exists
# 2488415fda1c: Pulling fs layer
# 09511c8fb203: Pulling fs layer
# a0c57fb89c4f: Pulling fs layer
# d0641dcc41d7: Pulling fs layer
# 315722fd9570: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# d0641dcc41d7: Waiting
# 315722fd9570: Waiting
# 4f4fb700ef54: Waiting
# 09511c8fb203: Verifying Checksum
# 09511c8fb203: Download complete
# a0c57fb89c4f: Verifying Checksum
# a0c57fb89c4f: Download complete
# d0641dcc41d7: Download complete
# 315722fd9570: Download complete
# 4f4fb700ef54: Verifying Checksum
# 4f4fb700ef54: Download complete
# 2488415fda1c: Verifying Checksum
# 2488415fda1c: Download complete
# 2488415fda1c: Pull complete
# 09511c8fb203: Pull complete
# a0c57fb89c4f: Pull complete
# d0641dcc41d7: Pull complete
# 315722fd9570: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:fb25f7a7190e3027ade20e97708a8665d41ab0e9ea9285fa242b2c83b0e5f43c
# Status: Downloaded newer image for apache/solr-nightly:10.0.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:10.0.0-SNAPSHOT-slim
# 180b9143cc2020eb54b4a1ec50f157ab819ee0bd8878fab0fcb4bfd40db0d184
# solr-data1
# solr-data2
# solr-data3
# Starting solr-node1 with embedded ZooKeeper
# e1b0d29d3d30e577dbf76dbbc02783720cc5ff12daf088ef228db86459cdc32f
# Starting solr-node2 connected to first node's ZooKeeper
# 1135ee30140a867bc6da1cb87db5a6251a2d982cafb8e2a24d30a274ea9fb688
# Started solr-node3 connected to first node's ZooKeeper
# 81754931140e2839a256f6daaebc391a83628d8f6d7d099de1e3f110e873cbf2
# Creating a Collection
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# WARN - 2026-06-13 12:26:28.917; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-13 12:26:28.952; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. "/solr"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [http://solr-node3:8985/solr] to [http://solr-node3:8985/].
# Created collection 'test-collection' with 3 shard(s), 1 replica(s) with config-set 'techproducts'
# Checking collection health
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_rolling_upgrade.bats: line 118: wait_for: command not found
# solr-node1
# solr-node2
# solr-node3
# solr-node1
# solr-node2
# solr-node3
# solr-data1
# solr-data2
# solr-data3
# solrcloud-test
ok 86 allowPaths - backup # in 15343 ms
ok 87 snapshot lifecycle # in 4821 ms
ok 88 snapshot list # in 3339 ms
ok 89 snapshot list via --solr-connection # in 3317 ms
ok 90 snapshot describe # in 3332 ms
ok 91 snapshot describe via --solr-connection # in 3466 ms
ok 92 start solr with ssl # in 16182 ms
ok 93 use different hostname when not checking peer-name # in 24822 ms
ok 94 start solr with ssl and auth # in 14735 ms
ok 95 start solr with client truststore and security manager # in 17064 ms
ok 96 start solr with mTLS needed # in 46409 ms
ok 97 start solr with mTLS wanted # in 35595 ms
ok 98 test keystore reload # in 59463 ms
ok 99 SOLR-11740 check 'solr stop' connection # in 16152 ms
ok 100 stop command for single port # in 18393 ms
ok 101 check stop command doesn't hang # in 224 ms
ok 102 SOLR-16976 solr starts with remote JMX enabled # in 8194 ms
not ok 103 deprecated system properties converted to modern properties # in 195110 ms
# (from function `assert_file_contains' in file .gradle/node/node_modules/bats-file/src/file.bash, line 566,
# in test file test/test_start_solr.bats, line 86)
# `assert_file_contains "${SOLR_LOGS_DIR}/solr.log" 'You are passing in deprecated system property disable.config.edit and should upgrade to using solr.api.config.edit.enabled instead.'' failed
# Waiting up to 180 seconds to see Solr running on port 38917 [|] [/] [-] [\] [|]
# Started Solr server on port 38917 (pid=2601704). Happy searching!
#
#
# -- file does not contain regex --
# path : /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/solr-home/logs/solr.log
# --
#
# Please find the SOLR_HOME snapshot for failed test #5 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/103-5
ok 104 start with custom jetty options # in 8299 ms
ok 105 -c flag prints no-op warning and still starts in cloud mode # in 9625 ms
not ok 106 bootstrapping a configset # in 9482 ms
# (from function `config_exists' in file test/bats_helper.bash, line 84,
# in test file test/test_start_solr.bats, line 116)
# `config_exists "techproducts"' failed
# Waiting up to 180 seconds to see Solr running on port 38917 [|] [/] [-] [\] [|]
# Started Solr server on port 38917 (pid=2606706). Happy searching!
#
# WARN - 2026-06-13 12:35:14.529; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-06-13 12:35:14.532; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# Please find the SOLR_HOME snapshot for failed test #8 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/106-8
ok 107 status detects locally running solr # in 9595 ms
ok 108 status with --solr-url from user # in 7886 ms
ok 109 status with --port from user # in 8050 ms
ok 110 multiple connection options are prevented # in 835 ms
ok 111 status with invalid --solr-url from user # in 1282 ms
ok 112 status with --short format # in 196112 ms
ok 113 searching solr via locally executed streaming expression # in 1732 ms
ok 114 searching solr via remotely executed streaming expression by solr-url # in 1255 ms
ok 115 searching solr via remotely executed streaming expression by solr connection # in 1134 ms
ok 116 variable interpolation # in 1702 ms
ok 117 searching solr without credentials fails with error # in 1112 ms
ok 118 --version returns Solr version # in 775 ms
ok 119 version as direct tool call still runs # in 836 ms
ok 120 short help # in 789 ms
ok 121 short help is inferred # in 766 ms
ok 122 long help # in 748 ms
ok 123 running subcommands with zk is prevented # in 720 ms
ok 124 listing out files # in 2115 ms
ok 125 connecting to solr via various solr urls and zk hosts # in 9122 ms
ok 126 copying files around # in 10982 ms
ok 127 upconfig # in 2510 ms
ok 128 SOLR-12429 test upconfig fails with symlink # in 1191 ms
ok 129 downconfig # in 1185 ms
ok 130 bin/solr zk cp gets 'solrhome' from '--solr-home' command line option # in 1412 ms
ok 131 env var ZK_HOST is honored # in 3159 ms
ok 132 Cleanup # in 1994 ms
> Task :solr:packaging:integrationTests FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build.gradle' line: 339
* What went wrong:
Execution failed for task ':solr:packaging:integrationTests'.
> Process 'command '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/.gradle/node/node_modules/bats/bin/bats'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/8.10/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 33m 10s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/voqti3c7huzjs
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 231 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/231/
No tests ran.
Build Log:
[...truncated 984 lines...]
BUILD FAILED in 31m 1s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/g235fcefimeo6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 230 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/230/
No tests ran.
Build Log:
[...truncated 906 lines...]
BUILD FAILED in 29m 48s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/nuesiyjybwrtq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 229 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/229/
No tests ran.
Build Log:
[...truncated 981 lines...]
BUILD FAILED in 29m 47s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/sw7nn4booqalm
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 228 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/228/
No tests ran.
Build Log:
[...truncated 983 lines...]
BUILD FAILED in 29m 26s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/abvfyr5262t6u
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 227 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/227/
No tests ran.
Build Log:
[...truncated 881 lines...]
BUILD FAILED in 31m
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/nvpfkfes52mba
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 226 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/226/
No tests ran.
Build Log:
[...truncated 881 lines...]
BUILD FAILED in 29m 26s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/riu2i7k3eqheq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 225 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/225/
No tests ran.
Build Log:
[...truncated 956 lines...]
BUILD FAILED in 30m 6s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/kqemboxba4uyo
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 224 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/224/
No tests ran.
Build Log:
[...truncated 959 lines...]
BUILD FAILED in 29m 12s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/uku3dms3w4lqm
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 223 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/223/
No tests ran.
Build Log:
[...truncated 958 lines...]
BUILD FAILED in 30m 28s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/4p7z5vbdyxjge
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 222 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/222/
No tests ran.
Build Log:
[...truncated 957 lines...]
BUILD FAILED in 31m 30s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/hoojki5rlfxb4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 221 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/221/
No tests ran.
Build Log:
[...truncated 877 lines...]
BUILD FAILED in 29m 13s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/iifzr3kingji6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 220 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/220/
No tests ran.
Build Log:
[...truncated 974 lines...]
BUILD FAILED in 39m 23s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/uvaxb4xzctoba
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 219 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/219/
No tests ran.
Build Log:
[...truncated 874 lines...]
BUILD FAILED in 28m 5s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/6yhhy4azwsc42
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 218 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/218/
No tests ran.
Build Log:
[...truncated 881 lines...]
BUILD FAILED in 30m 4s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/giq742fit5llg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 217 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/217/
No tests ran.
Build Log:
[...truncated 375 lines...]
# ERROR: Solr is not running on url http://localhost:38713 after 5 seconds
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/4-2
# Last output:
# Waiting up to 180 seconds to see Solr running on port 38713 [|] [/] [-] [\] [|]
# Started Solr server on port 38713 (pid=85186). Happy searching!
#
#
ok 5 auth rejects block-unknown option with invalid boolean # in 1828 ms
ok 6 auth rejects update-include-file-only option with invalid boolean # in 1867 ms
not ok 7 auth enable/disable lifecycle # in 20591 ms
# (in test file test/test_auth.bats, line 40)
# `solr auth enable --type basicAuth --credentials name:password' failed
# Waiting up to 180 seconds to see Solr running on port 38713 [|] [/] [-] [\] [|]
# Started Solr server on port 38713 (pid=87460). Happy searching!
#
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
# Couldn't get ZooKeeper host. Please make sure Solr is running in cloud mode, or a zk-host has been passed in.
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
#
# ERROR: Server refused connection at: http://localhost:38713/solr/admin/info/system
not ok 8 enable auth connects via zookeeper # in 1310 ms
# (in test file test/test_auth.bats, line 53)
# `solr start' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
#
# ERROR: Error from server at http://localhost:38713/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 9 create, config, api, and delete with basic auth # in 16695 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-05-26 16:31:45.427; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-26 16:31:45.431; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 10 post, postlogs and export with basic auth # in 16748 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/10-2
# Last output:
# WARN - 2026-05-26 16:32:02.198; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-26 16:32:02.202; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 11 package with basic auth # in 16636 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #3 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/11-3
# Last output:
# WARN - 2026-05-26 16:32:18.864; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-26 16:32:18.867; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 12 admin UI static content is excepted from basic auth # in 16664 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #4 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/12-4
# Last output:
# WARN - 2026-05-26 16:32:35.548; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-26 16:32:35.552; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 13 setup_file failed
# (from function `setup_file' in test file test/test_bats.bats, line 26)
# `solr start --verbose' failed
# Using Solr root directory: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT
# Using Java: /home/jenkins/tools/java/latest21/bin/java
# openjdk version "21.0.4" 2024-07-16 LTS
# OpenJDK Runtime Environment Temurin-21.0.4+7 (build 21.0.4+7-LTS)
# OpenJDK 64-Bit Server VM Temurin-21.0.4+7 (build 21.0.4+7-LTS, mixed mode, sharing)
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
not ok 14 setup_file failed
# (from function `setup_file' in test file test/test_compression.bats, line 22)
# `solr start -e films' failed
#
# Starting up Solr on port 38713 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 38713 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 38713 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test # at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/-
not ok 17 setup_file failed
# (from function `setup_file' in test file test/test_config.bats, line 22)
# `solr start' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
not ok 20 create for non cloud mode # in 1452 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_create.bats, line 34)
# `assert_output --partial "Created new core 'COLL_NAME'"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Created new core 'COLL_NAME'
# output (17 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
#
# ERROR: Error from server at http://localhost:38713/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/20-1
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
#
# ERROR: Error from server at http://localhost:38713/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 21 create for cloud mode # in 1454 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_create.bats, line 40)
# `assert_output --partial "Created collection 'COLL_NAME'"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Created collection 'COLL_NAME'
# output (17 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
#
# ERROR: Error from server at http://localhost:38713/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/21-2
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
#
# ERROR: Error from server at http://localhost:38713/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
ok 22 multiple connection options are prevented # in 851 ms
not ok 23 setup_file failed
# (from function `setup_file' in test file test/test_create_collection.bats, line 22)
# `solr start' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
not ok 34 setup_file failed
# (from function `setup_file' in test file test/test_delete_collection.bats, line 22)
# `solr start' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
not ok 39 start -e cloud works with --jvm-opts # in 1479 ms
# (in test file test/test_example.bats, line 32)
# `solr start -e cloud --no-prompt --jvm-opts "-Dcustom.prop=helloworld"' failed
#
# Welcome to the SolrCloud example!
#
# Starting up 2 Solr nodes for your example SolrCloud cluster.
#
# Creating Solr home directory /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr
# Cloning /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1 into
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node2
#
# Starting up Solr on port 38713 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 38713 --solr-home "build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server" --jvm-opts "-Dcustom.prop=helloworld"
#
#
# ERROR: Port 38713 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/39-1
not ok 40 start -e cloud works with --no-prompt # in 1257 ms
# (in test file test/test_example_noprompt.bats, line 32)
# `solr start -e cloud --no-prompt' failed
#
# Welcome to the SolrCloud example!
#
# Starting up 2 Solr nodes for your example SolrCloud cluster.
#
# Solr home directory /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr already exists.
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node2 already exists.
#
# Starting up Solr on port 38713 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 38713 --solr-home "build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 38713 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/40-1
not ok 41 Check export command # in 2955 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_export.bats, line 36)
# `assert_output --partial 'Export complete'' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Export complete
# output (48 lines):
# WARN - 2026-05-26 16:33:06.026; org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider; Attempt to fetch live_nodes from http://localhost:38713/solr failed.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:38713/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1194)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.submitClusterStateRequest(BaseHttpClusterStateProvider.java:221)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:293)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.updateLiveNodes(BaseHttpClusterStateProvider.java:280)
# at java.base/java.util.stream.MatchOps$1MatchSink.accept(MatchOps.java:90)
# at java.base/java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1685)
# at java.base/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129)
# at java.base/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527)
# at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
# at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:230)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:196)
# at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
# at java.base/java.util.stream.ReferencePipeline.anyMatch(ReferencePipeline.java:632)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:268)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getLiveNodes(BaseHttpClusterStateProvider.java:231)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getState(BaseHttpClusterStateProvider.java:109)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionRef(CloudSolrClient.java:1723)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.loadDocCollection(CloudSolrClient.java:1699)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$triggerCollectionRefresh$6(CloudSolrClient.java:1664)
# at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1768)
# at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
# at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
# at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
# at java.base/java.lang.Thread.run(Thread.java:1583)
#
# ERROR: Error refreshing state for collection techproducts
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/41-1
# Last output:
# WARN - 2026-05-26 16:33:06.026; org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider; Attempt to fetch live_nodes from http://localhost:38713/solr failed.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:38713/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1194)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.submitClusterStateRequest(BaseHttpClusterStateProvider.java:221)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:293)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.updateLiveNodes(BaseHttpClusterStateProvider.java:280)
# at java.base/java.util.stream.MatchOps$1MatchSink.accept(MatchOps.java:90)
# at java.base/java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1685)
# at java.base/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129)
# at java.base/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527)
# at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
# at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:230)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:196)
# at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
# at java.base/java.util.stream.ReferencePipeline.anyMatch(ReferencePipeline.java:632)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:268)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getLiveNodes(BaseHttpClusterStateProvider.java:231)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getState(BaseHttpClusterStateProvider.java:109)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionRef(CloudSolrClient.java:1723)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.loadDocCollection(CloudSolrClient.java:1699)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$triggerCollectionRefresh$6(CloudSolrClient.java:1664)
# at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1768)
# at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
# at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
# at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
# at java.base/java.lang.Thread.run(Thread.java:1583)
#
# ERROR: Error refreshing state for collection techproducts
ok 42 export fails on non cloud mode # in 2769 ms
Waiting for Tika Server to be ready on port 38718
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 43 using curl to extract a single pdf file # in 16566 ms
WARNING: Docker not available; bypassing test.
ok 44 using the bin/solr post tool to extract content from pdf # in 16556 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to crawl web site # in 16565 ms
Stopping Tika Server container
not ok 46 healthcheck on cloud solr # in 1292 ms
# (in test file test/test_healthcheck.bats, line 32)
# `solr start -e films' failed
#
# Starting up Solr on port 38713 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 38713 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 38713 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/46-1
not ok 47 healthcheck errors on standalone solr # in 1366 ms
# (in test file test/test_healthcheck.bats, line 39)
# `solr start --user-managed -e films' failed
#
# Starting up Solr on port 38713 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start --user-managed -p 38713 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 38713 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/47-2
ok 48 solr help flag prints help # in 736 ms
ok 49 solr with no flags prints help # in 750 ms
ok 50 start help flag prints help # in 157 ms
ok 51 start h flag prints help # in 154 ms
ok 52 stop help flag prints help # in 144 ms
ok 53 restart help flag prints help # in 159 ms
ok 54 status help flag prints help # in 823 ms
ok 55 healthcheck help flag prints help # in 755 ms
ok 56 create help flag prints help # in 762 ms
ok 57 delete help flag prints help # in 786 ms
ok 58 zk help flag prints help # in 780 ms
ok 59 auth help flag prints help # in 754 ms
ok 60 assert help flag prints help # in 758 ms
ok 61 post help flag prints help # in 752 ms
not ok 62 SQL Module # in 18945 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_modules.bats, line 36)
# `assert_output --partial '"docs":'' failed
#
# -- output does not contain substring --
# substring (1 lines):
# "docs":
# output (16 lines):
#
# ERROR: Error from server at http://localhost:38713/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json: Expected mime type in [application/json] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/62-1
# WARN - 2026-05-26 16:34:14.387; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-26 16:34:14.391; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-26 16:34:30.119; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
#
# ERROR: Error from server at http://localhost:38713/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json: Expected mime type in [application/json] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 63 icu collation in analysis-extras module # in 17919 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_modules.bats, line 49)
# `assert_output --partial "Created collection 'COLL_NAME'"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Created collection 'COLL_NAME'
# output (17 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
#
# ERROR: Error from server at http://localhost:38713/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/63-2
# WARN - 2026-05-26 16:34:32.338; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-26 16:34:32.342; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-26 16:34:48.071; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
#
# ERROR: Error from server at http://localhost:38713/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 64 Check lifecycle of sentiment classification # in 10320 ms
# (in test file test/test_opennlp.bats, line 63)
# `solr start -m 4g -Dsolr.modules=analysis-extras -Dsolr.packages.enabled=true' failed
# Downloading onnx model and vocab...
# % Total % Received % Xferd Average Speed Time Time Time Current
# Dload Upload Total Spent Left Speed
#
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 160M 0 432k 0 0 346k 0 0:07:54 0:00:01 0:07:53 346k
8 160M 8 13.2M 0 0 6155k 0 0:00:26 0:00:02 0:00:24 6155k
21 160M 21 34.3M 0 0 10.7M 0 0:00:14 0:00:03 0:00:11 10.7M
35 160M 35 56.8M 0 0 13.5M 0 0:00:11 0:00:04 0:00:07 13.5M
49 160M 49 78.9M 0 0 15.2M 0 0:00:10 0:00:05 0:00:05 15.8M
62 160M 62 100M 0 0 16.2M 0 0:00:09 0:00:06 0:00:03 20.3M
76 160M 76 123M 0 0 17.1M 0 0:00:09 0:00:07 0:00:02 22.1M
90 160M 90 145M 0 0 17.7M 0 0:00:09 0:00:08 0:00:01 22.2M
100 160M 100 160M 0 0 18.1M 0 0:00:08 0:00:08 --:--:-- 22.2M
# % Total % Received % Xferd Average Speed Time Time Time Current
# Dload Upload Total Spent Left Speed
#
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 851k 100 851k 0 0 665k 0 0:00:01 0:00:01 --:--:-- 665k
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/64-1
# Last output:
# total 64M
# drwxr-xr-x 2 jenkins jenkins 4.0K May 26 16:20 .
# drwxr-xr-x 3 jenkins jenkins 4.0K May 26 16:20 ..
# -rw-r--r-- 1 jenkins jenkins 14M May 26 16:20 icu4j-77.1.jar
# -rw-r--r-- 1 jenkins jenkins 83K May 26 16:20 lucene-analysis-icu-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 30K May 26 16:20 lucene-analysis-morfologik-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 40K May 26 16:20 lucene-analysis-opennlp-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 3.5M May 26 16:20 lucene-analysis-smartcn-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 508K May 26 16:20 lucene-analysis-stempel-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 19K May 26 16:20 morfologik-fsa-2.1.9.jar
# -rw-r--r-- 1 jenkins jenkins 1.8M May 26 16:20 morfologik-polish-2.1.9.jar
# -rw-r--r-- 1 jenkins jenkins 51K May 26 16:20 morfologik-stemming-2.1.9.jar
# -rw-r--r-- 1 jenkins jenkins 4.0M May 26 16:20 morfologik-ukrainian-search-4.9.1.jar
# -rw-r--r-- 1 jenkins jenkins 39M May 26 16:20 onnxruntime-1.24.3.jar
# -rw-r--r-- 1 jenkins jenkins 28K May 26 16:20 opennlp-dl-2.5.8.jar
# -rw-r--r-- 1 jenkins jenkins 1.4M May 26 16:20 opennlp-tools-2.5.8.jar
# -rw-r--r-- 1 jenkins jenkins 39K May 26 16:20 solr-analysis-extras-10.1.0-SNAPSHOT.jar
not ok 65 lifecycle of package # in 2140 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_packages.bats, line 38)
# `assert_output --partial "Available packages:"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Available packages:
# output (45 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:38713/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:348)
# at org.apache.solr.cli.StatusTool.reportStatus(StatusTool.java:300)
# at org.apache.solr.cli.CLIUtils.getZkHost(CLIUtils.java:248)
# at org.apache.solr.cli.PackageTool.runImpl(PackageTool.java:123)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
#
# ERROR: Error from server at http://localhost:38713/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/65-1
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:38713/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:348)
# at org.apache.solr.cli.StatusTool.reportStatus(StatusTool.java:300)
# at org.apache.solr.cli.CLIUtils.getZkHost(CLIUtils.java:248)
# at org.apache.solr.cli.PackageTool.runImpl(PackageTool.java:123)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
#
# ERROR: Error from server at http://localhost:38713/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 66 deploying and undeploying a collection level package # in 1409 ms
# (in test file test/test_packages.bats, line 44)
# `solr create -c foo-1.2' failed
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:38713.
#
# ERROR: Error from server at http://localhost:38713/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:38713/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/66-2
# Last output:
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
ok 67 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
not ok 68 Affinity placement plugin using sysprop # in 21815 ms
# (in test file test/test_placement_plugin.bats, line 34)
# `solr assert --cloud http://localhost:${SOLR_PORT} --timeout 3000' failed with status 100
#
# ERROR: Solr is not running on url http://localhost:38713 after 3 seconds
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/68-1
# WARN - 2026-05-26 16:35:08.537; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-26 16:35:08.540; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-26 16:35:24.289; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
not ok 69 Random placement plugin using ENV # in 21717 ms
# (in test file test/test_placement_plugin.bats, line 43)
# `solr assert --cloud http://localhost:${SOLR_PORT} --timeout 3000' failed with status 100
#
# ERROR: Solr is not running on url http://localhost:38713 after 3 seconds
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/69-2
# WARN - 2026-05-26 16:35:30.297; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-26 16:35:30.301; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-26 16:35:46.024; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
not ok 70 setup_file failed
# (from function `setup_file' in test file test/test_post.bats, line 22)
# `solr start -Dsolr.modules=extraction' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
not ok 81 setup_file failed
# (from function `setup_file' in test file test/test_postlogs.bats, line 22)
# `solr start' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 16 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-05-26T16:35:59.108416708Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-26T16:35:59.108461725Z It should be set to 65000 to avoid operational disruption.
2026-05-26T16:35:59.108464909Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-26T16:35:59.108696950Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-26T16:35:59.108702463Z
2026-05-26T16:35:59.119646151Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-26T16:35:59.127948623Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-26T16:35:59.204362519Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-26T16:35:59.236017019Z WARNING: A command line option has enabled the Security Manager
2026-05-26T16:35:59.236042263Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-26T16:36:00.467256876Z 2026-05-26 16:36:00.463 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-26T16:36:00.742234048Z 2026-05-26 16:36:00.741 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-26T16:36:00.767482907Z 2026-05-26 16:36:00.766 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-26T16:36:00.774296663Z 2026-05-26 16:36:00.773 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-26T16:36:00.774360430Z 2026-05-26 16:36:00.773 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-05-26T16:36:00.774479463Z 2026-05-26 16:36:00.774 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-26T16:36:00.774579072Z 2026-05-26 16:36:00.774 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-26T16:36:00.774075394Z
2026-05-26T16:36:00.777266804Z 2026-05-26 16:36:00.776 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-26T16:36:00.798303419Z 2026-05-26 16:36:00.797 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-26T16:36:00.811573451Z 2026-05-26 16:36:00.811 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-26T16:36:00.811812746Z 2026-05-26 16:36:00.811 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-26T16:36:00.846089050Z 2026-05-26 16:36:00.845 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-26T16:36:01.615148467Z 2026-05-26 16:36:01.614 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-26T16:36:01.889679899Z 2026-05-26 16:36:01.889 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-26T16:36:01.895834034Z 2026-05-26 16:36:01.895 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-05-26T16:36:01.895859749Z 2026-05-26 16:36:01.895 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-05-26T16:36:01.898854380Z 2026-05-26 16:36:01.898 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-05-26T16:36:01.898947925Z 2026-05-26 16:36:01.898 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-05-26T16:36:01.960808217Z 2026-05-26 16:36:01.960 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-05-26T16:36:02.401224458Z 2026-05-26 16:36:02.400 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-05-26T16:36:02.413777621Z 2026-05-26 16:36:02.413 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-26T16:36:02.450179751Z 2026-05-26 16:36:02.449 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-26T16:36:02.478157861Z 2026-05-26 16:36:02.477 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-26T16:36:02.478235551Z 2026-05-26 16:36:02.477 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-26T16:36:02.595381863Z 2026-05-26 16:36:02.594 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-26T16:36:02.617279742Z 2026-05-26 16:36:02.616 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-26T16:36:02.631403598Z 2026-05-26 16:36:02.630 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-05-26T16:36:02.642622168Z 2026-05-26 16:36:02.642 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72122242099511296-solr-node1:8983_solr-n_0000000000) starting
2026-05-26T16:36:02.712368569Z 2026-05-26 16:36:02.711 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-05-26T16:36:02.713215046Z 2026-05-26 16:36:02.712 INFO (OverseerStateUpdate-72122242099511296-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-05-26T16:36:02.723214967Z 2026-05-26 16:36:02.722 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-26T16:36:02.763798540Z 2026-05-26 16:36:02.763 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-26T16:36:03.155926434Z 2026-05-26 16:36:03.155 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-26T16:36:03.330731860Z 2026-05-26 16:36:03.330 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-26T16:36:03.618806129Z 2026-05-26 16:36:03.618 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-26T16:36:03.986022190Z 2026-05-26 16:36:03.985 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-26T16:36:04.079117588Z 2026-05-26 16:36:04.077 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_26.request.log
2026-05-26T16:36:04.082929796Z 2026-05-26 16:36:04.081 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-05-26T16:36:04.084546654Z 2026-05-26 16:36:04.084 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4957ms
2026-05-26T16:36:04.435678877Z 2026-05-26 16:36:04.435 INFO (qtp985324122-27-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=75
2026-05-26T16:36:04.652299556Z 2026-05-26 16:36:04.651 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=25
2026-05-26T16:36:08.179673429Z 2026-05-26 16:36:08.179 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-05-26T16:36:08.209597741Z 2026-05-26 16:36:08.209 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-26T16:36:11.981906117Z 2026-05-26 16:36:11.980 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=13
2026-05-26T16:36:12.017549104Z 2026-05-26 16:36:12.015 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=15
2026-05-26T16:36:12.046269732Z 2026-05-26 16:36:12.045 INFO (qtp985324122-27-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=2
2026-05-26T16:36:12.293482464Z 2026-05-26 16:36:12.293 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-05-26T16:36:12.312941208Z 2026-05-26 16:36:12.312 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-05-26T16:36:12.324616382Z 2026-05-26 16:36:12.324 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 173] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-26T16:36:12.361781935Z 2026-05-26 16:36:12.361 INFO (OverseerStateUpdate-72122242099511296-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-26T16:36:12.361806608Z "core":"test-collection_shard3_replica_n1",
2026-05-26T16:36:12.361810252Z "node_name":"solr-node1:8983_solr",
2026-05-26T16:36:12.361813099Z "base_url":"http://solr-node1:8983/solr",
2026-05-26T16:36:12.361816249Z "collection":"test-collection",
2026-05-26T16:36:12.361819167Z "shard":"shard3",
2026-05-26T16:36:12.361822302Z "state":"down",
2026-05-26T16:36:12.361824959Z "type":"NRT",
2026-05-26T16:36:12.361827622Z "operation":"ADDREPLICA",
2026-05-26T16:36:12.361830528Z "waitForFinalState":"false"}
2026-05-26T16:36:12.370914902Z 2026-05-26 16:36:12.370 INFO (OverseerStateUpdate-72122242099511296-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-26T16:36:12.370932581Z "core":"test-collection_shard2_replica_n2",
2026-05-26T16:36:12.370935234Z "node_name":"solr-node3:8985_solr",
2026-05-26T16:36:12.370937891Z "base_url":"http://solr-node3:8985/solr",
2026-05-26T16:36:12.370943471Z "collection":"test-collection",
2026-05-26T16:36:12.370945500Z "shard":"shard2",
2026-05-26T16:36:12.370947571Z "state":"down",
2026-05-26T16:36:12.370949520Z "type":"NRT",
2026-05-26T16:36:12.370951490Z "operation":"ADDREPLICA",
2026-05-26T16:36:12.370953595Z "waitForFinalState":"false"}
2026-05-26T16:36:12.375700134Z 2026-05-26 16:36:12.375 INFO (OverseerStateUpdate-72122242099511296-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-26T16:36:12.375711294Z "core":"test-collection_shard1_replica_n4",
2026-05-26T16:36:12.375713476Z "node_name":"solr-node2:8984_solr",
2026-05-26T16:36:12.375726737Z "base_url":"http://solr-node2:8984/solr",
2026-05-26T16:36:12.375729033Z "collection":"test-collection",
2026-05-26T16:36:12.375730955Z "shard":"shard1",
2026-05-26T16:36:12.375732833Z "state":"down",
2026-05-26T16:36:12.375734685Z "type":"NRT",
2026-05-26T16:36:12.375736619Z "operation":"ADDREPLICA",
2026-05-26T16:36:12.375738524Z "waitForFinalState":"false"}
2026-05-26T16:36:12.484928818Z 2026-05-26 16:36:12.484 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 186] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-26T16:36:12.562870099Z 2026-05-26 16:36:12.561 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-05-26T16:36:12.653657612Z 2026-05-26 16:36:12.653 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-26T16:36:12.715678348Z 2026-05-26 16:36:12.715 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-05-26T16:36:13.191283243Z 2026-05-26 16:36:13.190 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-26T16:36:13.191308033Z 2026-05-26 16:36:13.190 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true
2026-05-26T16:36:13.238408689Z 2026-05-26 16:36:13.237 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/]
2026-05-26T16:36:14.574690039Z 2026-05-26 16:36:14.573 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-26T16:36:14.610637780Z 2026-05-26 16:36:14.610 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-26T16:36:14.618320906Z 2026-05-26 16:36:14.617 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-26T16:36:14.619432354Z 2026-05-26 16:36:14.618 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-26T16:36:14.619490546Z 2026-05-26 16:36:14.619 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-26T16:36:14.721466450Z 2026-05-26 16:36:14.720 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-26T16:36:14.723835377Z 2026-05-26 16:36:14.723 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-26T16:36:14.726091402Z 2026-05-26 16:36:14.725 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-26T16:36:14.728380441Z 2026-05-26 16:36:14.727 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-26T16:36:14.747019213Z 2026-05-26 16:36:14.746 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-05-26T16:36:14.755154180Z 2026-05-26 16:36:14.754 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-26T16:36:14.768222658Z 2026-05-26 16:36:14.767 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-26T16:36:14.768300448Z 2026-05-26 16:36:14.767 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-26T16:36:14.778927775Z 2026-05-26 16:36:14.778 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-26T16:36:14.782054849Z 2026-05-26 16:36:14.781 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-05-26T16:36:14.783364912Z 2026-05-26 16:36:14.783 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-05-26T16:36:14.794958924Z 2026-05-26 16:36:14.794 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-05-26T16:36:14.794977129Z 2026-05-26 16:36:14.794 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-05-26T16:36:14.795074790Z 2026-05-26 16:36:14.794 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/
2026-05-26T16:36:14.797018252Z 2026-05-26 16:36:14.796 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-05-26T16:36:14.797735005Z 2026-05-26 16:36:14.797 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas
2026-05-26T16:36:14.799683161Z 2026-05-26 16:36:14.799 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72122242099511296-core_node3-n_0000000000
2026-05-26T16:36:14.805272068Z 2026-05-26 16:36:14.805 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3
2026-05-26T16:36:14.912508679Z 2026-05-26 16:36:14.912 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 238] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-26T16:36:14.916945009Z 2026-05-26 16:36:14.916 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-05-26T16:36:14.931529334Z 2026-05-26 16:36:14.931 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2371
2026-05-26T16:36:15.024481822Z 2026-05-26 16:36:15.023 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 95973,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-05-26T16:35:58.855584853Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-05-26T16:36:05.130774633Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-26T16:36:05.130810816Z It should be set to 65000 to avoid operational disruption.
2026-05-26T16:36:05.130815160Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-26T16:36:05.131123536Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-26T16:36:05.131141785Z
2026-05-26T16:36:05.142422834Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-26T16:36:05.150246562Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-26T16:36:05.196796574Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-26T16:36:05.234973752Z WARNING: A command line option has enabled the Security Manager
2026-05-26T16:36:05.234995070Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-26T16:36:06.480755001Z 2026-05-26 16:36:06.477 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-26T16:36:06.754197939Z 2026-05-26 16:36:06.753 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-26T16:36:06.781147122Z 2026-05-26 16:36:06.780 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-26T16:36:06.788881164Z 2026-05-26 16:36:06.788 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-26T16:36:06.788912365Z 2026-05-26 16:36:06.788 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-05-26T16:36:06.789118014Z 2026-05-26 16:36:06.788 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-26T16:36:06.789326158Z 2026-05-26 16:36:06.788 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-26T16:36:06.788409132Z
2026-05-26T16:36:06.792015319Z 2026-05-26 16:36:06.791 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-26T16:36:06.817245732Z 2026-05-26 16:36:06.816 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-26T16:36:06.842442944Z 2026-05-26 16:36:06.841 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-26T16:36:06.990374490Z 2026-05-26 16:36:06.989 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-05-26T16:36:07.012083147Z 2026-05-26 16:36:07.011 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-26T16:36:07.012192152Z 2026-05-26 16:36:07.011 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-26T16:36:07.012291509Z 2026-05-26 16:36:07.011 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-26T16:36:07.122464710Z 2026-05-26 16:36:07.121 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-05-26T16:36:07.122784150Z 2026-05-26 16:36:07.122 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-26T16:36:07.122869483Z 2026-05-26 16:36:07.122 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-26T16:36:07.157683951Z 2026-05-26 16:36:07.157 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-26T16:36:07.922560110Z 2026-05-26 16:36:07.921 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-26T16:36:08.070692318Z 2026-05-26 16:36:08.070 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-26T16:36:08.072231567Z 2026-05-26 16:36:08.070 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-05-26T16:36:08.082989613Z 2026-05-26 16:36:08.082 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-26T16:36:08.089322061Z 2026-05-26 16:36:08.088 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-26T16:36:08.093657087Z 2026-05-26 16:36:08.093 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-26T16:36:08.093790520Z 2026-05-26 16:36:08.093 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-26T16:36:08.132742492Z 2026-05-26 16:36:08.132 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-26T16:36:08.143491124Z 2026-05-26 16:36:08.142 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-26T16:36:08.173350225Z 2026-05-26 16:36:08.172 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-26T16:36:08.180515663Z 2026-05-26 16:36:08.180 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-05-26T16:36:08.188159780Z 2026-05-26 16:36:08.187 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-05-26T16:36:08.199376444Z 2026-05-26 16:36:08.198 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-05-26T16:36:08.209721082Z 2026-05-26 16:36:08.209 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-26T16:36:08.305630000Z 2026-05-26 16:36:08.305 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-26T16:36:08.663429829Z 2026-05-26 16:36:08.662 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-26T16:36:08.840761705Z 2026-05-26 16:36:08.840 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-26T16:36:09.117896544Z 2026-05-26 16:36:09.117 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-26T16:36:09.249661265Z 2026-05-26 16:36:09.248 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-26T16:36:09.255373762Z 2026-05-26 16:36:09.255 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_26.request.log
2026-05-26T16:36:09.259008525Z 2026-05-26 16:36:09.258 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-05-26T16:36:09.260510510Z 2026-05-26 16:36:09.260 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4111ms
2026-05-26T16:36:09.425552886Z 2026-05-26 16:36:09.425 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=47
2026-05-26T16:36:09.479386394Z 2026-05-26 16:36:09.478 INFO (qtp985324122-56-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=14
2026-05-26T16:36:12.563439929Z 2026-05-26 16:36:12.562 INFO (qtp985324122-58-solr-node1-6) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-05-26T16:36:12.612384422Z 2026-05-26 16:36:12.611 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-26T16:36:12.670320079Z 2026-05-26 16:36:12.669 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-05-26T16:36:13.104769969Z 2026-05-26 16:36:13.104 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-26T16:36:13.104853109Z 2026-05-26 16:36:13.104 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-05-26T16:36:13.124093534Z 2026-05-26 16:36:13.123 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-05-26T16:36:13.567599795Z 2026-05-26 16:36:13.567 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-26T16:36:13.586827941Z 2026-05-26 16:36:13.586 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-26T16:36:13.590721355Z 2026-05-26 16:36:13.590 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-26T16:36:13.591866710Z 2026-05-26 16:36:13.591 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-26T16:36:13.591896313Z 2026-05-26 16:36:13.591 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-26T16:36:13.638657479Z 2026-05-26 16:36:13.638 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-26T16:36:13.642365788Z 2026-05-26 16:36:13.642 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-26T16:36:13.645275898Z 2026-05-26 16:36:13.644 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-26T16:36:13.648048398Z 2026-05-26 16:36:13.647 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-26T16:36:13.661166076Z 2026-05-26 16:36:13.660 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4}
2026-05-26T16:36:13.669386964Z 2026-05-26 16:36:13.668 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-26T16:36:13.676294774Z 2026-05-26 16:36:13.675 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-26T16:36:13.676313939Z 2026-05-26 16:36:13.675 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-26T16:36:13.685885086Z 2026-05-26 16:36:13.685 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Register
[...truncated too long message...]
ntry
# Generating 2,048 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 9,999 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #7 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/93-7
# Last output:
# INFO - 2026-05-26 16:37:19.435; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.keyStorePassword
# INFO - 2026-05-26 16:37:19.438; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.trustStorePassword
# WARN - 2026-05-26 16:37:19.450; org.apache.solr.common.util.EnvUtils; You are passing in deprecated system property host and should upgrade to using solr.host.advertise instead. The deprecated property support will be removed in future version of Solr.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is https://localhost:38713.
#
# ERROR: IOException occurred when talking to server at: https://localhost:38713/solr/admin/info/system
not ok 94 SOLR-11740 check 'solr stop' connection # in 231 ms
# (in test file test/test_start_solr.bats, line 32)
# `solr start' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/94-1
not ok 95 stop command for single port # in 230 ms
# (in test file test/test_start_solr.bats, line 45)
# `solr start' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/95-2
ok 96 check stop command doesn't hang # in 218 ms
not ok 97 SOLR-16976 solr starts with remote JMX enabled # in 228 ms
# (in test file test/test_start_solr.bats, line 77)
# `solr start' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #4 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/97-4
not ok 98 deprecated system properties converted to modern properties # in 233 ms
# (in test file test/test_start_solr.bats, line 85)
# `solr start -Ddisable.config.edit=true' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #5 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/98-5
not ok 99 start with custom jetty options # in 236 ms
# (in test file test/test_start_solr.bats, line 93)
# `solr start --jettyconfig "--module=server"' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #6 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/99-6
not ok 100 bootstrapping a configset # in 234 ms
# (in test file test/test_start_solr.bats, line 104)
# `solr start -Dbootstrap_confdir="${confdir_path}" -Dcollection.configName=techproducts' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #7 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/100-7
not ok 101 status detects locally running solr # in 1018 ms
# (in test file test/test_status.bats, line 34)
# `solr start' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/101-1
# Last output:
#
# No Solr nodes are running.
not ok 102 status with --solr-url from user # in 226 ms
# (in test file test/test_status.bats, line 43)
# `solr start' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/102-2
not ok 103 status with --port from user # in 232 ms
# (in test file test/test_status.bats, line 50)
# `solr start' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #3 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/103-3
ok 104 multiple connection options are prevented # in 881 ms
ok 105 status with invalid --solr-url from user # in 1279 ms
not ok 106 status with --short format # in 230 ms
# (in test file test/test_status.bats, line 67)
# `solr start' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #6 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/106-6
not ok 107 setup_file failed
# (from function `setup_file' in test file test/test_stream.bats, line 22)
# `solr start -e techproducts' failed
#
# Starting up Solr on port 38713 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 38713 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server" -Dsolr.modules=clustering,extraction,langid,ltr,scripting -Dsolr.ltr.enabled=true -Dsolr.clustering.enabled=true
#
#
# ERROR: Port 38713 is already being used by another process.
#
ok 111 --version returns Solr version # in 749 ms
ok 112 version as direct tool call still runs # in 746 ms
not ok 113 setup_file failed
# (from function `setup_file' in test file test/test_zk.bats, line 22)
# `solr start -DminStateByteLenForCompression=0' failed
#
# Port 38713 is already being used by another process (pid: 87460)
# Please choose a different port using the -p option.
#
ok 125 Cleanup # in 1872 ms
FATAL: command execution failed
java.io.IOException: Backing channel 'lucene-solr-4' is disconnected.
at hudson.remoting.RemoteInvocationHandler.channelOrFail(RemoteInvocationHandler.java:227)
at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:306)
at jdk.proxy3/jdk.proxy3.$Proxy270.isAlive(Unknown Source)
at hudson.Launcher$RemoteLauncher$ProcImpl.isAlive(Launcher.java:1212)
at hudson.Launcher$RemoteLauncher$ProcImpl.join(Launcher.java:1204)
at hudson.Launcher$ProcStarter.join(Launcher.java:521)
at PluginClassLoader for gradle//hudson.plugins.gradle.Gradle.perform(Gradle.java:321)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:818)
at hudson.model.Build$BuildExecution.build(Build.java:199)
at hudson.model.Build$BuildExecution.doRun(Build.java:164)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:527)
at hudson.model.Run.execute(Run.java:1833)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:44)
at hudson.model.ResourceController.execute(ResourceController.java:101)
at hudson.model.Executor.run(Executor.java:446)
Caused by: java.io.IOException: Unexpected termination of the channel
at hudson.remoting.SynchronousCommandTransport$ReaderThread.run(SynchronousCommandTransport.java:80)
Caused by: java.io.EOFException
at java.base/java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2915)
at java.base/java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:3410)
at java.base/java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:954)
at java.base/java.io.ObjectInputStream.<init>(ObjectInputStream.java:392)
at hudson.remoting.ObjectInputStreamEx.<init>(ObjectInputStreamEx.java:50)
at hudson.remoting.Command.readFrom(Command.java:141)
at hudson.remoting.Command.readFrom(Command.java:127)
at hudson.remoting.AbstractSynchronousByteArrayCommandTransport.read(AbstractSynchronousByteArrayCommandTransport.java:35)
at hudson.remoting.SynchronousCommandTransport$ReaderThread.run(SynchronousCommandTransport.java:62)
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
ERROR: Step ‘Archive the artifacts’ failed: no workspace for Solr/Solr-TestIntegration-10.x #217
ERROR: lucene-solr-4 is offline; cannot locate jdk_21_latest
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
ERROR: lucene-solr-4 is offline; cannot locate jdk_21_latest
ERROR: lucene-solr-4 is offline; cannot locate jdk_21_latest
ERROR: lucene-solr-4 is offline; cannot locate jdk_21_latest
ERROR: lucene-solr-4 is offline; cannot locate jdk_21_latest
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 216 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/216/
No tests ran.
Build Log:
[...truncated 373 lines...]
# ERROR: Solr is not running on url http://localhost:46437 after 5 seconds
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/4-2
# Last output:
# Waiting up to 180 seconds to see Solr running on port 46437 [|] [/] [-] [\] [|]
# Started Solr server on port 46437 (pid=2012562). Happy searching!
#
#
ok 5 auth rejects block-unknown option with invalid boolean # in 1798 ms
ok 6 auth rejects update-include-file-only option with invalid boolean # in 1770 ms
not ok 7 auth enable/disable lifecycle # in 20605 ms
# (in test file test/test_auth.bats, line 40)
# `solr auth enable --type basicAuth --credentials name:password' failed
# Waiting up to 180 seconds to see Solr running on port 46437 [|] [/] [-] [\] [|]
# Started Solr server on port 46437 (pid=2014834). Happy searching!
#
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
# Couldn't get ZooKeeper host. Please make sure Solr is running in cloud mode, or a zk-host has been passed in.
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
#
# ERROR: Server refused connection at: http://localhost:46437/solr/admin/info/system
not ok 8 enable auth connects via zookeeper # in 1196 ms
# (in test file test/test_auth.bats, line 53)
# `solr start' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
#
# ERROR: Error from server at http://localhost:46437/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 9 create, config, api, and delete with basic auth # in 16678 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-05-22 16:50:50.552; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-22 16:50:50.556; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 10 post, postlogs and export with basic auth # in 16638 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/10-2
# Last output:
# WARN - 2026-05-22 16:51:07.238; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-22 16:51:07.241; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 11 package with basic auth # in 16665 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #3 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/11-3
# Last output:
# WARN - 2026-05-22 16:51:23.925; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-22 16:51:23.928; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 12 admin UI static content is excepted from basic auth # in 16671 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #4 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/12-4
# Last output:
# WARN - 2026-05-22 16:51:40.596; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-22 16:51:40.599; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 13 setup_file failed
# (from function `setup_file' in test file test/test_bats.bats, line 26)
# `solr start --verbose' failed
# Using Solr root directory: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT
# Using Java: /home/jenkins/tools/java/latest21/bin/java
# openjdk version "21.0.4" 2024-07-16 LTS
# OpenJDK Runtime Environment Temurin-21.0.4+7 (build 21.0.4+7-LTS)
# OpenJDK 64-Bit Server VM Temurin-21.0.4+7 (build 21.0.4+7-LTS, mixed mode, sharing)
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
not ok 14 setup_file failed
# (from function `setup_file' in test file test/test_compression.bats, line 22)
# `solr start -e films' failed
#
# Starting up Solr on port 46437 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 46437 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 46437 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test # at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/-
not ok 17 setup_file failed
# (from function `setup_file' in test file test/test_config.bats, line 22)
# `solr start' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
not ok 20 create for non cloud mode # in 1518 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_create.bats, line 34)
# `assert_output --partial "Created new core 'COLL_NAME'"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Created new core 'COLL_NAME'
# output (17 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
#
# ERROR: Error from server at http://localhost:46437/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/20-1
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
#
# ERROR: Error from server at http://localhost:46437/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 21 create for cloud mode # in 1451 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_create.bats, line 40)
# `assert_output --partial "Created collection 'COLL_NAME'"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Created collection 'COLL_NAME'
# output (17 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
#
# ERROR: Error from server at http://localhost:46437/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/21-2
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
#
# ERROR: Error from server at http://localhost:46437/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
ok 22 multiple connection options are prevented # in 847 ms
not ok 23 setup_file failed
# (from function `setup_file' in test file test/test_create_collection.bats, line 22)
# `solr start' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
not ok 34 setup_file failed
# (from function `setup_file' in test file test/test_delete_collection.bats, line 22)
# `solr start' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
not ok 39 start -e cloud works with --jvm-opts # in 1275 ms
# (in test file test/test_example.bats, line 32)
# `solr start -e cloud --no-prompt --jvm-opts "-Dcustom.prop=helloworld"' failed
#
# Welcome to the SolrCloud example!
#
# Starting up 2 Solr nodes for your example SolrCloud cluster.
#
# Creating Solr home directory /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr
# Cloning /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1 into
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node2
#
# Starting up Solr on port 46437 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 46437 --solr-home "build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server" --jvm-opts "-Dcustom.prop=helloworld"
#
#
# ERROR: Port 46437 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/39-1
not ok 40 start -e cloud works with --no-prompt # in 1247 ms
# (in test file test/test_example_noprompt.bats, line 32)
# `solr start -e cloud --no-prompt' failed
#
# Welcome to the SolrCloud example!
#
# Starting up 2 Solr nodes for your example SolrCloud cluster.
#
# Solr home directory /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr already exists.
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node2 already exists.
#
# Starting up Solr on port 46437 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 46437 --solr-home "build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 46437 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/40-1
not ok 41 Check export command # in 3002 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_export.bats, line 36)
# `assert_output --partial 'Export complete'' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Export complete
# output (48 lines):
# WARN - 2026-05-22 16:52:10.986; org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider; Attempt to fetch live_nodes from http://localhost:46437/solr failed.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:46437/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1194)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.submitClusterStateRequest(BaseHttpClusterStateProvider.java:221)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:293)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.updateLiveNodes(BaseHttpClusterStateProvider.java:280)
# at java.base/java.util.stream.MatchOps$1MatchSink.accept(MatchOps.java:90)
# at java.base/java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1685)
# at java.base/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129)
# at java.base/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527)
# at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
# at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:230)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:196)
# at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
# at java.base/java.util.stream.ReferencePipeline.anyMatch(ReferencePipeline.java:632)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:268)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getLiveNodes(BaseHttpClusterStateProvider.java:231)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getState(BaseHttpClusterStateProvider.java:109)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionRef(CloudSolrClient.java:1723)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.loadDocCollection(CloudSolrClient.java:1699)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$triggerCollectionRefresh$6(CloudSolrClient.java:1664)
# at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1768)
# at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
# at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
# at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
# at java.base/java.lang.Thread.run(Thread.java:1583)
#
# ERROR: Error refreshing state for collection techproducts
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/41-1
# Last output:
# WARN - 2026-05-22 16:52:10.986; org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider; Attempt to fetch live_nodes from http://localhost:46437/solr failed.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:46437/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1194)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.submitClusterStateRequest(BaseHttpClusterStateProvider.java:221)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:293)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.updateLiveNodes(BaseHttpClusterStateProvider.java:280)
# at java.base/java.util.stream.MatchOps$1MatchSink.accept(MatchOps.java:90)
# at java.base/java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1685)
# at java.base/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129)
# at java.base/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527)
# at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
# at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:230)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:196)
# at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
# at java.base/java.util.stream.ReferencePipeline.anyMatch(ReferencePipeline.java:632)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:268)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getLiveNodes(BaseHttpClusterStateProvider.java:231)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getState(BaseHttpClusterStateProvider.java:109)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionRef(CloudSolrClient.java:1723)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.loadDocCollection(CloudSolrClient.java:1699)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$triggerCollectionRefresh$6(CloudSolrClient.java:1664)
# at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1768)
# at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
# at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
# at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
# at java.base/java.lang.Thread.run(Thread.java:1583)
#
# ERROR: Error refreshing state for collection techproducts
ok 42 export fails on non cloud mode # in 2865 ms
Waiting for Tika Server to be ready on port 46442
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 43 using curl to extract a single pdf file # in 16594 ms
WARNING: Docker not available; bypassing test.
ok 44 using the bin/solr post tool to extract content from pdf # in 16549 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to crawl web site # in 16537 ms
Stopping Tika Server container
not ok 46 healthcheck on cloud solr # in 1259 ms
# (in test file test/test_healthcheck.bats, line 32)
# `solr start -e films' failed
#
# Starting up Solr on port 46437 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 46437 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 46437 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/46-1
not ok 47 healthcheck errors on standalone solr # in 1255 ms
# (in test file test/test_healthcheck.bats, line 39)
# `solr start --user-managed -e films' failed
#
# Starting up Solr on port 46437 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start --user-managed -p 46437 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 46437 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/47-2
ok 48 solr help flag prints help # in 740 ms
ok 49 solr with no flags prints help # in 759 ms
ok 50 start help flag prints help # in 147 ms
ok 51 start h flag prints help # in 149 ms
ok 52 stop help flag prints help # in 144 ms
ok 53 restart help flag prints help # in 158 ms
ok 54 status help flag prints help # in 805 ms
ok 55 healthcheck help flag prints help # in 749 ms
ok 56 create help flag prints help # in 759 ms
ok 57 delete help flag prints help # in 789 ms
ok 58 zk help flag prints help # in 788 ms
ok 59 auth help flag prints help # in 761 ms
ok 60 assert help flag prints help # in 740 ms
ok 61 post help flag prints help # in 749 ms
not ok 62 SQL Module # in 18885 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_modules.bats, line 36)
# `assert_output --partial '"docs":'' failed
#
# -- output does not contain substring --
# substring (1 lines):
# "docs":
# output (16 lines):
#
# ERROR: Error from server at http://localhost:46437/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json: Expected mime type in [application/json] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/62-1
# WARN - 2026-05-22 16:53:19.275; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-22 16:53:19.278; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-22 16:53:35.006; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
#
# ERROR: Error from server at http://localhost:46437/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json: Expected mime type in [application/json] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 63 icu collation in analysis-extras module # in 17864 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_modules.bats, line 49)
# `assert_output --partial "Created collection 'COLL_NAME'"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Created collection 'COLL_NAME'
# output (17 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
#
# ERROR: Error from server at http://localhost:46437/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/63-2
# WARN - 2026-05-22 16:53:37.170; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-22 16:53:37.173; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-22 16:53:52.896; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
#
# ERROR: Error from server at http://localhost:46437/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 64 Check lifecycle of sentiment classification # in 10777 ms
# (in test file test/test_opennlp.bats, line 63)
# `solr start -m 4g -Dsolr.modules=analysis-extras -Dsolr.packages.enabled=true' failed
# Downloading onnx model and vocab...
# % Total % Received % Xferd Average Speed Time Time Time Current
# Dload Upload Total Spent Left Speed
#
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 160M 0 208k 0 0 180k 0 0:15:09 0:00:01 0:15:08 180k
7 160M 7 12.6M 0 0 6077k 0 0:00:27 0:00:02 0:00:25 6076k
20 160M 20 33.2M 0 0 10.2M 0 0:00:15 0:00:03 0:00:12 10.2M
34 160M 34 54.7M 0 0 12.8M 0 0:00:12 0:00:04 0:00:08 12.8M
47 160M 47 75.8M 0 0 14.4M 0 0:00:11 0:00:05 0:00:06 15.2M
60 160M 60 96.5M 0 0 15.5M 0 0:00:10 0:00:06 0:00:04 19.0M
72 160M 72 117M 0 0 16.2M 0 0:00:09 0:00:07 0:00:02 20.6M
85 160M 85 137M 0 0 16.8M 0 0:00:09 0:00:08 0:00:01 21.2M
97 160M 97 156M 0 0 17.0M 0 0:00:09 0:00:09 --:--:-- 20.7M
100 160M 100 160M 0 0 17.2M 0 0:00:09 0:00:09 --:--:-- 20.8M
# % Total % Received % Xferd Average Speed Time Time Time Current
# Dload Upload Total Spent Left Speed
#
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
11 851k 11 98304 0 0 102k 0 0:00:08 --:--:-- 0:00:08 102k
100 851k 100 851k 0 0 668k 0 0:00:01 0:00:01 --:--:-- 668k
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/64-1
# Last output:
# total 64M
# drwxr-xr-x 2 jenkins jenkins 4.0K May 22 16:39 .
# drwxr-xr-x 3 jenkins jenkins 4.0K May 22 16:39 ..
# -rw-r--r-- 1 jenkins jenkins 14M May 22 16:39 icu4j-77.1.jar
# -rw-r--r-- 1 jenkins jenkins 83K May 22 16:39 lucene-analysis-icu-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 30K May 22 16:39 lucene-analysis-morfologik-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 40K May 22 16:39 lucene-analysis-opennlp-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 3.5M May 22 16:39 lucene-analysis-smartcn-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 508K May 22 16:39 lucene-analysis-stempel-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 19K May 22 16:39 morfologik-fsa-2.1.9.jar
# -rw-r--r-- 1 jenkins jenkins 1.8M May 22 16:39 morfologik-polish-2.1.9.jar
# -rw-r--r-- 1 jenkins jenkins 51K May 22 16:39 morfologik-stemming-2.1.9.jar
# -rw-r--r-- 1 jenkins jenkins 4.0M May 22 16:39 morfologik-ukrainian-search-4.9.1.jar
# -rw-r--r-- 1 jenkins jenkins 39M May 22 16:39 onnxruntime-1.24.3.jar
# -rw-r--r-- 1 jenkins jenkins 28K May 22 16:39 opennlp-dl-2.5.8.jar
# -rw-r--r-- 1 jenkins jenkins 1.4M May 22 16:39 opennlp-tools-2.5.8.jar
# -rw-r--r-- 1 jenkins jenkins 39K May 22 16:39 solr-analysis-extras-10.1.0-SNAPSHOT.jar
not ok 65 lifecycle of package # in 2283 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_packages.bats, line 38)
# `assert_output --partial "Available packages:"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Available packages:
# output (45 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:46437/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:348)
# at org.apache.solr.cli.StatusTool.reportStatus(StatusTool.java:300)
# at org.apache.solr.cli.CLIUtils.getZkHost(CLIUtils.java:248)
# at org.apache.solr.cli.PackageTool.runImpl(PackageTool.java:123)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
#
# ERROR: Error from server at http://localhost:46437/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/65-1
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:46437/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:348)
# at org.apache.solr.cli.StatusTool.reportStatus(StatusTool.java:300)
# at org.apache.solr.cli.CLIUtils.getZkHost(CLIUtils.java:248)
# at org.apache.solr.cli.PackageTool.runImpl(PackageTool.java:123)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
#
# ERROR: Error from server at http://localhost:46437/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 66 deploying and undeploying a collection level package # in 1295 ms
# (in test file test/test_packages.bats, line 44)
# `solr create -c foo-1.2' failed
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46437.
#
# ERROR: Error from server at http://localhost:46437/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46437/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/66-2
# Last output:
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
ok 67 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
not ok 68 Affinity placement plugin using sysprop # in 21796 ms
# (in test file test/test_placement_plugin.bats, line 34)
# `solr assert --cloud http://localhost:${SOLR_PORT} --timeout 3000' failed with status 100
#
# ERROR: Solr is not running on url http://localhost:46437 after 3 seconds
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/68-1
# WARN - 2026-05-22 16:54:13.843; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-22 16:54:13.846; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-22 16:54:29.580; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
not ok 69 Random placement plugin using ENV # in 21791 ms
# (in test file test/test_placement_plugin.bats, line 43)
# `solr assert --cloud http://localhost:${SOLR_PORT} --timeout 3000' failed with status 100
#
# ERROR: Solr is not running on url http://localhost:46437 after 3 seconds
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/69-2
# WARN - 2026-05-22 16:54:35.669; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-22 16:54:35.672; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-22 16:54:51.394; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
not ok 70 setup_file failed
# (from function `setup_file' in test file test/test_post.bats, line 22)
# `solr start -Dsolr.modules=extraction' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
not ok 81 setup_file failed
# (from function `setup_file' in test file test/test_postlogs.bats, line 22)
# `solr start' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 17 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-05-22T16:54:59.614966733Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-22T16:54:59.615010017Z It should be set to 65000 to avoid operational disruption.
2026-05-22T16:54:59.615014615Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-22T16:54:59.615342237Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-22T16:54:59.615351837Z
2026-05-22T16:54:59.626437644Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-22T16:54:59.634790203Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-22T16:54:59.696675922Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-22T16:54:59.733483719Z WARNING: A command line option has enabled the Security Manager
2026-05-22T16:54:59.733507280Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-22T16:55:00.967462521Z 2026-05-22 16:55:00.963 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-22T16:55:01.250619921Z 2026-05-22 16:55:01.250 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-22T16:55:01.277185495Z 2026-05-22 16:55:01.276 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-22T16:55:01.285473517Z 2026-05-22 16:55:01.284 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-22T16:55:01.285586361Z 2026-05-22 16:55:01.284 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-05-22T16:55:01.285856282Z 2026-05-22 16:55:01.285 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-22T16:55:01.285999840Z 2026-05-22 16:55:01.285 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-22T16:55:01.285191419Z
2026-05-22T16:55:01.289402011Z 2026-05-22 16:55:01.288 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-22T16:55:01.312479852Z 2026-05-22 16:55:01.311 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-22T16:55:01.326752706Z 2026-05-22 16:55:01.326 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-22T16:55:01.326831042Z 2026-05-22 16:55:01.326 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-22T16:55:01.363745242Z 2026-05-22 16:55:01.362 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-22T16:55:02.383646292Z 2026-05-22 16:55:02.382 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-22T16:55:02.651634950Z 2026-05-22 16:55:02.651 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-22T16:55:02.656666509Z 2026-05-22 16:55:02.656 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-05-22T16:55:02.656737271Z 2026-05-22 16:55:02.656 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-05-22T16:55:02.659398754Z 2026-05-22 16:55:02.659 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-05-22T16:55:02.659467657Z 2026-05-22 16:55:02.659 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-05-22T16:55:02.706256296Z 2026-05-22 16:55:02.705 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-05-22T16:55:03.161900667Z 2026-05-22 16:55:03.161 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-05-22T16:55:03.174895522Z 2026-05-22 16:55:03.174 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-22T16:55:03.210964018Z 2026-05-22 16:55:03.210 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-22T16:55:03.239099653Z 2026-05-22 16:55:03.238 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-22T16:55:03.239165246Z 2026-05-22 16:55:03.238 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-22T16:55:03.358411922Z 2026-05-22 16:55:03.357 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-22T16:55:03.379202699Z 2026-05-22 16:55:03.378 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-22T16:55:03.390473736Z 2026-05-22 16:55:03.390 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-05-22T16:55:03.397092876Z 2026-05-22 16:55:03.396 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72099667617579008-solr-node1:8983_solr-n_0000000000) starting
2026-05-22T16:55:03.456309100Z 2026-05-22 16:55:03.455 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-05-22T16:55:03.457173535Z 2026-05-22 16:55:03.456 INFO (OverseerStateUpdate-72099667617579008-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-05-22T16:55:03.466069958Z 2026-05-22 16:55:03.465 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-22T16:55:03.506117388Z 2026-05-22 16:55:03.505 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-22T16:55:03.923644813Z 2026-05-22 16:55:03.923 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-22T16:55:04.117432096Z 2026-05-22 16:55:04.116 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-22T16:55:04.378630848Z 2026-05-22 16:55:04.378 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-22T16:55:04.504977092Z 2026-05-22 16:55:04.504 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-22T16:55:04.510640076Z 2026-05-22 16:55:04.510 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_22.request.log
2026-05-22T16:55:04.514281820Z 2026-05-22 16:55:04.513 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-05-22T16:55:04.515307419Z 2026-05-22 16:55:04.515 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4882ms
2026-05-22T16:55:04.967627018Z 2026-05-22 16:55:04.966 INFO (qtp985324122-27-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=57
2026-05-22T16:55:05.303547575Z 2026-05-22 16:55:05.302 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=14
2026-05-22T16:55:08.702827449Z 2026-05-22 16:55:08.702 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-05-22T16:55:08.975431597Z 2026-05-22 16:55:08.974 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-22T16:55:12.615820514Z 2026-05-22 16:55:12.614 INFO (qtp985324122-100-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=13
2026-05-22T16:55:12.723192990Z 2026-05-22 16:55:12.722 INFO (qtp985324122-100-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=79
2026-05-22T16:55:12.750936067Z 2026-05-22 16:55:12.750 INFO (qtp985324122-27-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=2
2026-05-22T16:55:13.415533728Z 2026-05-22 16:55:13.414 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-05-22T16:55:13.488192619Z 2026-05-22 16:55:13.487 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-05-22T16:55:13.506750307Z 2026-05-22 16:55:13.505 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 174] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-22T16:55:13.584812755Z 2026-05-22 16:55:13.583 INFO (OverseerStateUpdate-72099667617579008-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-22T16:55:13.584834360Z "core":"test-collection_shard3_replica_n1",
2026-05-22T16:55:13.584838783Z "node_name":"solr-node1:8983_solr",
2026-05-22T16:55:13.584842205Z "base_url":"http://solr-node1:8983/solr",
2026-05-22T16:55:13.584849567Z "collection":"test-collection",
2026-05-22T16:55:13.584852930Z "shard":"shard3",
2026-05-22T16:55:13.584856001Z "state":"down",
2026-05-22T16:55:13.584859127Z "type":"NRT",
2026-05-22T16:55:13.584862424Z "operation":"ADDREPLICA",
2026-05-22T16:55:13.584865617Z "waitForFinalState":"false"}
2026-05-22T16:55:13.615254983Z 2026-05-22 16:55:13.614 INFO (OverseerStateUpdate-72099667617579008-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-22T16:55:13.615273997Z "core":"test-collection_shard2_replica_n2",
2026-05-22T16:55:13.615276587Z "node_name":"solr-node3:8985_solr",
2026-05-22T16:55:13.615278790Z "base_url":"http://solr-node3:8985/solr",
2026-05-22T16:55:13.615281122Z "collection":"test-collection",
2026-05-22T16:55:13.615283238Z "shard":"shard2",
2026-05-22T16:55:13.615285262Z "state":"down",
2026-05-22T16:55:13.615287290Z "type":"NRT",
2026-05-22T16:55:13.615293007Z "operation":"ADDREPLICA",
2026-05-22T16:55:13.615295057Z "waitForFinalState":"false"}
2026-05-22T16:55:13.621389771Z 2026-05-22 16:55:13.620 INFO (OverseerStateUpdate-72099667617579008-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-22T16:55:13.621406547Z "core":"test-collection_shard1_replica_n4",
2026-05-22T16:55:13.621409293Z "node_name":"solr-node2:8984_solr",
2026-05-22T16:55:13.621425234Z "base_url":"http://solr-node2:8984/solr",
2026-05-22T16:55:13.621427528Z "collection":"test-collection",
2026-05-22T16:55:13.621429876Z "shard":"shard1",
2026-05-22T16:55:13.621431947Z "state":"down",
2026-05-22T16:55:13.621433858Z "type":"NRT",
2026-05-22T16:55:13.621435921Z "operation":"ADDREPLICA",
2026-05-22T16:55:13.621437869Z "waitForFinalState":"false"}
2026-05-22T16:55:13.733700666Z 2026-05-22 16:55:13.733 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 186] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-22T16:55:13.861071771Z 2026-05-22 16:55:13.860 INFO (qtp985324122-100-solr-node1-6) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-05-22T16:55:13.931719158Z 2026-05-22 16:55:13.930 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-22T16:55:14.005552969Z 2026-05-22 16:55:14.004 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-05-22T16:55:14.497900169Z 2026-05-22 16:55:14.497 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-22T16:55:14.498362018Z 2026-05-22 16:55:14.497 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true
2026-05-22T16:55:14.522895142Z 2026-05-22 16:55:14.522 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/]
2026-05-22T16:55:15.601362391Z 2026-05-22 16:55:15.600 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-22T16:55:15.661214223Z 2026-05-22 16:55:15.660 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-22T16:55:15.672291346Z 2026-05-22 16:55:15.671 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-22T16:55:15.674044840Z 2026-05-22 16:55:15.673 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-22T16:55:15.674058205Z 2026-05-22 16:55:15.673 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-22T16:55:15.800338957Z 2026-05-22 16:55:15.799 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-22T16:55:15.806330026Z 2026-05-22 16:55:15.805 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-22T16:55:15.809085002Z 2026-05-22 16:55:15.808 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-22T16:55:15.822935144Z 2026-05-22 16:55:15.822 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-22T16:55:15.841950536Z 2026-05-22 16:55:15.841 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01}
2026-05-22T16:55:15.855698638Z 2026-05-22 16:55:15.855 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-22T16:55:15.863548728Z 2026-05-22 16:55:15.863 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-22T16:55:15.863571766Z 2026-05-22 16:55:15.863 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-22T16:55:15.884887055Z 2026-05-22 16:55:15.884 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-05-22T16:55:15.885992589Z 2026-05-22 16:55:15.885 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-22T16:55:15.886257038Z 2026-05-22 16:55:15.885 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-05-22T16:55:15.915975996Z 2026-05-22 16:55:15.915 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-05-22T16:55:15.915996061Z 2026-05-22 16:55:15.915 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-05-22T16:55:15.916000723Z 2026-05-22 16:55:15.915 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/
2026-05-22T16:55:15.917245991Z 2026-05-22 16:55:15.916 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-05-22T16:55:15.918251588Z 2026-05-22 16:55:15.918 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas
2026-05-22T16:55:15.919333739Z 2026-05-22 16:55:15.919 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72099667617579008-core_node3-n_0000000000
2026-05-22T16:55:15.927833068Z 2026-05-22 16:55:15.927 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3
2026-05-22T16:55:16.032416987Z 2026-05-22 16:55:16.031 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 240] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-22T16:55:16.036422778Z 2026-05-22 16:55:16.036 INFO (qtp985324122-100-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-05-22T16:55:16.043241969Z 2026-05-22 16:55:16.042 INFO (qtp985324122-100-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2184
2026-05-22T16:55:16.144673223Z 2026-05-22 16:55:16.144 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 246] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 2023020,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-05-22T16:54:59.326387951Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-05-22T16:55:05.640388142Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-22T16:55:05.640412124Z It should be set to 65000 to avoid operational disruption.
2026-05-22T16:55:05.640415059Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-22T16:55:05.640752986Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-22T16:55:05.640765249Z
2026-05-22T16:55:05.650549457Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-22T16:55:05.658139379Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-22T16:55:05.729484755Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-22T16:55:05.761550942Z WARNING: A command line option has enabled the Security Manager
2026-05-22T16:55:05.761588322Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-22T16:55:06.903049235Z 2026-05-22 16:55:06.897 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-22T16:55:07.197422025Z 2026-05-22 16:55:07.196 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-22T16:55:07.223287773Z 2026-05-22 16:55:07.222 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-22T16:55:07.230399017Z 2026-05-22 16:55:07.229 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-22T16:55:07.230536022Z 2026-05-22 16:55:07.230 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-05-22T16:55:07.230553396Z 2026-05-22 16:55:07.230 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-22T16:55:07.230708812Z 2026-05-22 16:55:07.230 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-22T16:55:07.230221595Z
2026-05-22T16:55:07.233667076Z 2026-05-22 16:55:07.233 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-22T16:55:07.255405062Z 2026-05-22 16:55:07.254 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-22T16:55:07.278610936Z 2026-05-22 16:55:07.278 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-22T16:55:07.420673164Z 2026-05-22 16:55:07.420 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-05-22T16:55:07.444819619Z 2026-05-22 16:55:07.444 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-22T16:55:07.444951474Z 2026-05-22 16:55:07.444 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-22T16:55:07.445113061Z 2026-05-22 16:55:07.444 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-22T16:55:07.557249553Z 2026-05-22 16:55:07.556 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-05-22T16:55:07.557484121Z 2026-05-22 16:55:07.557 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-22T16:55:07.557707282Z 2026-05-22 16:55:07.557 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-22T16:55:07.603653847Z 2026-05-22 16:55:07.603 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-22T16:55:08.420722243Z 2026-05-22 16:55:08.420 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-22T16:55:08.579854907Z 2026-05-22 16:55:08.579 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-22T16:55:08.581630968Z 2026-05-22 16:55:08.579 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-05-22T16:55:08.590115186Z 2026-05-22 16:55:08.589 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-22T16:55:08.596378229Z 2026-05-22 16:55:08.595 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-22T16:55:08.599945885Z 2026-05-22 16:55:08.599 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-22T16:55:08.600017169Z 2026-05-22 16:55:08.599 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-22T16:55:08.636259502Z 2026-05-22 16:55:08.635 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-22T16:55:08.645052577Z 2026-05-22 16:55:08.644 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-22T16:55:08.667645930Z 2026-05-22 16:55:08.667 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-22T16:55:08.680694602Z 2026-05-22 16:55:08.680 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-05-22T16:55:08.691767097Z 2026-05-22 16:55:08.691 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-05-22T16:55:08.704577854Z 2026-05-22 16:55:08.704 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-05-22T16:55:08.756862195Z 2026-05-22 16:55:08.756 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-22T16:55:08.975459244Z 2026-05-22 16:55:08.974 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-22T16:55:09.069824907Z 2026-05-22 16:55:09.069 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-22T16:55:09.261696392Z 2026-05-22 16:55:09.261 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-22T16:55:09.543595807Z 2026-05-22 16:55:09.543 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-22T16:55:09.680418921Z 2026-05-22 16:55:09.679 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-22T16:55:09.686344039Z 2026-05-22 16:55:09.685 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_22.request.log
2026-05-22T16:55:09.690409937Z 2026-05-22 16:55:09.689 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-05-22T16:55:09.691711506Z 2026-05-22 16:55:09.691 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4034ms
2026-05-22T16:55:09.860620500Z 2026-05-22 16:55:09.859 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=47
2026-05-22T16:55:09.919198214Z 2026-05-22 16:55:09.918 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=16
2026-05-22T16:55:13.859629524Z 2026-05-22 16:55:13.858 INFO (qtp985324122-58-solr-node1-6) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-05-22T16:55:13.921664660Z 2026-05-22 16:55:13.921 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-22T16:55:13.975853361Z 2026-05-22 16:55:13.975 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-05-22T16:55:14.298678219Z 2026-05-22 16:55:14.297 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-22T16:55:14.298702285Z 2026-05-22 16:55:14.298 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-05-22T16:55:14.317726570Z 2026-05-22 16:55:14.317 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-05-22T16:55:14.774255171Z 2026-05-22 16:55:14.773 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-22T16:55:14.792786027Z 2026-05-22 16:55:14.792 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-22T16:55:14.796243010Z 2026-05-22 16:55:14.795 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-22T16:55:14.797294755Z 2026-05-22 16:55:14.796 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-22T16:55:14.797348882Z 2026-05-22 16:55:14.797 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-22T16:55:14.839631279Z 2026-05-22 16:55:14.839 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-22T16:55:14.844353084Z 2026-05-22 16:55:14.843 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-22T16:55:14.846470765Z 2026-05-22 16:55:14.846 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-22T16:55:14.848404456Z 2026-05-22 16:55:14.848 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-22T16:55:14.860604905Z 2026-05-22 16:55:14.859 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-05-22T16:55:14.869185386Z 2026-05-22 16:55:14.868 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-22T16:55:14.875449687Z 2026-05-22 16:55:14.874 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-22T16:55:14.875525058Z 2026-05-22 16:55:14.875 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-22T16:55:14.882215720Z 2026-05-22 16:55:14.881 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_so
[...truncated too long message...]
r and self-signed certificate (SHA384withRSA) with a validity of 90 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Certificate was added to keystore
# Certificate reply was installed in keystore
# Generating 3,072 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 90 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Certificate was added to keystore
# Certificate was added to keystore
# Certificate reply was installed in keystore
# Certificate was added to keystore
# Certificate was added to keystore
# Generating 3,072 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 90 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Certificate was added to keystore
# Certificate was added to keystore
# Certificate reply was installed in keystore
# Certificate was added to keystore
# Certificate was added to keystore
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #6 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/92-6
# Last output:
# INFO - 2026-05-22 16:56:17.909; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.keyStorePassword
# INFO - 2026-05-22 16:56:17.912; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.trustStorePassword
# WARN - 2026-05-22 16:56:17.924; org.apache.solr.common.util.EnvUtils; You are passing in deprecated system property host and should upgrade to using solr.host.advertise instead. The deprecated property support will be removed in future version of Solr.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is https://localhost:46437.
#
# ERROR: IOException occurred when talking to server at: https://localhost:46437/solr/admin/info/system
not ok 93 test keystore reload # in 2772 ms
# (in test file test/test_ssl.bats, line 529)
# `solr start --jvm-opts "-Dsolr.jetty.ssl.context.reload.scan.interval.secs=1 -DsocketTimeout=5000"' failed
# Generating 2,048 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 9,999 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Generating 2,048 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 9,999 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #7 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/93-7
# Last output:
# INFO - 2026-05-22 16:56:20.738; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.keyStorePassword
# INFO - 2026-05-22 16:56:20.741; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.trustStorePassword
# WARN - 2026-05-22 16:56:20.753; org.apache.solr.common.util.EnvUtils; You are passing in deprecated system property host and should upgrade to using solr.host.advertise instead. The deprecated property support will be removed in future version of Solr.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is https://localhost:46437.
#
# ERROR: IOException occurred when talking to server at: https://localhost:46437/solr/admin/info/system
not ok 94 SOLR-11740 check 'solr stop' connection # in 232 ms
# (in test file test/test_start_solr.bats, line 32)
# `solr start' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/94-1
not ok 95 stop command for single port # in 238 ms
# (in test file test/test_start_solr.bats, line 45)
# `solr start' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/95-2
ok 96 check stop command doesn't hang # in 239 ms
not ok 97 SOLR-16976 solr starts with remote JMX enabled # in 236 ms
# (in test file test/test_start_solr.bats, line 77)
# `solr start' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #4 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/97-4
not ok 98 deprecated system properties converted to modern properties # in 239 ms
# (in test file test/test_start_solr.bats, line 85)
# `solr start -Ddisable.config.edit=true' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #5 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/98-5
not ok 99 start with custom jetty options # in 236 ms
# (in test file test/test_start_solr.bats, line 93)
# `solr start --jettyconfig "--module=server"' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #6 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/99-6
not ok 100 bootstrapping a configset # in 227 ms
# (in test file test/test_start_solr.bats, line 104)
# `solr start -Dbootstrap_confdir="${confdir_path}" -Dcollection.configName=techproducts' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #7 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/100-7
not ok 101 status detects locally running solr # in 980 ms
# (in test file test/test_status.bats, line 34)
# `solr start' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/101-1
# Last output:
#
# No Solr nodes are running.
not ok 102 status with --solr-url from user # in 223 ms
# (in test file test/test_status.bats, line 43)
# `solr start' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/102-2
not ok 103 status with --port from user # in 228 ms
# (in test file test/test_status.bats, line 50)
# `solr start' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #3 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/103-3
ok 104 multiple connection options are prevented # in 848 ms
ok 105 status with invalid --solr-url from user # in 1240 ms
not ok 106 status with --short format # in 225 ms
# (in test file test/test_status.bats, line 67)
# `solr start' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #6 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/106-6
not ok 107 setup_file failed
# (from function `setup_file' in test file test/test_stream.bats, line 22)
# `solr start -e techproducts' failed
#
# Starting up Solr on port 46437 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 46437 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server" -Dsolr.modules=clustering,extraction,langid,ltr,scripting -Dsolr.ltr.enabled=true -Dsolr.clustering.enabled=true
#
#
# ERROR: Port 46437 is already being used by another process.
#
ok 111 --version returns Solr version # in 773 ms
ok 112 version as direct tool call still runs # in 777 ms
not ok 113 setup_file failed
# (from function `setup_file' in test file test/test_zk.bats, line 22)
# `solr start -DminStateByteLenForCompression=0' failed
#
# Port 46437 is already being used by another process (pid: 2014834)
# Please choose a different port using the -p option.
#
ok 125 Cleanup # in 1871 ms
Build timed out (after 5,175 minutes). Marking the build as aborted.
Build timed out (after 5,175 minutes). Marking the build as failed.
Build was aborted
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 215 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/215/
No tests ran.
Build Log:
[...truncated 372 lines...]
# ERROR: Solr is not running on url http://localhost:46519 after 5 seconds
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/4-2
# Last output:
# Waiting up to 180 seconds to see Solr running on port 46519 [|] [/] [-] [\] [|]
# Started Solr server on port 46519 (pid=1319852). Happy searching!
#
#
ok 5 auth rejects block-unknown option with invalid boolean # in 1836 ms
ok 6 auth rejects update-include-file-only option with invalid boolean # in 1831 ms
not ok 7 auth enable/disable lifecycle # in 20823 ms
# (in test file test/test_auth.bats, line 40)
# `solr auth enable --type basicAuth --credentials name:password' failed
# Waiting up to 180 seconds to see Solr running on port 46519 [|] [/] [-] [\] [|]
# Started Solr server on port 46519 (pid=1324609). Happy searching!
#
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
# Couldn't get ZooKeeper host. Please make sure Solr is running in cloud mode, or a zk-host has been passed in.
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
#
# ERROR: Server refused connection at: http://localhost:46519/solr/admin/info/system
not ok 8 enable auth connects via zookeeper # in 1294 ms
# (in test file test/test_auth.bats, line 53)
# `solr start' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
#
# ERROR: Error from server at http://localhost:46519/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 9 create, config, api, and delete with basic auth # in 16669 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-05-21 08:52:37.184; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-21 08:52:37.188; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 10 post, postlogs and export with basic auth # in 16653 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/10-2
# Last output:
# WARN - 2026-05-21 08:52:53.859; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-21 08:52:53.862; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 11 package with basic auth # in 16683 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #3 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/11-3
# Last output:
# WARN - 2026-05-21 08:53:10.566; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-21 08:53:10.570; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 12 admin UI static content is excepted from basic auth # in 16665 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #4 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/12-4
# Last output:
# WARN - 2026-05-21 08:53:27.249; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-21 08:53:27.253; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 13 setup_file failed
# (from function `setup_file' in test file test/test_bats.bats, line 26)
# `solr start --verbose' failed
# Using Solr root directory: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT
# Using Java: /home/jenkins/tools/java/latest21/bin/java
# openjdk version "21.0.4" 2024-07-16 LTS
# OpenJDK Runtime Environment Temurin-21.0.4+7 (build 21.0.4+7-LTS)
# OpenJDK 64-Bit Server VM Temurin-21.0.4+7 (build 21.0.4+7-LTS, mixed mode, sharing)
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
not ok 14 setup_file failed
# (from function `setup_file' in test file test/test_compression.bats, line 22)
# `solr start -e films' failed
#
# Starting up Solr on port 46519 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 46519 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 46519 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test # at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/-
not ok 17 setup_file failed
# (from function `setup_file' in test file test/test_config.bats, line 22)
# `solr start' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
not ok 20 create for non cloud mode # in 1467 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_create.bats, line 34)
# `assert_output --partial "Created new core 'COLL_NAME'"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Created new core 'COLL_NAME'
# output (17 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
#
# ERROR: Error from server at http://localhost:46519/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/20-1
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
#
# ERROR: Error from server at http://localhost:46519/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 21 create for cloud mode # in 1495 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_create.bats, line 40)
# `assert_output --partial "Created collection 'COLL_NAME'"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Created collection 'COLL_NAME'
# output (17 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
#
# ERROR: Error from server at http://localhost:46519/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/21-2
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
#
# ERROR: Error from server at http://localhost:46519/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
ok 22 multiple connection options are prevented # in 822 ms
not ok 23 setup_file failed
# (from function `setup_file' in test file test/test_create_collection.bats, line 22)
# `solr start' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
not ok 34 setup_file failed
# (from function `setup_file' in test file test/test_delete_collection.bats, line 22)
# `solr start' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
not ok 39 start -e cloud works with --jvm-opts # in 1305 ms
# (in test file test/test_example.bats, line 32)
# `solr start -e cloud --no-prompt --jvm-opts "-Dcustom.prop=helloworld"' failed
#
# Welcome to the SolrCloud example!
#
# Starting up 2 Solr nodes for your example SolrCloud cluster.
#
# Creating Solr home directory /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr
# Cloning /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1 into
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node2
#
# Starting up Solr on port 46519 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 46519 --solr-home "build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server" --jvm-opts "-Dcustom.prop=helloworld"
#
#
# ERROR: Port 46519 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/39-1
not ok 40 start -e cloud works with --no-prompt # in 1299 ms
# (in test file test/test_example_noprompt.bats, line 32)
# `solr start -e cloud --no-prompt' failed
#
# Welcome to the SolrCloud example!
#
# Starting up 2 Solr nodes for your example SolrCloud cluster.
#
# Solr home directory /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr already exists.
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node2 already exists.
#
# Starting up Solr on port 46519 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 46519 --solr-home "build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 46519 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/40-1
not ok 41 Check export command # in 2981 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_export.bats, line 36)
# `assert_output --partial 'Export complete'' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Export complete
# output (48 lines):
# WARN - 2026-05-21 08:53:57.684; org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider; Attempt to fetch live_nodes from http://localhost:46519/solr failed.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:46519/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1194)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.submitClusterStateRequest(BaseHttpClusterStateProvider.java:221)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:293)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.updateLiveNodes(BaseHttpClusterStateProvider.java:280)
# at java.base/java.util.stream.MatchOps$1MatchSink.accept(MatchOps.java:90)
# at java.base/java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1685)
# at java.base/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129)
# at java.base/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527)
# at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
# at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:230)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:196)
# at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
# at java.base/java.util.stream.ReferencePipeline.anyMatch(ReferencePipeline.java:632)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:268)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getLiveNodes(BaseHttpClusterStateProvider.java:231)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getState(BaseHttpClusterStateProvider.java:109)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionRef(CloudSolrClient.java:1723)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.loadDocCollection(CloudSolrClient.java:1699)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$triggerCollectionRefresh$6(CloudSolrClient.java:1664)
# at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1768)
# at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
# at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
# at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
# at java.base/java.lang.Thread.run(Thread.java:1583)
#
# ERROR: Error refreshing state for collection techproducts
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/41-1
# Last output:
# WARN - 2026-05-21 08:53:57.684; org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider; Attempt to fetch live_nodes from http://localhost:46519/solr failed.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:46519/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1194)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.submitClusterStateRequest(BaseHttpClusterStateProvider.java:221)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:293)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.updateLiveNodes(BaseHttpClusterStateProvider.java:280)
# at java.base/java.util.stream.MatchOps$1MatchSink.accept(MatchOps.java:90)
# at java.base/java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1685)
# at java.base/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129)
# at java.base/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527)
# at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
# at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:230)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:196)
# at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
# at java.base/java.util.stream.ReferencePipeline.anyMatch(ReferencePipeline.java:632)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:268)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getLiveNodes(BaseHttpClusterStateProvider.java:231)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getState(BaseHttpClusterStateProvider.java:109)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionRef(CloudSolrClient.java:1723)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.loadDocCollection(CloudSolrClient.java:1699)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$triggerCollectionRefresh$6(CloudSolrClient.java:1664)
# at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1768)
# at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
# at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
# at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
# at java.base/java.lang.Thread.run(Thread.java:1583)
#
# ERROR: Error refreshing state for collection techproducts
ok 42 export fails on non cloud mode # in 2743 ms
Waiting for Tika Server to be ready on port 46524
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 43 using curl to extract a single pdf file # in 16620 ms
WARNING: Docker not available; bypassing test.
ok 44 using the bin/solr post tool to extract content from pdf # in 16582 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to crawl web site # in 16585 ms
Stopping Tika Server container
not ok 46 healthcheck on cloud solr # in 1323 ms
# (in test file test/test_healthcheck.bats, line 32)
# `solr start -e films' failed
#
# Starting up Solr on port 46519 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 46519 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 46519 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/46-1
not ok 47 healthcheck errors on standalone solr # in 1315 ms
# (in test file test/test_healthcheck.bats, line 39)
# `solr start --user-managed -e films' failed
#
# Starting up Solr on port 46519 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start --user-managed -p 46519 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 46519 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/47-2
ok 48 solr help flag prints help # in 795 ms
ok 49 solr with no flags prints help # in 748 ms
ok 50 start help flag prints help # in 151 ms
ok 51 start h flag prints help # in 159 ms
ok 52 stop help flag prints help # in 152 ms
ok 53 restart help flag prints help # in 154 ms
ok 54 status help flag prints help # in 809 ms
ok 55 healthcheck help flag prints help # in 766 ms
ok 56 create help flag prints help # in 787 ms
ok 57 delete help flag prints help # in 802 ms
ok 58 zk help flag prints help # in 847 ms
ok 59 auth help flag prints help # in 794 ms
ok 60 assert help flag prints help # in 803 ms
ok 61 post help flag prints help # in 780 ms
not ok 62 SQL Module # in 19050 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_modules.bats, line 36)
# `assert_output --partial '"docs":'' failed
#
# -- output does not contain substring --
# substring (1 lines):
# "docs":
# output (16 lines):
#
# ERROR: Error from server at http://localhost:46519/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json: Expected mime type in [application/json] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/62-1
# WARN - 2026-05-21 08:55:17.158; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-21 08:55:17.161; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-21 08:55:32.902; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
#
# ERROR: Error from server at http://localhost:46519/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json: Expected mime type in [application/json] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 63 icu collation in analysis-extras module # in 17941 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_modules.bats, line 49)
# `assert_output --partial "Created collection 'COLL_NAME'"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Created collection 'COLL_NAME'
# output (17 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
#
# ERROR: Error from server at http://localhost:46519/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/63-2
# WARN - 2026-05-21 08:55:35.150; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-21 08:55:35.153; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-21 08:55:50.877; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
#
# ERROR: Error from server at http://localhost:46519/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 64 Check lifecycle of sentiment classification # in 11373 ms
# (in test file test/test_opennlp.bats, line 63)
# `solr start -m 4g -Dsolr.modules=analysis-extras -Dsolr.packages.enabled=true' failed
# Downloading onnx model and vocab...
# % Total % Received % Xferd Average Speed Time Time Time Current
# Dload Upload Total Spent Left Speed
#
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 160M 0 512k 0 0 378k 0 0:07:15 0:00:01 0:07:14 377k
8 160M 8 13.8M 0 0 6046k 0 0:00:27 0:00:02 0:00:25 6045k
21 160M 21 34.3M 0 0 10.2M 0 0:00:15 0:00:03 0:00:12 10.2M
33 160M 33 53.2M 0 0 12.2M 0 0:00:13 0:00:04 0:00:09 12.2M
45 160M 45 72.9M 0 0 13.5M 0 0:00:11 0:00:05 0:00:06 14.4M
57 160M 57 92.7M 0 0 14.5M 0 0:00:11 0:00:06 0:00:05 18.3M
69 160M 69 111M 0 0 15.1M 0 0:00:10 0:00:07 0:00:03 19.4M
80 160M 80 129M 0 0 15.5M 0 0:00:10 0:00:08 0:00:02 19.2M
92 160M 92 149M 0 0 16.0M 0 0:00:10 0:00:09 0:00:01 19.3M
100 160M 100 160M 0 0 16.3M 0 0:00:09 0:00:09 --:--:-- 19.6M
# % Total % Received % Xferd Average Speed Time Time Time Current
# Dload Upload Total Spent Left Speed
#
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 851k 100 851k 0 0 641k 0 0:00:01 0:00:01 --:--:-- 642k
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/64-1
# Last output:
# total 64M
# drwxr-xr-x 2 jenkins jenkins 4.0K May 21 08:41 .
# drwxr-xr-x 3 jenkins jenkins 4.0K May 21 08:41 ..
# -rw-r--r-- 1 jenkins jenkins 14M May 21 08:41 icu4j-77.1.jar
# -rw-r--r-- 1 jenkins jenkins 83K May 21 08:41 lucene-analysis-icu-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 30K May 21 08:41 lucene-analysis-morfologik-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 40K May 21 08:41 lucene-analysis-opennlp-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 3.5M May 21 08:41 lucene-analysis-smartcn-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 508K May 21 08:41 lucene-analysis-stempel-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 19K May 21 08:41 morfologik-fsa-2.1.9.jar
# -rw-r--r-- 1 jenkins jenkins 1.8M May 21 08:41 morfologik-polish-2.1.9.jar
# -rw-r--r-- 1 jenkins jenkins 51K May 21 08:41 morfologik-stemming-2.1.9.jar
# -rw-r--r-- 1 jenkins jenkins 4.0M May 21 08:41 morfologik-ukrainian-search-4.9.1.jar
# -rw-r--r-- 1 jenkins jenkins 39M May 21 08:41 onnxruntime-1.24.3.jar
# -rw-r--r-- 1 jenkins jenkins 28K May 21 08:41 opennlp-dl-2.5.8.jar
# -rw-r--r-- 1 jenkins jenkins 1.4M May 21 08:41 opennlp-tools-2.5.8.jar
# -rw-r--r-- 1 jenkins jenkins 39K May 21 08:41 solr-analysis-extras-10.1.0-SNAPSHOT.jar
not ok 65 lifecycle of package # in 2333 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_packages.bats, line 38)
# `assert_output --partial "Available packages:"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Available packages:
# output (45 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:46519/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:348)
# at org.apache.solr.cli.StatusTool.reportStatus(StatusTool.java:300)
# at org.apache.solr.cli.CLIUtils.getZkHost(CLIUtils.java:248)
# at org.apache.solr.cli.PackageTool.runImpl(PackageTool.java:123)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
#
# ERROR: Error from server at http://localhost:46519/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/65-1
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:46519/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:348)
# at org.apache.solr.cli.StatusTool.reportStatus(StatusTool.java:300)
# at org.apache.solr.cli.CLIUtils.getZkHost(CLIUtils.java:248)
# at org.apache.solr.cli.PackageTool.runImpl(PackageTool.java:123)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
#
# ERROR: Error from server at http://localhost:46519/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 66 deploying and undeploying a collection level package # in 1356 ms
# (in test file test/test_packages.bats, line 44)
# `solr create -c foo-1.2' failed
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:46519.
#
# ERROR: Error from server at http://localhost:46519/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:46519/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/66-2
# Last output:
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
ok 67 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
not ok 68 Affinity placement plugin using sysprop # in 21752 ms
# (in test file test/test_placement_plugin.bats, line 34)
# `solr assert --cloud http://localhost:${SOLR_PORT} --timeout 3000' failed with status 100
#
# ERROR: Solr is not running on url http://localhost:46519 after 3 seconds
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/68-1
# WARN - 2026-05-21 08:56:12.534; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-21 08:56:12.537; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-21 08:56:28.263; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
not ok 69 Random placement plugin using ENV # in 21772 ms
# (in test file test/test_placement_plugin.bats, line 43)
# `solr assert --cloud http://localhost:${SOLR_PORT} --timeout 3000' failed with status 100
#
# ERROR: Solr is not running on url http://localhost:46519 after 3 seconds
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/69-2
# WARN - 2026-05-21 08:56:34.319; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-21 08:56:34.322; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-21 08:56:50.055; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
not ok 70 setup_file failed
# (from function `setup_file' in test file test/test_post.bats, line 22)
# `solr start -Dsolr.modules=extraction' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
not ok 81 setup_file failed
# (from function `setup_file' in test file test/test_postlogs.bats, line 22)
# `solr start' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 16 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-05-21T08:56:57.571658252Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-21T08:56:57.571712536Z It should be set to 65000 to avoid operational disruption.
2026-05-21T08:56:57.571715985Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-21T08:56:57.572029254Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-21T08:56:57.572042732Z
2026-05-21T08:56:57.581837356Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-21T08:56:57.590465470Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-21T08:56:57.650778476Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-21T08:56:57.683130688Z WARNING: A command line option has enabled the Security Manager
2026-05-21T08:56:57.683156081Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-21T08:56:58.861549729Z 2026-05-21 08:56:58.858 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-21T08:56:59.140770255Z 2026-05-21 08:56:59.140 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-21T08:56:59.166627004Z 2026-05-21 08:56:59.166 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-21T08:56:59.173750996Z 2026-05-21 08:56:59.173 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-21T08:56:59.174145948Z 2026-05-21 08:56:59.173 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-05-21T08:56:59.174160178Z 2026-05-21 08:56:59.173 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-21T08:56:59.174163951Z 2026-05-21 08:56:59.173 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-21T08:56:59.173400226Z
2026-05-21T08:56:59.176380629Z 2026-05-21 08:56:59.175 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-21T08:56:59.197253763Z 2026-05-21 08:56:59.196 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-21T08:56:59.210235754Z 2026-05-21 08:56:59.209 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-21T08:56:59.210591261Z 2026-05-21 08:56:59.209 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-21T08:56:59.250499163Z 2026-05-21 08:56:59.249 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-21T08:57:00.037152309Z 2026-05-21 08:57:00.036 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-21T08:57:00.285093751Z 2026-05-21 08:57:00.284 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-21T08:57:00.290969653Z 2026-05-21 08:57:00.290 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-05-21T08:57:00.291157357Z 2026-05-21 08:57:00.290 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-05-21T08:57:00.294112556Z 2026-05-21 08:57:00.293 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-05-21T08:57:00.294127468Z 2026-05-21 08:57:00.293 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-05-21T08:57:00.335654014Z 2026-05-21 08:57:00.335 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-05-21T08:57:00.796335816Z 2026-05-21 08:57:00.795 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-05-21T08:57:00.809078525Z 2026-05-21 08:57:00.808 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-21T08:57:00.850885005Z 2026-05-21 08:57:00.850 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-21T08:57:00.875136108Z 2026-05-21 08:57:00.874 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-21T08:57:00.875276863Z 2026-05-21 08:57:00.874 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-21T08:57:01.015577945Z 2026-05-21 08:57:01.014 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-21T08:57:01.039137982Z 2026-05-21 08:57:01.038 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-21T08:57:01.050633379Z 2026-05-21 08:57:01.050 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-05-21T08:57:01.056876804Z 2026-05-21 08:57:01.056 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72092193407959040-solr-node1:8983_solr-n_0000000000) starting
2026-05-21T08:57:01.123396989Z 2026-05-21 08:57:01.122 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-05-21T08:57:01.124318924Z 2026-05-21 08:57:01.123 INFO (OverseerStateUpdate-72092193407959040-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-05-21T08:57:01.133353914Z 2026-05-21 08:57:01.132 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-21T08:57:01.173471325Z 2026-05-21 08:57:01.172 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-21T08:57:01.544525227Z 2026-05-21 08:57:01.543 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-21T08:57:01.727365288Z 2026-05-21 08:57:01.726 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-21T08:57:02.061734910Z 2026-05-21 08:57:02.061 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-21T08:57:02.334944530Z 2026-05-21 08:57:02.334 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-21T08:57:02.406982030Z 2026-05-21 08:57:02.405 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_21.request.log
2026-05-21T08:57:02.411163182Z 2026-05-21 08:57:02.410 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-05-21T08:57:02.412742192Z 2026-05-21 08:57:02.412 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4824ms
2026-05-21T08:57:02.701331609Z 2026-05-21 08:57:02.700 INFO (qtp985324122-27-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=47
2026-05-21T08:57:02.830570026Z 2026-05-21 08:57:02.830 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=17
2026-05-21T08:57:06.127792265Z 2026-05-21 08:57:06.127 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-05-21T08:57:06.304416521Z 2026-05-21 08:57:06.303 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-21T08:57:10.144341349Z 2026-05-21 08:57:10.143 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=42
2026-05-21T08:57:10.204235217Z 2026-05-21 08:57:10.203 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=40
2026-05-21T08:57:10.234897408Z 2026-05-21 08:57:10.234 INFO (qtp985324122-27-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=4
2026-05-21T08:57:10.837908752Z 2026-05-21 08:57:10.836 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-05-21T08:57:10.865216713Z 2026-05-21 08:57:10.864 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-05-21T08:57:10.884509640Z 2026-05-21 08:57:10.884 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 174] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-21T08:57:10.958131630Z 2026-05-21 08:57:10.957 INFO (OverseerStateUpdate-72092193407959040-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-21T08:57:10.958151462Z "core":"test-collection_shard1_replica_n1",
2026-05-21T08:57:10.958154547Z "node_name":"solr-node2:8984_solr",
2026-05-21T08:57:10.958156804Z "base_url":"http://solr-node2:8984/solr",
2026-05-21T08:57:10.958159126Z "collection":"test-collection",
2026-05-21T08:57:10.958161260Z "shard":"shard1",
2026-05-21T08:57:10.958164924Z "state":"down",
2026-05-21T08:57:10.958167536Z "type":"NRT",
2026-05-21T08:57:10.958169486Z "operation":"ADDREPLICA",
2026-05-21T08:57:10.958171423Z "waitForFinalState":"false"}
2026-05-21T08:57:10.966633337Z 2026-05-21 08:57:10.966 INFO (OverseerStateUpdate-72092193407959040-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-21T08:57:10.966659930Z "core":"test-collection_shard2_replica_n2",
2026-05-21T08:57:10.966664732Z "node_name":"solr-node3:8985_solr",
2026-05-21T08:57:10.966686728Z "base_url":"http://solr-node3:8985/solr",
2026-05-21T08:57:10.966692113Z "collection":"test-collection",
2026-05-21T08:57:10.966695081Z "shard":"shard2",
2026-05-21T08:57:10.966697866Z "state":"down",
2026-05-21T08:57:10.966700368Z "type":"NRT",
2026-05-21T08:57:10.966703931Z "operation":"ADDREPLICA",
2026-05-21T08:57:10.966706780Z "waitForFinalState":"false"}
2026-05-21T08:57:10.973893357Z 2026-05-21 08:57:10.973 INFO (OverseerStateUpdate-72092193407959040-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-21T08:57:10.973910966Z "core":"test-collection_shard3_replica_n4",
2026-05-21T08:57:10.973914397Z "node_name":"solr-node1:8983_solr",
2026-05-21T08:57:10.973926208Z "base_url":"http://solr-node1:8983/solr",
2026-05-21T08:57:10.973928372Z "collection":"test-collection",
2026-05-21T08:57:10.973931319Z "shard":"shard3",
2026-05-21T08:57:10.973933272Z "state":"down",
2026-05-21T08:57:10.973935188Z "type":"NRT",
2026-05-21T08:57:10.973937157Z "operation":"ADDREPLICA",
2026-05-21T08:57:10.973939112Z "waitForFinalState":"false"}
2026-05-21T08:57:11.085648237Z 2026-05-21 08:57:11.084 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 186] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-21T08:57:11.154712388Z 2026-05-21 08:57:11.153 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard3_replica_n4&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-05-21T08:57:11.206330026Z 2026-05-21 08:57:11.205 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-21T08:57:11.253817594Z 2026-05-21 08:57:11.253 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-05-21T08:57:11.647484677Z 2026-05-21 08:57:11.645 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-21T08:57:11.647555449Z 2026-05-21 08:57:11.645 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n4' using configuration from configset techproducts, trusted=true
2026-05-21T08:57:11.671336334Z 2026-05-21 08:57:11.670 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n4], dataDir=[/var/solr/data/test-collection_shard3_replica_n4/data/]
2026-05-21T08:57:12.875081487Z 2026-05-21 08:57:12.873 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-21T08:57:12.937974742Z 2026-05-21 08:57:12.936 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-21T08:57:12.946126191Z 2026-05-21 08:57:12.945 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-21T08:57:12.948180106Z 2026-05-21 08:57:12.947 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-21T08:57:12.948223473Z 2026-05-21 08:57:12.947 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-21T08:57:13.085623966Z 2026-05-21 08:57:13.084 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-21T08:57:13.092305062Z 2026-05-21 08:57:13.091 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-21T08:57:13.095866746Z 2026-05-21 08:57:13.095 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-21T08:57:13.098626703Z 2026-05-21 08:57:13.098 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-21T08:57:13.156102730Z 2026-05-21 08:57:13.155 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01}
2026-05-21T08:57:13.164008883Z 2026-05-21 08:57:13.163 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-21T08:57:13.170768095Z 2026-05-21 08:57:13.170 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n4 solr-node1-6 core_node6 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-21T08:57:13.170787955Z 2026-05-21 08:57:13.170 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n4 solr-node1-6 core_node6 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-21T08:57:13.240042258Z 2026-05-21 08:57:13.239 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n4 solr-node1-6 core_node6 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-21T08:57:13.248849673Z 2026-05-21 08:57:13.247 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-05-21T08:57:13.255383625Z 2026-05-21 08:57:13.254 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-05-21T08:57:13.271535263Z 2026-05-21 08:57:13.271 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-05-21T08:57:13.271565257Z 2026-05-21 08:57:13.271 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-05-21T08:57:13.271645365Z 2026-05-21 08:57:13.271 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n4/
2026-05-21T08:57:13.272973246Z 2026-05-21 08:57:13.272 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-05-21T08:57:13.273401842Z 2026-05-21 08:57:13.273 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n4/ has no replicas
2026-05-21T08:57:13.274501862Z 2026-05-21 08:57:13.274 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72092193407959040-core_node6-n_0000000000
2026-05-21T08:57:13.281868293Z 2026-05-21 08:57:13.281 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n4/ shard3
2026-05-21T08:57:13.387113452Z 2026-05-21 08:57:13.386 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 242] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-21T08:57:13.391341407Z 2026-05-21 08:57:13.391 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node6 x:test-collection_shard3_replica_n4 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-05-21T08:57:13.400087613Z 2026-05-21 08:57:13.399 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2248
2026-05-21T08:57:13.500060179Z 2026-05-21 08:57:13.499 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 248] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1332815,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-05-21T08:56:57.353845593Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-05-21T08:57:03.155515056Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-21T08:57:03.155547316Z It should be set to 65000 to avoid operational disruption.
2026-05-21T08:57:03.155551185Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-21T08:57:03.155905331Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-21T08:57:03.155916156Z
2026-05-21T08:57:03.165850288Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-21T08:57:03.173323952Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-21T08:57:03.227416386Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-21T08:57:03.268785199Z WARNING: A command line option has enabled the Security Manager
2026-05-21T08:57:03.268803694Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-21T08:57:04.567818363Z 2026-05-21 08:57:04.564 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-21T08:57:04.844454722Z 2026-05-21 08:57:04.843 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-21T08:57:04.869681412Z 2026-05-21 08:57:04.869 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-21T08:57:04.876998650Z 2026-05-21 08:57:04.876 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-21T08:57:04.877053439Z 2026-05-21 08:57:04.876 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-05-21T08:57:04.877216189Z 2026-05-21 08:57:04.876 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-21T08:57:04.877332532Z 2026-05-21 08:57:04.876 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-21T08:57:04.876763113Z
2026-05-21T08:57:04.879873173Z 2026-05-21 08:57:04.879 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-21T08:57:04.903721002Z 2026-05-21 08:57:04.903 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-21T08:57:04.932281780Z 2026-05-21 08:57:04.931 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-21T08:57:05.077244957Z 2026-05-21 08:57:05.076 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-05-21T08:57:05.096203737Z 2026-05-21 08:57:05.095 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-21T08:57:05.096298118Z 2026-05-21 08:57:05.095 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-21T08:57:05.096452972Z 2026-05-21 08:57:05.095 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-21T08:57:05.206145509Z 2026-05-21 08:57:05.205 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-05-21T08:57:05.206270417Z 2026-05-21 08:57:05.205 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-21T08:57:05.206437626Z 2026-05-21 08:57:05.206 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-21T08:57:05.241570114Z 2026-05-21 08:57:05.241 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-21T08:57:06.026352052Z 2026-05-21 08:57:06.025 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-21T08:57:06.178969334Z 2026-05-21 08:57:06.178 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-21T08:57:06.180900670Z 2026-05-21 08:57:06.178 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-05-21T08:57:06.191756993Z 2026-05-21 08:57:06.191 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-21T08:57:06.197052417Z 2026-05-21 08:57:06.196 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-21T08:57:06.201833236Z 2026-05-21 08:57:06.201 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-21T08:57:06.201854574Z 2026-05-21 08:57:06.201 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-21T08:57:06.240269772Z 2026-05-21 08:57:06.239 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-21T08:57:06.249866872Z 2026-05-21 08:57:06.249 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
2026-05-21T08:57:06.273441837Z 2026-05-21 08:57:06.272 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-21T08:57:06.286008017Z 2026-05-21 08:57:06.285 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-05-21T08:57:06.296175502Z 2026-05-21 08:57:06.295 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-05-21T08:57:06.305478117Z 2026-05-21 08:57:06.304 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-21T08:57:06.352221247Z 2026-05-21 08:57:06.351 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-21T08:57:06.638046181Z 2026-05-21 08:57:06.637 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-21T08:57:06.810384537Z 2026-05-21 08:57:06.809 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-21T08:57:07.096762216Z 2026-05-21 08:57:07.096 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-21T08:57:07.237460181Z 2026-05-21 08:57:07.236 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-21T08:57:07.242823318Z 2026-05-21 08:57:07.242 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_21.request.log
2026-05-21T08:57:07.246021266Z 2026-05-21 08:57:07.245 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-05-21T08:57:07.246954828Z 2026-05-21 08:57:07.246 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4074ms
2026-05-21T08:57:07.404078498Z 2026-05-21 08:57:07.403 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=45
2026-05-21T08:57:07.462255901Z 2026-05-21 08:57:07.461 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=13
2026-05-21T08:57:11.151244808Z 2026-05-21 08:57:11.150 INFO (qtp985324122-58-solr-node1-6) [c: s: r: x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard1_replica_n1&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-05-21T08:57:11.205941757Z 2026-05-21 08:57:11.205 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-21T08:57:11.275041608Z 2026-05-21 08:57:11.274 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-05-21T08:57:11.666958666Z 2026-05-21 08:57:11.666 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-21T08:57:11.667051166Z 2026-05-21 08:57:11.666 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n1' using configuration from configset techproducts, trusted=true
2026-05-21T08:57:11.687814217Z 2026-05-21 08:57:11.687 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n1], dataDir=[/var/solr/data/test-collection_shard1_replica_n1/data/]
2026-05-21T08:57:12.147704749Z 2026-05-21 08:57:12.147 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-21T08:57:12.169102946Z 2026-05-21 08:57:12.168 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-21T08:57:12.173076232Z 2026-05-21 08:57:12.172 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-21T08:57:12.174317632Z 2026-05-21 08:57:12.173 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-21T08:57:12.174335157Z 2026-05-21 08:57:12.173 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-21T08:57:12.216039289Z 2026-05-21 08:57:12.215 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-21T08:57:12.219327312Z 2026-05-21 08:57:12.219 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-21T08:57:12.221595099Z 2026-05-21 08:57:12.221 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-21T08:57:12.223598000Z 2026-05-21 08:57:12.223 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-21T08:57:12.236413325Z 2026-05-21 08:57:12.235 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4}
2026-05-21T08:57:12.245808473Z 2026-05-21 08:57:12.245 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-21T08:57:12.253307538Z 2026-05-21 08:57:12.252 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n1 solr-node1-6 core_node3 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-21T08:57:12.253326160Z 2026-05-21 08:57:12.252 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n1 solr-node1-6 core_node3 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-21T08:57:12.259542492Z 2026-05-21 08:57:12.259 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n1 solr-node1-6 core_node3 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node3 x:test-collection_shard1_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-21T08:57:12.2737
[...truncated too long message...]
igned certificate (SHA384withRSA) with a validity of 90 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Certificate was added to keystore
# Certificate reply was installed in keystore
# Generating 3,072 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 90 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Certificate was added to keystore
# Certificate was added to keystore
# Certificate reply was installed in keystore
# Certificate was added to keystore
# Certificate was added to keystore
# Generating 3,072 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 90 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Certificate was added to keystore
# Certificate was added to keystore
# Certificate reply was installed in keystore
# Certificate was added to keystore
# Certificate was added to keystore
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #6 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/92-6
# Last output:
# INFO - 2026-05-21 08:58:15.611; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.keyStorePassword
# INFO - 2026-05-21 08:58:15.614; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.trustStorePassword
# WARN - 2026-05-21 08:58:15.625; org.apache.solr.common.util.EnvUtils; You are passing in deprecated system property host and should upgrade to using solr.host.advertise instead. The deprecated property support will be removed in future version of Solr.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is https://localhost:46519.
#
# ERROR: IOException occurred when talking to server at: https://localhost:46519/solr/admin/info/system
not ok 93 test keystore reload # in 2709 ms
# (in test file test/test_ssl.bats, line 529)
# `solr start --jvm-opts "-Dsolr.jetty.ssl.context.reload.scan.interval.secs=1 -DsocketTimeout=5000"' failed
# Generating 2,048 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 9,999 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Generating 2,048 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 9,999 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #7 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/93-7
# Last output:
# INFO - 2026-05-21 08:58:18.338; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.keyStorePassword
# INFO - 2026-05-21 08:58:18.341; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.trustStorePassword
# WARN - 2026-05-21 08:58:18.352; org.apache.solr.common.util.EnvUtils; You are passing in deprecated system property host and should upgrade to using solr.host.advertise instead. The deprecated property support will be removed in future version of Solr.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is https://localhost:46519.
#
# ERROR: IOException occurred when talking to server at: https://localhost:46519/solr/admin/info/system
not ok 94 SOLR-11740 check 'solr stop' connection # in 244 ms
# (in test file test/test_start_solr.bats, line 32)
# `solr start' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/94-1
not ok 95 stop command for single port # in 272 ms
# (in test file test/test_start_solr.bats, line 45)
# `solr start' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/95-2
ok 96 check stop command doesn't hang # in 240 ms
not ok 97 SOLR-16976 solr starts with remote JMX enabled # in 250 ms
# (in test file test/test_start_solr.bats, line 77)
# `solr start' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #4 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/97-4
not ok 98 deprecated system properties converted to modern properties # in 250 ms
# (in test file test/test_start_solr.bats, line 85)
# `solr start -Ddisable.config.edit=true' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #5 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/98-5
not ok 99 start with custom jetty options # in 247 ms
# (in test file test/test_start_solr.bats, line 93)
# `solr start --jettyconfig "--module=server"' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #6 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/99-6
not ok 100 bootstrapping a configset # in 243 ms
# (in test file test/test_start_solr.bats, line 104)
# `solr start -Dbootstrap_confdir="${confdir_path}" -Dcollection.configName=techproducts' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #7 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/100-7
not ok 101 status detects locally running solr # in 1018 ms
# (in test file test/test_status.bats, line 34)
# `solr start' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/101-1
# Last output:
#
# No Solr nodes are running.
not ok 102 status with --solr-url from user # in 228 ms
# (in test file test/test_status.bats, line 43)
# `solr start' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/102-2
not ok 103 status with --port from user # in 230 ms
# (in test file test/test_status.bats, line 50)
# `solr start' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #3 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/103-3
ok 104 multiple connection options are prevented # in 894 ms
ok 105 status with invalid --solr-url from user # in 1301 ms
not ok 106 status with --short format # in 234 ms
# (in test file test/test_status.bats, line 67)
# `solr start' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #6 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/106-6
not ok 107 setup_file failed
# (from function `setup_file' in test file test/test_stream.bats, line 22)
# `solr start -e techproducts' failed
#
# Starting up Solr on port 46519 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 46519 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server" -Dsolr.modules=clustering,extraction,langid,ltr,scripting -Dsolr.ltr.enabled=true -Dsolr.clustering.enabled=true
#
#
# ERROR: Port 46519 is already being used by another process.
#
ok 111 --version returns Solr version # in 773 ms
ok 112 version as direct tool call still runs # in 770 ms
not ok 113 setup_file failed
# (from function `setup_file' in test file test/test_zk.bats, line 22)
# `solr start -DminStateByteLenForCompression=0' failed
#
# Port 46519 is already being used by another process (pid: 1324609)
# Please choose a different port using the -p option.
#
ok 125 Cleanup # in 1899 ms
Build timed out (after 1,217 minutes). Marking the build as aborted.
Build timed out (after 1,217 minutes). Marking the build as failed.
Build was aborted
Archiving artifacts
Terminated
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 214 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/214/
No tests ran.
Build Log:
[...truncated 372 lines...]
# ERROR: Solr is not running on url http://localhost:35101 after 5 seconds
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/4-2
# Last output:
# Waiting up to 180 seconds to see Solr running on port 35101 [|] [/] [-] [\] [|]
# Started Solr server on port 35101 (pid=1188051). Happy searching!
#
#
ok 5 auth rejects block-unknown option with invalid boolean # in 1780 ms
ok 6 auth rejects update-include-file-only option with invalid boolean # in 1768 ms
not ok 7 auth enable/disable lifecycle # in 20618 ms
# (in test file test/test_auth.bats, line 40)
# `solr auth enable --type basicAuth --credentials name:password' failed
# Waiting up to 180 seconds to see Solr running on port 35101 [|] [/] [-] [\] [|]
# Started Solr server on port 35101 (pid=1192605). Happy searching!
#
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
# Couldn't get ZooKeeper host. Please make sure Solr is running in cloud mode, or a zk-host has been passed in.
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
#
# ERROR: Server refused connection at: http://localhost:35101/solr/admin/info/system
not ok 8 enable auth connects via zookeeper # in 1216 ms
# (in test file test/test_auth.bats, line 53)
# `solr start' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
#
# ERROR: Error from server at http://localhost:35101/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 9 create, config, api, and delete with basic auth # in 16635 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-05-19 22:08:10.193; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-19 22:08:10.196; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 10 post, postlogs and export with basic auth # in 16615 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/10-2
# Last output:
# WARN - 2026-05-19 22:08:26.837; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-19 22:08:26.840; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 11 package with basic auth # in 16646 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #3 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/11-3
# Last output:
# WARN - 2026-05-19 22:08:43.497; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-19 22:08:43.500; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 12 admin UI static content is excepted from basic auth # in 16654 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #4 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/12-4
# Last output:
# WARN - 2026-05-19 22:09:00.166; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-19 22:09:00.169; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
not ok 13 setup_file failed
# (from function `setup_file' in test file test/test_bats.bats, line 26)
# `solr start --verbose' failed
# Using Solr root directory: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT
# Using Java: /home/jenkins/tools/java/latest21/bin/java
# openjdk version "21.0.4" 2024-07-16 LTS
# OpenJDK Runtime Environment Temurin-21.0.4+7 (build 21.0.4+7-LTS)
# OpenJDK 64-Bit Server VM Temurin-21.0.4+7 (build 21.0.4+7-LTS, mixed mode, sharing)
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
not ok 14 setup_file failed
# (from function `setup_file' in test file test/test_compression.bats, line 22)
# `solr start -e films' failed
#
# Starting up Solr on port 35101 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 35101 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 35101 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test # at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/-
not ok 17 setup_file failed
# (from function `setup_file' in test file test/test_config.bats, line 22)
# `solr start' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
not ok 20 create for non cloud mode # in 1424 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_create.bats, line 34)
# `assert_output --partial "Created new core 'COLL_NAME'"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Created new core 'COLL_NAME'
# output (17 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
#
# ERROR: Error from server at http://localhost:35101/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/20-1
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
#
# ERROR: Error from server at http://localhost:35101/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 21 create for cloud mode # in 1387 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_create.bats, line 40)
# `assert_output --partial "Created collection 'COLL_NAME'"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Created collection 'COLL_NAME'
# output (17 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
#
# ERROR: Error from server at http://localhost:35101/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/21-2
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
#
# ERROR: Error from server at http://localhost:35101/solr/admin/info/system?wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
ok 22 multiple connection options are prevented # in 808 ms
not ok 23 setup_file failed
# (from function `setup_file' in test file test/test_create_collection.bats, line 22)
# `solr start' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
not ok 34 setup_file failed
# (from function `setup_file' in test file test/test_delete_collection.bats, line 22)
# `solr start' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
not ok 39 start -e cloud works with --jvm-opts # in 1223 ms
# (in test file test/test_example.bats, line 32)
# `solr start -e cloud --no-prompt --jvm-opts "-Dcustom.prop=helloworld"' failed
#
# Welcome to the SolrCloud example!
#
# Starting up 2 Solr nodes for your example SolrCloud cluster.
#
# Creating Solr home directory /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr
# Cloning /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1 into
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node2
#
# Starting up Solr on port 35101 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 35101 --solr-home "build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server" --jvm-opts "-Dcustom.prop=helloworld"
#
#
# ERROR: Port 35101 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/39-1
not ok 40 start -e cloud works with --no-prompt # in 1236 ms
# (in test file test/test_example_noprompt.bats, line 32)
# `solr start -e cloud --no-prompt' failed
#
# Welcome to the SolrCloud example!
#
# Starting up 2 Solr nodes for your example SolrCloud cluster.
#
# Solr home directory /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr already exists.
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/example/cloud/node2 already exists.
#
# Starting up Solr on port 35101 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 35101 --solr-home "build/solr-10.1.0-SNAPSHOT/example/cloud/node1/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 35101 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/40-1
not ok 41 Check export command # in 2865 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_export.bats, line 36)
# `assert_output --partial 'Export complete'' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Export complete
# output (48 lines):
# WARN - 2026-05-19 22:09:29.988; org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider; Attempt to fetch live_nodes from http://localhost:35101/solr failed.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:35101/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1194)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.submitClusterStateRequest(BaseHttpClusterStateProvider.java:221)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:293)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.updateLiveNodes(BaseHttpClusterStateProvider.java:280)
# at java.base/java.util.stream.MatchOps$1MatchSink.accept(MatchOps.java:90)
# at java.base/java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1685)
# at java.base/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129)
# at java.base/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527)
# at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
# at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:230)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:196)
# at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
# at java.base/java.util.stream.ReferencePipeline.anyMatch(ReferencePipeline.java:632)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:268)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getLiveNodes(BaseHttpClusterStateProvider.java:231)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getState(BaseHttpClusterStateProvider.java:109)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionRef(CloudSolrClient.java:1723)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.loadDocCollection(CloudSolrClient.java:1699)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$triggerCollectionRefresh$6(CloudSolrClient.java:1664)
# at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1768)
# at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
# at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
# at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
# at java.base/java.lang.Thread.run(Thread.java:1583)
#
# ERROR: Error refreshing state for collection techproducts
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/41-1
# Last output:
# WARN - 2026-05-19 22:09:29.988; org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider; Attempt to fetch live_nodes from http://localhost:35101/solr failed.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:35101/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin: Expected mime type in [application/vnd.apache.solr.javabin, application/octet-stream] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/collections?action=CLUSTERSTATUS&includeAll=false&liveNodes=true&prs=true&wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1194)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.submitClusterStateRequest(BaseHttpClusterStateProvider.java:221)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:293)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.updateLiveNodes(BaseHttpClusterStateProvider.java:280)
# at java.base/java.util.stream.MatchOps$1MatchSink.accept(MatchOps.java:90)
# at java.base/java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1685)
# at java.base/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129)
# at java.base/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527)
# at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
# at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:230)
# at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:196)
# at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
# at java.base/java.util.stream.ReferencePipeline.anyMatch(ReferencePipeline.java:632)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.fetchLiveNodes(BaseHttpClusterStateProvider.java:268)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getLiveNodes(BaseHttpClusterStateProvider.java:231)
# at org.apache.solr.client.solrj.impl.BaseHttpClusterStateProvider.getState(BaseHttpClusterStateProvider.java:109)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionRef(CloudSolrClient.java:1723)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.loadDocCollection(CloudSolrClient.java:1699)
# at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$triggerCollectionRefresh$6(CloudSolrClient.java:1664)
# at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1768)
# at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
# at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
# at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
# at java.base/java.lang.Thread.run(Thread.java:1583)
#
# ERROR: Error refreshing state for collection techproducts
ok 42 export fails on non cloud mode # in 2699 ms
Waiting for Tika Server to be ready on port 35106
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 43 using curl to extract a single pdf file # in 16589 ms
WARNING: Docker not available; bypassing test.
ok 44 using the bin/solr post tool to extract content from pdf # in 16539 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to crawl web site # in 16516 ms
Stopping Tika Server container
not ok 46 healthcheck on cloud solr # in 1228 ms
# (in test file test/test_healthcheck.bats, line 32)
# `solr start -e films' failed
#
# Starting up Solr on port 35101 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 35101 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 35101 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/46-1
not ok 47 healthcheck errors on standalone solr # in 1254 ms
# (in test file test/test_healthcheck.bats, line 39)
# `solr start --user-managed -e films' failed
#
# Starting up Solr on port 35101 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start --user-managed -p 35101 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server"
#
#
# ERROR: Port 35101 is already being used by another process.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/47-2
ok 48 solr help flag prints help # in 714 ms
ok 49 solr with no flags prints help # in 717 ms
ok 50 start help flag prints help # in 143 ms
ok 51 start h flag prints help # in 143 ms
ok 52 stop help flag prints help # in 142 ms
ok 53 restart help flag prints help # in 144 ms
ok 54 status help flag prints help # in 784 ms
ok 55 healthcheck help flag prints help # in 737 ms
ok 56 create help flag prints help # in 731 ms
ok 57 delete help flag prints help # in 745 ms
ok 58 zk help flag prints help # in 754 ms
ok 59 auth help flag prints help # in 730 ms
ok 60 assert help flag prints help # in 741 ms
ok 61 post help flag prints help # in 747 ms
not ok 62 SQL Module # in 18841 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_modules.bats, line 36)
# `assert_output --partial '"docs":'' failed
#
# -- output does not contain substring --
# substring (1 lines):
# "docs":
# output (16 lines):
#
# ERROR: Error from server at http://localhost:35101/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json: Expected mime type in [application/json] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/62-1
# WARN - 2026-05-19 22:10:37.958; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-19 22:10:37.961; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-19 22:10:53.686; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
#
# ERROR: Error from server at http://localhost:35101/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json: Expected mime type in [application/json] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/COLL_NAME/sql?stmt=select+id+from+COLL_NAME+limit+10&wt=json</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 63 icu collation in analysis-extras module # in 17798 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_modules.bats, line 49)
# `assert_output --partial "Created collection 'COLL_NAME'"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Created collection 'COLL_NAME'
# output (17 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
#
# ERROR: Error from server at http://localhost:35101/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/63-2
# WARN - 2026-05-19 22:10:55.784; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-19 22:10:55.787; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
#
# ERROR: Error from server at http://localhost:35101/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 64 Check lifecycle of sentiment classification # in 11468 ms
# (in test file test/test_opennlp.bats, line 63)
# `solr start -m 4g -Dsolr.modules=analysis-extras -Dsolr.packages.enabled=true' failed
# Downloading onnx model and vocab...
# % Total % Received % Xferd Average Speed Time Time Time Current
# Dload Upload Total Spent Left Speed
#
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 160M 0 112k 0 0 108k 0 0:25:16 0:00:01 0:25:15 108k
4 160M 4 7536k 0 0 3758k 0 0:00:43 0:00:02 0:00:41 3760k
17 160M 17 27.5M 0 0 9359k 0 0:00:17 0:00:03 0:00:14 9359k
29 160M 29 46.6M 0 0 11.6M 0 0:00:13 0:00:04 0:00:09 11.6M
41 160M 41 66.5M 0 0 13.2M 0 0:00:12 0:00:05 0:00:07 13.2M
53 160M 53 86.7M 0 0 14.3M 0 0:00:11 0:00:06 0:00:05 17.3M
65 160M 65 105M 0 0 15.0M 0 0:00:10 0:00:07 0:00:03 19.6M
78 160M 78 125M 0 0 15.6M 0 0:00:10 0:00:08 0:00:02 19.6M
89 160M 89 144M 0 0 15.9M 0 0:00:10 0:00:09 0:00:01 19.4M
100 160M 100 160M 0 0 16.3M 0 0:00:09 0:00:09 --:--:-- 19.4M
# % Total % Received % Xferd Average Speed Time Time Time Current
# Dload Upload Total Spent Left Speed
#
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
24 851k 24 208k 0 0 176k 0 0:00:04 0:00:01 0:00:03 175k
100 851k 100 851k 0 0 593k 0 0:00:01 0:00:01 --:--:-- 593k
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/64-1
# Last output:
# total 64M
# drwxr-xr-x 2 jenkins jenkins 4.0K May 19 21:56 .
# drwxr-xr-x 3 jenkins jenkins 4.0K May 19 21:56 ..
# -rw-r--r-- 1 jenkins jenkins 14M May 19 21:56 icu4j-77.1.jar
# -rw-r--r-- 1 jenkins jenkins 83K May 19 21:56 lucene-analysis-icu-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 30K May 19 21:56 lucene-analysis-morfologik-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 40K May 19 21:56 lucene-analysis-opennlp-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 3.5M May 19 21:56 lucene-analysis-smartcn-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 508K May 19 21:56 lucene-analysis-stempel-10.4.0.jar
# -rw-r--r-- 1 jenkins jenkins 19K May 19 21:56 morfologik-fsa-2.1.9.jar
# -rw-r--r-- 1 jenkins jenkins 1.8M May 19 21:56 morfologik-polish-2.1.9.jar
# -rw-r--r-- 1 jenkins jenkins 51K May 19 21:56 morfologik-stemming-2.1.9.jar
# -rw-r--r-- 1 jenkins jenkins 4.0M May 19 21:56 morfologik-ukrainian-search-4.9.1.jar
# -rw-r--r-- 1 jenkins jenkins 39M May 19 21:56 onnxruntime-1.24.3.jar
# -rw-r--r-- 1 jenkins jenkins 28K May 19 21:56 opennlp-dl-2.5.8.jar
# -rw-r--r-- 1 jenkins jenkins 1.4M May 19 21:56 opennlp-tools-2.5.8.jar
# -rw-r--r-- 1 jenkins jenkins 39K May 19 21:56 solr-analysis-extras-10.1.0-SNAPSHOT.jar
not ok 65 lifecycle of package # in 2174 ms
# (from function `assert_output' in file .gradle/node/node_modules/bats-assert/src/assert.bash, line 247,
# in test file test/test_packages.bats, line 38)
# `assert_output --partial "Available packages:"' failed
#
# -- output does not contain substring --
# substring (1 lines):
# Available packages:
# output (45 lines):
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:35101/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:348)
# at org.apache.solr.cli.StatusTool.reportStatus(StatusTool.java:300)
# at org.apache.solr.cli.CLIUtils.getZkHost(CLIUtils.java:248)
# at org.apache.solr.cli.PackageTool.runImpl(PackageTool.java:123)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
#
# ERROR: Error from server at http://localhost:35101/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
# --
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/65-1
# Last output:
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
# org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://localhost:35101/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
# at org.apache.solr.client.solrj.impl.HttpSolrClient.checkContentType(HttpSolrClient.java:308)
# at org.apache.solr.client.solrj.impl.HttpSolrClient.processErrorsAndResponse(HttpSolrClient.java:228)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.processErrorsAndResponse(HttpJettySolrClient.java:595)
# at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:489)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
# at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:348)
# at org.apache.solr.cli.StatusTool.reportStatus(StatusTool.java:300)
# at org.apache.solr.cli.CLIUtils.getZkHost(CLIUtils.java:248)
# at org.apache.solr.cli.PackageTool.runImpl(PackageTool.java:123)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
#
# ERROR: Error from server at http://localhost:35101/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
not ok 66 deploying and undeploying a collection level package # in 1274 ms
# (in test file test/test_packages.bats, line 44)
# `solr create -c foo-1.2' failed
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is http://localhost:35101.
#
# ERROR: Error from server at http://localhost:35101/solr/admin/info/system?wt=javabin: Expected mime type in [application/octet-stream, application/vnd.apache.solr.javabin] but got text/html. <html>
# <head>
# <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
# <title>Error 503 Service Unavailable</title>
# </head>
# <body>
# <h2>HTTP ERROR 503 Service Unavailable</h2>
# <table>
# <tr><th>URI:</th><td>http://localhost:35101/solr/admin/info/system?wt=javabin</td></tr>
# <tr><th>STATUS:</th><td>503</td></tr>
# <tr><th>MESSAGE:</th><td>Service Unavailable</td></tr>
# </table>
#
# </body>
# </html>
#
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/66-2
# Last output:
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
ok 67 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
not ok 68 Affinity placement plugin using sysprop # in 21752 ms
# (in test file test/test_placement_plugin.bats, line 34)
# `solr assert --cloud http://localhost:${SOLR_PORT} --timeout 3000' failed with status 100
#
# ERROR: Solr is not running on url http://localhost:35101 after 3 seconds
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/68-1
# WARN - 2026-05-19 22:11:32.951; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-19 22:11:32.954; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# ERROR - 2026-05-19 22:11:48.687; org.apache.solr.cli.ZkLsTool; Could not complete ls operation for reason:
# org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
# at org.apache.solr.common.cloud.SolrZkClient$Builder.build(SolrZkClient.java:1188)
# at org.apache.solr.cli.CLIUtils.getSolrZkClient(CLIUtils.java:273)
# at org.apache.solr.cli.ZkLsTool.runImpl(ZkLsTool.java:58)
# at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81)
# at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117)
# Caused by: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
# at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:203)
# ... 6 more
# Last output:
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
not ok 69 Random placement plugin using ENV # in 21788 ms
# (in test file test/test_placement_plugin.bats, line 43)
# `solr assert --cloud http://localhost:${SOLR_PORT} --timeout 3000' failed with status 100
#
# ERROR: Solr is not running on url http://localhost:35101 after 3 seconds
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/69-2
# WARN - 2026-05-19 22:11:54.775; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-19 22:11:54.778; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: java.util.concurrent.TimeoutException: Timeout while waiting for Zookeeper Client to connect: 15000 ms
#
# Last output:
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
not ok 70 setup_file failed
# (from function `setup_file' in test file test/test_post.bats, line 22)
# `solr start -Dsolr.modules=extraction' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
not ok 81 setup_file failed
# (from function `setup_file' in test file test/test_postlogs.bats, line 22)
# `solr start' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 16 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-05-19T22:12:20.650402259Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-19T22:12:20.650439705Z It should be set to 65000 to avoid operational disruption.
2026-05-19T22:12:20.650443052Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-19T22:12:20.650756848Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-19T22:12:20.650767947Z
2026-05-19T22:12:20.660161603Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-19T22:12:20.668316661Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-19T22:12:20.715530731Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-19T22:12:20.747390613Z WARNING: A command line option has enabled the Security Manager
2026-05-19T22:12:20.747413929Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-19T22:12:21.960701095Z 2026-05-19 22:12:21.957 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-19T22:12:22.229809982Z 2026-05-19 22:12:22.229 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-19T22:12:22.255431166Z 2026-05-19 22:12:22.254 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-19T22:12:22.262756937Z 2026-05-19 22:12:22.262 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-19T22:12:22.262893482Z 2026-05-19 22:12:22.262 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-05-19T22:12:22.262980795Z 2026-05-19 22:12:22.262 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-19T22:12:22.263140984Z 2026-05-19 22:12:22.262 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-19T22:12:22.262525981Z
2026-05-19T22:12:22.265819503Z 2026-05-19 22:12:22.265 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-19T22:12:22.288571097Z 2026-05-19 22:12:22.288 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-19T22:12:22.301706382Z 2026-05-19 22:12:22.301 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-19T22:12:22.301780631Z 2026-05-19 22:12:22.301 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-19T22:12:22.336539438Z 2026-05-19 22:12:22.335 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-19T22:12:23.253447544Z 2026-05-19 22:12:23.252 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-19T22:12:23.486222181Z 2026-05-19 22:12:23.485 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-19T22:12:23.491062067Z 2026-05-19 22:12:23.490 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-05-19T22:12:23.491171915Z 2026-05-19 22:12:23.490 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-05-19T22:12:23.493683999Z 2026-05-19 22:12:23.493 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-05-19T22:12:23.493857809Z 2026-05-19 22:12:23.493 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-05-19T22:12:23.528167890Z 2026-05-19 22:12:23.527 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-05-19T22:12:23.997662455Z 2026-05-19 22:12:23.996 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-05-19T22:12:24.011465276Z 2026-05-19 22:12:24.011 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-19T22:12:24.047821438Z 2026-05-19 22:12:24.047 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-19T22:12:24.139315218Z 2026-05-19 22:12:24.138 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-19T22:12:24.139442391Z 2026-05-19 22:12:24.138 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-19T22:12:24.270256052Z 2026-05-19 22:12:24.269 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-19T22:12:24.293999262Z 2026-05-19 22:12:24.293 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-19T22:12:24.306384587Z 2026-05-19 22:12:24.305 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-05-19T22:12:24.312547596Z 2026-05-19 22:12:24.312 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72083928546934784-solr-node1:8983_solr-n_0000000000) starting
2026-05-19T22:12:24.378155556Z 2026-05-19 22:12:24.377 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-05-19T22:12:24.379465990Z 2026-05-19 22:12:24.379 INFO (OverseerStateUpdate-72083928546934784-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-05-19T22:12:24.388303930Z 2026-05-19 22:12:24.387 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-19T22:12:24.430201594Z 2026-05-19 22:12:24.429 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-19T22:12:24.806071611Z 2026-05-19 22:12:24.803 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-19T22:12:24.982754690Z 2026-05-19 22:12:24.982 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-19T22:12:25.246245520Z 2026-05-19 22:12:25.245 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-19T22:12:25.377732296Z 2026-05-19 22:12:25.376 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-19T22:12:25.382977736Z 2026-05-19 22:12:25.382 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_19.request.log
2026-05-19T22:12:25.387124477Z 2026-05-19 22:12:25.386 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-05-19T22:12:25.388758728Z 2026-05-19 22:12:25.388 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4722ms
2026-05-19T22:12:25.773981575Z 2026-05-19 22:12:25.772 INFO (qtp985324122-27-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=139
2026-05-19T22:12:25.851373628Z 2026-05-19 22:12:25.850 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=18
2026-05-19T22:12:29.120638201Z 2026-05-19 22:12:29.120 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-05-19T22:12:29.282099130Z 2026-05-19 22:12:29.281 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-19T22:12:32.945220271Z 2026-05-19 22:12:32.944 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=42
2026-05-19T22:12:32.974500448Z 2026-05-19 22:12:32.973 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=10
2026-05-19T22:12:33.001126228Z 2026-05-19 22:12:33.000 INFO (qtp985324122-27-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=2
2026-05-19T22:12:33.240084618Z 2026-05-19 22:12:33.239 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=5
2026-05-19T22:12:33.268132262Z 2026-05-19 22:12:33.267 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-05-19T22:12:33.287399109Z 2026-05-19 22:12:33.286 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-05-19T22:12:33.293921422Z 2026-05-19 22:12:33.293 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 174] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-19T22:12:33.338977784Z 2026-05-19 22:12:33.338 INFO (OverseerStateUpdate-72083928546934784-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-19T22:12:33.338999437Z "core":"test-collection_shard3_replica_n1",
2026-05-19T22:12:33.339002014Z "node_name":"solr-node1:8983_solr",
2026-05-19T22:12:33.339004732Z "base_url":"http://solr-node1:8983/solr",
2026-05-19T22:12:33.339006848Z "collection":"test-collection",
2026-05-19T22:12:33.339008911Z "shard":"shard3",
2026-05-19T22:12:33.339010877Z "state":"down",
2026-05-19T22:12:33.339012893Z "type":"NRT",
2026-05-19T22:12:33.339014789Z "operation":"ADDREPLICA",
2026-05-19T22:12:33.339016919Z "waitForFinalState":"false"}
2026-05-19T22:12:33.345043975Z 2026-05-19 22:12:33.344 INFO (OverseerStateUpdate-72083928546934784-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-19T22:12:33.345060485Z "core":"test-collection_shard2_replica_n2",
2026-05-19T22:12:33.345062953Z "node_name":"solr-node3:8985_solr",
2026-05-19T22:12:33.345065153Z "base_url":"http://solr-node3:8985/solr",
2026-05-19T22:12:33.345067179Z "collection":"test-collection",
2026-05-19T22:12:33.345069216Z "shard":"shard2",
2026-05-19T22:12:33.345071243Z "state":"down",
2026-05-19T22:12:33.345073125Z "type":"NRT",
2026-05-19T22:12:33.345075149Z "operation":"ADDREPLICA",
2026-05-19T22:12:33.345077143Z "waitForFinalState":"false"}
2026-05-19T22:12:33.349220674Z 2026-05-19 22:12:33.348 INFO (OverseerStateUpdate-72083928546934784-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-19T22:12:33.349245585Z "core":"test-collection_shard1_replica_n4",
2026-05-19T22:12:33.349248382Z "node_name":"solr-node2:8984_solr",
2026-05-19T22:12:33.349254933Z "base_url":"http://solr-node2:8984/solr",
2026-05-19T22:12:33.349257016Z "collection":"test-collection",
2026-05-19T22:12:33.349259147Z "shard":"shard1",
2026-05-19T22:12:33.349261170Z "state":"down",
2026-05-19T22:12:33.349263156Z "type":"NRT",
2026-05-19T22:12:33.349265088Z "operation":"ADDREPLICA",
2026-05-19T22:12:33.349267150Z "waitForFinalState":"false"}
2026-05-19T22:12:33.457452923Z 2026-05-19 22:12:33.457 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 186] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-19T22:12:33.540219204Z 2026-05-19 22:12:33.539 INFO (qtp985324122-109-solr-node1-8) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-05-19T22:12:33.629798779Z 2026-05-19 22:12:33.629 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-19T22:12:33.690390903Z 2026-05-19 22:12:33.689 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-05-19T22:12:34.151572215Z 2026-05-19 22:12:34.151 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-19T22:12:34.151693348Z 2026-05-19 22:12:34.151 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true
2026-05-19T22:12:34.190371714Z 2026-05-19 22:12:34.189 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/]
2026-05-19T22:12:35.453905708Z 2026-05-19 22:12:35.453 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-19T22:12:35.509244863Z 2026-05-19 22:12:35.508 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-19T22:12:35.518987157Z 2026-05-19 22:12:35.518 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-19T22:12:35.521358940Z 2026-05-19 22:12:35.520 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-19T22:12:35.521373642Z 2026-05-19 22:12:35.521 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-19T22:12:35.604772235Z 2026-05-19 22:12:35.604 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-19T22:12:35.647193179Z 2026-05-19 22:12:35.646 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-19T22:12:35.650102534Z 2026-05-19 22:12:35.649 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-19T22:12:35.652403563Z 2026-05-19 22:12:35.652 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-19T22:12:35.667120261Z 2026-05-19 22:12:35.666 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-05-19T22:12:35.675052460Z 2026-05-19 22:12:35.674 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-19T22:12:35.681145601Z 2026-05-19 22:12:35.680 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-19T22:12:35.681189251Z 2026-05-19 22:12:35.680 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-19T22:12:35.689456392Z 2026-05-19 22:12:35.689 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-19T22:12:35.695763371Z 2026-05-19 22:12:35.695 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-05-19T22:12:35.698485297Z 2026-05-19 22:12:35.698 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-05-19T22:12:35.711527868Z 2026-05-19 22:12:35.711 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-05-19T22:12:35.711567242Z 2026-05-19 22:12:35.711 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-05-19T22:12:35.711638583Z 2026-05-19 22:12:35.711 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/
2026-05-19T22:12:35.713473741Z 2026-05-19 22:12:35.713 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-05-19T22:12:35.714055056Z 2026-05-19 22:12:35.713 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas
2026-05-19T22:12:35.722091802Z 2026-05-19 22:12:35.721 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72083928546934784-core_node3-n_0000000000
2026-05-19T22:12:35.731659678Z 2026-05-19 22:12:35.731 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3
2026-05-19T22:12:35.836074794Z 2026-05-19 22:12:35.835 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 240] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-19T22:12:35.840169130Z 2026-05-19 22:12:35.839 INFO (qtp985324122-109-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-05-19T22:12:35.848947791Z 2026-05-19 22:12:35.848 INFO (qtp985324122-109-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2310
2026-05-19T22:12:35.858735565Z 2026-05-19 22:12:35.858 INFO (qtp985324122-101-solr-node1-7) [c:test-collection s: r: x: t:solr-node1-7] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
2026-05-19T22:12:35.948653554Z 2026-05-19 22:12:35.948 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 246] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-19T22:12:35.951115590Z 2026-05-19 22:12:35.950 INFO (qtp985324122-101-solr-node1-7) [c:test-collection s: r: x: t:solr-node1-7] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=2699
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1200844,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-05-19T22:12:20.448436377Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-05-19T22:12:26.149586451Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-19T22:12:26.149618137Z It should be set to 65000 to avoid operational disruption.
2026-05-19T22:12:26.149621114Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-19T22:12:26.149973846Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-19T22:12:26.149989473Z
2026-05-19T22:12:26.159656735Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-19T22:12:26.167533260Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-19T22:12:26.218720341Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-19T22:12:26.252778945Z WARNING: A command line option has enabled the Security Manager
2026-05-19T22:12:26.252796819Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-19T22:12:27.401321453Z 2026-05-19 22:12:27.397 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-19T22:12:27.676509006Z 2026-05-19 22:12:27.675 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-19T22:12:27.702346710Z 2026-05-19 22:12:27.701 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-19T22:12:27.710013392Z 2026-05-19 22:12:27.709 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-19T22:12:27.710034828Z 2026-05-19 22:12:27.709 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-05-19T22:12:27.710037708Z 2026-05-19 22:12:27.709 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-19T22:12:27.710050226Z 2026-05-19 22:12:27.709 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-19T22:12:27.709504888Z
2026-05-19T22:12:27.712960307Z 2026-05-19 22:12:27.712 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-19T22:12:27.735849388Z 2026-05-19 22:12:27.735 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-19T22:12:27.760147962Z 2026-05-19 22:12:27.759 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-19T22:12:27.900750413Z 2026-05-19 22:12:27.900 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-05-19T22:12:27.919192007Z 2026-05-19 22:12:27.918 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-19T22:12:27.919275063Z 2026-05-19 22:12:27.918 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-19T22:12:27.919398377Z 2026-05-19 22:12:27.918 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-19T22:12:28.029270090Z 2026-05-19 22:12:28.028 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-05-19T22:12:28.029620036Z 2026-05-19 22:12:28.029 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-19T22:12:28.029767883Z 2026-05-19 22:12:28.029 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-19T22:12:28.066228375Z 2026-05-19 22:12:28.065 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-19T22:12:28.805546061Z 2026-05-19 22:12:28.804 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-19T22:12:28.987939804Z 2026-05-19 22:12:28.986 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-19T22:12:28.989630883Z 2026-05-19 22:12:28.986 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-05-19T22:12:28.999131343Z 2026-05-19 22:12:28.998 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-19T22:12:29.007478745Z 2026-05-19 22:12:29.007 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-19T22:12:29.022767372Z 2026-05-19 22:12:29.022 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-19T22:12:29.022931942Z 2026-05-19 22:12:29.022 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-19T22:12:29.054228155Z 2026-05-19 22:12:29.053 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-19T22:12:29.063033403Z 2026-05-19 22:12:29.062 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-19T22:12:29.085509164Z 2026-05-19 22:12:29.084 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-19T22:12:29.098261407Z 2026-05-19 22:12:29.097 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-05-19T22:12:29.110123366Z 2026-05-19 22:12:29.109 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-05-19T22:12:29.121612223Z 2026-05-19 22:12:29.121 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-05-19T22:12:29.164764854Z 2026-05-19 22:12:29.164 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-19T22:12:29.282742160Z 2026-05-19 22:12:29.282 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-19T22:12:29.484396018Z 2026-05-19 22:12:29.483 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-19T22:12:29.683041148Z 2026-05-19 22:12:29.682 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-19T22:12:29.954326008Z 2026-05-19 22:12:29.953 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-19T22:12:30.106404492Z 2026-05-19 22:12:30.105 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-19T22:12:30.112513092Z 2026-05-19 22:12:30.112 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_19.request.log
2026-05-19T22:12:30.117038210Z 2026-05-19 22:12:30.116 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-05-19T22:12:30.118727652Z 2026-05-19 22:12:30.118 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @3952ms
2026-05-19T22:12:30.292278958Z 2026-05-19 22:12:30.291 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=45
2026-05-19T22:12:30.358641821Z 2026-05-19 22:12:30.357 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=16
2026-05-19T22:12:33.539339469Z 2026-05-19 22:12:33.538 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-05-19T22:12:33.600846183Z 2026-05-19 22:12:33.600 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-19T22:12:33.671787136Z 2026-05-19 22:12:33.671 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config
2026-05-19T22:12:33.978136343Z 2026-05-19 22:12:33.977 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-19T22:12:33.978160853Z 2026-05-19 22:12:33.977 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-05-19T22:12:33.996910999Z 2026-05-19 22:12:33.996 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-05-19T22:12:34.433932444Z 2026-05-19 22:12:34.433 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-19T22:12:34.453771654Z 2026-05-19 22:12:34.453 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-19T22:12:34.457289781Z 2026-05-19 22:12:34.456 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-19T22:12:34.458470036Z 2026-05-19 22:12:34.457 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-19T22:12:34.458484698Z 2026-05-19 22:12:34.458 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-19T22:12:34.501474342Z 2026-05-19 22:12:34.501 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-19T22:12:34.504499282Z 2026-05-19 22:12:34.504 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-19T22:12:34.506506555Z 2026-05-19 22:12:34.506 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-19T22:12:34.508359172Z 2026-05-19 22:12:34.508 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-19T22:12:34.520640891Z 2026-05-19 22:12:34.520 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-05-19T22:12:34.528660701Z 2026-05-19 22:12:34.528 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-19T22:12:34.536151011Z 2026-05-19 22:12:34.535 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-19T22:12:34.536170911Z 2026-05-19 22:12:34.535 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-19T22:12:34.542622238Z 2026-05-19 22:12:34.542 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-19T22:12:34.554274395Z 2026-05-19 22:12:34.553 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-05-19T22:12:34.555698378Z 2026-05-19 22:12:34.555 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1
2026-05-19T22:12:34.571558131Z 2026-05-19 22:12:34.571 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to
[...truncated too long message...]
pair and self-signed certificate (SHA384withRSA) with a validity of 90 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Certificate was added to keystore
# Certificate reply was installed in keystore
# Generating 3,072 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 90 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Certificate was added to keystore
# Certificate was added to keystore
# Certificate reply was installed in keystore
# Certificate was added to keystore
# Certificate was added to keystore
# Generating 3,072 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 90 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Certificate was added to keystore
# Certificate was added to keystore
# Certificate reply was installed in keystore
# Certificate was added to keystore
# Certificate was added to keystore
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #6 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/92-6
# Last output:
# INFO - 2026-05-19 22:13:38.120; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.keyStorePassword
# INFO - 2026-05-19 22:13:38.123; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.trustStorePassword
# WARN - 2026-05-19 22:13:38.134; org.apache.solr.common.util.EnvUtils; You are passing in deprecated system property host and should upgrade to using solr.host.advertise instead. The deprecated property support will be removed in future version of Solr.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is https://localhost:35101.
#
# ERROR: IOException occurred when talking to server at: https://localhost:35101/solr/admin/info/system
not ok 93 test keystore reload # in 2579 ms
# (in test file test/test_ssl.bats, line 529)
# `solr start --jvm-opts "-Dsolr.jetty.ssl.context.reload.scan.interval.secs=1 -DsocketTimeout=5000"' failed
# Generating 2,048 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 9,999 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
# Generating 2,048 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 9,999 days
# for: CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #7 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/93-7
# Last output:
# INFO - 2026-05-19 22:13:40.715; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.keyStorePassword
# INFO - 2026-05-19 22:13:40.718; org.apache.solr.util.configuration.SSLConfigurations; Setting javax.net.ssl.trustStorePassword
# WARN - 2026-05-19 22:13:40.730; org.apache.solr.common.util.EnvUtils; You are passing in deprecated system property host and should upgrade to using solr.host.advertise instead. The deprecated property support will be removed in future version of Solr.
# Neither --zk-host or --solr-url parameters, nor ZK_HOST env var provided, so assuming solr url is https://localhost:35101.
#
# ERROR: IOException occurred when talking to server at: https://localhost:35101/solr/admin/info/system
not ok 94 SOLR-11740 check 'solr stop' connection # in 224 ms
# (in test file test/test_start_solr.bats, line 32)
# `solr start' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/94-1
not ok 95 stop command for single port # in 232 ms
# (in test file test/test_start_solr.bats, line 45)
# `solr start' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/95-2
ok 96 check stop command doesn't hang # in 225 ms
not ok 97 SOLR-16976 solr starts with remote JMX enabled # in 233 ms
# (in test file test/test_start_solr.bats, line 77)
# `solr start' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #4 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/97-4
not ok 98 deprecated system properties converted to modern properties # in 233 ms
# (in test file test/test_start_solr.bats, line 85)
# `solr start -Ddisable.config.edit=true' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #5 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/98-5
not ok 99 start with custom jetty options # in 227 ms
# (in test file test/test_start_solr.bats, line 93)
# `solr start --jettyconfig "--module=server"' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #6 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/99-6
not ok 100 bootstrapping a configset # in 233 ms
# (in test file test/test_start_solr.bats, line 104)
# `solr start -Dbootstrap_confdir="${confdir_path}" -Dcollection.configName=techproducts' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #7 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/100-7
not ok 101 status detects locally running solr # in 965 ms
# (in test file test/test_status.bats, line 34)
# `solr start' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/101-1
# Last output:
#
# No Solr nodes are running.
not ok 102 status with --solr-url from user # in 222 ms
# (in test file test/test_status.bats, line 43)
# `solr start' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #2 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/102-2
not ok 103 status with --port from user # in 227 ms
# (in test file test/test_status.bats, line 50)
# `solr start' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #3 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/103-3
ok 104 multiple connection options are prevented # in 881 ms
ok 105 status with invalid --solr-url from user # in 1307 ms
not ok 106 status with --short format # in 234 ms
# (in test file test/test_status.bats, line 67)
# `solr start' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #6 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/106-6
not ok 107 setup_file failed
# (from function `setup_file' in test file test/test_stream.bats, line 22)
# `solr start -e techproducts' failed
#
# Starting up Solr on port 35101 using command:
# "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/bin/solr" start -p 35101 --solr-home "build/solr-10.1.0-SNAPSHOT/server/solr" --server-dir "/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/solr-10.1.0-SNAPSHOT/server" -Dsolr.modules=clustering,extraction,langid,ltr,scripting -Dsolr.ltr.enabled=true -Dsolr.clustering.enabled=true
#
#
# ERROR: Port 35101 is already being used by another process.
#
ok 111 --version returns Solr version # in 747 ms
ok 112 version as direct tool call still runs # in 751 ms
not ok 113 setup_file failed
# (from function `setup_file' in test file test/test_zk.bats, line 22)
# `solr start -DminStateByteLenForCompression=0' failed
#
# Port 35101 is already being used by another process (pid: 1192605)
# Please choose a different port using the -p option.
#
ok 125 Cleanup # in 1887 ms
Build timed out (after 303 minutes). Marking the build as aborted.
Build timed out (after 303 minutes). Marking the build as failed.
Build was aborted
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 213 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/213/
No tests ran.
Build Log:
[...truncated 347 lines...]
# ERROR: KeeperErrorCode = NoNode for /security.json
# --
#
# Last output:
# WARN - 2026-05-18 17:44:37.788; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-18 17:44:37.792; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-18 17:44:38.258; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-18 17:44:38.258; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: KeeperErrorCode = NoNode for /security.json
not ok 9 create, config, api, and delete with basic auth # in 1325 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 46655 is already being used by another process (pid: 1491086)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-05-18 17:44:39.463; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-18 17:44:39.467; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
ok 10 post, postlogs and export with basic auth # in 17715 ms
ok 11 package with basic auth # in 12368 ms
ok 12 admin UI static content is excepted from basic auth # in 10737 ms
ok 13 nothing # in 10 ms
ok 14 server does not compress response without Accept-Encoding header # in 145 ms
ok 15 server compresses response when Accept-Encoding: gzip is requested # in 78 ms
ok 16 compressed response can be decompressed and parsed # in 79 ms
ok 17 setting property # in 11208 ms
ok 18 short form of setting property # in 6561 ms
ok 19 connecting to solr via various solr urls and zk hosts # in 9308 ms
ok 20 create for non cloud mode # in 7879 ms
ok 21 create for cloud mode # in 9774 ms
ok 22 multiple connection options are prevented # in 799 ms
ok 23 create collection # in 7605 ms
ok 24 create collection using solrUrl # in 5264 ms
ok 25 create collection using legacy solrUrl # in 5406 ms
ok 26 create collection using Zookeeper # in 5252 ms
ok 27 reject d option with invalid config dir # in 2695 ms
ok 28 accept d option with builtin config # in 5121 ms
ok 29 accept d option with explicit path to config # in 5245 ms
ok 30 accept n option as config name # in 5159 ms
ok 31 allow config reuse when n option specifies same config # in 8983 ms
ok 32 create multisharded collections when s provided # in 5078 ms
ok 33 create replicated collections when rf provided # in 6050 ms
ok 34 can delete collections # in 10574 ms
ok 35 can delete collections with solr-url # in 8321 ms
ok 36 collection delete also deletes zk config # in 7234 ms
ok 37 deletes accompanying zk config with nondefault name # in 7243 ms
ok 38 delete-config option can opt to leave config in zk # in 7023 ms
ok 39 start -e cloud works with --jvm-opts # in 23929 ms
ok 40 start -e cloud works with --no-prompt # in 46856 ms
ok 41 Check export command # in 19496 ms
ok 42 export fails on non cloud mode # in 9775 ms
Waiting for Tika Server to be ready on port 46660
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 43 using curl to extract a single pdf file # in 16639 ms
WARNING: Docker not available; bypassing test.
ok 44 using the bin/solr post tool to extract content from pdf # in 16569 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to crawl web site # in 16585 ms
Stopping Tika Server container
ok 46 healthcheck on cloud solr # in 10581 ms
ok 47 healthcheck errors on standalone solr # in 11288 ms
ok 48 solr help flag prints help # in 739 ms
ok 49 solr with no flags prints help # in 745 ms
ok 50 start help flag prints help # in 146 ms
ok 51 start h flag prints help # in 147 ms
ok 52 stop help flag prints help # in 144 ms
ok 53 restart help flag prints help # in 147 ms
ok 54 status help flag prints help # in 794 ms
ok 55 healthcheck help flag prints help # in 754 ms
ok 56 create help flag prints help # in 769 ms
ok 57 delete help flag prints help # in 760 ms
ok 58 zk help flag prints help # in 785 ms
ok 59 auth help flag prints help # in 761 ms
ok 60 assert help flag prints help # in 761 ms
ok 61 post help flag prints help # in 762 ms
ok 62 SQL Module # in 15730 ms
ok 63 icu collation in analysis-extras module # in 12577 ms
ok 64 Check lifecycle of sentiment classification # in 23712 ms
ok 65 lifecycle of package # in 8842 ms
ok 66 deploying and undeploying a collection level package # in 13660 ms
ok 67 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
ok 68 Affinity placement plugin using sysprop # in 15332 ms
ok 69 Random placement plugin using ENV # in 15351 ms
ok 70 Check help command # in 2445 ms
ok 71 basic post with a type specified # in 5431 ms
ok 72 basic post with collection instead of url parameter # in 4070 ms
ok 73 basic post with solr-url and collection # in 4054 ms
ok 74 basic post WITHOUT a type specified # in 11569 ms
ok 75 crawling a directory as a dry-run # in 792 ms
ok 76 crawling a directory # in 4454 ms
ok 77 skipcommit and optimize and delete # in 3890 ms
ok 78 args mode # in 6917 ms
ok 79 stdin mode # in 3512 ms
ok 80 verbose echo the Solr response # in 3855 ms
ok 81 post solr log into solr via cli # in 9354 ms
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 10 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 16 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-05-18T17:54:53.686231180Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-18T17:54:53.686268053Z It should be set to 65000 to avoid operational disruption.
2026-05-18T17:54:53.686271123Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-18T17:54:53.686552110Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-18T17:54:53.686564062Z
2026-05-18T17:54:53.696322220Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-18T17:54:53.703857425Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-18T17:54:53.748370063Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-18T17:54:53.782413146Z WARNING: A command line option has enabled the Security Manager
2026-05-18T17:54:53.782439175Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-18T17:54:54.881051302Z 2026-05-18 17:54:54.877 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-18T17:54:55.149476988Z 2026-05-18 17:54:55.148 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-18T17:54:55.174271480Z 2026-05-18 17:54:55.173 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-18T17:54:55.181097296Z 2026-05-18 17:54:55.180 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-18T17:54:55.181150739Z 2026-05-18 17:54:55.180 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-05-18T17:54:55.181296354Z 2026-05-18 17:54:55.180 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-18T17:54:55.181393996Z 2026-05-18 17:54:55.180 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-18T17:54:55.180890227Z
2026-05-18T17:54:55.183856000Z 2026-05-18 17:54:55.183 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-18T17:54:55.204698339Z 2026-05-18 17:54:55.204 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-18T17:54:55.217514135Z 2026-05-18 17:54:55.217 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-18T17:54:55.217615905Z 2026-05-18 17:54:55.217 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-18T17:54:55.251169392Z 2026-05-18 17:54:55.250 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-18T17:54:55.970084748Z 2026-05-18 17:54:55.969 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-18T17:54:56.211204373Z 2026-05-18 17:54:56.210 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-18T17:54:56.216123931Z 2026-05-18 17:54:56.215 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-05-18T17:54:56.216174087Z 2026-05-18 17:54:56.215 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-05-18T17:54:56.218568815Z 2026-05-18 17:54:56.218 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-05-18T17:54:56.218648932Z 2026-05-18 17:54:56.218 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-05-18T17:54:56.254239808Z 2026-05-18 17:54:56.253 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-05-18T17:54:56.744245252Z 2026-05-18 17:54:56.742 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-05-18T17:54:56.761366154Z 2026-05-18 17:54:56.760 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-18T17:54:56.808236732Z 2026-05-18 17:54:56.807 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-18T17:54:56.854070099Z 2026-05-18 17:54:56.853 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-18T17:54:56.854137446Z 2026-05-18 17:54:56.853 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-18T17:54:56.991117997Z 2026-05-18 17:54:56.990 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-18T17:54:57.019969053Z 2026-05-18 17:54:57.019 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-18T17:54:57.037341974Z 2026-05-18 17:54:57.036 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-05-18T17:54:57.043767485Z 2026-05-18 17:54:57.043 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72077321711386624-solr-node1:8983_solr-n_0000000000) starting
2026-05-18T17:54:57.113514654Z 2026-05-18 17:54:57.112 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-05-18T17:54:57.114006597Z 2026-05-18 17:54:57.113 INFO (OverseerStateUpdate-72077321711386624-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-05-18T17:54:57.127536057Z 2026-05-18 17:54:57.127 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-18T17:54:57.171319104Z 2026-05-18 17:54:57.170 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-18T17:54:57.558244263Z 2026-05-18 17:54:57.556 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-18T17:54:57.755461485Z 2026-05-18 17:54:57.754 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-18T17:54:58.031023488Z 2026-05-18 17:54:58.030 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-18T17:54:58.230448553Z 2026-05-18 17:54:58.229 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-18T17:54:58.310541276Z 2026-05-18 17:54:58.310 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_18.request.log
2026-05-18T17:54:58.314896830Z 2026-05-18 17:54:58.314 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-05-18T17:54:58.315719700Z 2026-05-18 17:54:58.315 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4613ms
2026-05-18T17:54:58.537260797Z 2026-05-18 17:54:58.536 INFO (qtp985324122-27-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=51
2026-05-18T17:54:58.604081089Z 2026-05-18 17:54:58.603 INFO (qtp985324122-97-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=18
2026-05-18T17:55:02.205195073Z 2026-05-18 17:55:02.204 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-05-18T17:55:02.341371875Z 2026-05-18 17:55:02.340 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-18T17:55:06.378794292Z 2026-05-18 17:55:06.377 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=11
2026-05-18T17:55:06.406475114Z 2026-05-18 17:55:06.406 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=9
2026-05-18T17:55:06.428603085Z 2026-05-18 17:55:06.428 INFO (qtp985324122-27-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=2
2026-05-18T17:55:07.034970450Z 2026-05-18 17:55:07.034 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-05-18T17:55:07.052261281Z 2026-05-18 17:55:07.051 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-05-18T17:55:07.057152304Z 2026-05-18 17:55:07.056 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 174] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T17:55:07.096433474Z 2026-05-18 17:55:07.095 INFO (OverseerStateUpdate-72077321711386624-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-18T17:55:07.096454887Z "core":"test-collection_shard3_replica_n1",
2026-05-18T17:55:07.096457601Z "node_name":"solr-node1:8983_solr",
2026-05-18T17:55:07.096459986Z "base_url":"http://solr-node1:8983/solr",
2026-05-18T17:55:07.096462534Z "collection":"test-collection",
2026-05-18T17:55:07.096464564Z "shard":"shard3",
2026-05-18T17:55:07.096466652Z "state":"down",
2026-05-18T17:55:07.096468646Z "type":"NRT",
2026-05-18T17:55:07.096470626Z "operation":"ADDREPLICA",
2026-05-18T17:55:07.096472643Z "waitForFinalState":"false"}
2026-05-18T17:55:07.123779621Z 2026-05-18 17:55:07.123 INFO (OverseerStateUpdate-72077321711386624-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-18T17:55:07.123801645Z "core":"test-collection_shard2_replica_n2",
2026-05-18T17:55:07.123804355Z "node_name":"solr-node3:8985_solr",
2026-05-18T17:55:07.123806744Z "base_url":"http://solr-node3:8985/solr",
2026-05-18T17:55:07.123808870Z "collection":"test-collection",
2026-05-18T17:55:07.123811075Z "shard":"shard2",
2026-05-18T17:55:07.123813046Z "state":"down",
2026-05-18T17:55:07.123814998Z "type":"NRT",
2026-05-18T17:55:07.123816909Z "operation":"ADDREPLICA",
2026-05-18T17:55:07.123818867Z "waitForFinalState":"false"}
2026-05-18T17:55:07.134052260Z 2026-05-18 17:55:07.133 INFO (OverseerStateUpdate-72077321711386624-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-18T17:55:07.134075800Z "core":"test-collection_shard1_replica_n4",
2026-05-18T17:55:07.134079918Z "node_name":"solr-node2:8984_solr",
2026-05-18T17:55:07.134095472Z "base_url":"http://solr-node2:8984/solr",
2026-05-18T17:55:07.134098556Z "collection":"test-collection",
2026-05-18T17:55:07.134100756Z "shard":"shard1",
2026-05-18T17:55:07.134102727Z "state":"down",
2026-05-18T17:55:07.134104630Z "type":"NRT",
2026-05-18T17:55:07.134106673Z "operation":"ADDREPLICA",
2026-05-18T17:55:07.134108654Z "waitForFinalState":"false"}
2026-05-18T17:55:07.255470862Z 2026-05-18 17:55:07.254 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 186] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T17:55:07.333274958Z 2026-05-18 17:55:07.332 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-05-18T17:55:07.451483591Z 2026-05-18 17:55:07.451 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-18T17:55:07.539721695Z 2026-05-18 17:55:07.539 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-05-18T17:55:07.952882258Z 2026-05-18 17:55:07.952 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-18T17:55:07.952984570Z 2026-05-18 17:55:07.952 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true
2026-05-18T17:55:07.999083497Z 2026-05-18 17:55:07.998 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/]
2026-05-18T17:55:08.864815124Z 2026-05-18 17:55:08.863 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-18T17:55:08.890159587Z 2026-05-18 17:55:08.889 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-18T17:55:08.893989232Z 2026-05-18 17:55:08.893 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-18T17:55:08.894757808Z 2026-05-18 17:55:08.894 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-18T17:55:08.894773442Z 2026-05-18 17:55:08.894 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-18T17:55:08.990866246Z 2026-05-18 17:55:08.989 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-18T17:55:08.995997559Z 2026-05-18 17:55:08.995 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-18T17:55:08.998115687Z 2026-05-18 17:55:08.997 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-18T17:55:09.008466588Z 2026-05-18 17:55:09.008 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-18T17:55:09.074497675Z 2026-05-18 17:55:09.073 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-05-18T17:55:09.083872190Z 2026-05-18 17:55:09.083 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-18T17:55:09.091482147Z 2026-05-18 17:55:09.091 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-18T17:55:09.091502814Z 2026-05-18 17:55:09.091 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-18T17:55:09.102962362Z 2026-05-18 17:55:09.102 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-18T17:55:09.114470976Z 2026-05-18 17:55:09.108 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-05-18T17:55:09.116162180Z 2026-05-18 17:55:09.115 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-05-18T17:55:09.129390148Z 2026-05-18 17:55:09.129 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-05-18T17:55:09.129408210Z 2026-05-18 17:55:09.129 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-05-18T17:55:09.129449080Z 2026-05-18 17:55:09.129 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/
2026-05-18T17:55:09.130718771Z 2026-05-18 17:55:09.130 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-05-18T17:55:09.131263539Z 2026-05-18 17:55:09.131 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas
2026-05-18T17:55:09.132329602Z 2026-05-18 17:55:09.132 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72077321711386624-core_node3-n_0000000000
2026-05-18T17:55:09.141171208Z 2026-05-18 17:55:09.140 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3
2026-05-18T17:55:09.248031757Z 2026-05-18 17:55:09.247 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 238] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T17:55:09.251495757Z 2026-05-18 17:55:09.251 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-05-18T17:55:09.259886925Z 2026-05-18 17:55:09.259 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1930
2026-05-18T17:55:09.360239712Z 2026-05-18 17:55:09.359 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1523611,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-05-18T17:54:53.441260974Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-05-18T17:54:58.929867108Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-18T17:54:58.929902702Z It should be set to 65000 to avoid operational disruption.
2026-05-18T17:54:58.929905759Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-18T17:54:58.930191646Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-18T17:54:58.930201938Z
2026-05-18T17:54:58.939747916Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-18T17:54:58.948651155Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-18T17:54:58.993830494Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-18T17:54:59.028273039Z WARNING: A command line option has enabled the Security Manager
2026-05-18T17:54:59.028305374Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-18T17:55:00.165091264Z 2026-05-18 17:55:00.161 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-18T17:55:00.440605034Z 2026-05-18 17:55:00.440 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-18T17:55:00.466814133Z 2026-05-18 17:55:00.466 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-18T17:55:00.474577085Z 2026-05-18 17:55:00.473 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-18T17:55:00.474709164Z 2026-05-18 17:55:00.474 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-05-18T17:55:00.474866447Z 2026-05-18 17:55:00.474 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-18T17:55:00.475031642Z 2026-05-18 17:55:00.474 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-18T17:55:00.474210137Z
2026-05-18T17:55:00.477997472Z 2026-05-18 17:55:00.477 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-18T17:55:00.501692610Z 2026-05-18 17:55:00.501 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-18T17:55:00.525860489Z 2026-05-18 17:55:00.525 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-18T17:55:00.673526853Z 2026-05-18 17:55:00.672 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-05-18T17:55:00.696689159Z 2026-05-18 17:55:00.696 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-18T17:55:00.696794271Z 2026-05-18 17:55:00.696 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-18T17:55:00.696916380Z 2026-05-18 17:55:00.696 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-18T17:55:00.807748927Z 2026-05-18 17:55:00.807 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-05-18T17:55:00.807982972Z 2026-05-18 17:55:00.807 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-18T17:55:00.808066733Z 2026-05-18 17:55:00.807 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-18T17:55:00.844633814Z 2026-05-18 17:55:00.843 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-18T17:55:01.839202236Z 2026-05-18 17:55:01.838 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-18T17:55:02.073865990Z 2026-05-18 17:55:02.073 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-18T17:55:02.076017125Z 2026-05-18 17:55:02.073 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-05-18T17:55:02.087053651Z 2026-05-18 17:55:02.086 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-18T17:55:02.092931604Z 2026-05-18 17:55:02.092 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-18T17:55:02.096944102Z 2026-05-18 17:55:02.096 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-18T17:55:02.096958320Z 2026-05-18 17:55:02.096 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-18T17:55:02.133648931Z 2026-05-18 17:55:02.133 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-18T17:55:02.142841675Z 2026-05-18 17:55:02.142 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-18T17:55:02.167813665Z 2026-05-18 17:55:02.167 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-18T17:55:02.182197454Z 2026-05-18 17:55:02.181 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-05-18T17:55:02.194771674Z 2026-05-18 17:55:02.194 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-05-18T17:55:02.208545400Z 2026-05-18 17:55:02.208 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-05-18T17:55:02.255649797Z 2026-05-18 17:55:02.255 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-18T17:55:02.340778399Z 2026-05-18 17:55:02.340 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-18T17:55:02.534789449Z 2026-05-18 17:55:02.534 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-18T17:55:02.724979929Z 2026-05-18 17:55:02.724 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-18T17:55:03.006618759Z 2026-05-18 17:55:03.006 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-18T17:55:03.178752878Z 2026-05-18 17:55:03.177 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-18T17:55:03.186574440Z 2026-05-18 17:55:03.186 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_18.request.log
2026-05-18T17:55:03.191038891Z 2026-05-18 17:55:03.190 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-05-18T17:55:03.193812224Z 2026-05-18 17:55:03.192 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4245ms
2026-05-18T17:55:03.415410459Z 2026-05-18 17:55:03.414 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=80
2026-05-18T17:55:03.470594967Z 2026-05-18 17:55:03.470 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=14
2026-05-18T17:55:07.332272081Z 2026-05-18 17:55:07.331 INFO (qtp985324122-58-solr-node1-6) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-05-18T17:55:07.388533494Z 2026-05-18 17:55:07.387 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-18T17:55:07.449137314Z 2026-05-18 17:55:07.448 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-05-18T17:55:07.728321097Z 2026-05-18 17:55:07.727 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-18T17:55:07.728350295Z 2026-05-18 17:55:07.727 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-05-18T17:55:07.748379827Z 2026-05-18 17:55:07.747 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-05-18T17:55:08.195958761Z 2026-05-18 17:55:08.195 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-18T17:55:08.214067524Z 2026-05-18 17:55:08.213 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-18T17:55:08.217500722Z 2026-05-18 17:55:08.217 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-18T17:55:08.218589353Z 2026-05-18 17:55:08.217 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-18T17:55:08.218689072Z 2026-05-18 17:55:08.218 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-18T17:55:08.347723784Z 2026-05-18 17:55:08.347 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-18T17:55:08.350869728Z 2026-05-18 17:55:08.350 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-18T17:55:08.352805901Z 2026-05-18 17:55:08.352 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-18T17:55:08.358080671Z 2026-05-18 17:55:08.357 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-18T17:55:08.370918380Z 2026-05-18 17:55:08.370 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01}
2026-05-18T17:55:08.379171434Z 2026-05-18 17:55:08.378 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-18T17:55:08.385337104Z 2026-05-18 17:55:08.384 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-18T17:55:08.385358826Z 2026-05-18 17:55:08.385 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-18T17:55:08.392417284Z 2026-05-18 17:55:08.392 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-18T17:55:08.403700644Z 2026-05-18 17:55:08.403 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-05-18T17:55:08.405138366Z 2026-05-18 17:55:08.404 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1
2026-05-18T17:55:08.418572930Z 2026-05-18 17:55:08.418 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-05-18T17:55:08.418591094Z 2026-05-18 17:55:08.418 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-05-18T17:55:08.418630189Z 2026-05-18 17:55:08.418 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node2:8984/solr/test-collection_shard1_replica_n4/
2026-05-18T17:55:08.419902953Z 2026-05-18 17:55:08.419 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-05-18T17:55:08.420428074Z 2026-05-18 17:55:08.420 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ has no replicas
2026-05-18T17:55:08.421520075Z 2026-05-18 17:55:08.421 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard1/leader after winning as /collections/test-collection/leader_elect/shard1/election/72077321711386627-core_node6-n_0000000000
2026-05-18T17:55:08.430352900Z 2026-05-18 17:55:08.430 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ shard1
2026-05-18T17:55:08.660118193Z 2026-05-18 17:55:08.659 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 219] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T17:55:08.671922012Z 2026-05-18 17:55:08.671 INFO (qtp985324122-58-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-05-18T17:55:08.681599879Z 2026-05-18 17:55:08.681 INFO (qtp985324122-58-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1350
2026-05-18T17:55:08.785791241Z 2026-05-18 17:55:08.785 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 223] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T17:55:09.248687339Z 2026-05-18 17:55:09.247 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 238] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T17:55:09.360481388Z 2026-05-18 17:55:09.360 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node2 ===
{
"Name": "/solr-node2",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1524007,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-05-18T17:54:58.696092154Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node3 ===
2026-05-18T17:54:59.166797714Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-18T17:54:59.166835540Z It should be set to 65000 to avoid operational disruption.
2026-05-18T17:54:59.166838783Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-18T17:54:59.166840983Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-18T17:54:59.166843039Z
2026-05-18T17:54:59.176043753Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-18T17:54:59.183584949Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-18T17:54:59.229451439Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-18T17:54:59.260472131Z WARNING: A command line option has enabled the Security Manager
2026-05-18T17:54:59.260498568Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-18T17:55:00.514307514Z 2026-05-18 17:55:00.510 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-18T17:55:00.789848095Z 2026-05-18 17:55:00.789 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-18T17:55:00.818102900Z 2026-05-18 17:55:00.817 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-18T17:55:00.826287890Z 2026-05-18 17:55:00.825 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-18T17:55:00.826318218Z 2026-05-18 17:55:00.825 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8985
2026-05-18T17:55:00.826485432Z 2026-05-18 17:55:00.825 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-18T17:55:00.826696919Z 2026-05-18 17:55:00.825 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-18T17:55:00.825833707Z
2026-05-18T17:55:00.829242963Z 2026-05-18 17:55:00.828 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-18T17:55:00.853005733Z 2026-05-18 17:55:00.852 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-18T17:55:00.881766925Z 2026-05-18 17:55:00.881 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-18T17:55:01.026128490Z 2026-05-18 17:55:01.025 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-05-18T17:55:01.045259261Z 2026-05-18 17:55:01.044 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-18T17:55:01.045408507Z 2026-05-18 17:55:01.044 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-18T17:55:01.045552339Z 2026-05-18 17:55:01.044 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-18T17:55:01.155496627Z 2026-05-18 17:55:01.154 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-05-18T17:55:01.155905292Z 2026-05-18 17:55:01.155 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-18T17:55:01.156036272Z 2026-05-18 17:55:01.155 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-18T17:55:01.194578953Z 2026-05-18 17:55:01.193 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-18T17:55:02.059298521Z 2026-05-18 17:55:02.058 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-18T17:55:02.208974260Z 2026-05-18 17:55:02.208 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-18T17:55:02.210447117Z 2026-05-18 17:55:02.208 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-05-18T17:55:02.219574345Z 2026-05-18 17:55:02.219 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-18T17:55:02.224503810Z 2026-05-18 17:55:02.224 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-18T17:55:02.228081261Z 2026-05-18 17:55:02.227 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-18T17:55:02.228165714Z 2026-05-18 17:55:02.227 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-18T17:55:02.274929428Z 2026-05-18 17:55:02.274 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-18T17:55:02.283245324Z 2026-05-18 17:55:02.282 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
2026-05-18T17:55:02.309065104Z 2026-05-18 17:55:02.308 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-18T17:55:02.320814848Z 2026-05-18 17:55:02.320 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node3:8985_solr as DOWN
2026-05-18T17:55:02.332017933Z 2026-05-18 17:55:02.331 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node3:8985_solr
2026-05-18T17:55:02.342359391Z 2026-05-18 17:55:02.341 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-18T17:55:02.386020214Z 2026-05-18 17:55:02.385 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-18T17:55:02.658646622Z 2026-05-18 17:55:02.658 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-18T17:55:02.841147285Z 2026-05-18 17:55:02.840 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-18T17:55:03.100850947Z 2026-05-18 17:55:03.100 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-18T17:55:03.238453855Z 2026-05-18 17:55:03.237 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-18T17:55:03.244777135Z 2026-05-18 17:55:03.244 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_18.request.log
2026-05-18T17:55:03.248836444Z 2026-05-18 17:55:03.248 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8985}
2026-05-18T17:55:03.249725051Z 2026-05-18 17:55:03.249 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4067ms
2026-05-18T17:55:05.210914081Z 2026-05-18 17:55:05.203 INFO (qtp985324122-27-solr-node3-1) [c: s: r: x: t:solr-node3-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=71
2026-05-18T17:55:05.281130200Z 2026-05-18 17:55:05.280 INFO (qtp985324122-56-solr-node3-2) [c: s: r: x: t:solr-node3-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=21
2026-05-18T17:55:07.002525708Z 2026-05-18 17:55:07.001 INFO (qtp985324122-58-solr-node3-3) [c: s: r: x: t:solr-node3-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=11
2026-05-18T17:55:07.333966271Z 2026-05-18 17:55:07.333 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node5&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&shard=shard2&wt=javabin
2026-05-18T17:55:07.437018456Z 2026-05-18 17:55:07.436 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-18T17:55:07.508169025Z 2026-05-18 17:55:07.507 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-05-18T17:55:07.824947566Z 2026-05-18 17:55:07.824 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-18T17:55:07.825013791Z 2026-05-18 17:55:07.824 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard2_replica_n2' using configuration from configset techproducts, trusted=true
2026-05-18T17:55:07.844378910Z 2026-05-18 17:55:07.843 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard2_replica_n2], dataDir=[/var/solr/data/test-collection_shard2_replica_n2/data/]
2026-05-18T17:55:08.307582600Z 2026-05-18 17:55:08.307 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-18T17:55:08.330406375Z 2026-05-18 17:55:08.329 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-18T17:55:08.338781013Z 2026-05-18 17:55:08.338 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-18T17:55:08.340149988Z 2026-05-18 17:55:08.339 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-18T17:55:08.340205396Z 2026-05-18 17:55:08.339 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-18T17:55:08.390636916Z 2026-05-18 17:55:08.390 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-18T17:55:08.393807635Z 2026-05-18 17:55:08.393 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-18T17:55:08.395645664Z 2026-05-18 17:55:08.395 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-18T17:55:08.398210937Z 2026-05-18 17:55:08.397 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-18T17:55:08.410731683Z 2026-05-18 17:55:08.410 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-05-18T17:55:08.419124009Z 2026-05-18 17:55:08.418 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-18T17:55:08.425980796Z 2026-05-18 17:55:08.425 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-18T17:55:08.426019363Z 2026-05-18 17:55:08.425 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-18T17:55:08.432156347Z 2026-05-18 17:55:08.431 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-18T17:55:08.441432526Z 2026-05-18 17:55:08.440 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard2 to Terms{values={core_node5=0}, version=0} for registerTerm
2026-05-18T17:55:08.443035910Z 2026-05-18 17:55:08.442 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard2
2026-05-18T17:55:08.510132545Z 2026-05-18 17:55:08.509 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-05-18T17:55:08.510186194Z 2026-05-18 17:55:08.509 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-05-18T17:55:08.510190663Z 2026-05-18 17:55:08.509 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node3:8985/solr/test-collection_shard2_replica_n2/
2026-05-18T17:55:08.512478153Z 2026-05-18 17:55:08.512 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-05-18T17:55:08.513282771Z 2026-05-18 17:55:08.512 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ has no replicas
2026-05-18T17:55:08.514805824Z 2026-05-18 17:55:08.514 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard2/leader after winning as /collections/test-collection/leader_elect/shard2/election/72077321711386628-core_node5-n_0000000000
2026-05-18T17:55:08.530894674Z 2026-05-18 17:55:08.530 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ shard2
2026-05-18T17:55:08.660117198Z 2026-05-18 17:55:08.659 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 219] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T17:55:08.671889710Z 2026-05-18 17:55:08.671 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-05-18T17:55:08.682514737Z 2026-05-18 17:55:08.682 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=techproducts&newCollection=true&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&collection=test-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1350
2026-05-18T17:55:08.785370234Z 2026-05-18 17:55:08.784 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 223] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T17:55:09.248721995Z 2026-05-18 17:55:09.247 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 238] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T17:55:09.274080687Z 2026-05-18 17:55:09.273 INFO (qtp985324122-59-solr-node3-4) [c:test-collection s: r: x: t:solr-node3-4] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
2026-05-18T17:55:09.360582411Z 2026-05-18 17:55:09.360 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 244] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T17:55:09.363506702Z 2026-05-18 17:55:09.363 INFO (qtp985324122-59-solr-node3-4) [c:test-collection s: r: x: t:solr-node3-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=2348
# === Docker inspect for solr-node3 ===
{
"Name": "/solr-node3",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1524190,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-05-18T17:54:58.92905747Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# Docker artifacts saved to: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/docker
not ok 82 Docker SolrCloud rolling upgrade # in 27353 ms
# (in test file test/test_rolling_upgrade.bats, line 118)
# `wait_for 30 1 docker exec solr-node1 solr healthcheck -c test-collection' failed with status 127
# 9.10.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# af6eca94c810: Pulling fs layer
# cb0efb96dabd: Pulling fs layer
# 3e9d91201f40: Pulling fs layer
# 66b76b382631: Pulling fs layer
# 601f2c23751f: Pulling fs layer
# 0ed719aef393: Pulling fs layer
# 69a09c96e8bf: Pulling fs layer
# c937aee9a870: Pulling fs layer
# 9cdfe5320de8: Pulling fs layer
# 3aa9120e46f5: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# 66b76b382631: Waiting
# 601f2c23751f: Waiting
# 0ed719aef393: Waiting
# 69a09c96e8bf: Waiting
# c937aee9a870: Waiting
# 9cdfe5320de8: Waiting
# 3aa9120e46f5: Waiting
# 4f4fb700ef54: Waiting
# cb0efb96dabd: Verifying Checksum
# cb0efb96dabd: Download complete
# af6eca94c810: Verifying Checksum
# af6eca94c810: Download complete
# 66b76b382631: Verifying Checksum
# 66b76b382631: Download complete
# 601f2c23751f: Verifying Checksum
# 601f2c23751f: Download complete
# 3e9d91201f40: Verifying Checksum
# 3e9d91201f40: Download complete
# 69a09c96e8bf: Download complete
# c937aee9a870: Download complete
# 9cdfe5320de8: Verifying Checksum
# 9cdfe5320de8: Download complete
# 3aa9120e46f5: Download complete
# 4f4fb700ef54: Verifying Checksum
# 4f4fb700ef54: Download complete
# 0ed719aef393: Verifying Checksum
# 0ed719aef393: Download complete
# af6eca94c810: Pull complete
# cb0efb96dabd: Pull complete
# 3e9d91201f40: Pull complete
# 66b76b382631: Pull complete
# 601f2c23751f: Pull complete
# 0ed719aef393: Pull complete
# 69a09c96e8bf: Pull complete
# c937aee9a870: Pull complete
# 9cdfe5320de8: Pull complete
# 3aa9120e46f5: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:15c8155ceffe1042f37c77bb537aa531804a3510702ff87f2af5fedf523f4203
# Status: Downloaded newer image for apache/solr-nightly:9.10.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:9.10.0-SNAPSHOT-slim
# 10.0.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# 40d16f30db40: Already exists
# 9d5a0b6cb36c: Already exists
# 49518c28c0b3: Already exists
# 41162acdcc29: Already exists
# 2ed0fd558ca3: Already exists
# 8b0357ffbe54: Pulling fs layer
# 0df26b0d5a69: Pulling fs layer
# a26fc87814fe: Pulling fs layer
# 67860c4a05fe: Pulling fs layer
# d1a380e5a7b1: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# 67860c4a05fe: Waiting
# d1a380e5a7b1: Waiting
# 4f4fb700ef54: Waiting
# 0df26b0d5a69: Verifying Checksum
# 0df26b0d5a69: Download complete
# a26fc87814fe: Verifying Checksum
# a26fc87814fe: Download complete
# 67860c4a05fe: Verifying Checksum
# 67860c4a05fe: Download complete
# d1a380e5a7b1: Verifying Checksum
# d1a380e5a7b1: Download complete
# 4f4fb700ef54: Verifying Checksum
# 4f4fb700ef54: Download complete
# 8b0357ffbe54: Verifying Checksum
# 8b0357ffbe54: Download complete
# 8b0357ffbe54: Pull complete
# 0df26b0d5a69: Pull complete
# a26fc87814fe: Pull complete
# 67860c4a05fe: Pull complete
# d1a380e5a7b1: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:16825c9cb8e05e7a6dd9d4209615781d30bc3137a6017a5254cec93560c7c8ba
# Status: Downloaded newer image for apache/solr-nightly:10.0.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:10.0.0-SNAPSHOT-slim
# e14dc15402a2f8778175fbfc836bdcca228d09aae753cd0b8d83afdfbd5c8776
# solr-data1
# solr-data2
# solr-data3
# Starting solr-node1 with embedded ZooKeeper
# 2d2f713ab0644af8419216c384cf9079af849dc1ede9dda2a47bf36434b01e2a
# Starting solr-node2 connected to first node's ZooKeeper
# b7839594e47ac596038e4430a76278579355b7bef12b9313bf38d7d903e10f50
# Started solr-node3 connected to first node's ZooKeeper
# bfcdd01f992baea4dab026706e43fee4c6a36f34f570fb3ef4d61cbb097796de
# Creating a Collection
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# WARN - 2026-05-18 17:55:06.470; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-18 17:55:06.506; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. "/solr"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [http://solr-node3:8985/solr] to [http://solr-node3:8985/].
# Created collection 'test-collection' with 3 shard(s), 1 replica(s) with config-set 'techproducts'
# Checking collection health
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_rolling_upgrade.bats: line 118: wait_for: command not found
# solr-node1
# solr-node2
# solr-node3
# solr-node1
# solr-node2
# solr-node3
# solr-data1
# solr-data2
# solr-data3
# solrcloud-test
ok 83 allowPaths - backup # in 15529 ms
ok 84 snapshot lifecycle # in 4866 ms
ok 85 snapshot list # in 3490 ms
ok 86 snapshot describe # in 3410 ms
ok 87 start solr with ssl # in 16704 ms
ok 88 use different hostname when not checking peer-name # in 25173 ms
ok 89 start solr with ssl and auth # in 14789 ms
ok 90 start solr with client truststore and security manager # in 17328 ms
ok 91 start solr with mTLS needed # in 48729 ms
ok 92 start solr with mTLS wanted # in 35751 ms
ok 93 test keystore reload # in 61815 ms
ok 94 SOLR-11740 check 'solr stop' connection # in 16097 ms
ok 95 stop command for single port # in 18820 ms
ok 96 check stop command doesn't hang # in 236 ms
ok 97 SOLR-16976 solr starts with remote JMX enabled # in 8790 ms
ok 98 deprecated system properties converted to modern properties # in 195243 ms
ok 99 start with custom jetty options # in 8429 ms
ok 100 bootstrapping a configset # in 9813 ms
ok 101 status detects locally running solr # in 10131 ms
ok 102 status with --solr-url from user # in 8405 ms
ok 103 status with --port from user # in 8280 ms
ok 104 multiple connection options are prevented # in 910 ms
ok 105 status with invalid --solr-url from user # in 1316 ms
ok 106 status with --short format # in 196063 ms
ok 107 searching solr via locally executed streaming expression # in 1717 ms
ok 108 searching solr via remotely executed streaming expression # in 1254 ms
ok 109 variable interpolation # in 1659 ms
ok 110 searching solr without credentials fails with error # in 1182 ms
ok 111 --version returns Solr version # in 754 ms
ok 112 version as direct tool call still runs # in 748 ms
ok 113 short help # in 845 ms
ok 114 short help is inferred # in 817 ms
ok 115 long help # in 809 ms
ok 116 running subcommands with zk is prevented # in 750 ms
ok 117 listing out files # in 2208 ms
ok 118 connecting to solr via various solr urls and zk hosts # in 8053 ms
ok 119 copying files around # in 11104 ms
ok 120 upconfig # in 2646 ms
ok 121 SOLR-12429 test upconfig fails with symlink # in 1131 ms
ok 122 downconfig # in 1195 ms
ok 123 bin/solr zk cp gets 'solrhome' from '--solr-home' command line option # in 1428 ms
ok 124 env var ZK_HOST is honored # in 3204 ms
ok 125 Cleanup # in 1885 ms
> Task :solr:packaging:integrationTests FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build.gradle' line: 339
* What went wrong:
Execution failed for task ':solr:packaging:integrationTests'.
> Process 'command '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/.gradle/node/node_modules/bats/bin/bats'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/8.10/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 29m 41s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/vagfyxc4bxbn6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 212 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/212/
No tests ran.
Build Log:
[...truncated 347 lines...]
# ERROR: KeeperErrorCode = NoNode for /security.json
# --
#
# Last output:
# WARN - 2026-05-18 06:16:54.507; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-18 06:16:54.511; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-18 06:16:54.975; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-18 06:16:54.975; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
#
# ERROR: KeeperErrorCode = NoNode for /security.json
not ok 9 create, config, api, and delete with basic auth # in 1295 ms
# (from function `setup' in test file test/test_basic_auth.bats, line 24)
# `solr start -Dsolr.packages.enabled=true' failed
# Starting Solr
#
# Port 41823 is already being used by another process (pid: 1702748)
# Please choose a different port using the -p option.
#
# Please find the SOLR_HOME snapshot for failed test #1 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/9-1
# Last output:
# WARN - 2026-05-18 06:16:56.140; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-18 06:16:56.143; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
ok 10 post, postlogs and export with basic auth # in 17731 ms
ok 11 package with basic auth # in 12023 ms
ok 12 admin UI static content is excepted from basic auth # in 10653 ms
ok 13 nothing # in 9 ms
ok 14 server does not compress response without Accept-Encoding header # in 145 ms
ok 15 server compresses response when Accept-Encoding: gzip is requested # in 79 ms
ok 16 compressed response can be decompressed and parsed # in 78 ms
ok 17 setting property # in 11019 ms
ok 18 short form of setting property # in 6532 ms
ok 19 connecting to solr via various solr urls and zk hosts # in 9305 ms
ok 20 create for non cloud mode # in 7688 ms
ok 21 create for cloud mode # in 9902 ms
ok 22 multiple connection options are prevented # in 830 ms
ok 23 create collection # in 7770 ms
ok 24 create collection using solrUrl # in 5372 ms
ok 25 create collection using legacy solrUrl # in 5198 ms
ok 26 create collection using Zookeeper # in 5264 ms
ok 27 reject d option with invalid config dir # in 2616 ms
ok 28 accept d option with builtin config # in 5132 ms
ok 29 accept d option with explicit path to config # in 5200 ms
ok 30 accept n option as config name # in 5161 ms
ok 31 allow config reuse when n option specifies same config # in 8958 ms
ok 32 create multisharded collections when s provided # in 5128 ms
ok 33 create replicated collections when rf provided # in 6113 ms
ok 34 can delete collections # in 10786 ms
ok 35 can delete collections with solr-url # in 8548 ms
ok 36 collection delete also deletes zk config # in 7524 ms
ok 37 deletes accompanying zk config with nondefault name # in 7317 ms
ok 38 delete-config option can opt to leave config in zk # in 7098 ms
ok 39 start -e cloud works with --jvm-opts # in 21953 ms
ok 40 start -e cloud works with --no-prompt # in 18483 ms
ok 41 Check export command # in 18774 ms
ok 42 export fails on non cloud mode # in 9326 ms
Waiting for Tika Server to be ready on port 41828
WARNING: Tika Server did not become ready in time; Tika-dependent tests will be bypassed.
WARNING: Docker not available; bypassing test.
ok 43 using curl to extract a single pdf file # in 16822 ms
WARNING: Docker not available; bypassing test.
ok 44 using the bin/solr post tool to extract content from pdf # in 16568 ms
WARNING: Docker not available; bypassing test.
ok 45 using the bin/solr post tool to crawl web site # in 16558 ms
Stopping Tika Server container
ok 46 healthcheck on cloud solr # in 10087 ms
ok 47 healthcheck errors on standalone solr # in 11208 ms
ok 48 solr help flag prints help # in 736 ms
ok 49 solr with no flags prints help # in 757 ms
ok 50 start help flag prints help # in 147 ms
ok 51 start h flag prints help # in 149 ms
ok 52 stop help flag prints help # in 143 ms
ok 53 restart help flag prints help # in 145 ms
ok 54 status help flag prints help # in 825 ms
ok 55 healthcheck help flag prints help # in 747 ms
ok 56 create help flag prints help # in 775 ms
ok 57 delete help flag prints help # in 748 ms
ok 58 zk help flag prints help # in 744 ms
ok 59 auth help flag prints help # in 759 ms
ok 60 assert help flag prints help # in 757 ms
ok 61 post help flag prints help # in 761 ms
ok 62 SQL Module # in 15465 ms
ok 63 icu collation in analysis-extras module # in 12179 ms
ok 64 Check lifecycle of sentiment classification # in 23948 ms
ok 65 lifecycle of package # in 8485 ms
ok 66 deploying and undeploying a collection level package # in 13186 ms
ok 67 deploying and undeploying a cluster level package # skip For developing package infra; requires a connection to github
ok 68 Affinity placement plugin using sysprop # in 15379 ms
ok 69 Random placement plugin using ENV # in 15941 ms
ok 70 Check help command # in 2333 ms
ok 71 basic post with a type specified # in 5506 ms
ok 72 basic post with collection instead of url parameter # in 4014 ms
ok 73 basic post with solr-url and collection # in 3987 ms
ok 74 basic post WITHOUT a type specified # in 11770 ms
ok 75 crawling a directory as a dry-run # in 823 ms
ok 76 crawling a directory # in 4411 ms
ok 77 skipcommit and optimize and delete # in 3867 ms
ok 78 args mode # in 7397 ms
ok 79 stdin mode # in 3621 ms
ok 80 verbose echo the Solr response # in 3874 ms
ok 81 post solr log into solr via cli # in 9358 ms
# Test failed - capturing Docker diagnostics
# === docker ps (summary) ===
NAMES STATUS IMAGE PORTS
solr-node3 Up 11 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node2 Up 11 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
solr-node1 Up 17 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp
# === Docker logs for solr-node1 ===
2026-05-18T06:26:39.982629565Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-18T06:26:39.982666502Z It should be set to 65000 to avoid operational disruption.
2026-05-18T06:26:39.982669757Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-18T06:26:39.982948127Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-18T06:26:39.982964632Z
2026-05-18T06:26:39.992020522Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-18T06:26:39.999912970Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-18T06:26:40.041246096Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-18T06:26:40.073174139Z WARNING: A command line option has enabled the Security Manager
2026-05-18T06:26:40.073193924Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-18T06:26:41.226955731Z 2026-05-18 06:26:41.223 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-18T06:26:41.492892271Z 2026-05-18 06:26:41.492 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-18T06:26:41.517416489Z 2026-05-18 06:26:41.516 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-18T06:26:41.524642572Z 2026-05-18 06:26:41.524 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-18T06:26:41.524735767Z 2026-05-18 06:26:41.524 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983
2026-05-18T06:26:41.524833683Z 2026-05-18 06:26:41.524 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-18T06:26:41.524966512Z 2026-05-18 06:26:41.524 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-18T06:26:41.524391144Z
2026-05-18T06:26:41.527557875Z 2026-05-18 06:26:41.527 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-18T06:26:41.549805793Z 2026-05-18 06:26:41.549 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-18T06:26:41.563084287Z 2026-05-18 06:26:41.562 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-18T06:26:41.563151518Z 2026-05-18 06:26:41.562 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-18T06:26:41.599526146Z 2026-05-18 06:26:41.598 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-18T06:26:42.283185356Z 2026-05-18 06:26:42.282 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-18T06:26:42.955247887Z 2026-05-18 06:26:42.919 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-18T06:26:42.957058906Z 2026-05-18 06:26:42.949 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default
2026-05-18T06:26:42.957155409Z 2026-05-18 06:26:42.949 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg
2026-05-18T06:26:42.961676620Z 2026-05-18 06:26:42.961 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0
2026-05-18T06:26:42.961773272Z 2026-05-18 06:26:42.961 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details.
2026-05-18T06:26:43.327565741Z 2026-05-18 06:26:43.327 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
2026-05-18T06:26:43.467109667Z 2026-05-18 06:26:43.466 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983
2026-05-18T06:26:43.493643733Z 2026-05-18 06:26:43.492 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-18T06:26:43.699413713Z 2026-05-18 06:26:43.695 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-18T06:26:43.736549649Z 2026-05-18 06:26:43.736 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-18T06:26:43.736671665Z 2026-05-18 06:26:43.736 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-18T06:26:44.043062831Z 2026-05-18 06:26:44.041 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-18T06:26:44.074349003Z 2026-05-18 06:26:44.073 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-18T06:26:44.102420911Z 2026-05-18 06:26:44.101 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr
2026-05-18T06:26:44.127790914Z 2026-05-18 06:26:44.127 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72074547711901696-solr-node1:8983_solr-n_0000000000) starting
2026-05-18T06:26:44.198275961Z 2026-05-18 06:26:44.197 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr
2026-05-18T06:26:44.198433510Z 2026-05-18 06:26:44.198 INFO (OverseerStateUpdate-72074547711901696-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr
2026-05-18T06:26:44.211721123Z 2026-05-18 06:26:44.209 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-18T06:26:44.253866743Z 2026-05-18 06:26:44.253 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-18T06:26:44.800560840Z 2026-05-18 06:26:44.800 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-18T06:26:44.983602966Z 2026-05-18 06:26:44.983 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-18T06:26:45.265003891Z 2026-05-18 06:26:45.264 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-18T06:26:45.428795058Z 2026-05-18 06:26:45.428 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-18T06:26:45.436096340Z 2026-05-18 06:26:45.435 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_18.request.log
2026-05-18T06:26:45.635059585Z 2026-05-18 06:26:45.488 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983}
2026-05-18T06:26:45.638479896Z 2026-05-18 06:26:45.637 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @5639ms
2026-05-18T06:26:45.852115585Z 2026-05-18 06:26:45.849 INFO (qtp985324122-27-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=45
2026-05-18T06:26:45.928206878Z 2026-05-18 06:26:45.925 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=13
2026-05-18T06:26:49.552699069Z 2026-05-18 06:26:49.552 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-05-18T06:26:49.593617767Z 2026-05-18 06:26:49.593 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-18T06:26:54.160149686Z 2026-05-18 06:26:54.159 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=8
2026-05-18T06:26:54.186396759Z 2026-05-18 06:26:54.185 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=9
2026-05-18T06:26:54.211380361Z 2026-05-18 06:26:54.210 INFO (qtp985324122-27-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=2
2026-05-18T06:26:54.524915383Z 2026-05-18 06:26:54.524 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection
2026-05-18T06:26:54.546740384Z 2026-05-18 06:26:54.545 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children
2026-05-18T06:26:54.553984472Z 2026-05-18 06:26:54.553 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 173] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T06:26:54.591395713Z 2026-05-18 06:26:54.590 INFO (OverseerStateUpdate-72074547711901696-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-18T06:26:54.591418618Z "core":"test-collection_shard3_replica_n1",
2026-05-18T06:26:54.591421821Z "node_name":"solr-node1:8983_solr",
2026-05-18T06:26:54.591424874Z "base_url":"http://solr-node1:8983/solr",
2026-05-18T06:26:54.591427525Z "collection":"test-collection",
2026-05-18T06:26:54.591430589Z "shard":"shard3",
2026-05-18T06:26:54.591433072Z "state":"down",
2026-05-18T06:26:54.591435554Z "type":"NRT",
2026-05-18T06:26:54.591437953Z "operation":"ADDREPLICA",
2026-05-18T06:26:54.591440673Z "waitForFinalState":"false"}
2026-05-18T06:26:54.598303052Z 2026-05-18 06:26:54.597 INFO (OverseerStateUpdate-72074547711901696-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-18T06:26:54.598321694Z "core":"test-collection_shard2_replica_n2",
2026-05-18T06:26:54.598324899Z "node_name":"solr-node3:8985_solr",
2026-05-18T06:26:54.598327638Z "base_url":"http://solr-node3:8985/solr",
2026-05-18T06:26:54.598330247Z "collection":"test-collection",
2026-05-18T06:26:54.598332853Z "shard":"shard2",
2026-05-18T06:26:54.598335284Z "state":"down",
2026-05-18T06:26:54.598337780Z "type":"NRT",
2026-05-18T06:26:54.598340301Z "operation":"ADDREPLICA",
2026-05-18T06:26:54.598342803Z "waitForFinalState":"false"}
2026-05-18T06:26:54.604887286Z 2026-05-18 06:26:54.604 INFO (OverseerStateUpdate-72074547711901696-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() {
2026-05-18T06:26:54.604902682Z "core":"test-collection_shard1_replica_n4",
2026-05-18T06:26:54.604906017Z "node_name":"solr-node2:8984_solr",
2026-05-18T06:26:54.604919591Z "base_url":"http://solr-node2:8984/solr",
2026-05-18T06:26:54.604921758Z "collection":"test-collection",
2026-05-18T06:26:54.604923693Z "shard":"shard1",
2026-05-18T06:26:54.604925551Z "state":"down",
2026-05-18T06:26:54.604927480Z "type":"NRT",
2026-05-18T06:26:54.604929411Z "operation":"ADDREPLICA",
2026-05-18T06:26:54.604931382Z "waitForFinalState":"false"}
2026-05-18T06:26:54.713891751Z 2026-05-18 06:26:54.713 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 185] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T06:26:54.770545679Z 2026-05-18 06:26:54.770 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin
2026-05-18T06:26:54.816628177Z 2026-05-18 06:26:54.816 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-18T06:26:54.918939338Z 2026-05-18 06:26:54.918 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-05-18T06:26:55.408713347Z 2026-05-18 06:26:55.406 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-18T06:26:55.408802428Z 2026-05-18 06:26:55.406 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true
2026-05-18T06:26:55.442290512Z 2026-05-18 06:26:55.441 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/]
2026-05-18T06:26:57.016861007Z 2026-05-18 06:26:57.015 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-18T06:26:57.100144349Z 2026-05-18 06:26:57.098 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-18T06:26:57.130166746Z 2026-05-18 06:26:57.128 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-18T06:26:57.132045222Z 2026-05-18 06:26:57.129 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-18T06:26:57.134076744Z 2026-05-18 06:26:57.129 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-18T06:26:57.202974559Z 2026-05-18 06:26:57.202 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-18T06:26:57.209962006Z 2026-05-18 06:26:57.209 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-18T06:26:57.216638259Z 2026-05-18 06:26:57.216 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-18T06:26:57.221623921Z 2026-05-18 06:26:57.221 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-18T06:26:57.243979656Z 2026-05-18 06:26:57.243 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4}
2026-05-18T06:26:57.251120493Z 2026-05-18 06:26:57.250 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-18T06:26:57.256155261Z 2026-05-18 06:26:57.255 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-18T06:26:57.257285773Z 2026-05-18 06:26:57.257 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-18T06:26:57.286755075Z 2026-05-18 06:26:57.286 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-6 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-18T06:26:57.288268128Z 2026-05-18 06:26:57.287 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm
2026-05-18T06:26:57.289636933Z 2026-05-18 06:26:57.289 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3
2026-05-18T06:26:57.307311662Z 2026-05-18 06:26:57.307 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-05-18T06:26:57.307335800Z 2026-05-18 06:26:57.307 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-05-18T06:26:57.307399094Z 2026-05-18 06:26:57.307 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/
2026-05-18T06:26:57.309663223Z 2026-05-18 06:26:57.309 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-05-18T06:26:57.309706290Z 2026-05-18 06:26:57.309 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas
2026-05-18T06:26:57.311143459Z 2026-05-18 06:26:57.310 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72074547711901696-core_node3-n_0000000000
2026-05-18T06:26:57.322379684Z 2026-05-18 06:26:57.322 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3
2026-05-18T06:26:57.430508255Z 2026-05-18 06:26:57.430 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 241] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T06:26:57.441321277Z 2026-05-18 06:26:57.440 INFO (qtp985324122-99-solr-node1-6) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-05-18T06:26:57.466518927Z 2026-05-18 06:26:57.465 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2697
2026-05-18T06:26:57.555919548Z 2026-05-18 06:26:57.555 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 247] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node1 ===
{
"Name": "/solr-node1",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1732269,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-05-18T06:26:39.741037448Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node2 ===
2026-05-18T06:26:46.291227183Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-18T06:26:46.291252327Z It should be set to 65000 to avoid operational disruption.
2026-05-18T06:26:46.291255547Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-18T06:26:46.291591742Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-18T06:26:46.291602117Z
2026-05-18T06:26:46.301320483Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-18T06:26:46.308916958Z [0.001s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-18T06:26:46.353058436Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-18T06:26:46.385379823Z WARNING: A command line option has enabled the Security Manager
2026-05-18T06:26:46.385404958Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-18T06:26:47.615832568Z 2026-05-18 06:26:47.612 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-18T06:26:47.888139175Z 2026-05-18 06:26:47.887 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-18T06:26:47.915222026Z 2026-05-18 06:26:47.914 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-18T06:26:47.922639826Z 2026-05-18 06:26:47.921 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-18T06:26:47.922659401Z 2026-05-18 06:26:47.921 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984
2026-05-18T06:26:47.922662177Z 2026-05-18 06:26:47.922 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-18T06:26:47.922664785Z 2026-05-18 06:26:47.922 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-18T06:26:47.922114970Z
2026-05-18T06:26:47.925378815Z 2026-05-18 06:26:47.924 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-18T06:26:47.948619723Z 2026-05-18 06:26:47.948 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-18T06:26:47.972977190Z 2026-05-18 06:26:47.972 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-18T06:26:48.113316539Z 2026-05-18 06:26:48.112 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-05-18T06:26:48.133389638Z 2026-05-18 06:26:48.132 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-18T06:26:48.133449647Z 2026-05-18 06:26:48.133 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-18T06:26:48.133546234Z 2026-05-18 06:26:48.133 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-18T06:26:48.243823493Z 2026-05-18 06:26:48.243 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-05-18T06:26:48.244233646Z 2026-05-18 06:26:48.243 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-18T06:26:48.244316227Z 2026-05-18 06:26:48.243 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-18T06:26:48.277718926Z 2026-05-18 06:26:48.277 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-18T06:26:49.142273972Z 2026-05-18 06:26:49.141 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-18T06:26:49.373578837Z 2026-05-18 06:26:49.372 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-18T06:26:49.375013502Z 2026-05-18 06:26:49.373 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-05-18T06:26:49.385541090Z 2026-05-18 06:26:49.385 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-18T06:26:49.396809706Z 2026-05-18 06:26:49.396 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-18T06:26:49.404916149Z 2026-05-18 06:26:49.404 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-18T06:26:49.405014834Z 2026-05-18 06:26:49.404 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-18T06:26:49.438576793Z 2026-05-18 06:26:49.437 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-18T06:26:49.452914683Z 2026-05-18 06:26:49.452 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-18T06:26:49.482703805Z 2026-05-18 06:26:49.482 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-18T06:26:49.519375017Z 2026-05-18 06:26:49.517 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN
2026-05-18T06:26:49.541112135Z 2026-05-18 06:26:49.539 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr
2026-05-18T06:26:49.560970579Z 2026-05-18 06:26:49.553 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-05-18T06:26:49.594011909Z 2026-05-18 06:26:49.593 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-18T06:26:49.679063832Z 2026-05-18 06:26:49.678 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-18T06:26:50.032979723Z 2026-05-18 06:26:50.030 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-18T06:26:50.219510925Z 2026-05-18 06:26:50.219 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-18T06:26:50.509325047Z 2026-05-18 06:26:50.504 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-18T06:26:50.650010839Z 2026-05-18 06:26:50.649 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-18T06:26:50.657650646Z 2026-05-18 06:26:50.657 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_18.request.log
2026-05-18T06:26:50.663136205Z 2026-05-18 06:26:50.662 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984}
2026-05-18T06:26:50.665144484Z 2026-05-18 06:26:50.664 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4357ms
2026-05-18T06:26:50.829052841Z 2026-05-18 06:26:50.828 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=42
2026-05-18T06:26:50.891855310Z 2026-05-18 06:26:50.891 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=16
2026-05-18T06:26:54.772921442Z 2026-05-18 06:26:54.772 INFO (qtp985324122-59-solr-node1-6) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin
2026-05-18T06:26:54.832719908Z 2026-05-18 06:26:54.832 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-18T06:26:54.888458876Z 2026-05-18 06:26:54.888 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-05-18T06:26:55.217247400Z 2026-05-18 06:26:55.216 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-18T06:26:55.217271837Z 2026-05-18 06:26:55.216 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true
2026-05-18T06:26:55.236934391Z 2026-05-18 06:26:55.236 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/]
2026-05-18T06:26:55.682356470Z 2026-05-18 06:26:55.681 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-18T06:26:55.702054909Z 2026-05-18 06:26:55.701 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-18T06:26:55.706107238Z 2026-05-18 06:26:55.705 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-18T06:26:55.706911376Z 2026-05-18 06:26:55.706 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-18T06:26:55.706975024Z 2026-05-18 06:26:55.706 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-18T06:26:55.753024244Z 2026-05-18 06:26:55.752 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-18T06:26:55.756173814Z 2026-05-18 06:26:55.755 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-18T06:26:55.758373274Z 2026-05-18 06:26:55.757 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-18T06:26:55.760027699Z 2026-05-18 06:26:55.759 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-18T06:26:55.771393128Z 2026-05-18 06:26:55.770 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-05-18T06:26:55.779311371Z 2026-05-18 06:26:55.778 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-18T06:26:55.785367321Z 2026-05-18 06:26:55.784 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-18T06:26:55.785457012Z 2026-05-18 06:26:55.785 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-18T06:26:55.794469884Z 2026-05-18 06:26:55.793 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-6 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-18T06:26:55.814374603Z 2026-05-18 06:26:55.813 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node6=0}, version=0} for registerTerm
2026-05-18T06:26:55.816162311Z 2026-05-18 06:26:55.815 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1
2026-05-18T06:26:55.841502070Z 2026-05-18 06:26:55.840 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-05-18T06:26:55.841534690Z 2026-05-18 06:26:55.841 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-05-18T06:26:55.841537787Z 2026-05-18 06:26:55.841 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node2:8984/solr/test-collection_shard1_replica_n4/
2026-05-18T06:26:55.843115259Z 2026-05-18 06:26:55.842 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-05-18T06:26:55.844132838Z 2026-05-18 06:26:55.843 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ has no replicas
2026-05-18T06:26:55.845506120Z 2026-05-18 06:26:55.845 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard1/leader after winning as /collections/test-collection/leader_elect/shard1/election/72074547711901699-core_node6-n_0000000000
2026-05-18T06:26:55.855484022Z 2026-05-18 06:26:55.855 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ shard1
2026-05-18T06:26:55.997193352Z 2026-05-18 06:26:55.996 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 205] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T06:26:56.001565531Z 2026-05-18 06:26:56.001 INFO (qtp985324122-59-solr-node1-6) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-05-18T06:26:56.014519301Z 2026-05-18 06:26:56.014 INFO (qtp985324122-59-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1243
2026-05-18T06:26:56.114865017Z 2026-05-18 06:26:56.114 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 208] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T06:26:56.657155983Z 2026-05-18 06:26:56.656 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 223] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T06:26:56.962207538Z 2026-05-18 06:26:56.961 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 226] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T06:26:57.429981660Z 2026-05-18 06:26:57.429 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 241] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T06:26:57.556233155Z 2026-05-18 06:26:57.555 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 247] for collection [test-collection] has occurred - updating... (live nodes size: [3])
# === Docker inspect for solr-node2 ===
{
"Name": "/solr-node2",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1732666,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-05-18T06:26:46.079487694Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# === Docker logs for solr-node3 ===
2026-05-18T06:26:46.526720670Z *** [WARN] *** Your open file limit is currently 1024.
2026-05-18T06:26:46.526757434Z It should be set to 65000 to avoid operational disruption.
2026-05-18T06:26:46.526760685Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh
2026-05-18T06:26:46.527040266Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag.
2026-05-18T06:26:46.527049947Z
2026-05-18T06:26:46.536769875Z Java 17 detected. Enabled workaround for SOLR-16463
2026-05-18T06:26:46.544755201Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system.
2026-05-18T06:26:46.594240014Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true
2026-05-18T06:26:46.625515997Z WARNING: A command line option has enabled the Security Manager
2026-05-18T06:26:46.625547524Z WARNING: The Security Manager is deprecated and will be removed in a future release
2026-05-18T06:26:47.724425098Z 2026-05-18 06:26:47.721 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8
2026-05-18T06:26:48.001984268Z 2026-05-18 06:26:48.001 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner
2026-05-18T06:26:48.026521664Z 2026-05-18 06:26:48.026 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
2026-05-18T06:26:48.033398446Z 2026-05-18 06:26:48.032 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted]
2026-05-18T06:26:48.033502982Z 2026-05-18 06:26:48.033 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8985
2026-05-18T06:26:48.033642319Z 2026-05-18 06:26:48.033 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim
2026-05-18T06:26:48.033675525Z 2026-05-18 06:26:48.033 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-18T06:26:48.033158572Z
2026-05-18T06:26:48.036436664Z 2026-05-18 06:26:48.036 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log
2026-05-18T06:26:48.058368611Z 2026-05-18 06:26:48.057 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home)
2026-05-18T06:26:48.082268095Z 2026-05-18 06:26:48.081 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-18T06:26:48.220605091Z 2026-05-18 06:26:48.220 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper
2026-05-18T06:26:48.238498764Z 2026-05-18 06:26:48.237 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-18T06:26:48.238572399Z 2026-05-18 06:26:48.238 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-18T06:26:48.238698848Z 2026-05-18 06:26:48.238 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
2026-05-18T06:26:48.347235805Z 2026-05-18 06:26:48.346 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper)
2026-05-18T06:26:48.347420419Z 2026-05-18 06:26:48.347 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default
2026-05-18T06:26:48.347547346Z 2026-05-18 06:26:48.347 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml
2026-05-18T06:26:48.383169039Z 2026-05-18 06:26:48.382 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib]
2026-05-18T06:26:49.316895453Z 2026-05-18 06:26:49.315 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled.
2026-05-18T06:26:49.467296109Z 2026-05-18 06:26:49.466 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure.
2026-05-18T06:26:49.468707964Z 2026-05-18 06:26:49.466 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983
2026-05-18T06:26:49.477399119Z 2026-05-18 06:26:49.476 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-18T06:26:49.482580226Z 2026-05-18 06:26:49.482 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper
2026-05-18T06:26:49.487222016Z 2026-05-18 06:26:49.486 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected
2026-05-18T06:26:49.487275352Z 2026-05-18 06:26:49.486 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2026-05-18T06:26:49.519080623Z 2026-05-18 06:26:49.518 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html
2026-05-18T06:26:49.534345418Z 2026-05-18 06:26:49.533 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
2026-05-18T06:26:49.553044515Z 2026-05-18 06:26:49.552 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
2026-05-18T06:26:49.558834365Z 2026-05-18 06:26:49.558 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates.
2026-05-18T06:26:49.573457043Z 2026-05-18 06:26:49.573 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node3:8985_solr as DOWN
2026-05-18T06:26:49.584619047Z 2026-05-18 06:26:49.584 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node3:8985_solr
2026-05-18T06:26:49.594234765Z 2026-05-18 06:26:49.593 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
2026-05-18T06:26:49.635734994Z 2026-05-18 06:26:49.635 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
2026-05-18T06:26:49.907393760Z 2026-05-18 06:26:49.906 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data
2026-05-18T06:26:50.088270919Z 2026-05-18 06:26:50.087 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available.
2026-05-18T06:26:50.358891398Z 2026-05-18 06:26:50.358 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-18T06:26:50.480063794Z 2026-05-18 06:26:50.479 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp}
2026-05-18T06:26:50.485606100Z 2026-05-18 06:26:50.485 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_18.request.log
2026-05-18T06:26:50.496813666Z 2026-05-18 06:26:50.496 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8985}
2026-05-18T06:26:50.497798800Z 2026-05-18 06:26:50.497 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @3954ms
2026-05-18T06:26:52.991636429Z 2026-05-18 06:26:52.978 INFO (qtp985324122-27-solr-node3-1) [c: s: r: x: t:solr-node3-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=160
2026-05-18T06:26:53.083987025Z 2026-05-18 06:26:53.083 INFO (qtp985324122-56-solr-node3-2) [c: s: r: x: t:solr-node3-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=23
2026-05-18T06:26:54.491467404Z 2026-05-18 06:26:54.490 INFO (qtp985324122-59-solr-node3-3) [c: s: r: x: t:solr-node3-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=21
2026-05-18T06:26:54.772866073Z 2026-05-18 06:26:54.771 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node5&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&shard=shard2&wt=javabin
2026-05-18T06:26:54.863325487Z 2026-05-18 06:26:54.862 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0
2026-05-18T06:26:54.976896139Z 2026-05-18 06:26:54.976 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Schema name=default-config
2026-05-18T06:26:55.376553681Z 2026-05-18 06:26:55.376 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id
2026-05-18T06:26:55.376578209Z 2026-05-18 06:26:55.376 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard2_replica_n2' using configuration from configset techproducts, trusted=true
2026-05-18T06:26:55.399103628Z 2026-05-18 06:26:55.398 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard2_replica_n2], dataDir=[/var/solr/data/test-collection_shard2_replica_n2/data/]
2026-05-18T06:26:56.204774511Z 2026-05-18 06:26:56.204 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
2026-05-18T06:26:56.231007723Z 2026-05-18 06:26:56.230 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
2026-05-18T06:26:56.239461501Z 2026-05-18 06:26:56.239 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
2026-05-18T06:26:56.302797344Z 2026-05-18 06:26:56.301 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
2026-05-18T06:26:56.302824936Z 2026-05-18 06:26:56.302 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
2026-05-18T06:26:56.456088214Z 2026-05-18 06:26:56.455 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts
2026-05-18T06:26:56.473678860Z 2026-05-18 06:26:56.473 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts
2026-05-18T06:26:56.476368636Z 2026-05-18 06:26:56.476 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml
2026-05-18T06:26:56.478096134Z 2026-05-18 06:26:56.477 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
2026-05-18T06:26:56.490586375Z 2026-05-18 06:26:56.490 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal}
2026-05-18T06:26:56.502696331Z 2026-05-18 06:26:56.502 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
2026-05-18T06:26:56.508314523Z 2026-05-18 06:26:56.507 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.QuerySenderListener QuerySenderListener done.
2026-05-18T06:26:56.508415514Z 2026-05-18 06:26:56.508 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
2026-05-18T06:26:56.516367511Z 2026-05-18 06:26:56.515 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-6 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
2026-05-18T06:26:56.523889785Z 2026-05-18 06:26:56.523 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard2 to Terms{values={core_node5=0}, version=0} for registerTerm
2026-05-18T06:26:56.525357906Z 2026-05-18 06:26:56.525 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard2
2026-05-18T06:26:56.538760871Z 2026-05-18 06:26:56.538 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
2026-05-18T06:26:56.538918442Z 2026-05-18 06:26:56.538 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
2026-05-18T06:26:56.538930065Z 2026-05-18 06:26:56.538 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync replicas to http://solr-node3:8985/solr/test-collection_shard2_replica_n2/
2026-05-18T06:26:56.540015700Z 2026-05-18 06:26:56.539 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
2026-05-18T06:26:56.540511722Z 2026-05-18 06:26:56.540 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.SyncStrategy http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ has no replicas
2026-05-18T06:26:56.541511862Z 2026-05-18 06:26:56.541 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard2/leader after winning as /collections/test-collection/leader_elect/shard2/election/72074547711901700-core_node5-n_0000000000
2026-05-18T06:26:56.552955758Z 2026-05-18 06:26:56.552 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ shard2
2026-05-18T06:26:56.657205737Z 2026-05-18 06:26:56.656 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 223] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T06:26:56.660970531Z 2026-05-18 06:26:56.660 INFO (qtp985324122-60-solr-node1-6) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-6] o.a.s.c.ZkController I am the leader, no recovery necessary
2026-05-18T06:26:56.669638100Z 2026-05-18 06:26:56.669 INFO (qtp985324122-60-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=techproducts&newCollection=true&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&collection=test-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1899
2026-05-18T06:26:56.962253564Z 2026-05-18 06:26:56.961 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 226] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T06:26:57.429915835Z 2026-05-18 06:26:57.429 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 241] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T06:26:57.497627925Z 2026-05-18 06:26:57.497 INFO (qtp985324122-58-solr-node3-4) [c:test-collection s: r: x: t:solr-node3-4] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
2026-05-18T06:26:57.556156688Z 2026-05-18 06:26:57.555 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 247] for collection [test-collection] has occurred - updating... (live nodes size: [3])
2026-05-18T06:26:57.560209224Z 2026-05-18 06:26:57.559 INFO (qtp985324122-58-solr-node3-4) [c:test-collection s: r: x: t:solr-node3-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=3054
# === Docker inspect for solr-node3 ===
{
"Name": "/solr-node3",
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1732849,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-05-18T06:26:46.309037033Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Ports": {
"8983/tcp": null
}
}
# Docker artifacts saved to: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/docker
not ok 82 Docker SolrCloud rolling upgrade # in 30464 ms
# (in test file test/test_rolling_upgrade.bats, line 118)
# `wait_for 30 1 docker exec solr-node1 solr healthcheck -c test-collection' failed with status 127
# 9.10.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# af6eca94c810: Pulling fs layer
# cb0efb96dabd: Pulling fs layer
# 3e9d91201f40: Pulling fs layer
# 66b76b382631: Pulling fs layer
# 601f2c23751f: Pulling fs layer
# 0ed719aef393: Pulling fs layer
# 69a09c96e8bf: Pulling fs layer
# c937aee9a870: Pulling fs layer
# 9cdfe5320de8: Pulling fs layer
# 3aa9120e46f5: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# 66b76b382631: Waiting
# 601f2c23751f: Waiting
# 0ed719aef393: Waiting
# 69a09c96e8bf: Waiting
# c937aee9a870: Waiting
# 4f4fb700ef54: Waiting
# 9cdfe5320de8: Waiting
# 3aa9120e46f5: Waiting
# cb0efb96dabd: Verifying Checksum
# cb0efb96dabd: Download complete
# 3e9d91201f40: Verifying Checksum
# 3e9d91201f40: Download complete
# af6eca94c810: Verifying Checksum
# af6eca94c810: Download complete
# 66b76b382631: Verifying Checksum
# 66b76b382631: Download complete
# 601f2c23751f: Verifying Checksum
# 601f2c23751f: Download complete
# 69a09c96e8bf: Download complete
# 9cdfe5320de8: Verifying Checksum
# 9cdfe5320de8: Download complete
# c937aee9a870: Verifying Checksum
# c937aee9a870: Download complete
# 4f4fb700ef54: Verifying Checksum
# 4f4fb700ef54: Download complete
# 3aa9120e46f5: Download complete
# 0ed719aef393: Verifying Checksum
# 0ed719aef393: Download complete
# af6eca94c810: Pull complete
# cb0efb96dabd: Pull complete
# 3e9d91201f40: Pull complete
# 66b76b382631: Pull complete
# 601f2c23751f: Pull complete
# 0ed719aef393: Pull complete
# 69a09c96e8bf: Pull complete
# c937aee9a870: Pull complete
# 9cdfe5320de8: Pull complete
# 3aa9120e46f5: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:15c8155ceffe1042f37c77bb537aa531804a3510702ff87f2af5fedf523f4203
# Status: Downloaded newer image for apache/solr-nightly:9.10.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:9.10.0-SNAPSHOT-slim
# 10.0.0-SNAPSHOT-slim: Pulling from apache/solr-nightly
# 40d16f30db40: Already exists
# 9d5a0b6cb36c: Already exists
# 49518c28c0b3: Already exists
# 41162acdcc29: Already exists
# 2ed0fd558ca3: Already exists
# 8b0357ffbe54: Pulling fs layer
# 0df26b0d5a69: Pulling fs layer
# a26fc87814fe: Pulling fs layer
# 67860c4a05fe: Pulling fs layer
# d1a380e5a7b1: Pulling fs layer
# 4f4fb700ef54: Pulling fs layer
# 67860c4a05fe: Waiting
# d1a380e5a7b1: Waiting
# 4f4fb700ef54: Waiting
# 0df26b0d5a69: Verifying Checksum
# 0df26b0d5a69: Download complete
# a26fc87814fe: Download complete
# 67860c4a05fe: Verifying Checksum
# 67860c4a05fe: Download complete
# 4f4fb700ef54: Verifying Checksum
# 4f4fb700ef54: Download complete
# d1a380e5a7b1: Verifying Checksum
# d1a380e5a7b1: Download complete
# 8b0357ffbe54: Verifying Checksum
# 8b0357ffbe54: Download complete
# 8b0357ffbe54: Pull complete
# 0df26b0d5a69: Pull complete
# a26fc87814fe: Pull complete
# 67860c4a05fe: Pull complete
# d1a380e5a7b1: Pull complete
# 4f4fb700ef54: Pull complete
# Digest: sha256:16825c9cb8e05e7a6dd9d4209615781d30bc3137a6017a5254cec93560c7c8ba
# Status: Downloaded newer image for apache/solr-nightly:10.0.0-SNAPSHOT-slim
# docker.io/apache/solr-nightly:10.0.0-SNAPSHOT-slim
# 01dd5fbe11936c0ad0879ff9d09132e17f2c83af655311d31c3056637c2b8ffb
# solr-data1
# solr-data2
# solr-data3
# Starting solr-node1 with embedded ZooKeeper
# e1e5c6418c709585da07ce9863ece39e8dec02f55fe116f4fec91659b5d38b57
# Starting solr-node2 connected to first node's ZooKeeper
# 613bca23bda59090f7f38e1f3e9197d6bf43a2e417576912830cd5f46d96f108
# Started solr-node3 connected to first node's ZooKeeper
# c4aff0d28e964a9cf439d026ea69e520f8f31cf2bd8a2c95ed497a1b66854c58
# Creating a Collection
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983.
# WARN - 2026-05-18 06:26:54.240; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARN - 2026-05-18 06:26:54.270; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes
# WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. "/solr"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [http://solr-node3:8985/solr] to [http://solr-node3:8985/].
# Created collection 'test-collection' with 3 shard(s), 1 replica(s) with config-set 'techproducts'
# Checking collection health
# /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_rolling_upgrade.bats: line 118: wait_for: command not found
# solr-node1
# solr-node2
# solr-node3
# solr-node1
# solr-node2
# solr-node3
# solr-data1
# solr-data2
# solr-data3
# solrcloud-test
ok 83 allowPaths - backup # in 15805 ms
ok 84 snapshot lifecycle # in 4844 ms
ok 85 snapshot list # in 3329 ms
ok 86 snapshot describe # in 3330 ms
ok 87 start solr with ssl # in 16533 ms
ok 88 use different hostname when not checking peer-name # in 24784 ms
ok 89 start solr with ssl and auth # in 14683 ms
ok 90 start solr with client truststore and security manager # in 16620 ms
ok 91 start solr with mTLS needed # in 45727 ms
ok 92 start solr with mTLS wanted # in 34310 ms
ok 93 test keystore reload # in 59102 ms
ok 94 SOLR-11740 check 'solr stop' connection # in 15902 ms
ok 95 stop command for single port # in 18284 ms
ok 96 check stop command doesn't hang # in 218 ms
ok 97 SOLR-16976 solr starts with remote JMX enabled # in 8105 ms
ok 98 deprecated system properties converted to modern properties # in 195021 ms
ok 99 start with custom jetty options # in 8218 ms
ok 100 bootstrapping a configset # in 9305 ms
ok 101 status detects locally running solr # in 9521 ms
ok 102 status with --solr-url from user # in 7935 ms
ok 103 status with --port from user # in 8170 ms
ok 104 multiple connection options are prevented # in 859 ms
ok 105 status with invalid --solr-url from user # in 1246 ms
ok 106 status with --short format # in 195976 ms
ok 107 searching solr via locally executed streaming expression # in 1723 ms
ok 108 searching solr via remotely executed streaming expression # in 1288 ms
ok 109 variable interpolation # in 1705 ms
ok 110 searching solr without credentials fails with error # in 1141 ms
ok 111 --version returns Solr version # in 736 ms
ok 112 version as direct tool call still runs # in 756 ms
ok 113 short help # in 789 ms
ok 114 short help is inferred # in 797 ms
ok 115 long help # in 798 ms
ok 116 running subcommands with zk is prevented # in 731 ms
ok 117 listing out files # in 2145 ms
ok 118 connecting to solr via various solr urls and zk hosts # in 7945 ms
ok 119 copying files around # in 11038 ms
ok 120 upconfig # in 2547 ms
ok 121 SOLR-12429 test upconfig fails with symlink # in 1206 ms
ok 122 downconfig # in 1207 ms
ok 123 bin/solr zk cp gets 'solrhome' from '--solr-home' command line option # in 1441 ms
ok 124 env var ZK_HOST is honored # in 3359 ms
ok 125 Cleanup # in 1873 ms
> Task :solr:packaging:integrationTests FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build.gradle' line: 339
* What went wrong:
Execution failed for task ':solr:packaging:integrationTests'.
> Process 'command '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/.gradle/node/node_modules/bats/bin/bats'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/8.10/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 29m 15s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/6r34x6y7xxcuy
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Solr » Solr-TestIntegration-10.x - Build # 211 - Still Failing!
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/211/
No tests ran.
Build Log:
[...truncated 923 lines...]
BUILD FAILED in 28m 11s
128 actionable tasks: 128 executed
Publishing Build Scan to Develocity...
https://develocity.apache.org/s/zmhc3yagdevgu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
Archiving artifacts
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any