You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by st...@apache.org on 2023/11/08 17:13:58 UTC

(solr) branch branch_9x updated: SOLR-17011 Add tracing spans to internal collection commands (#1979) (#2067)

This is an automated email from the ASF dual-hosted git repository.

stillalex pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git


The following commit(s) were added to refs/heads/branch_9x by this push:
     new 7dcd6b845f8 SOLR-17011 Add tracing spans to internal collection commands (#1979) (#2067)
7dcd6b845f8 is described below

commit 7dcd6b845f89d36726a3411dc6f127ab0a92d0ca
Author: Alex D <st...@apache.org>
AuthorDate: Wed Nov 8 09:13:49 2023 -0800

    SOLR-17011 Add tracing spans to internal collection commands (#1979) (#2067)
---
 solr/CHANGES.txt                                   |   2 +
 .../solr/cloud/api/collections/CollApiCmds.java    |  45 ++++++-
 .../java/org/apache/solr/handler/IndexFetcher.java |   9 +-
 .../org/apache/solr/update/UpdateShardHandler.java |   9 --
 .../apache/solr/util/tracing/SimplePropagator.java |   4 +-
 .../org/apache/solr/util/tracing/TraceUtils.java   |  12 ++
 .../solr/util/tracing/TestDistributedTracing.java  | 130 +++++++++++++++++++--
 .../TestSimplePropagatorDistributedTracing.java    |  17 +++
 8 files changed, 201 insertions(+), 27 deletions(-)

diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 42e53988f3c..9e85323ad58 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -27,6 +27,8 @@ Improvements
 * SOLR-16397: Swap core v2 endpoints have been updated to be more REST-ful.
   SWAP is now available at `POST /api/cores/coreName/swap` (Sanjay Dutt via Jason Gerlowski)
 
+* SOLR-17011: Add tracing spans to internal collection commands (Alex Deparvu)
+
 Optimizations
 ---------------------
 (No changes)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java
index 1595904bc4f..78437421118 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java
@@ -69,10 +69,15 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.SP
 import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
 import static org.apache.solr.common.params.CommonParams.NAME;
 
+import io.opentracing.Span;
+import io.opentracing.Tracer;
+import io.opentracing.noop.NoopTracer;
+import io.opentracing.util.GlobalTracer;
 import java.lang.invoke.MethodHandles;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import org.apache.solr.cloud.DistributedClusterStateUpdater;
@@ -94,6 +99,7 @@ import org.apache.solr.common.util.SuppressForbidden;
 import org.apache.solr.common.util.Utils;
 import org.apache.solr.handler.component.ShardHandler;
 import org.apache.solr.handler.component.ShardRequest;
+import org.apache.solr.util.tracing.TraceUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -183,7 +189,44 @@ public class CollApiCmds {
     }
 
     CollApiCmds.CollectionApiCommand getActionCommand(CollectionParams.CollectionAction action) {
-      return commandMap.get(action);
+      var command = commandMap.get(action);
+      if (command != null) {
+        return new TraceAwareCommand(commandMap.get(action));
+      } else {
+        return command;
+      }
+    }
+  }
+
+  public static class TraceAwareCommand implements CollectionApiCommand {
+
+    private final CollectionApiCommand command;
+    private final Tracer tracer = GlobalTracer.get();
+
+    public TraceAwareCommand(CollectionApiCommand command) {
+      this.command = command;
+    }
+
+    @Override
+    public void call(ClusterState state, ZkNodeProps message, NamedList<Object> results)
+        throws Exception {
+      if (tracer instanceof NoopTracer) {
+        command.call(state, message, results);
+      } else {
+        String collection =
+            Optional.ofNullable(message.getStr(COLLECTION_PROP, message.getStr(NAME)))
+                .orElse("unknown");
+        boolean isAsync = message.containsKey(ASYNC);
+        Span localSpan =
+            TraceUtils.startCollectionApiCommandSpan(
+                tracer, command.getClass().getSimpleName(), collection, isAsync);
+        try (var scope = tracer.scopeManager().activate(localSpan)) {
+          assert scope != null; // prevent javac warning about scope being unused
+          command.call(state, message, results);
+        } finally {
+          localSpan.finish();
+        }
+      }
     }
   }
 
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index 3015afb6900..1df9d1e101c 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -32,7 +32,6 @@ import static org.apache.solr.handler.ReplicationHandler.EXTERNAL;
 import static org.apache.solr.handler.ReplicationHandler.FETCH_FROM_LEADER;
 import static org.apache.solr.handler.ReplicationHandler.FILE;
 import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
-import static org.apache.solr.handler.ReplicationHandler.FileInfo;
 import static org.apache.solr.handler.ReplicationHandler.GENERATION;
 import static org.apache.solr.handler.ReplicationHandler.INTERNAL;
 import static org.apache.solr.handler.ReplicationHandler.LEADER_URL;
@@ -121,6 +120,7 @@ import org.apache.solr.core.DirectoryFactory;
 import org.apache.solr.core.DirectoryFactory.DirContext;
 import org.apache.solr.core.IndexDeletionPolicyWrapper;
 import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.ReplicationHandler.FileInfo;
 import org.apache.solr.handler.admin.api.CoreReplicationAPI;
 import org.apache.solr.request.LocalSolrQueryRequest;
 import org.apache.solr.request.SolrQueryRequest;
@@ -132,6 +132,7 @@ import org.apache.solr.util.PropertiesOutputStream;
 import org.apache.solr.util.RTimer;
 import org.apache.solr.util.RefCounted;
 import org.apache.solr.util.TestInjection;
+import org.apache.solr.util.stats.InstrumentedHttpRequestExecutor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -265,11 +266,13 @@ public class IndexFetcher {
     httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_USER, httpBasicAuthUser);
     httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, httpBasicAuthPassword);
     httpClientParams.set(HttpClientUtil.PROP_ALLOW_COMPRESSION, useCompression);
-
+    // no metrics, just tracing
+    InstrumentedHttpRequestExecutor executor = new InstrumentedHttpRequestExecutor(null);
     return HttpClientUtil.createClient(
         httpClientParams,
         core.getCoreContainer().getUpdateShardHandler().getRecoveryOnlyConnectionManager(),
-        true);
+        true,
+        executor);
   }
 
   public IndexFetcher(
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
index 3a52fe87fb9..b3ab8cb9156 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
@@ -21,7 +21,6 @@ import static org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.KNOWN_M
 import com.google.common.annotations.VisibleForTesting;
 import java.lang.invoke.MethodHandles;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadFactory;
@@ -80,8 +79,6 @@ public class UpdateShardHandler implements SolrInfoBean {
 
   private final CloseableHttpClient defaultClient;
 
-  private final InstrumentedPoolingHttpClientConnectionManager updateOnlyConnectionManager;
-
   private final InstrumentedPoolingHttpClientConnectionManager recoveryOnlyConnectionManager;
 
   private final InstrumentedPoolingHttpClientConnectionManager defaultConnectionManager;
@@ -90,16 +87,12 @@ public class UpdateShardHandler implements SolrInfoBean {
 
   private final InstrumentedHttpListenerFactory updateHttpListenerFactory;
 
-  private final Set<String> metricNames = ConcurrentHashMap.newKeySet();
   private SolrMetricsContext solrMetricsContext;
 
   private int socketTimeout = HttpClientUtil.DEFAULT_SO_TIMEOUT;
   private int connectionTimeout = HttpClientUtil.DEFAULT_CONNECT_TIMEOUT;
 
   public UpdateShardHandler(UpdateShardHandlerConfig cfg) {
-    updateOnlyConnectionManager =
-        new InstrumentedPoolingHttpClientConnectionManager(
-            HttpClientUtil.getSocketFactoryRegistryProvider().getSocketFactoryRegistry());
     recoveryOnlyConnectionManager =
         new InstrumentedPoolingHttpClientConnectionManager(
             HttpClientUtil.getSocketFactoryRegistryProvider().getSocketFactoryRegistry());
@@ -108,8 +101,6 @@ public class UpdateShardHandler implements SolrInfoBean {
             HttpClientUtil.getSocketFactoryRegistryProvider().getSocketFactoryRegistry());
     ModifiableSolrParams clientParams = new ModifiableSolrParams();
     if (cfg != null) {
-      updateOnlyConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
-      updateOnlyConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
       recoveryOnlyConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
       recoveryOnlyConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
       defaultConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
diff --git a/solr/core/src/java/org/apache/solr/util/tracing/SimplePropagator.java b/solr/core/src/java/org/apache/solr/util/tracing/SimplePropagator.java
index b33d98d78c6..49c34f926f2 100644
--- a/solr/core/src/java/org/apache/solr/util/tracing/SimplePropagator.java
+++ b/solr/core/src/java/org/apache/solr/util/tracing/SimplePropagator.java
@@ -20,9 +20,9 @@ import io.opentracing.Scope;
 import io.opentracing.ScopeManager;
 import io.opentracing.Span;
 import io.opentracing.SpanContext;
-import io.opentracing.Tracer;
 import io.opentracing.Tracer.SpanBuilder;
 import io.opentracing.noop.NoopSpan;
+import io.opentracing.noop.NoopTracer;
 import io.opentracing.propagation.Format;
 import io.opentracing.propagation.TextMap;
 import io.opentracing.tag.Tag;
@@ -64,7 +64,7 @@ public class SimplePropagator {
    *
    * <p>Heavily inspired from JaegerTracer, NoopTracer
    */
-  static class SimplePropagatorTracer implements Tracer {
+  static class SimplePropagatorTracer implements NoopTracer {
 
     private final ScopeManager scopeManager = new ThreadLocalScopeManager();
 
diff --git a/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java b/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java
index 0cc9ed43bb3..aec30e6a1fd 100644
--- a/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java
+++ b/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java
@@ -18,6 +18,7 @@ package org.apache.solr.util.tracing;
 
 import io.opentracing.Span;
 import io.opentracing.Tracer;
+import io.opentracing.Tracer.SpanBuilder;
 import io.opentracing.noop.NoopSpan;
 import io.opentracing.propagation.Format;
 import io.opentracing.tag.Tags;
@@ -75,4 +76,15 @@ public class TraceUtils {
       req.getSpan().setTag("class", clazz);
     }
   }
+
+  public static Span startCollectionApiCommandSpan(
+      Tracer tracer, String name, String collection, boolean isAsync) {
+    SpanBuilder spanBuilder =
+        tracer
+            .buildSpan(name)
+            .asChildOf(tracer.activeSpan())
+            .withTag(Tags.SPAN_KIND, isAsync ? Tags.SPAN_KIND_PRODUCER : Tags.SPAN_KIND_CLIENT)
+            .withTag(Tags.DB_INSTANCE, collection);
+    return spanBuilder.start();
+  }
 }
diff --git a/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java b/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java
index 3c56a45d999..2a8ef839c2f 100644
--- a/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java
+++ b/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java
@@ -22,7 +22,9 @@ import io.opentracing.mock.MockTracer;
 import io.opentracing.util.GlobalTracer;
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrRequest;
@@ -39,6 +41,7 @@ import org.apache.solr.util.LogLevel;
 import org.hamcrest.MatcherAssert;
 import org.hamcrest.Matchers;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -64,6 +67,11 @@ public class TestDistributedTracing extends SolrCloudTestCase {
     cluster.waitForActiveCollection(COLLECTION, 2, 4);
   }
 
+  @Before
+  private void resetSpanData() {
+    getAndClearSpans();
+  }
+
   @AfterClass
   public static void afterTest() {
     tracer = null;
@@ -80,12 +88,14 @@ public class TestDistributedTracing extends SolrCloudTestCase {
     // Indexing
     cloudClient.add(COLLECTION, sdoc("id", "1"));
     finishedSpans = getAndClearSpans();
-    finishedSpans.removeIf(x -> !x.tags().get("http.url").toString().endsWith("/update"));
+
+    finishedSpans.removeIf(
+        x -> x.tags() != null && !x.tags().get("http.url").toString().endsWith("/update"));
     assertEquals(2, finishedSpans.size());
     assertOneSpanIsChildOfAnother(finishedSpans);
     // core because cloudClient routes to core
     assertEquals("post:/{core}/update", finishedSpans.get(0).operationName());
-    assertDbInstanceCore(finishedSpans.get(0));
+    assertDbInstanceCore(finishedSpans.get(0), COLLECTION);
 
     cloudClient.add(COLLECTION, sdoc("id", "2"));
     cloudClient.add(COLLECTION, sdoc("id", "3"));
@@ -96,7 +106,8 @@ public class TestDistributedTracing extends SolrCloudTestCase {
     // Searching
     cloudClient.query(COLLECTION, new SolrQuery("*:*"));
     finishedSpans = getAndClearSpans();
-    finishedSpans.removeIf(x -> !x.tags().get("http.url").toString().endsWith("/select"));
+    finishedSpans.removeIf(
+        x -> x.tags() != null && !x.tags().get("http.url").toString().endsWith("/select"));
     // one from client to server, 2 for execute query, 2 for fetching documents
     assertEquals(5, finishedSpans.size());
     var parentId = getRootTraceId(finishedSpans);
@@ -106,7 +117,7 @@ public class TestDistributedTracing extends SolrCloudTestCase {
       }
     }
     assertEquals("get:/{core}/select", finishedSpans.get(0).operationName());
-    assertDbInstanceCore(finishedSpans.get(0));
+    assertDbInstanceCore(finishedSpans.get(0), COLLECTION);
   }
 
   @Test
@@ -136,7 +147,7 @@ public class TestDistributedTracing extends SolrCloudTestCase {
         .process(cloudClient);
     finishedSpans = getAndClearSpans();
     assertEquals("post:/collections/{collection}/reload", finishedSpans.get(0).operationName());
-    assertDbInstanceColl(finishedSpans.get(0));
+    assertDbInstanceColl(finishedSpans.get(0), COLLECTION);
 
     new V2Request.Builder("/c/" + COLLECTION + "/update/json")
         .withMethod(SolrRequest.METHOD.POST)
@@ -146,7 +157,7 @@ public class TestDistributedTracing extends SolrCloudTestCase {
         .process(cloudClient);
     finishedSpans = getAndClearSpans();
     assertEquals("post:/c/{collection}/update/json", finishedSpans.get(0).operationName());
-    assertDbInstanceColl(finishedSpans.get(0));
+    assertDbInstanceColl(finishedSpans.get(0), COLLECTION);
 
     final V2Response v2Response =
         new V2Request.Builder("/c/" + COLLECTION + "/select")
@@ -156,7 +167,7 @@ public class TestDistributedTracing extends SolrCloudTestCase {
             .process(cloudClient);
     finishedSpans = getAndClearSpans();
     assertEquals("get:/c/{collection}/select", finishedSpans.get(0).operationName());
-    assertDbInstanceColl(finishedSpans.get(0));
+    assertDbInstanceColl(finishedSpans.get(0), COLLECTION);
     assertEquals(1, ((SolrDocumentList) v2Response.getResponse().get("response")).getNumFound());
   }
 
@@ -167,7 +178,6 @@ public class TestDistributedTracing extends SolrCloudTestCase {
    */
   @Test
   public void testApacheClient() throws Exception {
-    getAndClearSpans(); // reset
     CollectionAdminRequest.ColStatus a1 = CollectionAdminRequest.collectionStatus(COLLECTION);
     CollectionAdminResponse r1 = a1.process(cluster.getSolrClient());
     assertEquals(0, r1.getStatus());
@@ -181,13 +191,109 @@ public class TestDistributedTracing extends SolrCloudTestCase {
     }
   }
 
-  private void assertDbInstanceColl(MockSpan mockSpan) {
-    MatcherAssert.assertThat(mockSpan.tags().get("db.instance"), Matchers.equalTo("collection1"));
+  @Test
+  public void testInternalCollectionApiCommands() throws Exception {
+    String collecton = "testInternalCollectionApiCommands";
+    verifyCollectionCreation(collecton);
+    verifyCollectionDeletion(collecton);
+  }
+
+  private void verifyCollectionCreation(String collection) throws Exception {
+    var a1 = CollectionAdminRequest.createCollection(collection, 2, 2);
+    CollectionAdminResponse r1 = a1.process(cluster.getSolrClient());
+    assertEquals(0, r1.getStatus());
+
+    // Expecting 8 spans:
+    // 1. api call 'operationName:"create:/admin/collections"',
+    // db.instance=testInternalCollectionApiCommands
+    // - unique traceId unrelated to the internal trace id generated for the operation
+    // 2. internal CollectionApiCommand 'operationName:"CreateCollectionCmd"'
+    // db.instance=testInternalCollectionApiCommands
+    // - this will be the parent span, all following spans will have the same traceId
+    //
+    // 3..6 (4 times) operationName:"post:/admin/cores"
+    // db.instance=testInternalCollectionApiCommands_shard1_replica_n2
+    // db.instance=testInternalCollectionApiCommands_shard2_replica_n4
+    // db.instance=testInternalCollectionApiCommands_shard2_replica_n1
+    // db.instance=testInternalCollectionApiCommands_shard1_replica_n6
+    //
+    // 7..8 (2 times) name=post:/{core}/get
+    // db.instance=testInternalCollectionApiCommands_shard2_replica_n4
+    // db.instance=testInternalCollectionApiCommands_shard1_replica_n2
+
+    var finishedSpans = getAndClearSpans();
+    var s0 = finishedSpans.remove(0);
+    assertDbInstanceColl(s0, collection);
+    assertEquals("create:/admin/collections", s0.operationName());
+
+    Map<String, Integer> ops = new HashMap<>();
+    assertEquals(7, finishedSpans.size());
+    System.err.println("finishedSpans " + finishedSpans);
+
+    var parentTraceId = finishedSpans.get(0).context().traceId();
+    var parentId = finishedSpans.get(0).context().spanId();
+    for (var span : finishedSpans) {
+      if (span.context().spanId() == parentId) {
+        assertDbInstanceColl(span, collection);
+      } else {
+        assertDbInstanceCore(span, collection);
+      }
+      assertEquals(span.context().traceId(), parentTraceId);
+      ops.put(span.operationName(), ops.getOrDefault(span.operationName(), 0) + 1);
+    }
+    var expectedOps =
+        Map.of("CreateCollectionCmd", 1, "post:/admin/cores", 4, "post:/{core}/get", 2);
+    assertEquals(expectedOps, ops);
+  }
+
+  private void verifyCollectionDeletion(String collection) throws Exception {
+    var a1 = CollectionAdminRequest.deleteCollection(collection);
+    CollectionAdminResponse r1 = a1.process(cluster.getSolrClient());
+    assertEquals(0, r1.getStatus());
+
+    // Expecting 6 spans:
+    // 1. api call 'operationName:"delete:/admin/collections"',
+    // db.instance=testInternalCollectionApiCommands
+    // - unique traceId unrelated to the internal trace id generated for the operation
+    // 2. internal CollectionApiCommand 'operationName:"DeleteCollectionCmd"'
+    // db.instance=testInternalCollectionApiCommands
+    // - this will be the parent span, all following spans will have the same traceId
+    //
+    // 3..6 (4 times) name=post:/admin/cores
+    // db.instance=testInternalCollectionApiCommands_shard2_replica_n1
+    // db.instance=testInternalCollectionApiCommands_shard1_replica_n2
+    // db.instance=testInternalCollectionApiCommands_shard2_replica_n4
+    // db.instance=testInternalCollectionApiCommands_shard1_replica_n6
+
+    var finishedSpans = getAndClearSpans();
+    var s0 = finishedSpans.remove(0);
+    assertDbInstanceColl(s0, collection);
+    assertEquals("delete:/admin/collections", s0.operationName());
+
+    Map<String, Integer> ops = new HashMap<>();
+    assertEquals(5, finishedSpans.size());
+    var parentTraceId = finishedSpans.get(0).context().traceId();
+    var parentId = finishedSpans.get(0).context().spanId();
+    for (var span : finishedSpans) {
+      if (span.context().spanId() == parentId) {
+        assertDbInstanceColl(span, collection);
+      } else {
+        assertDbInstanceCore(span, collection);
+      }
+      assertEquals(span.context().traceId(), parentTraceId);
+      ops.put(span.operationName(), ops.getOrDefault(span.operationName(), 0) + 1);
+    }
+    var expectedOps = Map.of("DeleteCollectionCmd", 1, "post:/admin/cores", 4);
+    assertEquals(expectedOps, ops);
+  }
+
+  private static void assertDbInstanceColl(MockSpan mockSpan, String collection) {
+    MatcherAssert.assertThat(mockSpan.tags().get("db.instance"), Matchers.equalTo(collection));
   }
 
-  private void assertDbInstanceCore(MockSpan mockSpan) {
+  private static void assertDbInstanceCore(MockSpan mockSpan, String collection) {
     MatcherAssert.assertThat(
-        (String) mockSpan.tags().get("db.instance"), Matchers.startsWith("collection1_"));
+        (String) mockSpan.tags().get("db.instance"), Matchers.startsWith(collection + "_"));
   }
 
   private void assertOneSpanIsChildOfAnother(List<MockSpan> finishedSpans) {
diff --git a/solr/core/src/test/org/apache/solr/util/tracing/TestSimplePropagatorDistributedTracing.java b/solr/core/src/test/org/apache/solr/util/tracing/TestSimplePropagatorDistributedTracing.java
index 6a8dee6d27d..4a1fc80e67e 100644
--- a/solr/core/src/test/org/apache/solr/util/tracing/TestSimplePropagatorDistributedTracing.java
+++ b/solr/core/src/test/org/apache/solr/util/tracing/TestSimplePropagatorDistributedTracing.java
@@ -30,10 +30,12 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.request.QueryRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
 import org.apache.solr.cloud.SolrCloudTestCase;
 import org.apache.solr.common.util.SuppressForbidden;
 import org.apache.solr.common.util.TimeSource;
 import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.admin.CoreAdminOperation;
 import org.apache.solr.logging.MDCLoggingContext;
 import org.apache.solr.update.processor.LogUpdateProcessorFactory;
 import org.apache.solr.util.LogListener;
@@ -182,4 +184,19 @@ public class TestSimplePropagatorDistributedTracing extends SolrCloudTestCase {
     client.connect();
     return client;
   }
+
+  @Test
+  public void testInternalCollectionApiCommands() throws Exception {
+    String collecton = "testInternalCollectionApiCommands";
+    verifyCollectionCreation(collecton);
+  }
+
+  private void verifyCollectionCreation(String collection) throws Exception {
+    try (LogListener reqLog = LogListener.info(CoreAdminOperation.class.getName())) {
+      var a1 = CollectionAdminRequest.createCollection(collection, 2, 2);
+      CollectionAdminResponse r1 = a1.process(cluster.getSolrClient());
+      assertEquals(0, r1.getStatus());
+      assertSameTraceId(reqLog, null);
+    }
+  }
 }