You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ds...@apache.org on 2021/06/03 15:24:00 UTC
[solr] branch main updated: SOLR-15392: Tracing span operation name
and tags (#115)
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new f38dbd3 SOLR-15392: Tracing span operation name and tags (#115)
f38dbd3 is described below
commit f38dbd343a10a53f3c941eea83f3251588f52d9c
Author: David Smiley <ds...@apache.org>
AuthorDate: Thu Jun 3 11:23:55 2021 -0400
SOLR-15392: Tracing span operation name and tags (#115)
The span operation name of a request is now composed of a command/verb
and a templated path. ex: "reload:/c/{collection}"
Add these span tags:
* http.method
* http.status_code
* http.params (non-standard; renamed from just "params")
* db.type ("solr")
* db.instance (collection or core name)
* db.user
And some span.log for relatively uncommon SolrDispatchFilter cases.
---
solr/CHANGES.txt | 6 +-
.../src/java/org/apache/solr/api/AnnotatedApi.java | 2 +
solr/core/src/java/org/apache/solr/api/ApiBag.java | 2 +-
.../src/java/org/apache/solr/api/V2HttpCall.java | 99 ++++++++++++++++++--
.../solr/handler/admin/CollectionsHandler.java | 5 +-
.../solr/handler/admin/CoreAdminHandler.java | 13 +--
.../org/apache/solr/request/SolrQueryRequest.java | 27 ++++--
.../apache/solr/request/SolrQueryRequestBase.java | 20 +++-
.../java/org/apache/solr/servlet/HttpSolrCall.java | 45 ++++++++-
.../apache/solr/servlet/SolrDispatchFilter.java | 32 +++++--
.../apache/solr/servlet/SolrRequestParsers.java | 7 +-
.../org/apache/solr/util/tracing/TraceUtils.java | 40 ++++++++
.../solr/util/tracing/TestDistributedTracing.java | 103 +++++++++++++++++----
13 files changed, 340 insertions(+), 61 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 7f44a2e..3496488 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -131,6 +131,10 @@ when told to. The admin UI now tells it to. (Nazerke Seidan, David Smiley)
* SOLR-15421: ConfigSet API also checks for solrconfig.xml when checking the existence of a configset (Andras Salamon via David Smiley)
+* SOLR-15392: Distributed Tracing request span operation names are now composed of a command/verb
+ and a templated path. The collection or core name is now in the db.instance tag.
+ (David Smiley)
+
Other Changes
----------------------
* SOLR-14656: Autoscaling framework removed (Ishan Chattopadhyaya, noble, Ilan Ginzburg)
@@ -281,7 +285,7 @@ Other Changes
(David Smiley, Christine Poerschke)
* SOLR-15283: Overhaul distributed-tracing. Solr's "samplePercentage" is gone; tracing is either
- enabled and you always have trace IDs or it isn't, based on wether you configure a tracing plugin.
+ enabled and you always have trace IDs or it isn't, based on whether you configure a tracing plugin.
Reporting of traces to a server like Zipkin is a configuration matter of the tracing plugin you
use. The Jaeger plugin is now completely configurable via system properties or environment vars,
not by values in solr.xml. (David Smiley)
diff --git a/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java b/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java
index 628aca7..6587d1f 100644
--- a/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java
+++ b/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java
@@ -49,6 +49,7 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.PermissionNameProvider;
import org.apache.solr.util.SolrJacksonAnnotationInspector;
+import org.apache.solr.util.tracing.TraceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -210,6 +211,7 @@ public class AnnotatedApi extends Api implements PermissionNameProvider , Closea
}
for (CommandOperation cmd : cmds) {
+ TraceUtils.ifNotNoop(req.getSpan(), (span) -> span.log("Command: " + cmd.name));
commands.get(cmd.name).invoke(req, rsp, cmd);
}
diff --git a/solr/core/src/java/org/apache/solr/api/ApiBag.java b/solr/core/src/java/org/apache/solr/api/ApiBag.java
index aa64cec..7efcdec 100644
--- a/solr/core/src/java/org/apache/solr/api/ApiBag.java
+++ b/solr/core/src/java/org/apache/solr/api/ApiBag.java
@@ -189,7 +189,7 @@ public class ApiBag {
result = specCopy;
}
if (isCoreSpecific) {
- List<String> pieces = req.getHttpSolrCall() == null ? null : ((V2HttpCall) req.getHttpSolrCall()).pieces;
+ List<String> pieces = req.getHttpSolrCall() == null ? null : ((V2HttpCall) req.getHttpSolrCall()).getPathSegments();
if (pieces != null) {
String prefix = "/" + pieces.get(0) + "/" + pieces.get(1);
List<String> paths = result.getMap("url", NOT_NULL).getList("paths", NOT_NULL);
diff --git a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java
index 504db1f..71755db 100644
--- a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java
+++ b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java
@@ -18,12 +18,15 @@
package org.apache.solr.api;
import com.google.common.collect.ImmutableSet;
+import io.opentracing.Span;
+import io.opentracing.tag.Tags;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.annotation.SolrThreadSafe;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.JsonSchemaValidator;
import org.apache.solr.common.util.PathTrie;
import org.apache.solr.common.util.ValidatingJsonMap;
@@ -48,7 +51,6 @@ import java.util.*;
import java.util.function.Supplier;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
-import static org.apache.solr.common.util.PathTrie.getPathSegments;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.*;
// class that handle the '/v2' path
@@ -56,7 +58,7 @@ import static org.apache.solr.servlet.SolrDispatchFilter.Action.*;
public class V2HttpCall extends HttpSolrCall {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private Api api;
- List<String> pieces;
+ private List<String> pathSegments;
private String prefix;
HashMap<String, String> parts = new HashMap<>();
static final Set<String> knownPrefixes = ImmutableSet.of("cluster", "node", "collections", "cores", "c");
@@ -70,8 +72,8 @@ public class V2HttpCall extends HttpSolrCall {
String path = this.path;
final String fullPath = path = path.substring(7);//strip off '/____v2'
try {
- pieces = getPathSegments(path);
- if (pieces.size() == 0 || (pieces.size() == 1 && path.endsWith(CommonParams.INTROSPECT))) {
+ pathSegments = PathTrie.getPathSegments(path);
+ if (pathSegments.size() == 0 || (pathSegments.size() == 1 && path.endsWith(CommonParams.INTROSPECT))) {
api = new Api(null) {
@Override
public void call(SolrQueryRequest req, SolrQueryResponse rsp) {
@@ -82,7 +84,7 @@ public class V2HttpCall extends HttpSolrCall {
initAdminRequest(path);
return;
} else {
- prefix = pieces.get(0);
+ prefix = pathSegments.get(0);
}
boolean isCompositeApi = false;
@@ -98,7 +100,7 @@ public class V2HttpCall extends HttpSolrCall {
}
if ("c".equals(prefix) || "collections".equals(prefix)) {
- origCorename = pieces.get(1);
+ origCorename = pathSegments.get(1);
DocCollection collection = resolveDocCollection(queryParams.get(COLLECTION_PROP, origCorename));
@@ -120,7 +122,7 @@ public class V2HttpCall extends HttpSolrCall {
}
}
} else if ("cores".equals(prefix)) {
- origCorename = pieces.get(1);
+ origCorename = pathSegments.get(1);
core = cores.getCore(origCorename);
} else {
api = getApiInfo(cores.getRequestHandlers(), path, req.getMethod(), fullPath, parts);
@@ -140,7 +142,7 @@ public class V2HttpCall extends HttpSolrCall {
}
}
- this.path = path = path.substring(prefix.length() + pieces.get(1).length() + 2);
+ this.path = path = path.substring(prefix.length() + pathSegments.get(1).length() + 2);
Api apiInfo = getApiInfo(core.getRequestHandlers(), path, req.getMethod(), fullPath, parts);
if (isCompositeApi && apiInfo instanceof CompositeApi) {
((CompositeApi) this.api).add(apiInfo);
@@ -312,6 +314,10 @@ public class V2HttpCall extends HttpSolrCall {
}
}
+ public List<String> getPathSegments() {
+ return pathSegments;
+ }
+
public static class CompositeApi extends Api {
private LinkedList<Api> apis = new LinkedList<>();
@@ -361,6 +367,83 @@ public class V2HttpCall extends HttpSolrCall {
}
@Override
+ protected void populateTracingSpan(Span span) {
+ // Set db.instance
+ String coreOrColName = this.origCorename;
+ if (coreOrColName == null) {
+ Map<String, String> pathTemplateValues = getUrlParts(); // == solrReq.getPathTemplateValues()
+ coreOrColName = pathTemplateValues.get("collection");
+ if (coreOrColName == null) {
+ coreOrColName = pathTemplateValues.get("core");
+ }
+ }
+ if (coreOrColName != null) {
+ span.setTag(Tags.DB_INSTANCE, coreOrColName);
+ }
+
+ // Get the templatize-ed path, ex: "/c/{collection}"
+ String path;
+ if (api instanceof AnnotatedApi) {
+ // ideal scenario; eventually everything might be AnnotatedApi?
+ path = ((AnnotatedApi) api).getEndPoint().path()[0]; // consider first to be primary
+ } else {
+ path = computeEndpointPath();
+ }
+
+ String verb = null;
+ // if this api has commands ...
+ final Map<String, JsonSchemaValidator> validators = getValidators(); // should be cached
+ if (validators != null && validators.isEmpty() == false && solrReq != null) {
+ boolean validateInput = true; // because getCommands caches it; and we want it validated later
+ // does this request have one command?
+ List<CommandOperation> cmds = solrReq.getCommands(validateInput);
+ if (cmds.size() == 1) {
+ verb = cmds.get(0).name;
+ }
+ }
+ if (verb == null) {
+ verb = req.getMethod().toLowerCase(Locale.ROOT);
+ }
+
+ span.setOperationName(verb + ":" + path);
+ }
+
+ /**
+ * Example:
+ * /c/collection1/ and template map collection->collection1 produces /c/{collection}.
+ */
+ private String computeEndpointPath() {
+ // It's not ideal to compute this; let's try to transition away from hitting this code path
+ // by using Annotation APIs
+ // collection -> myColName
+ final Map<String, String> pathTemplateKeyVal = getUrlParts();
+ // myColName -> collection
+ final Map<String, String> pathTemplateValKey;
+ if (pathTemplateKeyVal.isEmpty()) { // typical
+ pathTemplateValKey = Collections.emptyMap();
+ } else if (pathTemplateKeyVal.size() == 1) { // typical
+ Map.Entry<String, String> entry = pathTemplateKeyVal.entrySet().iterator().next();
+ pathTemplateValKey = Map.of(entry.getValue(), entry.getKey());
+ } else { // uncommon
+ pathTemplateValKey = new HashMap<>();
+ for (Map.Entry<String, String> entry : pathTemplateKeyVal.entrySet()) {
+ pathTemplateValKey.put(entry.getValue(), entry.getKey());
+ }
+ }
+ final StringBuilder builder = new StringBuilder();
+ for (String segment : pathSegments) {
+ builder.append('/');
+ String key = pathTemplateValKey.get(segment);
+ if (key == null) {
+ builder.append(segment);
+ } else {
+ builder.append('{').append(key).append('}');
+ }
+ }
+ return builder.toString();
+ }
+
+ @Override
protected Object _getHandler() {
return api;
}
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 8891817..3310700 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -82,6 +82,7 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.PermissionNameProvider;
+import org.apache.solr.util.tracing.TraceUtils;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -205,11 +206,13 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
if (action == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown action: " + a);
}
+ final String collection = params.get(COLLECTION);
+ MDCLoggingContext.setCollection(collection);
+ TraceUtils.setDbInstance(req, collection);
CollectionOperation operation = CollectionOperation.get(action);
if (log.isDebugEnabled()) {
log.debug("Invoked Collection Action: {} with params {}", action.toLower(), req.getParamString());
}
- MDCLoggingContext.setCollection(req.getParams().get(COLLECTION));
invokeAction(req, rsp, cores, action, operation);
} else {
throw new SolrException(ErrorCode.BAD_REQUEST, "action is a required param");
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
index 0ef3ebb..69706ac 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
@@ -41,6 +41,7 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.handler.RequestHandlerBase;
@@ -51,8 +52,8 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.PermissionNameProvider;
-import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.util.stats.MetricUtils;
+import org.apache.solr.util.tracing.TraceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
@@ -165,18 +166,18 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa
}
// Pick the action
- CoreAdminOperation op = opMap.get(req.getParams().get(ACTION, STATUS.toString()).toLowerCase(Locale.ROOT));
+ final String action = req.getParams().get(ACTION, STATUS.toString()).toLowerCase(Locale.ROOT);
+ CoreAdminOperation op = opMap.get(action);
if (op == null) {
handleCustomAction(req, rsp);
return;
}
final CallInfo callInfo = new CallInfo(this, req, rsp, op);
- String coreName = req.getParams().get(CoreAdminParams.CORE);
- if (coreName == null) {
- coreName = req.getParams().get(CoreAdminParams.NAME);
- }
+ final String coreName =
+ req.getParams().get(CoreAdminParams.CORE, req.getParams().get(CoreAdminParams.NAME));
MDCLoggingContext.setCoreName(coreName);
+ TraceUtils.setDbInstance(req, coreName);
if (taskId == null) {
callInfo.call();
} else {
diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java
index 3947648..f472040 100644
--- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java
+++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java
@@ -16,22 +16,24 @@
*/
package org.apache.solr.request;
+import java.security.Principal;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import io.opentracing.Span;
import io.opentracing.Tracer;
+import io.opentracing.noop.NoopSpan;
import io.opentracing.util.GlobalTracer;
-import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.schema.IndexSchema;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.servlet.HttpSolrCall;
-import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.util.RTimerTree;
-import java.security.Principal;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
/**
* <p>Container for a request to execute a query.</p>
* <p><code>SolrQueryRequest</code> is not thread safe.</p>
@@ -142,6 +144,15 @@ public interface SolrQueryRequest extends AutoCloseable {
default Tracer getTracer() {
return GlobalTracer.get(); // default impl is only for some tests
}
+
+ /**
+ * The distributed tracing Span for the request itself; never null. This is useful for adding tags
+ * or updating the operation name of the request span. If you need the current span, which might
+ * not necessarily be the request span, do this instead: {@code tracer.activeSpan()}.
+ */
+ default Span getSpan() {
+ return NoopSpan.INSTANCE;
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java
index bc72db8..0d86906 100644
--- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java
+++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java
@@ -23,7 +23,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import io.opentracing.Span;
import io.opentracing.Tracer;
+import io.opentracing.noop.NoopSpan;
import io.opentracing.util.GlobalTracer;
import org.apache.solr.api.ApiBag;
import org.apache.solr.common.SolrException;
@@ -144,9 +146,6 @@ public abstract class SolrQueryRequestBase implements SolrQueryRequest, Closeabl
@Override
public Tracer getTracer() {
- if (core != null) {
- return core.getCoreContainer().getTracer(); // this is the common path
- }
final HttpSolrCall call = getHttpSolrCall();
if (call != null) {
final Tracer tracer = (Tracer) call.getReq().getAttribute(Tracer.class.getName());
@@ -154,10 +153,25 @@ public abstract class SolrQueryRequestBase implements SolrQueryRequest, Closeabl
return tracer;
}
}
+ if (core != null) {
+ return core.getCoreContainer().getTracer();
+ }
return GlobalTracer.get(); // this way is not ideal (particularly in tests) but it's okay
}
@Override
+ public Span getSpan() {
+ final HttpSolrCall call = getHttpSolrCall();
+ if (call != null) {
+ final Span span = (Span) call.getReq().getAttribute(Span.class.getName());
+ if (span != null) {
+ return span;
+ }
+ }
+ return NoopSpan.INSTANCE;
+ }
+
+ @Override
public void updateSchemaToLatest() {
schema = core.getLatestSchema();
}
diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
index 892230e..f04c0b1 100644
--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
@@ -33,12 +33,15 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import io.opentracing.Span;
+import io.opentracing.tag.Tags;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.Header;
@@ -70,6 +73,7 @@ import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
@@ -109,6 +113,7 @@ import org.apache.solr.servlet.cache.Method;
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.apache.solr.util.RTimerTree;
import org.apache.solr.util.TimeOut;
+import org.apache.solr.util.tracing.TraceUtils;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -500,9 +505,7 @@ public class HttpSolrCall {
*/
public Action call() throws IOException {
MDCLoggingContext.reset();
- Span activeSpan = (Span) req.getAttribute(Span.class.getName()); // never null
- MDCLoggingContext.setTracerId(activeSpan.context().toTraceId()); // handles empty string
-
+ MDCLoggingContext.setTracerId(getSpan().context().toTraceId()); // handles empty string
MDCLoggingContext.setNode(cores);
if (cores == null) {
@@ -521,6 +524,8 @@ public class HttpSolrCall {
try {
init();
+ TraceUtils.ifNotNoop(getSpan(), this::populateTracingSpan);
+
// Perform authorization here, if:
// (a) Authorization is enabled, and
// (b) The requested resource is not a known static file
@@ -603,6 +608,40 @@ public class HttpSolrCall {
}
+ /** Get the span for this request. Not null. */
+ protected Span getSpan() {
+ // Span was put into the request by SolrDispatchFilter
+ return (Span) Objects.requireNonNull(req.getAttribute(Span.class.getName()));
+ }
+
+ // called after init().
+ protected void populateTracingSpan(Span span) {
+ // Set db.instance
+ String coreOrColName = HttpSolrCall.this.origCorename;
+ if (coreOrColName == null && getCore() != null) {
+ coreOrColName = getCore().getName();
+ }
+ if (coreOrColName != null) {
+ span.setTag(Tags.DB_INSTANCE, coreOrColName);
+ }
+
+ // Set operation name.
+ String path = getPath();
+ if (coreOrColName != null) {
+ // prefix path by core or collection name
+ if (getCore() != null && getCore().getName().equals(coreOrColName)) {
+ path = "/{core}" + path;
+ } else {
+ path = "/{collection}" + path;
+ }
+ }
+ String verb =
+ getQueryParams()
+ .get(CoreAdminParams.ACTION, req.getMethod())
+ .toLowerCase(Locale.ROOT);
+ span.setOperationName(verb + ":" + path);
+ }
+
private boolean shouldAudit() {
return cores.getAuditLoggerPlugin() != null;
}
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
index 05d4be4..2816727 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
@@ -60,6 +60,8 @@ import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
import com.google.common.annotations.VisibleForTesting;
import io.opentracing.Span;
import io.opentracing.Tracer;
+import io.opentracing.noop.NoopSpan;
+import io.opentracing.noop.NoopTracer;
import io.opentracing.propagation.Format;
import io.opentracing.tag.Tags;
import io.opentracing.util.GlobalTracer;
@@ -449,12 +451,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
}
Tracer tracer = cores == null ? GlobalTracer.get() : cores.getTracer();
- String hostAndPort = request.getServerName() + "_" + request.getServerPort();
- Span span = tracer.buildSpan(request.getMethod() + ":" + hostAndPort)
- .asChildOf(tracer.extract(Format.Builtin.HTTP_HEADERS, new HttpServletCarrier(request)))
- .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER)
- .withTag(Tags.HTTP_URL, request.getRequestURL().toString())
- .start();
+ Span span = buildSpan(tracer, request);
request.setAttribute(Tracer.class.getName(), tracer);
request.setAttribute(Span.class.getName(), span);
boolean accepted = false;
@@ -500,6 +497,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
if (log.isDebugEnabled()) {
log.debug("User principal: {}", request.getUserPrincipal());
}
+ span.setTag(Tags.DB_USER, String.valueOf(request.getUserPrincipal()));
}
HttpSolrCall call = getHttpSolrCall(request, response, retry);
@@ -508,12 +506,15 @@ public class SolrDispatchFilter extends BaseSolrFilter {
Action result = call.call();
switch (result) {
case PASSTHROUGH:
+ span.log("SolrDispatchFilter PASSTHROUGH");
chain.doFilter(request, response);
break;
case RETRY:
+ span.log("SolrDispatchFilter RETRY");
doFilter(request, response, chain, true); // RECURSION
break;
case FORWARD:
+ span.log("SolrDispatchFilter FORWARD");
request.getRequestDispatcher(call.getPath()).forward(request, response);
break;
case ADMIN:
@@ -534,10 +535,27 @@ public class SolrDispatchFilter extends BaseSolrFilter {
if (accepted) {
rateLimitManager.decrementActiveRequests(request);
}
+ span.setTag(Tags.HTTP_STATUS, response.getStatus());
span.finish();
}
}
-
+
+ protected Span buildSpan(Tracer tracer, HttpServletRequest request) {
+ if (tracer instanceof NoopTracer) {
+ return NoopSpan.INSTANCE;
+ }
+ Tracer.SpanBuilder spanBuilder = tracer.buildSpan("http.request") // will be changed later
+ .asChildOf(tracer.extract(Format.Builtin.HTTP_HEADERS, new HttpServletCarrier(request)))
+ .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER)
+ .withTag(Tags.HTTP_METHOD, request.getMethod())
+ .withTag(Tags.HTTP_URL, request.getRequestURL().toString());
+ if (request.getQueryString() != null) {
+ spanBuilder.withTag("http.params", request.getQueryString());
+ }
+ spanBuilder.withTag(Tags.DB_TYPE, "solr");
+ return spanBuilder.start();
+ }
+
// we make sure we read the full client request so that the client does
// not hit a connection reset and we can reuse the
// connection - see SOLR-8453 and SOLR-8683
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
index 8bf1e02..8685191 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
@@ -43,8 +43,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import io.opentracing.Span;
-import io.opentracing.noop.NoopSpan;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.api.V2HttpCall;
@@ -173,10 +171,6 @@ public class SolrRequestParsers {
ArrayList<ContentStream> streams = new ArrayList<>(1);
SolrParams params = parser.parseParamsAndFillStreams( req, streams );
- Span span = (Span) req.getAttribute(Span.class.getName()); // not null but maybe in some tests?
- if (span != null && !(span instanceof NoopSpan)) {
- span.setTag("params", params.toString());
- }
SolrQueryRequest sreq = buildRequestFrom(core, params, streams, getRequestTimer(req), req);
// Handlers and login will want to know the path. If it contains a ':'
@@ -190,6 +184,7 @@ public class SolrRequestParsers {
return sreq;
}
+ /** For embedded Solr use; not related to HTTP. */
public SolrQueryRequest buildRequestFrom(SolrCore core, SolrParams params, Collection<ContentStream> streams) throws Exception {
return buildRequestFrom(core, params, streams, new RTimerTree(), null);
}
diff --git a/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java b/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java
new file mode 100644
index 0000000..ea00235
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util.tracing;
+
+import java.util.function.Consumer;
+
+import io.opentracing.Span;
+import io.opentracing.noop.NoopSpan;
+import io.opentracing.tag.Tags;
+import org.apache.solr.request.SolrQueryRequest;
+
+/** Utilities for distributed tracing. */
+public class TraceUtils {
+
+ public static void setDbInstance(SolrQueryRequest req, String coreOrColl) {
+ if (coreOrColl != null) {
+ ifNotNoop(req.getSpan(), (span) -> span.setTag(Tags.DB_INSTANCE, coreOrColl));
+ }
+ }
+
+ public static void ifNotNoop(Span span, Consumer<Span> consumer) {
+ if (span != null && !(span instanceof NoopSpan)) {
+ consumer.accept(span);
+ }
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java b/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java
index 8d1ebc2..0e9a9a6 100644
--- a/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java
+++ b/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java
@@ -18,7 +18,7 @@
package org.apache.solr.util.tracing;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
@@ -27,16 +27,22 @@ import io.opentracing.mock.MockSpan;
import io.opentracing.mock.MockTracer;
import io.opentracing.util.GlobalTracer;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.V2Request;
+import org.apache.solr.client.solrj.response.V2Response;
import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.util.LogLevel;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-
@LogLevel("org.apache.solr.core.TracerConfigurator=trace")
public class TestDistributedTracing extends SolrCloudTestCase {
private static final String COLLECTION = "collection1";
@@ -69,29 +75,28 @@ public class TestDistributedTracing extends SolrCloudTestCase {
// and what we assert it looks like in a structured visual way.
CloudSolrClient cloudClient = cluster.getSolrClient();
- List<MockSpan> allSpans = tracer.finishedSpans();
+ List<MockSpan> finishedSpans;
+ // Indexing
cloudClient.add(COLLECTION, sdoc("id", "1"));
- List<MockSpan> finishedSpans = getRecentSpans(allSpans);
+ finishedSpans = getAndClearSpans();
finishedSpans.removeIf(x ->
!x.tags().get("http.url").toString().endsWith("/update"));
assertEquals(2, finishedSpans.size());
assertOneSpanIsChildOfAnother(finishedSpans);
+ // core because cloudClient routes to core
+ assertEquals("post:/{core}/update", finishedSpans.get(0).operationName());
+ assertDbInstanceCore(finishedSpans.get(0));
cloudClient.add(COLLECTION, sdoc("id", "2"));
- finishedSpans = getRecentSpans(allSpans);
- finishedSpans.removeIf(x ->
- !x.tags().get("http.url").toString().endsWith("/update"));
- assertEquals(2, finishedSpans.size());
- assertOneSpanIsChildOfAnother(finishedSpans);
-
cloudClient.add(COLLECTION, sdoc("id", "3"));
cloudClient.add(COLLECTION, sdoc("id", "4"));
cloudClient.commit(COLLECTION);
+ getAndClearSpans();
- getRecentSpans(allSpans);
+ // Searching
cloudClient.query(COLLECTION, new SolrQuery("*:*"));
- finishedSpans = getRecentSpans(allSpans);
+ finishedSpans = getAndClearSpans();
finishedSpans.removeIf(x ->
!x.tags().get("http.url").toString().endsWith("/select"));
// one from client to server, 2 for execute query, 2 for fetching documents
@@ -106,6 +111,71 @@ public class TestDistributedTracing extends SolrCloudTestCase {
fail("All spans must belong to single span, but:"+finishedSpans);
}
}
+ assertEquals("get:/{collection}/select", finishedSpans.get(0).operationName());
+ assertDbInstanceColl(finishedSpans.get(0));
+ }
+
+ @Test
+ public void testAdminApi() throws Exception {
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+ List<MockSpan> finishedSpans;
+
+ // Admin API call
+ cloudClient.request(new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/metrics", params()));
+ finishedSpans = getAndClearSpans();
+ assertEquals("get:/admin/metrics", finishedSpans.get(0).operationName());
+
+ CollectionAdminRequest.listCollections(cloudClient);
+ finishedSpans = getAndClearSpans();
+ assertEquals("list:/admin/collections", finishedSpans.get(0).operationName());
+ }
+
+ @Test
+ public void testV2Api() throws Exception {
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+ List<MockSpan> finishedSpans;
+
+ new V2Request.Builder("/c/" + COLLECTION)
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload("{\n" +
+ " \"reload\" : {}\n" +
+ "}")
+ .build()
+ .process(cloudClient);
+ finishedSpans = getAndClearSpans();
+ assertEquals("reload:/c/{collection}", finishedSpans.get(0).operationName());
+ assertDbInstanceColl(finishedSpans.get(0));
+
+ new V2Request.Builder("/c/" + COLLECTION + "/update/json")
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload("{\n" +
+ " \"id\" : \"9\"\n" +
+ "}")
+ .withParams(params("commit", "true"))
+ .build()
+ .process(cloudClient);
+ finishedSpans = getAndClearSpans();
+ assertEquals("post:/c/{collection}/update/json", finishedSpans.get(0).operationName());
+ assertDbInstanceColl(finishedSpans.get(0));
+
+ final V2Response v2Response = new V2Request.Builder("/c/" + COLLECTION + "/select")
+ .withMethod(SolrRequest.METHOD.GET)
+ .withParams(params("q", "id:9"))
+ .build()
+ .process(cloudClient);
+ finishedSpans = getAndClearSpans();
+ assertEquals("get:/c/{collection}/select", finishedSpans.get(0).operationName());
+ assertDbInstanceColl(finishedSpans.get(0));
+ assertEquals(1, ((SolrDocumentList)v2Response.getResponse().get("response")).getNumFound());
+ }
+
+ private void assertDbInstanceColl(MockSpan mockSpan) {
+ MatcherAssert.assertThat(mockSpan.tags().get("db.instance"), Matchers.equalTo("collection1"));
+ }
+
+ private void assertDbInstanceCore(MockSpan mockSpan) {
+ MatcherAssert.assertThat(
+ (String) mockSpan.tags().get("db.instance"), Matchers.startsWith("collection1_"));
}
private void assertOneSpanIsChildOfAnother(List<MockSpan> finishedSpans) {
@@ -120,11 +190,10 @@ public class TestDistributedTracing extends SolrCloudTestCase {
assertEquals(child.parentId(), parent.context().spanId());
}
- private List<MockSpan> getRecentSpans(List<MockSpan> allSpans) {
- List<MockSpan> result = new ArrayList<>(tracer.finishedSpans());
- result.removeAll(allSpans);
- allSpans.clear();
- allSpans.addAll(tracer.finishedSpans());
+ private List<MockSpan> getAndClearSpans() {
+ List<MockSpan> result = tracer.finishedSpans(); // returns a mutable copy
+ Collections.reverse(result); // nicer to see spans chronologically
+ tracer.reset();
return result;
}
}