You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2024/09/06 14:01:27 UTC
(skywalking) branch master updated: Add self observability metrics for otel handler (#12598)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 78483991b7 Add self observability metrics for otel handler (#12598)
78483991b7 is described below
commit 78483991b780152a00f7f4032e0012638a1aba93
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Fri Sep 6 22:01:22 2024 +0800
Add self observability metrics for otel handler (#12598)
---
docs/en/changes/changes.md | 1 +
.../otel/otlp/OpenTelemetryLogHandler.java | 24 +++-
.../otlp/OpenTelemetryMetricRequestProcessor.java | 75 ++++++-----
.../otel/otlp/OpenTelemetryTraceHandler.java | 64 ++++++----
.../src/main/resources/otel-rules/oap.yaml | 35 +++--
.../so11y_oap/so11y-instance.json | 142 +++++++++++++++++++++
6 files changed, 276 insertions(+), 65 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 6fa5460bb2..db4d10bbf1 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -58,6 +58,7 @@
* Fix the compatibility with Grafana 11 when using label_values query variables.
* Nacos as config server and cluster coordinator supports configuration contextPath.
* Update the endpoint name format to `<Method>:<Path>` in eBPF Access Log Receiver.
+* Add self-observability metrics for OpenTelemetry receiver.
#### UI
diff --git a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandler.java b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandler.java
index 3d6c6bc9db..35c75b7ace 100644
--- a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandler.java
+++ b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandler.java
@@ -27,6 +27,7 @@ import io.opentelemetry.proto.common.v1.KeyValue;
import io.opentelemetry.proto.logs.v1.LogRecord;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
+import lombok.Getter;
import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair;
import org.apache.skywalking.apm.network.logging.v3.LogData;
import org.apache.skywalking.apm.network.logging.v3.LogDataBody;
@@ -39,6 +40,10 @@ import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.apache.skywalking.oap.server.receiver.otel.Handler;
import org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule;
+import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
+import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsTag;
import java.util.Map;
import java.util.stream.Collectors;
@@ -54,6 +59,17 @@ public class OpenTelemetryLogHandler
private ILogAnalyzerService logAnalyzerService;
+ @Getter(lazy = true)
+ private final MetricsCreator metricsCreator = manager.find(TelemetryModule.NAME).provider().getService(MetricsCreator.class);
+
+ @Getter(lazy = true)
+ private final HistogramMetrics processHistogram = getMetricsCreator().createHistogramMetric(
+ "otel_logs_latency",
+ "The latency to process the logs request",
+ MetricsTag.EMPTY_KEY,
+ MetricsTag.EMPTY_VALUE
+ );
+
@Override
public String type() {
return "otlp-logs";
@@ -87,9 +103,11 @@ public class OpenTelemetryLogHandler
.getScopeLogsList()
.stream()
.flatMap(it -> it.getLogRecordsList().stream())
- .forEach(
- logRecord ->
- doAnalysisQuietly(service, layer, serviceInstance, logRecord));
+ .forEach(logRecord -> {
+ try (final var timer = getProcessHistogram().createTimer()) {
+ doAnalysisQuietly(service, layer, serviceInstance, logRecord);
+ }
+ });
responseObserver.onNext(ExportLogsServiceResponse.getDefaultInstance());
responseObserver.onCompleted();
});
diff --git a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryMetricRequestProcessor.java b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryMetricRequestProcessor.java
index 0be3aaaf2c..7ef39c0a8f 100644
--- a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryMetricRequestProcessor.java
+++ b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryMetricRequestProcessor.java
@@ -25,6 +25,7 @@ import io.opentelemetry.proto.common.v1.KeyValue;
import io.opentelemetry.proto.metrics.v1.Sum;
import io.opentelemetry.proto.metrics.v1.SummaryDataPoint;
import io.vavr.Function1;
+import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.meter.analyzer.MetricConvert;
@@ -42,6 +43,10 @@ import org.apache.skywalking.oap.server.library.util.prometheus.metrics.Histogra
import org.apache.skywalking.oap.server.library.util.prometheus.metrics.Metric;
import org.apache.skywalking.oap.server.library.util.prometheus.metrics.Summary;
import org.apache.skywalking.oap.server.receiver.otel.OtelMetricReceiverConfig;
+import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
+import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsTag;
import java.io.IOException;
import java.util.HashMap;
@@ -73,39 +78,51 @@ public class OpenTelemetryMetricRequestProcessor implements Service {
.build();
private List<PrometheusMetricConverter> converters;
- public void processMetricsRequest(final ExportMetricsServiceRequest requests) {
- requests.getResourceMetricsList().forEach(request -> {
- if (log.isDebugEnabled()) {
- log.debug("Resource attributes: {}", request.getResource().getAttributesList());
- }
+ @Getter(lazy = true)
+ private final MetricsCreator metricsCreator = manager.find(TelemetryModule.NAME).provider().getService(MetricsCreator.class);
- final Map<String, String> nodeLabels =
- request
- .getResource()
- .getAttributesList()
- .stream()
- .collect(toMap(
- it -> LABEL_MAPPINGS
- .getOrDefault(it.getKey(), it.getKey())
- .replaceAll("\\.", "_"),
- it -> anyValueToString(it.getValue()),
- (v1, v2) -> v1
- ));
+ @Getter(lazy = true)
+ private final HistogramMetrics processHistogram = getMetricsCreator().createHistogramMetric(
+ "otel_metrics_latency",
+ "The latency to process the metrics request",
+ MetricsTag.EMPTY_KEY,
+ MetricsTag.EMPTY_VALUE
+ );
- converters
- .forEach(convert -> convert.toMeter(
+ public void processMetricsRequest(final ExportMetricsServiceRequest requests) {
+ try (final var unused = getProcessHistogram().createTimer()) {
+ requests.getResourceMetricsList().forEach(request -> {
+ if (log.isDebugEnabled()) {
+ log.debug("Resource attributes: {}", request.getResource().getAttributesList());
+ }
+
+ final Map<String, String> nodeLabels =
request
- .getScopeMetricsList().stream()
- .flatMap(scopeMetrics -> scopeMetrics
- .getMetricsList().stream()
- .flatMap(metric -> adaptMetrics(nodeLabels, metric))
- .map(Function1.liftTry(Function.identity()))
- .flatMap(tryIt -> MetricConvert.log(
- tryIt,
- "Convert OTEL metric to prometheus metric"
- )))));
- });
+ .getResource()
+ .getAttributesList()
+ .stream()
+ .collect(toMap(
+ it -> LABEL_MAPPINGS
+ .getOrDefault(it.getKey(), it.getKey())
+ .replaceAll("\\.", "_"),
+ it -> anyValueToString(it.getValue()),
+ (v1, v2) -> v1
+ ));
+ converters
+ .forEach(convert -> convert.toMeter(
+ request
+ .getScopeMetricsList().stream()
+ .flatMap(scopeMetrics -> scopeMetrics
+ .getMetricsList().stream()
+ .flatMap(metric -> adaptMetrics(nodeLabels, metric))
+ .map(Function1.liftTry(Function.identity()))
+ .flatMap(tryIt -> MetricConvert.log(
+ tryIt,
+ "Convert OTEL metric to prometheus metric"
+ )))));
+ });
+ }
}
public void start() throws ModuleStartException {
diff --git a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryTraceHandler.java b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryTraceHandler.java
index 18e76bee34..f1b687ba50 100644
--- a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryTraceHandler.java
+++ b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryTraceHandler.java
@@ -30,6 +30,7 @@ import io.opentelemetry.proto.common.v1.KeyValue;
import io.opentelemetry.proto.resource.v1.Resource;
import io.opentelemetry.proto.trace.v1.ScopeSpans;
import io.opentelemetry.proto.trace.v1.Status;
+import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister;
@@ -40,6 +41,10 @@ import org.apache.skywalking.oap.server.receiver.otel.Handler;
import org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule;
import org.apache.skywalking.oap.server.receiver.zipkin.SpanForwardService;
import org.apache.skywalking.oap.server.receiver.zipkin.ZipkinReceiverModule;
+import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
+import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsTag;
import zipkin2.Endpoint;
import zipkin2.Span;
@@ -64,6 +69,17 @@ public class OpenTelemetryTraceHandler
private final ModuleManager manager;
private SpanForwardService forwardService;
+ @Getter(lazy = true)
+ private final MetricsCreator metricsCreator = manager.find(TelemetryModule.NAME).provider().getService(MetricsCreator.class);
+
+ @Getter(lazy = true)
+ private final HistogramMetrics processHistogram = getMetricsCreator().createHistogramMetric(
+ "otel_spans_latency",
+ "The latency to process the span request",
+ MetricsTag.EMPTY_KEY,
+ MetricsTag.EMPTY_VALUE
+ );
+
@Override
public String type() {
return "otlp-traces";
@@ -80,34 +96,36 @@ public class OpenTelemetryTraceHandler
@Override
public void export(ExportTraceServiceRequest request, StreamObserver<ExportTraceServiceResponse> responseObserver) {
final ArrayList<Span> result = new ArrayList<>();
- request.getResourceSpansList().forEach(resourceSpans -> {
- final Resource resource = resourceSpans.getResource();
- final List<ScopeSpans> scopeSpansList = resourceSpans.getScopeSpansList();
- if (resource.getAttributesCount() == 0 && scopeSpansList.size() == 0) {
- return;
- }
- final Map<String, String> resourceTags = convertAttributeToMap(resource.getAttributesList());
- String serviceName = extractZipkinServiceName(resourceTags);
- if (StringUtil.isEmpty(serviceName)) {
- log.warn("No service name found in resource attributes, discarding the trace");
- return;
- }
+ try (final var unused = getProcessHistogram().createTimer()) {
+ request.getResourceSpansList().forEach(resourceSpans -> {
+ final Resource resource = resourceSpans.getResource();
+ final List<ScopeSpans> scopeSpansList = resourceSpans.getScopeSpansList();
+ if (resource.getAttributesCount() == 0 && scopeSpansList.size() == 0) {
+ return;
+ }
+ final Map<String, String> resourceTags = convertAttributeToMap(resource.getAttributesList());
+ String serviceName = extractZipkinServiceName(resourceTags);
+ if (StringUtil.isEmpty(serviceName)) {
+ log.warn("No service name found in resource attributes, discarding the trace");
+ return;
+ }
- try {
- for (ScopeSpans scopeSpans : scopeSpansList) {
- extractScopeTag(scopeSpans.getScope(), resourceTags);
- for (io.opentelemetry.proto.trace.v1.Span span : scopeSpans.getSpansList()) {
- Span zipkinSpan = convertSpan(span, serviceName, resourceTags);
- result.add(zipkinSpan);
+ try {
+ for (ScopeSpans scopeSpans : scopeSpansList) {
+ extractScopeTag(scopeSpans.getScope(), resourceTags);
+ for (io.opentelemetry.proto.trace.v1.Span span : scopeSpans.getSpansList()) {
+ Span zipkinSpan = convertSpan(span, serviceName, resourceTags);
+ result.add(zipkinSpan);
+ }
}
+ } catch (Exception e) {
+ log.warn("convert span error, discarding the span: {}", e.getMessage());
}
- } catch (Exception e) {
- log.warn("convert span error, discarding the span: {}", e.getMessage());
- }
- });
+ });
+ getForwardService().send(result);
+ }
- getForwardService().send(result);
responseObserver.onNext(ExportTraceServiceResponse.getDefaultInstance());
responseObserver.onCompleted();
}
diff --git a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml
index 6de0e00b53..a8aa11ee2e 100644
--- a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml
+++ b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml
@@ -37,15 +37,17 @@ metricsRules:
- name: instance_jvm_memory_bytes_used
exp: jvm_memory_bytes_used.sum(['service', 'host_name'])
- name: instance_jvm_gc_count
- exp: "jvm_gc_collection_seconds_count.tagMatch('gc', 'PS Scavenge|Copy|ParNew|G1 Young Generation|PS MarkSweep|MarkSweepCompact|ConcurrentMarkSweep|G1 Old Generation')
- .sum(['service', 'host_name', 'gc']).increase('PT1M')
- .tag({tags -> if (tags['gc'] == 'PS Scavenge' || tags['gc'] == 'Copy' || tags['gc'] == 'ParNew' || tags['gc'] == 'G1 Young Generation') {tags.gc = 'young_gc_count'} })
- .tag({tags -> if (tags['gc'] == 'PS MarkSweep' || tags['gc'] == 'MarkSweepCompact' || tags['gc'] == 'ConcurrentMarkSweep' || tags['gc'] == 'G1 Old Generation') {tags.gc = 'old_gc_count'} })"
+ exp: >
+ jvm_gc_collection_seconds_count.tagMatch('gc', 'PS Scavenge|Copy|ParNew|G1 Young Generation|PS MarkSweep|MarkSweepCompact|ConcurrentMarkSweep|G1 Old Generation')
+ .sum(['service', 'host_name', 'gc']).increase('PT1M')
+ .tag({tags -> if (tags['gc'] == 'PS Scavenge' || tags['gc'] == 'Copy' || tags['gc'] == 'ParNew' || tags['gc'] == 'G1 Young Generation') {tags.gc = 'young_gc_count'} })
+ .tag({tags -> if (tags['gc'] == 'PS MarkSweep' || tags['gc'] == 'MarkSweepCompact' || tags['gc'] == 'ConcurrentMarkSweep' || tags['gc'] == 'G1 Old Generation') {tags.gc = 'old_gc_count'} })
- name: instance_jvm_gc_time
- exp: "(jvm_gc_collection_seconds_sum * 1000).tagMatch('gc', 'PS Scavenge|Copy|ParNew|G1 Young Generation|PS MarkSweep|MarkSweepCompact|ConcurrentMarkSweep|G1 Old Generation')
- .sum(['service', 'host_name', 'gc']).increase('PT1M')
- .tag({tags -> if (tags['gc'] == 'PS Scavenge' || tags['gc'] == 'Copy' || tags['gc'] == 'ParNew' || tags['gc'] == 'G1 Young Generation') {tags.gc = 'young_gc_time'} })
- .tag({tags -> if (tags['gc'] == 'PS MarkSweep' || tags['gc'] == 'MarkSweepCompact' || tags['gc'] == 'ConcurrentMarkSweep' || tags['gc'] == 'G1 Old Generation') {tags.gc = 'old_gc_time'} })"
+ exp: >
+ (jvm_gc_collection_seconds_sum * 1000).tagMatch('gc', 'PS Scavenge|Copy|ParNew|G1 Young Generation|PS MarkSweep|MarkSweepCompact|ConcurrentMarkSweep|G1 Old Generation')
+ .sum(['service', 'host_name', 'gc']).increase('PT1M')
+ .tag({tags -> if (tags['gc'] == 'PS Scavenge' || tags['gc'] == 'Copy' || tags['gc'] == 'ParNew' || tags['gc'] == 'G1 Young Generation') {tags.gc = 'young_gc_time'} })
+ .tag({tags -> if (tags['gc'] == 'PS MarkSweep' || tags['gc'] == 'MarkSweepCompact' || tags['gc'] == 'ConcurrentMarkSweep' || tags['gc'] == 'G1 Old Generation') {tags.gc = 'old_gc_time'} })
- name: instance_trace_count
exp: trace_in_latency_count.sum(['service', 'host_name']).increase('PT1M')
- name: instance_trace_latency_percentile
@@ -59,8 +61,9 @@ metricsRules:
- name: instance_mesh_analysis_error_count
exp: mesh_analysis_error_count.sum(['service', 'host_name']).increase('PT1M')
- name: instance_metrics_aggregation
- exp: "metrics_aggregation.tagEqual('dimensionality', 'minute').sum(['service', 'host_name', 'level']).increase('PT1M')
- .tag({tags -> if (tags['level'] == '1') {tags.level = 'L1 aggregation'} }).tag({tags -> if (tags['level'] == '2') {tags.level = 'L2 aggregation'} })"
+ exp: >
+ metrics_aggregation.tagEqual('dimensionality', 'minute').sum(['service', 'host_name', 'level']).increase('PT1M')
+ .tag({tags -> if (tags['level'] == '1') {tags.level = 'L1 aggregation'} }).tag({tags -> if (tags['level'] == '2') {tags.level = 'L2 aggregation'} })
- name: instance_persistence_execute_percentile
exp: persistence_timer_bulk_execute_latency.sum(['le', 'service', 'host_name']).increase('PT5M').histogram().histogram_percentile([50,70,90,99])
- name: instance_persistence_prepare_percentile
@@ -99,3 +102,15 @@ metricsRules:
exp: k8s_als_drop_count.sum(['service', 'host_name']).increase('PT1M')
- name: instance_k8s_als_latency_percentile
exp: k8s_als_in_latency.sum(['le', 'service', 'host_name']).increase('PT1M').histogram().histogram_percentile([50,70,90,99])
+ - name: otel_metrics_received
+ exp: otel_metrics_latency_count.sum(['service', 'host_name']).increase('PT1M')
+ - name: otel_logs_received
+ exp: otel_logs_latency_count.sum(['service', 'host_name']).increase('PT1M')
+ - name: otel_spans_received
+ exp: otel_spans_latency_count.sum(['service', 'host_name']).increase('PT1M')
+ - name: otel_metrics_latency_percentile
+ exp: otel_metrics_latency.sum(['le', 'service', 'host_name']).increase('PT1M').histogram().histogram_percentile([50,70,90,99])
+ - name: otel_logs_latency_percentile
+ exp: otel_logs_latency.sum(['le', 'service', 'host_name']).increase('PT1M').histogram().histogram_percentile([50,70,90,99])
+ - name: otel_spans_latency_percentile
+ exp: otel_spans_latency.sum(['le', 'service', 'host_name']).increase('PT1M').histogram().histogram_percentile([50,70,90,99])
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json
index 090d3de9e2..0752732cb5 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json
@@ -467,6 +467,148 @@
"expressions": [
"relabels(meter_oap_instance_k8s_als_latency_percentile{p='50,75,90,95,99'},p='50,75,90,95,99',percentile='50,75,90,95,99')"
]
+ },
+ {
+ "x": 12,
+ "y": 13,
+ "w": 6,
+ "h": 13,
+ "i": "19",
+ "type": "Widget",
+ "expressions": [
+ "meter_oap_otel_metrics_received"
+ ],
+ "graph": {
+ "type": "Line",
+ "step": false,
+ "smooth": false,
+ "showSymbol": true,
+ "showXAxis": true,
+ "showYAxis": true
+ },
+ "widget": {
+ "title": "OpenTelemetry Metrics (Requests / Second)"
+ },
+ "metricConfig": [
+ {
+ "label": "Received Requests"
+ }
+ ]
+ },
+ {
+ "x": 12,
+ "y": 26,
+ "w": 6,
+ "h": 13,
+ "i": "20",
+ "type": "Widget",
+ "expressions": [
+ "meter_oap_otel_logs_received"
+ ],
+ "graph": {
+ "type": "Line",
+ "step": false,
+ "smooth": false,
+ "showSymbol": true,
+ "showXAxis": true,
+ "showYAxis": true
+ },
+ "widget": {
+ "title": "OpenTelemetry Logs (Requests / Second)"
+ }
+ },
+ {
+ "x": 0,
+ "y": 26,
+ "w": 6,
+ "h": 13,
+ "i": "21",
+ "type": "Widget",
+ "expressions": [
+ "meter_oap_otel_spans_received"
+ ],
+ "graph": {
+ "type": "Line",
+ "step": false,
+ "smooth": false,
+ "showSymbol": true,
+ "showXAxis": true,
+ "showYAxis": true
+ },
+ "widget": {
+ "title": "OpenTelemetry Spans (Requests / Second)"
+ },
+ "metricConfig": [
+ {
+ "unit": "Requests / Second"
+ }
+ ]
+ },
+ {
+ "x": 18,
+ "y": 13,
+ "w": 6,
+ "h": 13,
+ "i": "22",
+ "type": "Widget",
+ "expressions": [
+ "relabels(meter_oap_otel_metrics_latency_percentile{p='50,75,90,95,99'},p='50,75,90,95,99',percentile='50,75,90,95,99')"
+ ],
+ "graph": {
+ "type": "Line",
+ "step": false,
+ "smooth": false,
+ "showSymbol": true,
+ "showXAxis": true,
+ "showYAxis": true
+ },
+ "widget": {
+ "title": "OpenTelemetry Metrics Latency (ms / min)"
+ }
+ },
+ {
+ "x": 6,
+ "y": 26,
+ "w": 6,
+ "h": 13,
+ "i": "23",
+ "type": "Widget",
+ "expressions": [
+ "relabels(meter_oap_otel_spans_latency_percentile{p='50,75,90,95,99'},p='50,75,90,95,99',percentile='50,75,90,95,99')"
+ ],
+ "graph": {
+ "type": "Line",
+ "step": false,
+ "smooth": false,
+ "showSymbol": true,
+ "showXAxis": true,
+ "showYAxis": true
+ },
+ "widget": {
+ "title": "OpenTelemetry Spans Latency (ms / min)"
+ }
+ },
+ {
+ "x": 18,
+ "y": 26,
+ "w": 6,
+ "h": 13,
+ "i": "24",
+ "type": "Widget",
+ "expressions": [
+ "relabels(meter_oap_otel_logs_latency_percentile{p='50,75,90,95,99'},p='50,75,90,95,99',percentile='50,75,90,95,99')"
+ ],
+ "graph": {
+ "type": "Line",
+ "step": false,
+ "smooth": false,
+ "showSymbol": true,
+ "showXAxis": true,
+ "showYAxis": true
+ },
+ "widget": {
+ "title": "OpenTelemetry Logs Latency (ms / min)"
+ }
}
]
}