You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2024/10/11 14:49:13 UTC
(skywalking) branch master updated: Polish eBPF so11y metrics and add error count for query metrics (#12692)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 29618d7f2e Polish eBPF so11y metrics and add error count for query metrics (#12692)
29618d7f2e is described below
commit 29618d7f2e795f1822d46015c23edc3f6b0c66be
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Fri Oct 11 22:49:08 2024 +0800
Polish eBPF so11y metrics and add error count for query metrics (#12692)
---
docs/en/changes/changes.md | 1 +
.../oap/query/graphql/GraphQLQueryHandler.java | 14 +++++
.../provider/handler/AccessLogServiceHandler.java | 16 +++---
.../src/main/resources/otel-rules/oap.yaml | 8 +++
.../so11y_oap/so11y-instance.json | 66 ++++++++++++++++++----
5 files changed, 88 insertions(+), 17 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index c8f88b8b5e..80abde735c 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -6,6 +6,7 @@
* Add self observability metrics for GraphQL query, `graphql_query_latency`.
* Reduce the count of process index and adding time range when query process index.
* Bump up Apache commons-io to 2.17.0.
+* Polish eBPF so11y metrics and add error count for query metrics.
#### UI
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryHandler.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryHandler.java
index 8ba0d730dc..bb51696637 100644
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryHandler.java
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryHandler.java
@@ -20,6 +20,7 @@ package org.apache.skywalking.oap.query.graphql;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
+import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics;
import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics;
import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;
import org.apache.skywalking.oap.server.telemetry.api.MetricsTag;
@@ -51,6 +52,16 @@ public class GraphQLQueryHandler {
"The processing latency of graphql query",
MetricsTag.EMPTY_KEY,
MetricsTag.EMPTY_VALUE);
+ @Getter(lazy = true)
+ private final CounterMetrics errorCount =
+ moduleManager.find(TelemetryModule.NAME)
+ .provider()
+ .getService(MetricsCreator.class)
+ .createCounter(
+ "graphql_query_error_count",
+ "The error count of graphql query",
+ MetricsTag.EMPTY_KEY,
+ MetricsTag.EMPTY_VALUE);
public GraphQLQueryHandler(
final ModuleManager moduleManager,
@@ -81,6 +92,9 @@ public class GraphQLQueryHandler {
final HttpRequest req) throws Exception {
try (final var ignored = getQueryHistogram().createTimer()) {
return graphqlService.serve(ctx, req);
+ } catch (Exception e) {
+ getErrorCount().inc();
+ throw e;
}
}
}
diff --git a/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/AccessLogServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/AccessLogServiceHandler.java
index 66f277719a..cd1421678a 100644
--- a/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/AccessLogServiceHandler.java
+++ b/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/AccessLogServiceHandler.java
@@ -91,6 +91,7 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess
protected final NamingControl namingControl;
private final CounterMetrics inCounter;
+ private final CounterMetrics errorStreamCounter;
private final HistogramMetrics processHistogram;
private final CounterMetrics dropCounter;
private final ConcurrentHashMap<String, DropDataReason> dropReasons = new ConcurrentHashMap<>();
@@ -103,13 +104,16 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess
.provider()
.getService(MetricsCreator.class);
this.inCounter = metricsCreator.createCounter(
- "k8s_als_in_count", "The count of envoy ALS message received", MetricsTag.EMPTY_KEY,
+ "k8s_als_in_count", "The count of eBPF log entries received", MetricsTag.EMPTY_KEY,
+ MetricsTag.EMPTY_VALUE);
+ this.errorStreamCounter = metricsCreator.createCounter(
+ "k8s_als_error_streams", "The error count of eBPF log streams that OAP failed to process", MetricsTag.EMPTY_KEY,
MetricsTag.EMPTY_VALUE);
this.processHistogram = metricsCreator.createHistogramMetric(
- "k8s_als_in_latency", "The process latency of envoy ALS message received", MetricsTag.EMPTY_KEY,
+ "k8s_als_in_latency", "The processing latency of eBPF log streams", MetricsTag.EMPTY_KEY,
MetricsTag.EMPTY_VALUE);
this.dropCounter = metricsCreator.createCounter(
- "k8s_als_drop_count", "The count of envoy ALS message dropped", MetricsTag.EMPTY_KEY,
+ "k8s_als_drop_count", "The count of eBPF log entries dropped", MetricsTag.EMPTY_KEY,
MetricsTag.EMPTY_VALUE);
// schedule to print the drop reasons(debug log)
@@ -125,8 +129,7 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess
@Override
public void onNext(EBPFAccessLogMessage logMessage) {
- final HistogramMetrics.Timer timer = processHistogram.createTimer();
- try {
+ try (final var ignored = processHistogram.createTimer()) {
if (isFirst || logMessage.hasNode()) {
isFirst = false;
node = new NodeInfo(logMessage.getNode());
@@ -158,8 +161,7 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess
}
} catch (Exception e) {
log.error("Access log service handler process error.", e);
- } finally {
- timer.finish();
+ errorStreamCounter.inc();
}
}
diff --git a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml
index 485a02d15f..72e6b92284 100644
--- a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml
+++ b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml
@@ -102,6 +102,10 @@ metricsRules:
exp: k8s_als_drop_count.sum(['service', 'host_name']).increase('PT1M')
- name: instance_k8s_als_latency_percentile
exp: k8s_als_in_latency.sum(['le', 'service', 'host_name']).increase('PT1M').histogram().histogram_percentile([50,70,90,99])
+ - name: instance_k8s_als_streams
+ exp: k8s_als_in_latency_count.sum(['service', 'host_name']).increase('PT1M')
+ - name: instance_k8s_als_error_streams
+ exp: k8s_als_error_streams.sum(['service', 'host_name']).increase('PT1M')
- name: otel_metrics_received
exp: otel_metrics_latency_count.sum(['service', 'host_name']).increase('PT1M')
- name: otel_logs_received
@@ -116,3 +120,7 @@ metricsRules:
exp: otel_spans_latency.sum(['le', 'service', 'host_name']).increase('PT1M').histogram().histogram_percentile([50,70,90,99])
- name: graphql_query_latency_percentile
exp: graphql_query_latency.sum(['le', 'service', 'host_name']).increase('PT5M').histogram().histogram_percentile([50,70,90,99])
+ - name: instance_graphql_query_count
+ exp: graphql_query_latency_count.sum(['service', 'host_name']).increase('PT1M')
+ - name: instance_graphql_query_error_count
+ exp: graphql_query_error_count.sum(['service', 'host_name']).increase('PT1M')
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json
index 34edf346c6..3a9403c260 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json
@@ -342,6 +342,33 @@
"unit": "ns"
}
]
+ },
+ {
+ "x": 6,
+ "y": 39,
+ "w": 6,
+ "h": 11,
+ "i": "17",
+ "type": "Widget",
+ "widget": {
+ "title": "GraphQL Query Count"
+ },
+ "graph": {
+ "type": "Bar",
+ "showBackground": true
+ },
+ "expressions": [
+ "meter_oap_instance_graphql_query_count",
+ "meter_oap_instance_graphql_query_error_count"
+ ],
+ "metricConfig": [
+ {
+ "label": "Total Count"
+ },
+ {
+ "label": "Error Count"
+ }
+ ]
}
]
},
@@ -474,7 +501,7 @@
]
},
{
- "x": 6,
+ "x": 12,
"y": 13,
"w": 6,
"h": 13,
@@ -496,8 +523,8 @@
]
},
{
- "x": 12,
- "y": 13,
+ "x": 6,
+ "y": 26,
"w": 6,
"h": 13,
"i": "19",
@@ -523,8 +550,8 @@
]
},
{
- "x": 12,
- "y": 26,
+ "x": 0,
+ "y": 39,
"w": 6,
"h": 13,
"i": "20",
@@ -545,7 +572,7 @@
}
},
{
- "x": 0,
+ "x": 12,
"y": 26,
"w": 6,
"h": 13,
@@ -572,8 +599,8 @@
]
},
{
- "x": 18,
- "y": 13,
+ "x": 0,
+ "y": 26,
"w": 6,
"h": 13,
"i": "22",
@@ -594,7 +621,7 @@
}
},
{
- "x": 6,
+ "x": 18,
"y": 26,
"w": 6,
"h": 13,
@@ -617,7 +644,7 @@
},
{
"x": 18,
- "y": 26,
+ "y": 13,
"w": 6,
"h": 13,
"i": "24",
@@ -636,6 +663,25 @@
"widget": {
"title": "OpenTelemetry Logs Latency (ms / min)"
}
+ },
+ {
+ "x": 6,
+ "y": 13,
+ "w": 6,
+ "h": 13,
+ "i": "25",
+ "type": "Widget",
+ "widget": {
+ "title": "K8S ALS Log Streams (Per Minute)"
+ },
+ "graph": {
+ "type": "Bar",
+ "showBackground": true
+ },
+ "expressions": [
+ "meter_oap_instance_k8s_als_streams",
+ "meter_oap_instance_k8s_als_error_streams"
+ ]
}
]
}