You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2021/10/13 09:15:41 UTC
[skywalking] branch mq created (now 81bd9ef)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a change to branch mq
in repository https://gitbox.apache.org/repos/asf/skywalking.git.
at 81bd9ef Message Queue performance and consuming latency monitoring
This branch includes the following new commits:
new 81bd9ef Message Queue performance and consuming latency monitoring
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[skywalking] 01/01: Message Queue performance and consuming latency
monitoring
Posted by wu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch mq
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 81bd9ef1fa75152a405028040c0f96cd27898f7a
Author: Wu Sheng <wu...@foxmail.com>
AuthorDate: Wed Oct 13 17:15:22 2021 +0800
Message Queue performance and consuming latency monitoring
---
CHANGES.md | 3 +
README.md | 3 +-
.../listener/MultiScopesAnalysisListener.java | 3 +
.../apache/skywalking/oal/rt/grammar/OALLexer.g4 | 1 +
.../apache/skywalking/oal/rt/grammar/OALParser.g4 | 8 +-
.../skywalking/oal/rt/parser/OALListener.java | 15 ++-
.../skywalking/oap/server/core/source/All.java | 2 +-
.../oap/server/core/source/Endpoint.java | 2 +-
.../oap/server/core/source/RequestType.java | 3 +-
.../skywalking/oap/server/core/source/Service.java | 2 +-
.../oap/server/core/source/ServiceInstance.java | 2 +-
.../server/receiver/trace/mock/ServiceCMock.java | 2 +
.../server-starter/src/main/resources/oal/core.oal | 4 +
.../resources/ui-initialized-templates/apm.yml | 110 +++++++++++++++------
14 files changed, 118 insertions(+), 42 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index b23d894..77ac2cb 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -27,6 +27,9 @@ Release Notes.
* Refactor the OAL compiler context to improve readability.
* Fix wrong generated codes of `hashCode` and `remoteHashCode` methods for numeric fields.
* Replace e2e cases to e2e-v2: Nodejs.
+* Support `!= null` in OAL engine.
+* Add `Message Queue Consuming Count` metric for MQ consuming service and endpoint.
+* Add `Message Queue Avg Consuming Latency` metric for MQ consuming service and endpoint.
#### UI
diff --git a/README.md b/README.md
index e53693e..118e387 100644
--- a/README.md
+++ b/README.md
@@ -22,10 +22,11 @@ The core features are following.
- Root cause analysis. Profile the code on the runtime
- Service topology map analysis
- Service, service instance and endpoint dependency analysis
-- Slow services and endpoints detected
+- Slow services and endpoints detecting
- Performance optimization
- Distributed tracing and context propagation
- Database access metrics. Detect slow database access statements(including SQL statements)
+- Message Queue performance and consuming latency monitoring
- Alarm
- Browser performance monitoring
- Infrastructure(VM, network, disk etc.) monitoring
diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/MultiScopesAnalysisListener.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/MultiScopesAnalysisListener.java
index 927fd0d..dcc3ed0 100644
--- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/MultiScopesAnalysisListener.java
+++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/MultiScopesAnalysisListener.java
@@ -264,6 +264,9 @@ public class MultiScopesAnalysisListener implements EntryAnalysisListener, ExitA
case Database:
sourceBuilder.setType(RequestType.DATABASE);
break;
+ case MQ:
+ sourceBuilder.setType(RequestType.MQ);
+ break;
default:
sourceBuilder.setType(RequestType.RPC);
break;
diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4
index 9f053f4..af0289c 100644
--- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4
+++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4
@@ -78,6 +78,7 @@ CONTAIN: 'contain';
NOT_CONTAIN: 'not contain';
// Literals
+NULL_LITERAL: 'null';
BOOL_LITERAL: 'true'
| 'false'
diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4
index 119b926..09b0844 100644
--- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4
+++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4
@@ -121,7 +121,7 @@ numberMatch
;
stringMatch
- : conditionAttributeStmt DUALEQUALS (stringConditionValue | enumConditionValue)
+ : conditionAttributeStmt DUALEQUALS (stringConditionValue | enumConditionValue | nullConditionValue)
;
greaterMatch
@@ -145,7 +145,7 @@ booleanNotEqualMatch
;
notEqualMatch
- : conditionAttributeStmt NOT_EQUAL (numberConditionValue | stringConditionValue | enumConditionValue)
+ : conditionAttributeStmt NOT_EQUAL (numberConditionValue | stringConditionValue | enumConditionValue | nullConditionValue)
;
likeMatch
@@ -188,6 +188,10 @@ numberConditionValue
: NUMBER_LITERAL
;
+nullConditionValue
+ : NULL_LITERAL
+ ;
+
sourceAttrCast
: castStmt
;
diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java
index 3621a06..5b7ef9f 100644
--- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java
+++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java
@@ -202,7 +202,7 @@ public class OALListener extends OALParserBaseListener {
@Override
public void enterEnumConditionValue(OALParser.EnumConditionValueContext ctx) {
- enterConditionValue(ctx.getText());
+ enterEnumConditionValue(ctx.getText());
}
@Override
@@ -212,18 +212,23 @@ public class OALListener extends OALParserBaseListener {
}
@Override
+ public void enterNullConditionValue(OALParser.NullConditionValueContext ctx) {
+ enterConditionValue(ctx.getText());
+ }
+
+ @Override
public void enterExpressionAttrCast(final OALParser.ExpressionAttrCastContext ctx) {
conditionExpression.setCastType(ctx.getText());
}
private void enterConditionValue(String value) {
- if (value.split("\\.").length == 2 && !value.startsWith("\"")) {
- // Value is an enum.
- value = sourcePackage + value;
- }
conditionExpression.addValue(value);
}
+ private void enterEnumConditionValue(String value) {
+ conditionExpression.addValue(sourcePackage + value);
+ }
+
/////////////
// Expression end.
////////////
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/All.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/All.java
index 8af8b69..291283b 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/All.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/All.java
@@ -73,6 +73,6 @@ public class All extends Source {
private Map<String, String> originalTags;
public String getTag(String key) {
- return originalTags.getOrDefault(key, Const.EMPTY_STRING);
+ return originalTags.getOrDefault(key, null);
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java
index f9c3e20..a7e5b8c 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java
@@ -100,6 +100,6 @@ public class Endpoint extends Source {
}
public String getTag(String key) {
- return originalTags.getOrDefault(key, Const.EMPTY_STRING);
+ return originalTags.getOrDefault(key, null);
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/RequestType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/RequestType.java
index 6eea865..fe0b0af 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/RequestType.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/RequestType.java
@@ -30,5 +30,6 @@ public enum RequestType {
* Logic request only.
*/
LOGIC,
- TCP
+ TCP,
+ MQ
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java
index a7534f7..87e4f68 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java
@@ -92,6 +92,6 @@ public class Service extends Source {
private TCPInfo tcpInfo = new TCPInfo();
public String getTag(String key) {
- return originalTags.getOrDefault(key, Const.EMPTY_STRING);
+ return originalTags.getOrDefault(key, null);
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java
index a00ca22..ca558a4 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java
@@ -102,6 +102,6 @@ public class ServiceInstance extends Source {
}
public String getTag(String key) {
- return originalTags.getOrDefault(key, Const.EMPTY_STRING);
+ return originalTags.getOrDefault(key, null);
}
}
diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/mock/ServiceCMock.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/mock/ServiceCMock.java
index 7bb7ac9..f984d2a 100644
--- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/mock/ServiceCMock.java
+++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/trace/mock/ServiceCMock.java
@@ -19,6 +19,7 @@
package org.apache.skywalking.oap.server.receiver.trace.mock;
import io.grpc.stub.StreamObserver;
+import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair;
import org.apache.skywalking.apm.network.language.agent.v3.RefType;
import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject;
import org.apache.skywalking.apm.network.language.agent.v3.SegmentReference;
@@ -61,6 +62,7 @@ class ServiceCMock {
span.setIsError(false);
span.addRefs(createReference(traceId, parentSegmentId));
span.setOperationName(ServiceBMock.ROCKET_MQ_ENDPOINT);
+ span.addTags(KeyStringValuePair.newBuilder().setKey("transmission.latency").setValue("100").build());
return span;
}
diff --git a/oap-server/server-starter/src/main/resources/oal/core.oal b/oap-server/server-starter/src/main/resources/oal/core.oal
index 16dc46d..d95af57 100755
--- a/oap-server/server-starter/src/main/resources/oal/core.oal
+++ b/oap-server/server-starter/src/main/resources/oal/core.oal
@@ -29,6 +29,8 @@ service_sla = from(Service.*).percent(status == true);
service_cpm = from(Service.*).cpm();
service_percentile = from(Service.latency).percentile(10); // Multiple values including p50, p75, p90, p95, p99
service_apdex = from(Service.latency).apdex(name, status);
+service_mq_consume_count = from(Service.*).filter(type == RequestType.MQ).count();
+service_mq_consume_latency = from((str->long)Service.tag["transmission.latency"]).filter(type == RequestType.MQ).filter(tag["transmission.latency"] != null).longAvg();
// Service relation scope metrics for topology
service_relation_client_cpm = from(ServiceRelation.*).filter(detectPoint == DetectPoint.CLIENT).cpm();
@@ -60,6 +62,8 @@ endpoint_cpm = from(Endpoint.*).cpm();
endpoint_avg = from(Endpoint.latency).longAvg();
endpoint_sla = from(Endpoint.*).percent(status == true);
endpoint_percentile = from(Endpoint.latency).percentile(10); // Multiple values including p50, p75, p90, p95, p99
+endpoint_mq_consume_count = from(Endpoint.*).filter(type == RequestType.MQ).count();
+endpoint_mq_consume_latency = from((str->long)Endpoint.tag["transmission.latency"]).filter(type == RequestType.MQ).filter(tag["transmission.latency"] != null).longAvg();
// Endpoint relation scope metrics
endpoint_relation_cpm = from(EndpointRelation.*).filter(detectPoint == DetectPoint.SERVER).cpm();
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/apm.yml b/oap-server/server-starter/src/main/resources/ui-initialized-templates/apm.yml
index 5854c07..9686901 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/apm.yml
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/apm.yml
@@ -121,9 +121,9 @@ templates:
"name": "Service",
"children": [
{
- "width": 3,
+ "width": 4,
"title": "Service Apdex",
- "height": "200",
+ "height": "100",
"entityType": "Service",
"independentSelector": false,
"metricType": "REGULAR_VALUE",
@@ -134,21 +134,9 @@ templates:
"aggregationNum": "10000"
},
{
- "width": 3,
- "title": "Service Avg Response Time",
- "height": "200",
- "entityType": "Service",
- "independentSelector": false,
- "metricType": "REGULAR_VALUE",
- "metricName": "service_resp_time",
- "queryMetricType": "readMetricsValues",
- "chartType": "ChartLine",
- "unit": "ms"
- },
- {
- "width": 3,
+ "width": 4,
"title": "Successful Rate",
- "height": "200",
+ "height": "100",
"entityType": "Service",
"independentSelector": false,
"metricType": "REGULAR_VALUE",
@@ -160,9 +148,9 @@ templates:
"aggregationNum": "100"
},
{
- "width": 3,
+ "width": 4,
"title": "Service Load",
- "height": "200",
+ "height": "100",
"entityType": "Service",
"independentSelector": false,
"metricType": "REGULAR_VALUE",
@@ -174,6 +162,18 @@ templates:
},
{
"width": 3,
+ "title": "Service Avg Response Time",
+ "height": "200",
+ "entityType": "Service",
+ "independentSelector": false,
+ "metricType": "REGULAR_VALUE",
+ "metricName": "service_resp_time",
+ "queryMetricType": "readMetricsValues",
+ "chartType": "ChartLine",
+ "unit": "ms"
+ },
+ {
+ "width": 3,
"title": "Service Apdex",
"height": "200",
"entityType": "Service",
@@ -240,7 +240,33 @@ templates:
"tips": "This metrics is only avaible for TCP services"
},
{
- "width": "3",
+ "width": 3,
+ "title": "Message Queue Consuming Count",
+ "height": "200",
+ "entityType": "Service",
+ "independentSelector": false,
+ "metricType": "REGULAR_VALUE",
+ "metricName": "service_mq_consume_count",
+ "queryMetricType": "readMetricsValues",
+ "chartType": "ChartLine",
+ "unit": "",
+ "tips": "The number of consumed messages."
+ },
+ {
+ "width": 3,
+ "title": "Message Queue Avg Consuming Latency",
+ "height": "200",
+ "entityType": "Service",
+ "independentSelector": false,
+ "metricType": "REGULAR_VALUE",
+ "metricName": "service_mq_consume_latency",
+ "queryMetricType": "readMetricsValues",
+ "chartType": "ChartLine",
+ "unit": "ms",
+ "tips": "The avg latency of message consuming. Latency = timestamp(received) - timestamp(producing)"
+ },
+ {
+ "width": "4",
"title": "Service Instances Load",
"height": "280",
"entityType": "ServiceInstance",
@@ -254,7 +280,7 @@ templates:
"tips": "For HTTP 1/2, gRPC, RPC services, this means Calls Per Minute (CPM), for TCP services, this means Packets Per Minute (PPM)"
},
{
- "width": "3",
+ "width": "4",
"title": "Slow Service Instance",
"height": "280",
"entityType": "ServiceInstance",
@@ -267,7 +293,7 @@ templates:
"unit": "ms"
},
{
- "width": "3",
+ "width": "4",
"title": "Service Instance Successful Rate",
"height": "280",
"entityType": "ServiceInstance",
@@ -519,9 +545,9 @@ templates:
"sortOrder": "ASC"
},
{
- "width": 3,
+ "width": 4,
"title": "Endpoint Load",
- "height": 350,
+ "height": 200,
"entityType": "Endpoint",
"independentSelector": false,
"metricType": "REGULAR_VALUE",
@@ -530,9 +556,9 @@ templates:
"chartType": "ChartLine"
},
{
- "width": 3,
+ "width": 4,
"title": "Endpoint Avg Response Time",
- "height": 350,
+ "height": 200,
"entityType": "Endpoint",
"independentSelector": false,
"metricType": "REGULAR_VALUE",
@@ -542,9 +568,9 @@ templates:
"unit": "ms"
},
{
- "width": 3,
+ "width": 4,
"title": "Endpoint Response Time Percentile",
- "height": 350,
+ "height": 200,
"entityType": "Endpoint",
"independentSelector": false,
"metricType": "LABELED_VALUE",
@@ -556,9 +582,9 @@ templates:
"unit": "ms"
},
{
- "width": 3,
+ "width": 4,
"title": "Endpoint Successful Rate",
- "height": 350,
+ "height": 200,
"entityType": "Endpoint",
"independentSelector": false,
"metricType": "REGULAR_VALUE",
@@ -568,6 +594,32 @@ templates:
"unit": "%",
"aggregation": "/",
"aggregationNum": "100"
+ },
+ {
+ "width": 4,
+ "title": "Message Queue Consuming Count",
+ "height": "200",
+ "entityType": "Endpoint",
+ "independentSelector": false,
+ "metricType": "REGULAR_VALUE",
+ "metricName": "endpoint_mq_consume_count",
+ "queryMetricType": "readMetricsValues",
+ "chartType": "ChartLine",
+ "unit": "",
+ "tips": "The number of consumed messages."
+ },
+ {
+ "width": 4,
+ "title": "Message Queue Avg Consuming Latency",
+ "height": "200",
+ "entityType": "Endpoint",
+ "independentSelector": false,
+ "metricType": "REGULAR_VALUE",
+ "metricName": "endpoint_mq_consume_latency",
+ "queryMetricType": "readMetricsValues",
+ "chartType": "ChartLine",
+ "unit": "ms",
+ "tips": "The avg latency of message consuming. Latency = timestamp(received) - timestamp(producing)"
}
]
}