You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2019/12/19 05:26:36 UTC

[skywalking] branch master updated: Add relation metrics verification in e2e test (#4080)

This is an automated email from the ASF dual-hosted git repository.

kezhenxu94 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git


The following commit(s) were added to refs/heads/master by this push:
     new 4c99480  Add  relation metrics verification in e2e test (#4080)
4c99480 is described below

commit 4c99480f70833010e64fc62d17b790f0e6f996d5
Author: zhang-wei <pk...@outlook.com>
AuthorDate: Thu Dec 19 13:26:27 2019 +0800

    Add  relation metrics verification in e2e test (#4080)
---
 .../core/register/ServiceInstanceInventory.java    |   8 +-
 .../skywalking/e2e/metrics/MetricsMatcher.java     |  71 +++++++++++++
 .../skywalking/e2e/metrics/MetricsQuery.java       |  32 ++++++
 .../skywalking/e2e/ClusterVerificationITCase.java  | 115 +++++++--------------
 .../skywalking/e2e/ClusterVerificationITCase.java  | 112 +++++++-------------
 .../skywalking/e2e/SampleVerificationITCase.java   |  36 ++++++-
 .../skywalking/e2e/SampleVerificationITCase.java   |  36 ++++++-
 7 files changed, 249 insertions(+), 161 deletions(-)

diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInstanceInventory.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInstanceInventory.java
index 0f9ffa1..a988a72 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInstanceInventory.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInstanceInventory.java
@@ -55,7 +55,7 @@ public class ServiceInstanceInventory extends RegisterSource {
     public static final String IS_ADDRESS = "is_address";
     private static final String ADDRESS_ID = "address_id";
     public static final String NODE_TYPE = "node_type";
-    public static final String MAPPING_SERVICE_ID = "mapping_service_instance_id";
+    public static final String MAPPING_SERVICE_INSTANCE_ID = "mapping_service_instance_id";
     public static final String PROPERTIES = "properties";
     private static final Gson GSON = new Gson();
 
@@ -65,7 +65,7 @@ public class ServiceInstanceInventory extends RegisterSource {
     @Setter @Getter @Column(columnName = IS_ADDRESS) private int isAddress;
     @Setter @Getter @Column(columnName = ADDRESS_ID) private int addressId;
     @Setter(AccessLevel.PRIVATE) @Getter(AccessLevel.PACKAGE) @Column(columnName = NODE_TYPE) private int nodeType;
-    @Setter @Getter @Column(columnName = MAPPING_SERVICE_ID) private int mappingServiceInstanceId;
+    @Setter @Getter @Column(columnName = MAPPING_SERVICE_INSTANCE_ID) private int mappingServiceInstanceId;
     @Getter(AccessLevel.PRIVATE) @Column(columnName = PROPERTIES) private String prop;
     @Getter private JsonObject properties;
 
@@ -239,7 +239,7 @@ public class ServiceInstanceInventory extends RegisterSource {
             inventory.setLastUpdateTime(((Number)dbMap.get(LAST_UPDATE_TIME)).longValue());
 
             inventory.setNodeType(((Number)dbMap.get(NODE_TYPE)).intValue());
-            inventory.setMappingServiceInstanceId(((Number)dbMap.get(MAPPING_SERVICE_ID)).intValue());
+            inventory.setMappingServiceInstanceId(((Number)dbMap.get(MAPPING_SERVICE_INSTANCE_ID)).intValue());
 
             inventory.setName((String)dbMap.get(NAME));
             inventory.setInstanceUUID((String)dbMap.get(INSTANCE_UUID));
@@ -259,7 +259,7 @@ public class ServiceInstanceInventory extends RegisterSource {
             map.put(LAST_UPDATE_TIME, storageData.getLastUpdateTime());
 
             map.put(NODE_TYPE, storageData.getNodeType());
-            map.put(MAPPING_SERVICE_ID, storageData.getMappingServiceInstanceId());
+            map.put(MAPPING_SERVICE_INSTANCE_ID, storageData.getMappingServiceInstanceId());
 
             map.put(NAME, storageData.getName());
             map.put(INSTANCE_UUID, storageData.getInstanceUUID());
diff --git a/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/MetricsMatcher.java b/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/MetricsMatcher.java
new file mode 100644
index 0000000..1e1bed8
--- /dev/null
+++ b/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/MetricsMatcher.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.metrics;
+
+import org.apache.skywalking.e2e.SimpleQueryClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+
+/**
+ * @author zhangwei
+ */
+public class MetricsMatcher {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(MetricsMatcher.class);
+
+
+    public static void verifyMetrics(SimpleQueryClient queryClient, String metricName, String id,
+                                        final LocalDateTime minutesAgo) throws Exception {
+        verifyMetrics(queryClient, metricName, id, minutesAgo, 0, null);
+    }
+
+    public static void verifyMetrics(SimpleQueryClient queryClient, String metricName, String id,
+                                        final LocalDateTime minutesAgo, long retryInterval, Runnable generateTraffic) throws Exception {
+        boolean valid = false;
+        while (!valid) {
+            Metrics metrics = queryClient.metrics(
+                    new MetricsQuery()
+                            .stepByMinute()
+                            .metricsName(metricName)
+                            .start(minutesAgo)
+                            .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+                            .id(id)
+            );
+            LOGGER.info("{}: {}", metricName, metrics);
+            AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
+            MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
+            greaterThanZero.setValue("gt 0");
+            instanceRespTimeMatcher.setValue(greaterThanZero);
+            try {
+                instanceRespTimeMatcher.verify(metrics);
+                valid = true;
+            } catch (Throwable e) {
+                if (generateTraffic != null) {
+                    generateTraffic.run();
+                    Thread.sleep(retryInterval);
+                } else {
+                    throw e;
+                }
+            }
+        }
+    }
+}
diff --git a/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java b/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java
index 2cd9555..62c0d76 100644
--- a/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java
+++ b/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java
@@ -61,6 +61,38 @@ public class MetricsQuery extends AbstractQuery<MetricsQuery> {
         SERVICE_INSTANCE_SLA
     };
 
+    public static String SERVICE_RELATION_CLIENT_CPM = "service_relation_client_cpm";
+    public static String SERVICE_RELATION_SERVER_CPM = "service_relation_server_cpm";
+    public static String SERVICE_RELATION_CLIENT_CALL_SLA = "service_relation_client_call_sla";
+    public static String SERVICE_RELATION_SERVER_CALL_SLA = "service_relation_server_call_sla";
+    public static String SERVICE_RELATION_CLIENT_RESP_TIME = "service_relation_client_resp_time";
+    public static String SERVICE_RELATION_SERVER_RESP_TIME = "service_relation_server_resp_time";
+    public static String SERVICE_RELATION_CLIENT_P99 = "service_relation_client_p99";
+    public static String SERVICE_RELATION_SERVER_P99 = "service_relation_server_p99";
+    public static String[] ALL_SERVICE_RELATION_CLIENT_METRICS = {
+        SERVICE_RELATION_CLIENT_CPM
+    };
+
+    public static String[] ALL_SERVICE_RELATION_SERVER_METRICS = {
+        SERVICE_RELATION_SERVER_CPM
+    };
+
+    public static String SERVICE_INSTANCE_RELATION_CLIENT_CPM = "service_instance_relation_client_cpm";
+    public static String SERVICE_INSTANCE_RELATION_SERVER_CPM = "service_instance_relation_server_cpm";
+    public static String SERVICE_INSTANCE_RELATION_CLIENT_CALL_SLA = "service_instance_relation_client_call_sla";
+    public static String SERVICE_INSTANCE_RELATION_SERVER_CALL_SLA = "service_instance_relation_server_call_sla";
+    public static String SERVICE_INSTANCE_RELATION_CLIENT_RESP_TIME = "service_instance_relation_client_resp_time";
+    public static String SERVICE_INSTANCE_RELATION_SERVER_RESP_TIME = "service_instance_relation_server_resp_time";
+    public static String SERVICE_INSTANCE_RELATION_CLIENT_P99 = "service_instance_relation_client_p99";
+    public static String SERVICE_INSTANCE_RELATION_SERVER_P99 = "service_instance_relation_server_p99";
+    public static String[] ALL_SERVICE_INSTANCE_RELATION_CLIENT_METRICS = {
+        SERVICE_INSTANCE_RELATION_CLIENT_CPM
+    };
+
+    public static String[] ALL_SERVICE_INSTANCE_RELATION_SERVER_METRICS = {
+        SERVICE_INSTANCE_RELATION_SERVER_CPM
+    };
+
     private String id;
     private String metricsName;
 
diff --git a/test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/test/java/org/apache/skywalking/e2e/ClusterVerificationITCase.java b/test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/test/java/org/apache/skywalking/e2e/ClusterVerificationITCase.java
index cbe5827..36fabde 100755
--- a/test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/test/java/org/apache/skywalking/e2e/ClusterVerificationITCase.java
+++ b/test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/test/java/org/apache/skywalking/e2e/ClusterVerificationITCase.java
@@ -18,10 +18,6 @@
 
 package org.apache.skywalking.e2e;
 
-import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher;
-import org.apache.skywalking.e2e.metrics.Metrics;
-import org.apache.skywalking.e2e.metrics.MetricsQuery;
-import org.apache.skywalking.e2e.metrics.MetricsValueMatcher;
 import org.apache.skywalking.e2e.service.Service;
 import org.apache.skywalking.e2e.service.ServicesMatcher;
 import org.apache.skywalking.e2e.service.ServicesQuery;
@@ -58,6 +54,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
+import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics;
 import static org.apache.skywalking.e2e.metrics.MetricsQuery.*;
 import static org.assertj.core.api.Assertions.fail;
 
@@ -137,6 +134,8 @@ public class ClusterVerificationITCase {
 
                 final TopoMatcher topoMatcher = yaml.loadAs(expectedInputStream, TopoMatcher.class);
                 topoMatcher.verify(topoData);
+                verifyServiceRelationMetrics(topoData.getCalls(), minutesAgo);
+
                 valid = true;
                 for (Node node : topoData.getNodes()) {
                     if (gateWayName.equals(node.getName())) {
@@ -176,6 +175,8 @@ public class ClusterVerificationITCase {
                         new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.serviceInstanceTopo.yml").getInputStream();
                 final ServiceInstanceTopoMatcher topoMatcher = yaml.loadAs(expectedInputStream, ServiceInstanceTopoMatcher.class);
                 topoMatcher.verify(topoData);
+                verifyServiceInstanceRelationMetrics(topoData.getCalls(), minutesAgo);
+
                 valid = true;
             }catch (Throwable t){
                 LOGGER.warn(t.getMessage(), t);
@@ -278,31 +279,8 @@ public class ClusterVerificationITCase {
         for (Instance instance : instances.getInstances()) {
             for (String metricsName : ALL_INSTANCE_METRICS) {
                 LOGGER.info("verifying service instance response time: {}", instance);
-
-                boolean valid = false;
-                while (!valid) {
-                    LOGGER.warn("instanceMetrics is null, will retry to query");
-                    Metrics instanceMetrics = queryClient.metrics(
-                            new MetricsQuery()
-                                .stepByMinute()
-                                .metricsName(metricsName)
-                                .start(minutesAgo)
-                                .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
-                                .id(instance.getKey())
-                        );
-                    AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
-                    MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
-                    greaterThanZero.setValue("gt 0");
-                    instanceRespTimeMatcher.setValue(greaterThanZero);
-                    try {
-                        instanceRespTimeMatcher.verify(instanceMetrics);
-                        valid = true;
-                    } catch (Throwable ignored) {
-                        generateTraffic();
-                        Thread.sleep(retryInterval);
-                    }
-                    LOGGER.info("{}: {}", metricsName, instanceMetrics);
-                }
+                LOGGER.warn("instanceMetrics is null, will retry to query");
+                verifyMetrics(queryClient, metricsName, instance.getKey(), minutesAgo, retryInterval, this::generateTraffic);
             }
         }
     }
@@ -314,30 +292,7 @@ public class ClusterVerificationITCase {
             }
             for (String metricName : ALL_ENDPOINT_METRICS) {
                 LOGGER.info("verifying endpoint {}, metrics: {}", endpoint, metricName);
-
-                boolean valid = false;
-                while (!valid) {
-                    Metrics endpointMetrics = queryClient.metrics(
-                        new MetricsQuery()
-                            .stepByMinute()
-                            .metricsName(metricName)
-                            .start(minutesAgo)
-                            .end(LocalDateTime.now(ZoneOffset.UTC))
-                            .id(endpoint.getKey())
-                    );
-                    AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
-                    MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
-                    greaterThanZero.setValue("gt 0");
-                    instanceRespTimeMatcher.setValue(greaterThanZero);
-                    try {
-                        instanceRespTimeMatcher.verify(endpointMetrics);
-                        valid = true;
-                    } catch (Throwable ignored) {
-                        generateTraffic();
-                        Thread.sleep(retryInterval);
-                    }
-                    LOGGER.info("{}: {}", metricName, endpointMetrics);
-                }
+                verifyMetrics(queryClient, metricName, endpoint.getKey(), minutesAgo, retryInterval, this::generateTraffic);
             }
         }
     }
@@ -345,30 +300,7 @@ public class ClusterVerificationITCase {
     private void verifyServiceMetrics(Service service, final LocalDateTime minutesAgo) throws Exception {
         for (String metricName : ALL_SERVICE_METRICS) {
             LOGGER.info("verifying service {}, metrics: {}", service, metricName);
-
-            boolean valid = false;
-            while (!valid) {
-                Metrics serviceMetrics = queryClient.metrics(
-                    new MetricsQuery()
-                        .stepByMinute()
-                        .metricsName(metricName)
-                        .start(minutesAgo)
-                        .end(LocalDateTime.now(ZoneOffset.UTC))
-                        .id(service.getKey())
-                );
-                AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
-                MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
-                greaterThanZero.setValue("gt 0");
-                instanceRespTimeMatcher.setValue(greaterThanZero);
-                try {
-                    instanceRespTimeMatcher.verify(serviceMetrics);
-                    valid = true;
-                } catch (Throwable ignored) {
-                    generateTraffic();
-                    Thread.sleep(retryInterval);
-                }
-                LOGGER.info("{}: {}", metricName, serviceMetrics);
-            }
+            verifyMetrics(queryClient, metricName, service.getKey(), minutesAgo, retryInterval, this::generateTraffic);
         }
     }
 
@@ -393,6 +325,35 @@ public class ClusterVerificationITCase {
         tracesMatcher.verifyLoosely(traces);
     }
 
+    private void verifyServiceInstanceRelationMetrics(List<Call> calls, final LocalDateTime minutesAgo) throws Exception {
+        verifyRelationMetrics(calls, minutesAgo, ALL_SERVICE_INSTANCE_RELATION_CLIENT_METRICS, ALL_SERVICE_INSTANCE_RELATION_SERVER_METRICS);
+    }
+
+    private void verifyServiceRelationMetrics(List<Call> calls, final LocalDateTime minutesAgo) throws Exception {
+        verifyRelationMetrics(calls, minutesAgo, ALL_SERVICE_RELATION_CLIENT_METRICS, ALL_SERVICE_RELATION_SERVER_METRICS);
+    }
+
+    private void verifyRelationMetrics(List<Call> calls, final LocalDateTime minutesAgo, String[] relationClientMetrics, String[] relationServerMetrics) throws Exception {
+        for (Call call : calls) {
+            for (String detectPoint : call.getDetectPoints()) {
+                switch (detectPoint) {
+                    case "CLIENT": {
+                        for (String metricName : relationClientMetrics) {
+                            verifyMetrics(queryClient, metricName, call.getId(), minutesAgo, retryInterval, this::generateTraffic);
+                        }
+                        break;
+                    }
+                    case "SERVER": {
+                        for (String metricName : relationServerMetrics) {
+                            verifyMetrics(queryClient, metricName, call.getId(), minutesAgo, retryInterval, this::generateTraffic);
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
     private void generateTraffic() {
         try {
             final Map<String, String> user = new HashMap<>();
diff --git a/test/e2e/e2e-cluster/e2e-cluster-test-runner/src/test/java/org/apache/skywalking/e2e/ClusterVerificationITCase.java b/test/e2e/e2e-cluster/e2e-cluster-test-runner/src/test/java/org/apache/skywalking/e2e/ClusterVerificationITCase.java
index 8661a0e..d3fac68 100755
--- a/test/e2e/e2e-cluster/e2e-cluster-test-runner/src/test/java/org/apache/skywalking/e2e/ClusterVerificationITCase.java
+++ b/test/e2e/e2e-cluster/e2e-cluster-test-runner/src/test/java/org/apache/skywalking/e2e/ClusterVerificationITCase.java
@@ -18,10 +18,6 @@
 
 package org.apache.skywalking.e2e;
 
-import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher;
-import org.apache.skywalking.e2e.metrics.Metrics;
-import org.apache.skywalking.e2e.metrics.MetricsQuery;
-import org.apache.skywalking.e2e.metrics.MetricsValueMatcher;
 import org.apache.skywalking.e2e.service.Service;
 import org.apache.skywalking.e2e.service.ServicesMatcher;
 import org.apache.skywalking.e2e.service.ServicesQuery;
@@ -58,6 +54,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
+import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics;
 import static org.apache.skywalking.e2e.metrics.MetricsQuery.*;
 import static org.assertj.core.api.Assertions.fail;
 
@@ -136,6 +133,8 @@ public class ClusterVerificationITCase {
 
                 final TopoMatcher topoMatcher = yaml.loadAs(expectedInputStream, TopoMatcher.class);
                 topoMatcher.verify(topoData);
+                verifyServiceRelationMetrics(topoData.getCalls(), minutesAgo);
+
                 valid = true;
                 for (Node node : topoData.getNodes()) {
                     if (node.getName().equals(providerName)) {
@@ -174,6 +173,7 @@ public class ClusterVerificationITCase {
                         new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.serviceInstanceTopo.yml").getInputStream();
                 final ServiceInstanceTopoMatcher topoMatcher = yaml.loadAs(expectedInputStream, ServiceInstanceTopoMatcher.class);
                 topoMatcher.verify(topoData);
+                verifyServiceInstanceRelationMetrics(topoData.getCalls(), minutesAgo);
                 valid = true;
             } catch (Throwable t) {
                 LOGGER.warn(t.getMessage(), t);
@@ -277,30 +277,8 @@ public class ClusterVerificationITCase {
             for (String metricsName : ALL_INSTANCE_METRICS) {
                 LOGGER.info("verifying service instance response time: {}", instance);
 
-                boolean valid = false;
-                while (!valid) {
-                    LOGGER.warn("instanceMetrics is null, will retry to query");
-                    Metrics instanceMetrics = queryClient.metrics(
-                            new MetricsQuery()
-                                    .stepByMinute()
-                                    .metricsName(metricsName)
-                                    .start(minutesAgo)
-                                    .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
-                                    .id(instance.getKey())
-                    );
-                    AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
-                    MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
-                    greaterThanZero.setValue("gt 0");
-                    instanceRespTimeMatcher.setValue(greaterThanZero);
-                    try {
-                        instanceRespTimeMatcher.verify(instanceMetrics);
-                        valid = true;
-                    } catch (Throwable ignored) {
-                        generateTraffic();
-                        Thread.sleep(retryInterval);
-                    }
-                    LOGGER.info("{}: {}", metricsName, instanceMetrics);
-                }
+                LOGGER.warn("instanceMetrics is null, will retry to query");
+                verifyMetrics(queryClient, metricsName, instance.getKey(), minutesAgo, retryInterval, this::generateTraffic);
             }
         }
     }
@@ -313,29 +291,7 @@ public class ClusterVerificationITCase {
             for (String metricName : ALL_ENDPOINT_METRICS) {
                 LOGGER.info("verifying endpoint {}, metrics: {}", endpoint, metricName);
 
-                boolean valid = false;
-                while (!valid) {
-                    Metrics endpointMetrics = queryClient.metrics(
-                            new MetricsQuery()
-                                    .stepByMinute()
-                                    .metricsName(metricName)
-                                    .start(minutesAgo)
-                                    .end(LocalDateTime.now(ZoneOffset.UTC))
-                                    .id(endpoint.getKey())
-                    );
-                    AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
-                    MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
-                    greaterThanZero.setValue("gt 0");
-                    instanceRespTimeMatcher.setValue(greaterThanZero);
-                    try {
-                        instanceRespTimeMatcher.verify(endpointMetrics);
-                        valid = true;
-                    } catch (Throwable ignored) {
-                        generateTraffic();
-                        Thread.sleep(retryInterval);
-                    }
-                    LOGGER.info("{}: {}", metricName, endpointMetrics);
-                }
+                verifyMetrics(queryClient, metricName, endpoint.getKey(), minutesAgo, retryInterval, this::generateTraffic);
             }
         }
     }
@@ -344,29 +300,7 @@ public class ClusterVerificationITCase {
         for (String metricName : ALL_SERVICE_METRICS) {
             LOGGER.info("verifying service {}, metrics: {}", service, metricName);
 
-            boolean valid = false;
-            while (!valid) {
-                Metrics serviceMetrics = queryClient.metrics(
-                        new MetricsQuery()
-                                .stepByMinute()
-                                .metricsName(metricName)
-                                .start(minutesAgo)
-                                .end(LocalDateTime.now(ZoneOffset.UTC))
-                                .id(service.getKey())
-                );
-                AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
-                MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
-                greaterThanZero.setValue("gt 0");
-                instanceRespTimeMatcher.setValue(greaterThanZero);
-                try {
-                    instanceRespTimeMatcher.verify(serviceMetrics);
-                    valid = true;
-                } catch (Throwable ignored) {
-                    generateTraffic();
-                    Thread.sleep(retryInterval);
-                }
-                LOGGER.info("{}: {}", metricName, serviceMetrics);
-            }
+            verifyMetrics(queryClient, metricName, service.getKey(), minutesAgo, retryInterval, this::generateTraffic);
         }
     }
 
@@ -391,6 +325,36 @@ public class ClusterVerificationITCase {
         tracesMatcher.verifyLoosely(traces);
     }
 
+    private void verifyServiceInstanceRelationMetrics(List<Call> calls, final LocalDateTime minutesAgo) throws Exception {
+        verifyRelationMetrics(calls, minutesAgo, ALL_SERVICE_INSTANCE_RELATION_CLIENT_METRICS, ALL_SERVICE_INSTANCE_RELATION_SERVER_METRICS);
+    }
+
+    private void verifyServiceRelationMetrics(List<Call> calls, final LocalDateTime minutesAgo) throws Exception {
+        verifyRelationMetrics(calls, minutesAgo, ALL_SERVICE_RELATION_CLIENT_METRICS, ALL_SERVICE_RELATION_SERVER_METRICS);
+    }
+
+    private void verifyRelationMetrics(List<Call> calls, final LocalDateTime minutesAgo, String[] relationClientMetrics, String[] relationServerMetrics) throws Exception {
+        for (Call call : calls) {
+            for (String detectPoint : call.getDetectPoints()) {
+                switch (detectPoint) {
+                    case "CLIENT": {
+                        for (String metricName : relationClientMetrics) {
+                            verifyMetrics(queryClient, metricName, call.getId(), minutesAgo, retryInterval, this::generateTraffic);
+                        }
+                        break;
+                    }
+                    case "SERVER": {
+                        for (String metricName : relationServerMetrics) {
+                            verifyMetrics(queryClient, metricName, call.getId(), minutesAgo, retryInterval, this::generateTraffic);
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+
     private void generateTraffic() {
         try {
             final Map<String, String> user = new HashMap<>();
diff --git a/test/e2e/e2e-mysql/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java b/test/e2e/e2e-mysql/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java
index fb381d5..cd592bb 100644
--- a/test/e2e/e2e-mysql/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java
+++ b/test/e2e/e2e-mysql/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java
@@ -57,9 +57,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_ENDPOINT_METRICS;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_METRICS;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_METRICS;
+import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics;
+import static org.apache.skywalking.e2e.metrics.MetricsQuery.*;
 import static org.assertj.core.api.Assertions.assertThat;
 
 /**
@@ -164,6 +163,7 @@ public class SampleVerificationITCase {
 
         final TopoMatcher topoMatcher = new Yaml().loadAs(expectedInputStream, TopoMatcher.class);
         topoMatcher.verify(topoData);
+        verifyServiceRelationMetrics(topoData.getCalls(), minutesAgo);
     }
 
     private void verifyServiceInstanceTopo(LocalDateTime minutesAgo) throws Exception {
@@ -184,6 +184,7 @@ public class SampleVerificationITCase {
 
         final ServiceInstanceTopoMatcher topoMatcher = new Yaml().loadAs(expectedInputStream, ServiceInstanceTopoMatcher.class);
         topoMatcher.verify(topoData);
+        verifyServiceInstanceRelationMetrics(topoData.getCalls(), minutesAgo);
     }
 
     private void verifyServices(LocalDateTime minutesAgo) throws Exception {
@@ -329,6 +330,35 @@ public class SampleVerificationITCase {
         tracesMatcher.verify(traces);
     }
 
+    private void verifyServiceInstanceRelationMetrics(List<Call> calls, final LocalDateTime minutesAgo) throws Exception {
+        verifyRelationMetrics(calls, minutesAgo, ALL_SERVICE_INSTANCE_RELATION_CLIENT_METRICS, ALL_SERVICE_INSTANCE_RELATION_SERVER_METRICS);
+    }
+
+    private void verifyServiceRelationMetrics(List<Call> calls, final LocalDateTime minutesAgo) throws Exception {
+        verifyRelationMetrics(calls, minutesAgo, ALL_SERVICE_RELATION_CLIENT_METRICS, ALL_SERVICE_RELATION_SERVER_METRICS);
+    }
+
+    private void verifyRelationMetrics(List<Call> calls, final LocalDateTime minutesAgo, String[] relationClientMetrics, String[] relationServerMetrics) throws Exception {
+        for (Call call : calls) {
+            for (String detectPoint : call.getDetectPoints()) {
+                switch (detectPoint) {
+                    case "CLIENT": {
+                        for (String metricName : relationClientMetrics) {
+                            verifyMetrics(queryClient, metricName, call.getId(), minutesAgo);
+                        }
+                        break;
+                    }
+                    case "SERVER": {
+                        for (String metricName : relationServerMetrics) {
+                            verifyMetrics(queryClient, metricName, call.getId(), minutesAgo);
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
     private void doRetryableVerification(Runnable runnable) throws InterruptedException {
         while (true) {
             try {
diff --git a/test/e2e/e2e-single-service/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java b/test/e2e/e2e-single-service/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java
index fb381d5..cd592bb 100644
--- a/test/e2e/e2e-single-service/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java
+++ b/test/e2e/e2e-single-service/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java
@@ -57,9 +57,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_ENDPOINT_METRICS;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_METRICS;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_METRICS;
+import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics;
+import static org.apache.skywalking.e2e.metrics.MetricsQuery.*;
 import static org.assertj.core.api.Assertions.assertThat;
 
 /**
@@ -164,6 +163,7 @@ public class SampleVerificationITCase {
 
         final TopoMatcher topoMatcher = new Yaml().loadAs(expectedInputStream, TopoMatcher.class);
         topoMatcher.verify(topoData);
+        verifyServiceRelationMetrics(topoData.getCalls(), minutesAgo);
     }
 
     private void verifyServiceInstanceTopo(LocalDateTime minutesAgo) throws Exception {
@@ -184,6 +184,7 @@ public class SampleVerificationITCase {
 
         final ServiceInstanceTopoMatcher topoMatcher = new Yaml().loadAs(expectedInputStream, ServiceInstanceTopoMatcher.class);
         topoMatcher.verify(topoData);
+        verifyServiceInstanceRelationMetrics(topoData.getCalls(), minutesAgo);
     }
 
     private void verifyServices(LocalDateTime minutesAgo) throws Exception {
@@ -329,6 +330,35 @@ public class SampleVerificationITCase {
         tracesMatcher.verify(traces);
     }
 
+    private void verifyServiceInstanceRelationMetrics(List<Call> calls, final LocalDateTime minutesAgo) throws Exception {
+        verifyRelationMetrics(calls, minutesAgo, ALL_SERVICE_INSTANCE_RELATION_CLIENT_METRICS, ALL_SERVICE_INSTANCE_RELATION_SERVER_METRICS);
+    }
+
+    private void verifyServiceRelationMetrics(List<Call> calls, final LocalDateTime minutesAgo) throws Exception {
+        verifyRelationMetrics(calls, minutesAgo, ALL_SERVICE_RELATION_CLIENT_METRICS, ALL_SERVICE_RELATION_SERVER_METRICS);
+    }
+
+    private void verifyRelationMetrics(List<Call> calls, final LocalDateTime minutesAgo, String[] relationClientMetrics, String[] relationServerMetrics) throws Exception {
+        for (Call call : calls) {
+            for (String detectPoint : call.getDetectPoints()) {
+                switch (detectPoint) {
+                    case "CLIENT": {
+                        for (String metricName : relationClientMetrics) {
+                            verifyMetrics(queryClient, metricName, call.getId(), minutesAgo);
+                        }
+                        break;
+                    }
+                    case "SERVER": {
+                        for (String metricName : relationServerMetrics) {
+                            verifyMetrics(queryClient, metricName, call.getId(), minutesAgo);
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
     private void doRetryableVerification(Runnable runnable) throws InterruptedException {
         while (true) {
             try {