You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2023/01/02 06:26:15 UTC
[shardingsphere] branch master updated: Add : add query test for agent matrics E2E (#23256) (#23257)
This is an automated email from the ASF dual-hosted git repository.
zhangliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 758021db68b Add : add query test for agent matrics E2E (#23256) (#23257)
758021db68b is described below
commit 758021db68b2774de2bc32aa18688a1527bef242
Author: 孙念君 Nianjun Sun <su...@apache.org>
AuthorDate: Mon Jan 2 14:26:07 2023 +0800
Add : add query test for agent matrics E2E (#23256) (#23257)
* Add : add query test for agent matrics E2E (#23256)
* Remove : remove duplicated build method (#23256)
* Refactor : rename the build url method (#23256)
* Refactor : change the data structure for metrics name (#23256)
* Refactor : refactor Gson data name (#23256)
* Refactor : refactor variables name (#23256)
* Refactor : revert a readme file (#23256)
* Refactor : refactor the error log message (#23256)
* Refactor : refactor variables name (#23256)
* Refactor : refactor the java doc (#23256)
* Refactor : refactor the error message (#23256)
---
...shardingsphere-proxy-agent-metrics-assembly.xml | 11 ++++-
.../test/e2e/agent/metrics/MetricsPluginE2EIT.java | 50 ++++++++++++++--------
...etricResult.java => MetricsMetaDataResult.java} | 9 +++-
.../{MetricResult.java => MetricsQueryResult.java} | 47 +++++++++++++++++---
.../test/resources/docker/agent/conf/agent.yaml | 4 +-
.../src/test/resources/docker/docker-compose.yml | 4 +-
.../docker/prometheus/conf/prometheus.yml | 2 +-
.../src/test/resources/env/engine-env.properties | 5 ++-
8 files changed, 96 insertions(+), 36 deletions(-)
diff --git a/test/e2e/agent/plugins/metrics/src/test/assembly/shardingsphere-proxy-agent-metrics-assembly.xml b/test/e2e/agent/plugins/metrics/src/test/assembly/shardingsphere-proxy-agent-metrics-assembly.xml
index 3835aa7e0ad..3bf60c6fd2c 100644
--- a/test/e2e/agent/plugins/metrics/src/test/assembly/shardingsphere-proxy-agent-metrics-assembly.xml
+++ b/test/e2e/agent/plugins/metrics/src/test/assembly/shardingsphere-proxy-agent-metrics-assembly.xml
@@ -25,7 +25,7 @@
<fileSets>
<fileSet>
- <directory>${project.basedir}/../../../../../agent/plugins/metrics/target/plugins</directory>
+ <directory>${project.basedir}/../../../../../agent/plugins/metrics/type/target/plugins</directory>
<outputDirectory>./agent/plugins</outputDirectory>
<includes>
<include>**.jar</include>
@@ -39,7 +39,14 @@
</includes>
</fileSet>
<fileSet>
- <directory>${project.basedir}/../../../../../agent/plugins/metrics/target/lib</directory>
+ <directory>${project.basedir}/../../../../../agent/plugins/metrics/core/target/lib</directory>
+ <outputDirectory>./agent/lib</outputDirectory>
+ <includes>
+ <include>**.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../../../../../agent/plugins/core/target/lib</directory>
<outputDirectory>./agent/lib</outputDirectory>
<includes>
<include>**.jar</include>
diff --git a/test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/MetricsPluginE2EIT.java b/test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/MetricsPluginE2EIT.java
index ddb4317a31a..16662c8cffc 100644
--- a/test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/MetricsPluginE2EIT.java
+++ b/test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/MetricsPluginE2EIT.java
@@ -21,17 +21,19 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.shardingsphere.test.e2e.agent.common.BasePluginE2EIT;
import org.apache.shardingsphere.test.e2e.agent.common.env.E2ETestEnvironment;
import org.apache.shardingsphere.test.e2e.agent.common.util.OkHttpUtils;
-import org.apache.shardingsphere.test.e2e.agent.metrics.result.MetricResult;
+import org.apache.shardingsphere.test.e2e.agent.metrics.result.MetricsMetaDataResult;
+import org.apache.shardingsphere.test.e2e.agent.metrics.result.MetricsQueryResult;
import org.junit.Test;
import java.io.IOException;
import java.util.Collection;
-import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.Properties;
import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertNotNull;
import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
@Slf4j
public final class MetricsPluginE2EIT extends BasePluginE2EIT {
@@ -40,17 +42,17 @@ public final class MetricsPluginE2EIT extends BasePluginE2EIT {
public static final String PROXY_COLLECTION = "proxy_connection_total";
- public static final String PROXY_EXECUTE_LATENCY_MILLIS = "proxy_execute_latency_millis";
+ public static final String PROXY_EXECUTE_LATENCY_MILLIS = "proxy_execute_latency_millis_bucket";
public static final String PROXY_EXECUTE_ERROR = "proxy_execute_error_total";
- public static final String SQL_SELECT = "sql_select_total";
+ public static final String SQL_SELECT = "route_sql_select_total";
- public static final String SQL_UPDATE = "sql_update_total";
+ public static final String SQL_UPDATE = "route_sql_update_total";
- public static final String SQL_DELETE = "sql_delete_total";
+ public static final String SQL_DELETE = "route_sql_delete_total";
- public static final String SQL_INSERT = "sql_insert_total";
+ public static final String SQL_INSERT = "route_sql_insert_total";
public static final String ROUTE_DATASOURCE = "route_datasource_total";
@@ -68,25 +70,23 @@ public final class MetricsPluginE2EIT extends BasePluginE2EIT {
Thread.sleep(Long.parseLong(props.getProperty("prometheus.waitMs", "60000")));
} catch (final InterruptedException ignore) {
}
- String url = props.getProperty("prometheus.url");
+ String metaDataURL = props.getProperty("prometheus.metadata.url");
+ String queryURL = props.getProperty("prometheus.query.url");
Collection<String> metricsNames = buildMetricsNames();
for (String each : metricsNames) {
- String metricURL = buildMetricURL(url, each);
+ String metaDataURLWithParam = buildURLWithParameter(metaDataURL, each);
+ String queryURLWithParam = buildURLWithParameter(queryURL, each);
try {
- assertResult(OkHttpUtils.getInstance().get(metricURL, MetricResult.class), each);
+ assertMetadata(OkHttpUtils.getInstance().get(metaDataURLWithParam, MetricsMetaDataResult.class));
+ assertQuery(OkHttpUtils.getInstance().get(queryURLWithParam, MetricsQueryResult.class));
} catch (final IOException ex) {
- log.info("http get prometheus is error :", ex);
+ log.info("Access prometheus HTTP RESTful API error: ", ex);
}
}
}
- private void assertResult(final MetricResult metricResult, final String metricsName) {
- assertThat(metricResult.getStatus(), is("success"));
- assertNotNull(metricResult.getData());
- }
-
private Collection<String> buildMetricsNames() {
- Collection<String> result = new HashSet<>();
+ Collection<String> result = new LinkedHashSet<>();
result.add(PROXY_REQUEST);
result.add(PROXY_COLLECTION);
result.add(PROXY_EXECUTE_LATENCY_MILLIS);
@@ -102,7 +102,19 @@ public final class MetricsPluginE2EIT extends BasePluginE2EIT {
return result;
}
- private String buildMetricURL(final String url, final String metricsName) {
+ private String buildURLWithParameter(final String url, final String metricsName) {
return String.join("", url, metricsName);
}
+
+ // TODO remove if metadata result is not detailed.
+ private void assertMetadata(final MetricsMetaDataResult metricsMetaDataResult) {
+ assertThat(metricsMetaDataResult.getStatus(), is("success"));
+ assertNotNull(metricsMetaDataResult.getData());
+ }
+
+ // TODO add more detailed assert
+ private static void assertQuery(final MetricsQueryResult metricsQueryResult) {
+ assertThat(metricsQueryResult.getStatus(), is("success"));
+ assertFalse(metricsQueryResult.getData().getResult().isEmpty());
+ }
}
diff --git a/test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/result/MetricResult.java b/test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/result/MetricsMetaDataResult.java
similarity index 91%
copy from test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/result/MetricResult.java
copy to test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/result/MetricsMetaDataResult.java
index 29001729d5a..4f1e45bd8eb 100644
--- a/test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/result/MetricResult.java
+++ b/test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/result/MetricsMetaDataResult.java
@@ -23,14 +23,21 @@ import lombok.Setter;
import java.util.List;
import java.util.Map;
+
+/**
+ * Metrics meta data result.
+ */
@Getter
@Setter
-public final class MetricResult {
+public final class MetricsMetaDataResult {
private String status;
private Map<String, List<Metric>> data;
+ /**
+ * Metric.
+ */
@Getter
@Setter
public static final class Metric {
diff --git a/test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/result/MetricResult.java b/test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/result/MetricsQueryResult.java
similarity index 56%
rename from test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/result/MetricResult.java
rename to test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/result/MetricsQueryResult.java
index 29001729d5a..a8e649477c5 100644
--- a/test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/result/MetricResult.java
+++ b/test/e2e/agent/plugins/metrics/src/test/java/org/apache/shardingsphere/test/e2e/agent/metrics/result/MetricsQueryResult.java
@@ -17,28 +17,61 @@
package org.apache.shardingsphere.test.e2e.agent.metrics.result;
+import com.google.gson.annotations.SerializedName;
import lombok.Getter;
import lombok.Setter;
import java.util.List;
-import java.util.Map;
+/**
+ * Metrics query result.
+ */
@Getter
@Setter
-public final class MetricResult {
+public final class MetricsQueryResult {
private String status;
- private Map<String, List<Metric>> data;
+ private QueryData data;
+ /**
+ * Query data.
+ */
@Getter
@Setter
- public static final class Metric {
+ public static final class QueryData {
- private String type;
+ private String resultType;
+
+ private List<QueryDataResult> result;
+ }
+
+ /**
+ * Query data result.
+ */
+ @Getter
+ @Setter
+ public static final class QueryDataResult {
- private String help;
+ private QueryDataResultMetric queryDataResultMetric;
- private String unit;
+ private List<String> value;
+ }
+
+ /**
+ * Query data result metric.
+ */
+ public static final class QueryDataResultMetric {
+
+ @SerializedName("__name__")
+ private String name;
+
+ private String instance;
+
+ private String job;
+
+ private String sql;
+
+ private String type;
}
}
diff --git a/test/e2e/agent/plugins/metrics/src/test/resources/docker/agent/conf/agent.yaml b/test/e2e/agent/plugins/metrics/src/test/resources/docker/agent/conf/agent.yaml
index 9e18f54da94..0fe50abd671 100644
--- a/test/e2e/agent/plugins/metrics/src/test/resources/docker/agent/conf/agent.yaml
+++ b/test/e2e/agent/plugins/metrics/src/test/resources/docker/agent/conf/agent.yaml
@@ -22,8 +22,8 @@ plugins:
level: "INFO"
metrics:
Prometheus:
- host: "localhost"
- port: 18090
+ host: "0.0.0.0"
+ port: 19099
props:
jvm-information-collector-enabled: "true"
tracing:
diff --git a/test/e2e/agent/plugins/metrics/src/test/resources/docker/docker-compose.yml b/test/e2e/agent/plugins/metrics/src/test/resources/docker/docker-compose.yml
index 2c67520c26c..e07cf877698 100644
--- a/test/e2e/agent/plugins/metrics/src/test/resources/docker/docker-compose.yml
+++ b/test/e2e/agent/plugins/metrics/src/test/resources/docker/docker-compose.yml
@@ -30,7 +30,7 @@ services:
- LANG=C.UTF-8
prometheus:
- image: prom/prometheus:latest
+ image: prom/prometheus:v2.41.0
container_name: agent-prometheus
restart: always
volumes:
@@ -49,7 +49,7 @@ services:
ports:
- "43070:3307"
- "43080:3308"
- - "18090:18090"
+ - "19099:19099"
links:
- "mysql:mysql.agent.metrics.host"
volumes:
diff --git a/test/e2e/agent/plugins/metrics/src/test/resources/docker/prometheus/conf/prometheus.yml b/test/e2e/agent/plugins/metrics/src/test/resources/docker/prometheus/conf/prometheus.yml
index 3cf7ad330ac..a732bb0ffcc 100644
--- a/test/e2e/agent/plugins/metrics/src/test/resources/docker/prometheus/conf/prometheus.yml
+++ b/test/e2e/agent/plugins/metrics/src/test/resources/docker/prometheus/conf/prometheus.yml
@@ -32,4 +32,4 @@ scrape_configs:
- job_name: 'shardingsphere-agent'
static_configs:
- - targets: ['shardingsphere.proxy.host:18090']
+ - targets: ['shardingsphere.proxy.host:19099']
diff --git a/test/e2e/agent/plugins/metrics/src/test/resources/env/engine-env.properties b/test/e2e/agent/plugins/metrics/src/test/resources/env/engine-env.properties
index f96292cda77..921cb273128 100644
--- a/test/e2e/agent/plugins/metrics/src/test/resources/env/engine-env.properties
+++ b/test/e2e/agent/plugins/metrics/src/test/resources/env/engine-env.properties
@@ -21,8 +21,9 @@ it.env.value=metrics
proxy.url=jdbc:mysql://127.0.0.1:43070/agent-metrics-db?serverTimezone=UTC&useSSL=false&useLocalSessionState=true&characterEncoding=utf-8
proxy.username=root
proxy.password=root
-proxy.retry=60
+proxy.retry=100
proxy.waitMs=1000
prometheus.waitMs=60000
-prometheus.url=http://127.0.0.1:19090/api/v1/metadata?metric=
+prometheus.metadata.url=http://127.0.0.1:19090/api/v1/metadata?metric=
+prometheus.query.url=http://127.0.0.1:19090/api/v1/query?query=