You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wa...@apache.org on 2022/08/03 13:18:43 UTC
[skywalking] branch master updated: Elasticsearch storage support merge all metrics/meter indices into one physical index (#9416)
This is an automated email from the ASF dual-hosted git repository.
wankai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 0a032ced18 Elasticsearch storage support merge all metrics/meter indices into one physical index (#9416)
0a032ced18 is described below
commit 0a032ced18af4be79772930b538d3c698d415f7d
Author: Wan Kai <wa...@foxmail.com>
AuthorDate: Wed Aug 3 21:18:30 2022 +0800
Elasticsearch storage support merge all metrics/meter indices into one physical index (#9416)
---
.github/workflows/skywalking.yaml | 2 +
docs/en/changes/changes.md | 4 +
docs/en/setup/backend/backend-storage.md | 5 +
docs/en/setup/backend/configuration-vocabulary.md | 3 +-
.../analysis/manual/endpoint/EndpointTraffic.java | 2 +-
.../analysis/manual/instance/InstanceTraffic.java | 2 +-
.../analysis/manual/service/ServiceTraffic.java | 2 +-
.../meter/function/PercentileFunction.java | 2 +-
.../meter/function/avg/AvgHistogramFunction.java | 4 +-
.../avg/AvgHistogramPercentileFunction.java | 6 +-
.../meter/function/avg/AvgLabeledFunction.java | 6 +-
.../sum/SumHistogramPercentileFunction.java | 6 +-
.../server/core/analysis/metrics/ApdexMetrics.java | 2 +-
.../core/analysis/metrics/DoubleAvgMetrics.java | 4 +-
.../core/analysis/metrics/PercentMetrics.java | 2 +-
.../core/analysis/metrics/PercentileMetrics.java | 2 +-
.../EBPFProcessProfilingScheduleDispatcher.java | 7 +-
.../ebpf/storage/EBPFProfilingScheduleRecord.java | 15 ++-
.../oap/server/core/storage/model/Model.java | 6 +-
.../server/core/storage/model/StorageModels.java | 3 +-
.../src/main/resources/application.yml | 2 +
.../banyandb/measure/BanyanDBTopologyQueryDAO.java | 3 +-
.../StorageModuleElasticsearchConfig.java | 8 ++
.../StorageModuleElasticsearchProvider.java | 2 +
.../plugin/elasticsearch/base/IndexController.java | 72 ++++++++++--
.../elasticsearch/base/StorageEsInstaller.java | 11 +-
.../cache/NetworkAddressAliasEsDAO.java | 22 ++--
.../elasticsearch/query/AggregationQueryEsDAO.java | 4 +-
.../query/EBPFProfilingScheduleEsDAO.java | 8 +-
.../elasticsearch/query/MetadataQueryEsDAO.java | 37 +++++-
.../elasticsearch/query/MetricsQueryEsDAO.java | 6 +-
.../elasticsearch/query/ServiceLabelEsDAO.java | 6 +-
.../query/TagAutoCompleteQueryDAO.java | 6 +
.../elasticsearch/query/TopologyQueryEsDAO.java | 69 +++++++----
.../query/zipkin/ZipkinQueryEsDAO.java | 9 ++
.../elasticsearch/base/TimeSeriesUtilsTest.java | 12 +-
.../ebpf/oncpu/opensearch/docker-compose.yml | 3 +-
test/e2e-v2/cases/storage/banyandb/e2e.yaml | 5 +-
test/e2e-v2/cases/storage/es/e2e.yaml | 5 +-
.../es/es-sharding}/docker-compose.yml | 50 ++++----
.../cases/storage/{h2 => es/es-sharding}/e2e.yaml | 9 +-
.../expected/dependency-endpoint-consumer.yml | 53 +++++++++
...rvices.yml => dependency-endpoint-provider.yml} | 23 ++--
...ndency-services.yml => dependency-instance.yml} | 22 ++--
...rvices.yml => dependency-services-consumer.yml} | 19 ++-
...rvices.yml => dependency-services-provider.yml} | 11 ++
...-value.yml => metrics-has-value-percentile.yml} | 32 ++++-
.../cases/storage/expected/metrics-has-value.yml | 2 +-
...-endpoint.yml => service-endpoint-consumer.yml} | 6 +-
...has-value.yml => service-endpoint-provider.yml} | 6 +-
...-instance.yml => service-instance-consumer.yml} | 10 +-
...-instance.yml => service-instance-provider.yml} | 6 +-
.../cases/storage/expected/trace-info-detail.yml | 68 -----------
.../cases/storage/expected/trace-users-detail.yml | 54 ++++++++-
test/e2e-v2/cases/storage/expected/traces-list.yml | 6 +-
test/e2e-v2/cases/storage/h2/e2e.yaml | 5 +-
test/e2e-v2/cases/storage/mysql/e2e.yaml | 5 +-
test/e2e-v2/cases/storage/opensearch/e2e.yaml | 5 +-
test/e2e-v2/cases/storage/postgres/e2e.yaml | 5 +-
test/e2e-v2/cases/storage/storage-cases.yaml | 129 +++++++++++++++++----
test/e2e-v2/cases/storage/tidb/e2e.yaml | 5 +-
61 files changed, 642 insertions(+), 264 deletions(-)
diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml
index 9adb41c716..8db48051ec 100644
--- a/.github/workflows/skywalking.yaml
+++ b/.github/workflows/skywalking.yaml
@@ -322,6 +322,8 @@ jobs:
- name: Storage OpenSearch 1.2.0
config: test/e2e-v2/cases/storage/opensearch/e2e.yaml
env: OPENSEARCH_VERSION=1.2.0
+ - name: Storage ES Sharding
+ config: test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml
- name: Alarm H2
config: test/e2e-v2/cases/alarm/h2/e2e.yaml
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 0ff13f0df7..7b190dc656 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -29,6 +29,10 @@
* Support `SUM_PER_MIN` downsampling in `MAL`.
* Support `sumHistogramPercentile` in `MAL`.
* Add `VIRTUAL_CACHE` to Layer, to fix conjectured Redis server, which icon can't show on the topology.
+* [Breaking Change] Elasticsearch storage merge all metrics/meter indices into one physical index template `metrics-all` on the default setting.
+ Provide system environment variable(`SW_STORAGE_ES_LOGIC_SHARDING`) to shard metrics indices into multi-physical indices as the previous versions(one index template per metric/meter aggregation function).
+ In the current one index mode, users still could choose to adjust ElasticSearch's shard number(`SW_STORAGE_ES_INDEX_SHARDS_NUMBER`) to scale out.
+* [Breaking Change] Many columns of metrics model names are changed, The H2/Mysql/Tidb/Postgres storage users are required to remove all metrics-related tables for OAP to re-create or use a new database instance.
#### UI
diff --git a/docs/en/setup/backend/backend-storage.md b/docs/en/setup/backend/backend-storage.md
index 32e374bb92..6f9e27b0ab 100644
--- a/docs/en/setup/backend/backend-storage.md
+++ b/docs/en/setup/backend/backend-storage.md
@@ -49,6 +49,10 @@ Since 8.8.0, SkyWalking rebuilds the ElasticSearch client on top of ElasticSearc
correct request formats according to the server-side version, hence you don't need to download different binaries
and don't need to configure different storage selectors for different ElasticSearch server-side versions anymore.
+Since 9.2.0, SkyWalking merges all metrics/meter indices into one physical index template `metrics-all` on the default setting.
+Provide system environment variable(`SW_STORAGE_ES_LOGIC_SHARDING`) to shard metrics indices into multi-physical indices as the previous versions(one index template per metric/meter aggregation function).
+In the current one index mode, users still could choose to adjust ElasticSearch's shard number(`SW_STORAGE_ES_INDEX_SHARDS_NUMBER`) to scale out.
+
For now, SkyWalking supports ElasticSearch 6.x, ElasticSearch 7.x, ElasticSearch 8.x, and OpenSearch 1.x, their
configurations are as follows:
@@ -83,6 +87,7 @@ storage:
oapAnalyzer: ${SW_STORAGE_ES_OAP_ANALYZER:"{\"analyzer\":{\"oap_analyzer\":{\"type\":\"stop\"}}}"} # the oap analyzer.
oapLogAnalyzer: ${SW_STORAGE_ES_OAP_LOG_ANALYZER:"{\"analyzer\":{\"oap_log_analyzer\":{\"type\":\"standard\"}}}"} # the oap log analyzer. It could be customized by the ES analyzer configuration to support more language log formats, such as Chinese log, Japanese log and etc.
advanced: ${SW_STORAGE_ES_ADVANCED:""}
+ logicSharding: ${SW_STORAGE_ES_LOGIC_SHARDING:false}
```
### ElasticSearch With Https SSL Encrypting communications.
diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md
index 85ec7ecec4..babb14c47d 100644
--- a/docs/en/setup/backend/configuration-vocabulary.md
+++ b/docs/en/setup/backend/configuration-vocabulary.md
@@ -114,6 +114,7 @@ The Configuration Vocabulary lists all available configurations provided by `app
| - | - | profileTaskQueryMaxSize | The maximum size of profile task per query. [...]
| - | - | profileDataQueryScrollBatchSize | The batch size of query profiling data. [...]
| - | - | advanced | All settings of ElasticSearch index creation. The value should be in JSON format. [...]
+| - | - | logicSharding | Shard metrics indices into multi-physical indices, one index template per metric/meter aggregation function. | S [...]
| - | h2 | - | H2 storage is designed for demonstration and running in short term (i.e. 1-2 hours) only. [...]
| - | - | driver | H2 JDBC driver. [...]
| - | - | url | H2 connection URL. Defaults to H2 memory mode. [...]
@@ -238,7 +239,7 @@ The Configuration Vocabulary lists all available configurations provided by `app
| - | - | enableLogTestTool | Enable the log testing API to test the LAL. **NOTE**: This API evaluates untrusted code on the OAP server. A malicious script can do significant damage (steal keys and secrets, remove files and directories, install malware, etc). As such, please enable this API only when you c [...]
| - | - | maxQueryComplexity | Maximum complexity allowed for the GraphQL query that can be used to abort a query if the total number of data fields queried exceeds the defined threshold. [...]
| - | - | enableUpdateUITemplate | Allow user add，disable and update UI template. [...]
-| - | - | enableOnDemandPodLog | Ondemand Pod log: fetch the Pod logs on users' demand, the logs are fetched and displayed in real time, and are not persisted in any kind. This is helpful when users want to do some experiments and monitor the logs and see what's happing inside the service. Note: if you print [...]
+| - | - | enableOnDemandPodLog | Ondemand Pod log: fetch the Pod logs on users' demand, the logs are fetched and displayed in real time, and are not persisted in any kind. This is helpful when users want to do some experiments and monitor the logs and see what's happing inside the service. Note: if you print [...]
| query | graphql | - | GraphQL query implementation. [...]
| - | - | restHost | Binding IP of RESTful services. [...]
| - | - | restPort | Binding port of RESTful services. [...]
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
index d0189e2c1f..fa67c3358d 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
@@ -45,7 +45,7 @@ public class EndpointTraffic extends Metrics {
public static final String INDEX_NAME = "endpoint_traffic";
public static final String SERVICE_ID = "service_id";
- public static final String NAME = "name";
+ public static final String NAME = "endpoint_traffic_name";
@Setter
@Getter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java
index ef38bc995f..14098da9ce 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java
@@ -48,7 +48,7 @@ import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SE
public class InstanceTraffic extends Metrics {
public static final String INDEX_NAME = "instance_traffic";
public static final String SERVICE_ID = "service_id";
- public static final String NAME = "name";
+ public static final String NAME = "instance_traffic_name";
public static final String LAST_PING_TIME_BUCKET = "last_ping";
public static final String PROPERTIES = "properties";
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java
index 9e12488318..c7bd000695 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java
@@ -49,7 +49,7 @@ import static org.apache.skywalking.oap.server.core.Const.DOUBLE_COLONS_SPLIT;
public class ServiceTraffic extends Metrics {
public static final String INDEX_NAME = "service_traffic";
- public static final String NAME = "name";
+ public static final String NAME = "service_traffic_name";
public static final String SHORT_NAME = "short_name";
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java
index 31734d8a8d..6fa1656838 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java
@@ -52,7 +52,7 @@ import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
public abstract class PercentileFunction extends Meter implements AcceptableValue<PercentileFunction.PercentileArgument>, MultiIntValuesHolder {
public static final String DATASET = "dataset";
public static final String RANKS = "ranks";
- public static final String VALUE = "value";
+ public static final String VALUE = "datatable_value";
@Setter
@Getter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java
index d71f352777..6c487013c7 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java
@@ -55,8 +55,8 @@ import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
@ToString
public abstract class AvgHistogramFunction extends Meter implements AcceptableValue<BucketedValues> {
public static final String DATASET = "dataset";
- protected static final String SUMMATION = "summation";
- protected static final String COUNT = "count";
+ protected static final String SUMMATION = "datatable_summation";
+ protected static final String COUNT = "datatable_count";
@Setter
@Getter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java
index e87a817033..71aa4c3bbb 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java
@@ -69,9 +69,9 @@ public abstract class AvgHistogramPercentileFunction extends Meter implements Ac
private static final String DEFAULT_GROUP = "pD";
public static final String DATASET = "dataset";
public static final String RANKS = "ranks";
- public static final String VALUE = "value";
- protected static final String SUMMATION = "summation";
- protected static final String COUNT = "count";
+ public static final String VALUE = "datatable_value";
+ protected static final String SUMMATION = "datatable_summation";
+ protected static final String COUNT = "datatable_count";
@Setter
@Getter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java
index 0bde64db62..b69c956b28 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java
@@ -43,9 +43,9 @@ import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
@MeterFunction(functionName = "avgLabeled")
@ToString
public abstract class AvgLabeledFunction extends Meter implements AcceptableValue<DataTable>, LabeledValueHolder {
- protected static final String SUMMATION = "summation";
- protected static final String COUNT = "count";
- protected static final String VALUE = "value";
+ protected static final String SUMMATION = "datatable_summation";
+ protected static final String COUNT = "datatable_count";
+ protected static final String VALUE = "datatable_value";
@Setter
@Getter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumHistogramPercentileFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumHistogramPercentileFunction.java
index 497c85a40b..b3750352ba 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumHistogramPercentileFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumHistogramPercentileFunction.java
@@ -61,8 +61,8 @@ public abstract class SumHistogramPercentileFunction extends Meter implements Ac
private static final String DEFAULT_GROUP = "pD";
public static final String DATASET = "dataset";
public static final String RANKS = "ranks";
- public static final String VALUE = "value";
- protected static final String SUMMATION = "summation";
+ public static final String VALUE = "datatable_value";
+ protected static final String SUMMATION = "datatable_summation";
@Setter
@Getter
@@ -340,4 +340,4 @@ public abstract class SumHistogramPercentileFunction extends Meter implements Ac
public int hashCode() {
return Objects.hash(entityId, getTimeBucket());
}
-}
\ No newline at end of file
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java
index 14b4750bc8..1fb724df30 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java
@@ -44,7 +44,7 @@ public abstract class ApdexMetrics extends Metrics implements IntValueHolder {
protected static final String S_NUM = "s_num";
// Level: tolerated
protected static final String T_NUM = "t_num";
- protected static final String VALUE = "value";
+ protected static final String VALUE = "int_value";
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java
index 9356c21198..f11036a36d 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java
@@ -30,9 +30,9 @@ import org.apache.skywalking.oap.server.core.storage.annotation.Column;
@MetricsFunction(functionName = "doubleAvg")
public abstract class DoubleAvgMetrics extends Metrics implements DoubleValueHolder {
- protected static final String SUMMATION = "summation";
+ protected static final String SUMMATION = "double_summation";
protected static final String COUNT = "count";
- protected static final String VALUE = "value";
+ protected static final String VALUE = "double_value";
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetrics.java
index aa6d32f96f..2efd2a6ee3 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetrics.java
@@ -34,7 +34,7 @@ public abstract class PercentMetrics extends Metrics implements IntValueHolder {
@Getter
@Setter
- @Column(columnName = TOTAL)
+ @Column(columnName = TOTAL, storageOnly = true)
private long total;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java
index 1f3ca142bc..3808330bee 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java
@@ -38,7 +38,7 @@ import org.apache.skywalking.oap.server.core.storage.annotation.Column;
@MetricsFunction(functionName = "percentile")
public abstract class PercentileMetrics extends Metrics implements MultiIntValuesHolder {
protected static final String DATASET = "dataset";
- protected static final String VALUE = "value";
+ protected static final String VALUE = "datatable_value";
protected static final String PRECISION = "precision";
private static final int[] RANKS = {
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProcessProfilingScheduleDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProcessProfilingScheduleDispatcher.java
index 10e3fd11dd..c19f1326bd 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProcessProfilingScheduleDispatcher.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProcessProfilingScheduleDispatcher.java
@@ -18,6 +18,8 @@
package org.apache.skywalking.oap.server.core.profiling.ebpf.storage;
+import com.google.common.hash.Hashing;
+import com.linecorp.armeria.internal.shaded.guava.base.Charsets;
import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
@@ -32,6 +34,9 @@ public class EBPFProcessProfilingScheduleDispatcher implements SourceDispatcher<
traffic.setStartTime(source.getStartTime());
traffic.setEndTime(source.getCurrentTime());
traffic.setTimeBucket(TimeBucket.getMinuteTimeBucket(source.getCurrentTime()));
+ traffic.setScheduleId(Hashing.sha256().newHasher()
+ .putString(String.format("%s_%s_%d", source.getTaskId(), source.getProcessId(), source.getStartTime()), Charsets.UTF_8)
+ .hash().toString());
MetricsStreamProcessor.getInstance().in(traffic);
}
-}
\ No newline at end of file
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingScheduleRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingScheduleRecord.java
index 494627e4e6..253ea9ccd0 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingScheduleRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingScheduleRecord.java
@@ -18,8 +18,6 @@
package org.apache.skywalking.oap.server.core.profiling.ebpf.storage;
-import com.google.common.hash.Hashing;
-import com.linecorp.armeria.internal.shaded.guava.base.Charsets;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
@@ -57,6 +55,7 @@ public class EBPFProfilingScheduleRecord extends Metrics {
public static final String PROCESS_ID = "process_id";
public static final String START_TIME = "start_time";
public static final String END_TIME = "end_time";
+ public static final String EBPF_PROFILING_SCHEDULE_ID = "ebpf_profiling_schedule_id";
@Column(columnName = TASK_ID, length = 600)
private String taskId;
@@ -66,6 +65,8 @@ public class EBPFProfilingScheduleRecord extends Metrics {
private long startTime;
@Column(columnName = END_TIME)
private long endTime;
+ @Column(columnName = EBPF_PROFILING_SCHEDULE_ID)
+ private String scheduleId;
@Override
public boolean combine(Metrics metrics) {
@@ -92,15 +93,14 @@ public class EBPFProfilingScheduleRecord extends Metrics {
@Override
protected String id0() {
- return Hashing.sha256().newHasher()
- .putString(String.format("%s_%s_%d", taskId, processId, startTime), Charsets.UTF_8)
- .hash().toString();
+ return scheduleId;
}
@Override
public void deserialize(RemoteData remoteData) {
setTaskId(remoteData.getDataStrings(0));
setProcessId(remoteData.getDataStrings(1));
+ setScheduleId(remoteData.getDataStrings(2));
setStartTime(remoteData.getDataLongs(0));
setEndTime(remoteData.getDataLongs(1));
setTimeBucket(remoteData.getDataLongs(2));
@@ -111,6 +111,7 @@ public class EBPFProfilingScheduleRecord extends Metrics {
final RemoteData.Builder builder = RemoteData.newBuilder();
builder.addDataStrings(taskId);
builder.addDataStrings(processId);
+ builder.addDataStrings(scheduleId);
builder.addDataLongs(startTime);
builder.addDataLongs(endTime);
builder.addDataLongs(getTimeBucket());
@@ -129,6 +130,7 @@ public class EBPFProfilingScheduleRecord extends Metrics {
final EBPFProfilingScheduleRecord executeTraffic = new EBPFProfilingScheduleRecord();
executeTraffic.setTaskId((String) converter.get(TASK_ID));
executeTraffic.setProcessId((String) converter.get(PROCESS_ID));
+ executeTraffic.setScheduleId((String) converter.get(EBPF_PROFILING_SCHEDULE_ID));
executeTraffic.setStartTime(((Number) converter.get(START_TIME)).longValue());
executeTraffic.setEndTime(((Number) converter.get(END_TIME)).longValue());
executeTraffic.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue());
@@ -139,9 +141,10 @@ public class EBPFProfilingScheduleRecord extends Metrics {
public void entity2Storage(final EBPFProfilingScheduleRecord storageData, final Convert2Storage converter) {
converter.accept(TASK_ID, storageData.getTaskId());
converter.accept(PROCESS_ID, storageData.getProcessId());
+ converter.accept(EBPF_PROFILING_SCHEDULE_ID, storageData.getScheduleId());
converter.accept(START_TIME, storageData.getStartTime());
converter.accept(END_TIME, storageData.getEndTime());
converter.accept(TIME_BUCKET, storageData.getTimeBucket());
}
}
-}
\ No newline at end of file
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java
index eea3a61f80..4e37d756f3 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java
@@ -36,7 +36,7 @@ public class Model {
private final boolean record;
private final boolean superDataset;
private final boolean isTimeSeries;
- private final String aggregationFunctionName;
+ private final Class<?> streamClass;
private final boolean timeRelativeID;
private final SQLDatabaseModelExtension sqlDBModelExtension;
@@ -46,7 +46,7 @@ public class Model {
final DownSampling downsampling,
final boolean record,
final boolean superDataset,
- final String aggregationFunctionName,
+ final Class<?> streamClass,
boolean timeRelativeID,
final SQLDatabaseModelExtension sqlDBModelExtension) {
this.name = name;
@@ -56,7 +56,7 @@ public class Model {
this.isTimeSeries = !DownSampling.None.equals(downsampling);
this.record = record;
this.superDataset = superDataset;
- this.aggregationFunctionName = aggregationFunctionName;
+ this.streamClass = streamClass;
this.timeRelativeID = timeRelativeID;
this.sqlDBModelExtension = sqlDBModelExtension;
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
index bbf223b8ce..28a47de1ea 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
@@ -24,7 +24,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
-import org.apache.skywalking.oap.server.core.analysis.FunctionCategory;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.StorageException;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
@@ -69,7 +68,7 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
storage.getDownsampling(),
record,
isSuperDatasetModel(aClass),
- FunctionCategory.uniqueFunctionName(aClass),
+ aClass,
storage.isTimeRelativeID(),
sqlDBModelExtension
);
diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml
index 632bc702a9..a5294394d4 100755
--- a/oap-server/server-starter/src/main/resources/application.yml
+++ b/oap-server/server-starter/src/main/resources/application.yml
@@ -166,6 +166,8 @@ storage:
oapAnalyzer: ${SW_STORAGE_ES_OAP_ANALYZER:"{\"analyzer\":{\"oap_analyzer\":{\"type\":\"stop\"}}}"} # the oap analyzer.
oapLogAnalyzer: ${SW_STORAGE_ES_OAP_LOG_ANALYZER:"{\"analyzer\":{\"oap_log_analyzer\":{\"type\":\"standard\"}}}"} # the oap log analyzer. It could be customized by the ES analyzer configuration to support more language log formats, such as Chinese log, Japanese log and etc.
advanced: ${SW_STORAGE_ES_ADVANCED:""}
+ # Enable shard metrics indices into multi-physical indices, one index template per metric/meter aggregation function.
+ logicSharding: ${SW_STORAGE_ES_LOGIC_SHARDING:false}
h2:
driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource}
url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db;DB_CLOSE_DELAY=-1}
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java
index 5dfd13d404..7afc5c80e2 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java
@@ -25,6 +25,7 @@ import org.apache.skywalking.banyandb.v1.client.TimestampRange;
import org.apache.skywalking.oap.server.core.UnexpectedException;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import org.apache.skywalking.oap.server.core.analysis.manual.relation.endpoint.EndpointRelationServerSideMetrics;
+import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationClientSideMetrics;
import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationServerSideMetrics;
import org.apache.skywalking.oap.server.core.analysis.manual.relation.process.ProcessRelationClientSideMetrics;
import org.apache.skywalking.oap.server.core.analysis.manual.relation.process.ProcessRelationServerSideMetrics;
@@ -169,7 +170,7 @@ public class BanyanDBTopologyQueryDAO extends AbstractBanyanDBDAO implements ITo
timestampRange = new TimestampRange(TimeBucket.getTimestamp(startTB), TimeBucket.getTimestamp(endTB));
}
final String modelName = detectPoint == DetectPoint.SERVER ? ServiceInstanceRelationServerSideMetrics.INDEX_NAME :
- ServiceRelationClientSideMetrics.INDEX_NAME;
+ ServiceInstanceRelationClientSideMetrics.INDEX_NAME;
final Map<String, Call.CallDetail> callMap = new HashMap<>();
for (final QueryBuilder<MeasureQuery> q : queryBuilderList) {
MeasureQueryResponse resp = query(modelName,
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java
index 50fccf6aab..8010ace25e 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java
@@ -142,4 +142,12 @@ public class StorageModuleElasticsearchConfig extends ModuleConfig {
* If the value is <= 0, the number of available processors will be used.
*/
private int numHttpClientThread;
+
+ /**
+ * If disabled, all metrics would be persistent in one physical index template, to reduce the number of physical indices.
+ * If enabled, shard metrics indices into multi-physical indices, one index template per metric/meter aggregation function.
+ *
+ * @since 9.2.0
+ */
+ private boolean logicSharding = false;
}
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java
index 679ba121a8..451c481f58 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java
@@ -60,6 +60,7 @@ import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedExcepti
import org.apache.skywalking.oap.server.library.util.MultipleFilesChangeMonitor;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAO;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.HistoryDeleteEsDAO;
+import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.StorageEsDAO;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.StorageEsInstaller;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.TimeSeriesUtils;
@@ -217,6 +218,7 @@ public class StorageModuleElasticsearchProvider extends ModuleProvider {
ITagAutoCompleteQueryDAO.class, new TagAutoCompleteQueryDAO(elasticSearchClient));
this.registerServiceImplementation(
IZipkinQueryDAO.class, new ZipkinQueryEsDAO(elasticSearchClient));
+ IndexController.INSTANCE.setLogicSharding(config.isLogicSharding());
}
@Override
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexController.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexController.java
index 90c05ed64c..96d8bb291a 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexController.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexController.java
@@ -18,13 +18,20 @@
package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
+import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
-import org.apache.skywalking.oap.server.library.util.StringUtil;
+import org.apache.skywalking.oap.server.core.analysis.FunctionCategory;
+import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
+import org.apache.skywalking.oap.server.core.storage.model.ModelColumn;
import org.apache.skywalking.oap.server.core.Const;
import org.apache.skywalking.oap.server.core.storage.model.Model;
+import org.apache.skywalking.oap.server.library.util.StringUtil;
/**
* The metrics data, that generated by OAL or MAL, would be partitioned to storage by the functions of the OAL or MAL.
@@ -33,9 +40,18 @@ import org.apache.skywalking.oap.server.core.storage.model.Model;
@Slf4j
public enum IndexController {
INSTANCE;
+ /**
+ * Init in StorageModuleElasticsearchProvider.prepare() and the value from the config.
+ */
+ @Setter
+ private boolean logicSharding = false;
public String getTableName(Model model) {
- return isMetricModel(model) ? model.getAggregationFunctionName() : model.getName();
+ if (!logicSharding) {
+ return isMetricModel(model) ? "metrics-all" : model.getName();
+ }
+ String aggFuncName = FunctionCategory.uniqueFunctionName(model.getStreamClass());
+ return StringUtil.isNotBlank(aggFuncName) ? aggFuncName : model.getName();
}
/**
@@ -46,6 +62,9 @@ public enum IndexController {
if (!isMetricModel(model)) {
return originalID;
}
+ if (logicSharding && !isFunctionMetric(model)) {
+ return originalID;
+ }
return this.generateDocId(model.getName(), originalID);
}
@@ -60,12 +79,16 @@ public enum IndexController {
* Check the mode of the Model definition.
*/
public boolean isMetricModel(Model model) {
- return StringUtil.isNotBlank(model.getAggregationFunctionName());
+ return Metrics.class.isAssignableFrom(model.getStreamClass());
+ }
+
+ public boolean isFunctionMetric(Model model) {
+ return StringUtil.isNotBlank(FunctionCategory.uniqueFunctionName(model.getStreamClass()));
}
/**
* When a model is the metric storage mode, a column named {@link LogicIndicesRegister#METRIC_TABLE_NAME} would be
- * append to the physical index. The value of the column is the original table name in other storages, such as the
+ * appended to the physical index. The value of the column is the original table name in other storages, such as the
* OAL name.
*/
public Map<String, Object> appendMetricTableColumn(Model model, Map<String, Object> columns) {
@@ -81,7 +104,10 @@ public enum IndexController {
/**
* The relations of the logic table and the physical table.
*/
- private static final Map<String, String> LOGIC_INDICES_CATALOG = new ConcurrentHashMap<>();
+ private static final Map<String, String> LOGIC_INDICES_CATALOG = new HashMap<>();
+
+ private static final Map<String/*physical index name*/, Map<String/*column name*/, ModelColumn>> PHYSICAL_INDICES_COLUMNS = new HashMap<>();
+
/**
* The metric table name in aggregation physical storage.
*/
@@ -91,12 +117,42 @@ public enum IndexController {
return Optional.ofNullable(LOGIC_INDICES_CATALOG.get(logicName)).orElse(logicName);
}
- public static void registerRelation(String logicName, String physicalName) {
- LOGIC_INDICES_CATALOG.put(logicName, physicalName);
+ public static void registerRelation(Model model, String physicalName) {
+ LOGIC_INDICES_CATALOG.put(model.getName(), physicalName);
+ Map<String, ModelColumn> columns = PHYSICAL_INDICES_COLUMNS.computeIfAbsent(
+ physicalName, v -> new ConcurrentHashMap<>());
+ model.getColumns().forEach(modelColumn -> {
+ String columnName = modelColumn.getColumnName().getName();
+ if (columns.containsKey(columnName)) {
+ checkModelColumnConflicts(columns.get(columnName), modelColumn, physicalName);
+ } else {
+ columns.put(columnName, modelColumn);
+ }
+ });
}
- public static boolean isMetricTable(String logicName) {
+ public static boolean isPhysicalTable(String logicName) {
return !getPhysicalTableName(logicName).equals(logicName);
}
+
+ public static List<ModelColumn> getPhysicalTableColumns(Model model) {
+ String tableName = getPhysicalTableName(model.getName());
+ return new ArrayList<>(PHYSICAL_INDICES_COLUMNS.get(tableName).values());
+ }
+
+ private static void checkModelColumnConflicts(ModelColumn mc1, ModelColumn mc2, String physicalName) {
+ if (!(mc1.isIndexOnly() == mc2.isIndexOnly())) {
+ throw new IllegalArgumentException(mc1.getColumnName() + " and " + mc2.getColumnName() + " isIndexOnly conflict in index: " + physicalName);
+ }
+ if (!(mc1.isStorageOnly() == mc2.isStorageOnly())) {
+ throw new IllegalArgumentException(mc1.getColumnName() + " and " + mc2.getColumnName() + " isStorageOnly conflict in index: " + physicalName);
+ }
+ if (!mc1.getType().equals(mc2.getType())) {
+ throw new IllegalArgumentException(mc1.getColumnName() + " and " + mc2.getColumnName() + " Class type conflict in index: " + physicalName);
+ }
+ if (!(mc1.getElasticSearchExtension().needMatchQuery() == mc2.getElasticSearchExtension().needMatchQuery())) {
+ throw new IllegalArgumentException(mc1.getColumnName() + " and " + mc2.getColumnName() + " needMatchQuery conflict in index: " + physicalName);
+ }
+ }
}
}
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
index 1256f3fea3..22f60003ed 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
@@ -44,6 +44,7 @@ public class StorageEsInstaller extends ModelInstaller {
private final StorageModuleElasticsearchConfig config;
protected final ColumnTypeEsMapping columnTypeEsMapping;
+
/**
* The mappings of the template .
*/
@@ -66,7 +67,7 @@ public class StorageEsInstaller extends ModelInstaller {
protected boolean isExists(Model model) {
ElasticSearchClient esClient = (ElasticSearchClient) client;
String tableName = IndexController.INSTANCE.getTableName(model);
- IndexController.LogicIndicesRegister.registerRelation(model.getName(), tableName);
+ IndexController.LogicIndicesRegister.registerRelation(model, tableName);
if (!model.isTimeSeries()) {
boolean exist = esClient.isExistsIndex(tableName);
if (exist) {
@@ -151,7 +152,7 @@ public class StorageEsInstaller extends ModelInstaller {
tableName, settings, structures.getMapping(tableName), config.getIndexTemplateOrder());
log.info("create {} index template finished, isAcknowledged: {}", tableName, isAcknowledged);
if (!isAcknowledged) {
- throw new IOException("create " + tableName + " index template failure, ");
+ throw new IOException("create " + tableName + " index template failure");
}
}
@@ -203,7 +204,8 @@ public class StorageEsInstaller extends ModelInstaller {
indexRefreshInterval = 5;
}
setting.put("index.refresh_interval", indexRefreshInterval + "s");
- setting.put("analysis", getAnalyzerSetting(model.getColumns()));
+ List<ModelColumn> columns = IndexController.LogicIndicesRegister.getPhysicalTableColumns(model);
+ setting.put("analysis", getAnalyzerSetting(columns));
if (!StringUtil.isEmpty(config.getAdvanced())) {
Map<String, Object> advancedSettings = gson.fromJson(config.getAdvanced(), Map.class);
setting.putAll(advancedSettings);
@@ -258,7 +260,8 @@ public class StorageEsInstaller extends ModelInstaller {
}
}
- if (IndexController.INSTANCE.isMetricModel(model)) {
+ if ((IndexController.INSTANCE.isMetricModel(model) && !config.isLogicSharding())
+ || (config.isLogicSharding() && IndexController.INSTANCE.isFunctionMetric(model))) {
Map<String, Object> column = new HashMap<>();
column.put("type", "keyword");
properties.put(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, column);
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/cache/NetworkAddressAliasEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/cache/NetworkAddressAliasEsDAO.java
index e81ef85479..69b713a91f 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/cache/NetworkAddressAliasEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/cache/NetworkAddressAliasEsDAO.java
@@ -23,6 +23,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder;
import org.apache.skywalking.library.elasticsearch.requests.search.Query;
import org.apache.skywalking.library.elasticsearch.requests.search.Search;
import org.apache.skywalking.library.elasticsearch.requests.search.SearchParams;
@@ -34,6 +35,7 @@ import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter;
import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO;
+import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController;
@Slf4j
public class NetworkAddressAliasEsDAO extends EsDAO implements INetworkAddressAliasDAO {
@@ -50,20 +52,24 @@ public class NetworkAddressAliasEsDAO extends EsDAO implements INetworkAddressAl
@Override
public List<NetworkAddressAlias> loadLastUpdate(long timeBucketInMinute) {
List<NetworkAddressAlias> networkAddressAliases = new ArrayList<>();
-
+ final String index =
+ IndexController.LogicIndicesRegister.getPhysicalTableName(NetworkAddressAlias.INDEX_NAME);
try {
final int batchSize = Math.min(resultWindowMaxSize, scrollingBatchSize);
- final Search search =
- Search.builder().query(
- Query.range(NetworkAddressAlias.LAST_UPDATE_TIME_BUCKET)
- .gte(timeBucketInMinute))
- .size(batchSize)
- .build();
+ final BoolQueryBuilder query = Query.bool();
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(NetworkAddressAlias.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, NetworkAddressAlias.INDEX_NAME));
+ }
+ query.must(Query.range(NetworkAddressAlias.LAST_UPDATE_TIME_BUCKET)
+ .gte(timeBucketInMinute));
+
+ final Search search = Search.builder().query(query).size(batchSize).build();
+
final SearchParams params = new SearchParams().scroll(SCROLL_CONTEXT_RETENTION);
final NetworkAddressAlias.Builder builder = new NetworkAddressAlias.Builder();
SearchResponse results =
- getClient().search(NetworkAddressAlias.INDEX_NAME, search, params);
+ getClient().search(index, search, params);
final Set<String> scrollIds = new HashSet<>();
try {
while (true) {
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AggregationQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AggregationQueryEsDAO.java
index d99d4b0d8d..77cafcaf6d 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AggregationQueryEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AggregationQueryEsDAO.java
@@ -62,7 +62,7 @@ public class AggregationQueryEsDAO extends EsDAO implements IAggregationQueryDAO
final boolean asc = condition.getOrder().equals(Order.ASC);
if (CollectionUtils.isEmpty(additionalConditions)
- && IndexController.LogicIndicesRegister.isMetricTable(condition.getName())) {
+ && IndexController.LogicIndicesRegister.isPhysicalTable(condition.getName())) {
final BoolQueryBuilder boolQuery =
Query.bool()
.must(basicQuery)
@@ -74,7 +74,7 @@ public class AggregationQueryEsDAO extends EsDAO implements IAggregationQueryDAO
} else if (CollectionUtils.isEmpty(additionalConditions)) {
search.query(basicQuery);
} else if (CollectionUtils.isNotEmpty(additionalConditions)
- && IndexController.LogicIndicesRegister.isMetricTable(condition.getName())) {
+ && IndexController.LogicIndicesRegister.isPhysicalTable(condition.getName())) {
final BoolQueryBuilder boolQuery =
Query.bool()
.must(Query.term(
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EBPFProfilingScheduleEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EBPFProfilingScheduleEsDAO.java
index af81702166..62f9cd69e4 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EBPFProfilingScheduleEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EBPFProfilingScheduleEsDAO.java
@@ -50,7 +50,9 @@ public class EBPFProfilingScheduleEsDAO extends EsDAO implements IEBPFProfilingS
final String index =
IndexController.LogicIndicesRegister.getPhysicalTableName(EBPFProfilingScheduleRecord.INDEX_NAME);
final BoolQueryBuilder query = Query.bool();
-
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(EBPFProfilingScheduleRecord.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, EBPFProfilingScheduleRecord.INDEX_NAME));
+ }
query.must(Query.term(EBPFProfilingScheduleRecord.TASK_ID, taskId));
query.must(Query.range(EBPFProfilingScheduleRecord.START_TIME));
final SearchBuilder search = Search.builder().query(query)
@@ -63,11 +65,11 @@ public class EBPFProfilingScheduleEsDAO extends EsDAO implements IEBPFProfilingS
private EBPFProfilingSchedule parseSchedule(final SearchHit hit) {
final EBPFProfilingSchedule schedule = new EBPFProfilingSchedule();
- schedule.setScheduleId(hit.getId());
+ schedule.setScheduleId((String) hit.getSource().get(EBPFProfilingScheduleRecord.EBPF_PROFILING_SCHEDULE_ID));
schedule.setTaskId((String) hit.getSource().get(EBPFProfilingScheduleRecord.TASK_ID));
schedule.setProcessId((String) hit.getSource().get(EBPFProfilingScheduleRecord.PROCESS_ID));
schedule.setStartTime(((Number) hit.getSource().get(EBPFProfilingScheduleRecord.START_TIME)).longValue());
schedule.setEndTime(((Number) hit.getSource().get(EBPFProfilingScheduleRecord.END_TIME)).longValue());
return schedule;
}
-}
\ No newline at end of file
+}
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java
index 5db92975fa..7265987077 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java
@@ -91,6 +91,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
if (StringUtil.isNotEmpty(group)) {
query.must(Query.term(ServiceTraffic.GROUP, group));
}
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ServiceTraffic.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ServiceTraffic.INDEX_NAME));
+ }
final SearchParams params = new SearchParams().scroll(SCROLL_CONTEXT_RETENTION);
final List<Service> services = new ArrayList<>();
@@ -128,6 +131,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
final BoolQueryBuilder query =
Query.bool()
.must(Query.term(ServiceTraffic.SERVICE_ID, serviceId));
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ServiceTraffic.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ServiceTraffic.INDEX_NAME));
+ }
final SearchBuilder search = Search.builder().query(query).size(queryMaxSize);
final SearchResponse response = getClient().search(index, search.build());
@@ -145,6 +151,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
Query.bool()
.must(Query.range(InstanceTraffic.LAST_PING_TIME_BUCKET).gte(minuteTimeBucket))
.must(Query.term(InstanceTraffic.SERVICE_ID, serviceId));
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(InstanceTraffic.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, InstanceTraffic.INDEX_NAME));
+ }
final int batchSize = Math.min(queryMaxSize, scrollingBatchSize);
final SearchBuilder search = Search.builder().query(query).size(batchSize);
@@ -168,9 +177,13 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
public ServiceInstance getInstance(final String instanceId) throws IOException {
final String index =
IndexController.LogicIndicesRegister.getPhysicalTableName(InstanceTraffic.INDEX_NAME);
+ String id = instanceId;
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(InstanceTraffic.INDEX_NAME)) {
+ id = IndexController.INSTANCE.generateDocId(InstanceTraffic.INDEX_NAME, instanceId);
+ }
final BoolQueryBuilder query =
Query.bool()
- .must(Query.term("_id", instanceId));
+ .must(Query.term("_id", id));
final SearchBuilder search = Search.builder().query(query).size(1);
final SearchResponse response = getClient().search(index, search.build());
@@ -193,6 +206,10 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
query.must(Query.match(matchCName, keyword));
}
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(EndpointTraffic.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, EndpointTraffic.INDEX_NAME));
+ }
+
final SearchBuilder search = Search.builder().query(query).size(limit);
final SearchResponse response = getClient().search(index, search.build());
@@ -219,6 +236,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME);
final BoolQueryBuilder query = Query.bool();
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ProcessTraffic.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ProcessTraffic.INDEX_NAME));
+ }
final SearchBuilder search = Search.builder().query(query).size(queryMaxSize);
appendProcessWhereQuery(query, serviceId, null, null, supportStatus, lastPingStartTimeBucket, lastPingEndTimeBucket);
final SearchResponse results = getClient().search(index, search.build());
@@ -232,6 +252,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME);
final BoolQueryBuilder query = Query.bool();
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ProcessTraffic.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ProcessTraffic.INDEX_NAME));
+ }
final SearchBuilder search = Search.builder().query(query).size(queryMaxSize);
appendProcessWhereQuery(query, null, serviceInstanceId, null, null, lastPingStartTimeBucket, lastPingEndTimeBucket);
final SearchResponse results = getClient().search(index, search.build());
@@ -245,6 +268,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME);
final BoolQueryBuilder query = Query.bool();
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ProcessTraffic.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ProcessTraffic.INDEX_NAME));
+ }
final SearchBuilder search = Search.builder().query(query).size(queryMaxSize);
appendProcessWhereQuery(query, null, null, agentId, null, 0, 0);
final SearchResponse results = getClient().search(index, search.build());
@@ -258,6 +284,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME);
final BoolQueryBuilder query = Query.bool();
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ProcessTraffic.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ProcessTraffic.INDEX_NAME));
+ }
final SearchBuilder search = Search.builder().query(query).size(0);
appendProcessWhereQuery(query, serviceId, null, null, profilingSupportStatus,
lastPingStartTimeBucket, lastPingEndTimeBucket);
@@ -272,6 +301,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME);
final BoolQueryBuilder query = Query.bool();
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ProcessTraffic.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ProcessTraffic.INDEX_NAME));
+ }
final SearchBuilder search = Search.builder().query(query).size(0);
appendProcessWhereQuery(query, null, instanceId, null, null, 0, 0);
final SearchResponse results = getClient().search(index, search.build());
@@ -313,6 +345,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME);
final BoolQueryBuilder query = Query.bool()
.must(Query.term("_id", processId));
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ProcessTraffic.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ProcessTraffic.INDEX_NAME));
+ }
final SearchBuilder search = Search.builder().query(query).size(queryMaxSize);
final SearchResponse response = getClient().search(index, search.build());
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
index 31cb215e76..6d194ee2ee 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
@@ -110,7 +110,7 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
final List<String> ids = pointOfTimes.stream().map(pointOfTime -> {
String id = pointOfTime.id(condition.getEntity().buildId());
- if (IndexController.LogicIndicesRegister.isMetricTable(condition.getName())) {
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(condition.getName())) {
id = IndexController.INSTANCE.generateDocId(condition.getName(), id);
}
String indexName = TimeSeriesUtils.queryIndexName(
@@ -255,7 +255,7 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
final String entityId = condition.getEntity().buildId();
if (entityId == null &&
- IndexController.LogicIndicesRegister.isMetricTable(condition.getName())) {
+ IndexController.LogicIndicesRegister.isPhysicalTable(condition.getName())) {
sourceBuilder.query(
Query.bool()
.must(rangeQueryBuilder)
@@ -266,7 +266,7 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
);
} else if (entityId == null) {
sourceBuilder.query(rangeQueryBuilder);
- } else if (IndexController.LogicIndicesRegister.isMetricTable(condition.getName())) {
+ } else if (IndexController.LogicIndicesRegister.isPhysicalTable(condition.getName())) {
sourceBuilder.query(
Query.bool()
.must(rangeQueryBuilder)
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ServiceLabelEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ServiceLabelEsDAO.java
index a024b396a7..0cb7ee5827 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ServiceLabelEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ServiceLabelEsDAO.java
@@ -46,7 +46,9 @@ public class ServiceLabelEsDAO extends EsDAO implements IServiceLabelDAO {
final String index =
IndexController.LogicIndicesRegister.getPhysicalTableName(ServiceLabelRecord.INDEX_NAME);
final BoolQueryBuilder query = Query.bool();
-
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ServiceLabelRecord.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ServiceLabelRecord.INDEX_NAME));
+ }
query.must(Query.term(ServiceLabelRecord.SERVICE_ID, serviceId));
final SearchBuilder search = Search.builder().query(query).size(maxSize);
@@ -57,4 +59,4 @@ public class ServiceLabelEsDAO extends EsDAO implements IServiceLabelDAO {
private String parseLabel(final SearchHit hit) {
return (String) hit.getSource().get(ServiceLabelRecord.LABEL);
}
-}
\ No newline at end of file
+}
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java
index b9872cac41..c87e1b6ff1 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java
@@ -52,6 +52,9 @@ public class TagAutoCompleteQueryDAO extends EsDAO implements ITagAutoCompleteQu
final long endSecondTB) throws IOException {
BoolQueryBuilder query = Query.bool();
query.must(Query.term(TagAutocompleteData.TAG_TYPE, tagType.name()));
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(TagAutocompleteData.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, TagAutocompleteData.INDEX_NAME));
+ }
final SearchBuilder search = Search.builder().query(query);
search.aggregation(Aggregation.terms(TagAutocompleteData.TAG_KEY)
.field(TagAutocompleteData.TAG_KEY)
@@ -85,6 +88,9 @@ public class TagAutoCompleteQueryDAO extends EsDAO implements ITagAutoCompleteQu
final long endSecondTB) throws IOException {
BoolQueryBuilder query = Query.bool().must(Query.term(TagAutocompleteData.TAG_KEY, tagKey));
query.must(Query.term(TagAutocompleteData.TAG_TYPE, tagType.name()));
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(TagAutocompleteData.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, TagAutocompleteData.INDEX_NAME));
+ }
final SearchBuilder search = Search.builder().query(query).size(limit);
final SearchResponse response = getClient().search(
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java
index 7e3880151a..a2d5fd7320 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java
@@ -60,7 +60,7 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO {
}
final SearchBuilder sourceBuilder = Search.builder().size(0);
- setQueryCondition(sourceBuilder, startTB, endTB, serviceIds);
+ setQueryCondition(sourceBuilder, startTB, endTB, serviceIds, ServiceRelationServerSideMetrics.INDEX_NAME);
return buildServiceRelation(
sourceBuilder, ServiceRelationServerSideMetrics.INDEX_NAME, DetectPoint.SERVER);
@@ -75,7 +75,7 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO {
}
final SearchBuilder sourceBuilder = Search.builder().size(0);
- setQueryCondition(sourceBuilder, startTB, endTB, serviceIds);
+ setQueryCondition(sourceBuilder, startTB, endTB, serviceIds, ServiceRelationClientSideMetrics.INDEX_NAME);
return buildServiceRelation(
sourceBuilder, ServiceRelationClientSideMetrics.INDEX_NAME, DetectPoint.CLIENT);
@@ -85,10 +85,14 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO {
public List<Call.CallDetail> loadServiceRelationsDetectedAtServerSide(long startTB,
long endTB) {
SearchBuilder sourceBuilder = Search.builder();
- sourceBuilder.query(Query.range(ServiceRelationServerSideMetrics.TIME_BUCKET)
- .gte(startTB)
- .lte(endTB))
- .size(0);
+ final BoolQueryBuilder query = Query.bool()
+ .must(Query.range(ServiceRelationServerSideMetrics.TIME_BUCKET)
+ .gte(startTB)
+ .lte(endTB));
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ServiceRelationServerSideMetrics.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ServiceRelationServerSideMetrics.INDEX_NAME));
+ }
+ sourceBuilder.query(query).size(0);
return buildServiceRelation(
sourceBuilder, ServiceRelationServerSideMetrics.INDEX_NAME, DetectPoint.SERVER);
@@ -98,10 +102,14 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO {
public List<Call.CallDetail> loadServiceRelationDetectedAtClientSide(long startTB,
long endTB) {
SearchBuilder sourceBuilder = Search.builder();
- sourceBuilder.query(Query.range(ServiceRelationServerSideMetrics.TIME_BUCKET)
- .gte(startTB)
- .lte(endTB))
- .size(0);
+ final BoolQueryBuilder query = Query.bool()
+ .must(Query.range(ServiceRelationClientSideMetrics.TIME_BUCKET)
+ .gte(startTB)
+ .lte(endTB));
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ServiceRelationClientSideMetrics.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ServiceRelationClientSideMetrics.INDEX_NAME));
+ }
+ sourceBuilder.query(query).size(0);
return buildServiceRelation(
sourceBuilder, ServiceRelationClientSideMetrics.INDEX_NAME, DetectPoint.CLIENT);
@@ -113,7 +121,7 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO {
long startTB,
long endTB) {
final SearchBuilder search = Search.builder().size(0);
- setInstanceQueryCondition(search, startTB, endTB, clientServiceId, serverServiceId);
+ setInstanceQueryCondition(search, startTB, endTB, clientServiceId, serverServiceId, ServiceInstanceRelationServerSideMetrics.INDEX_NAME);
return buildInstanceRelation(
search, ServiceInstanceRelationServerSideMetrics.INDEX_NAME, DetectPoint.SERVER);
@@ -125,14 +133,14 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO {
long startTB,
long endTB) {
final SearchBuilder search = Search.builder().size(0);
- setInstanceQueryCondition(search, startTB, endTB, clientServiceId, serverServiceId);
+ setInstanceQueryCondition(search, startTB, endTB, clientServiceId, serverServiceId, ServiceInstanceRelationClientSideMetrics.INDEX_NAME);
return buildInstanceRelation(
search, ServiceInstanceRelationClientSideMetrics.INDEX_NAME, DetectPoint.CLIENT);
}
private void setInstanceQueryCondition(SearchBuilder search, long startTB, long endTB,
- String clientServiceId, String serverServiceId) {
+ String clientServiceId, String serverServiceId, String indexName) {
final BoolQueryBuilder serverRelationBoolQuery =
Query.bool()
.must(
@@ -170,7 +178,9 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO {
.gte(startTB)
.lte(endTB))
.must(serviceIdBoolQuery);
-
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(indexName)) {
+ boolQuery.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, indexName));
+ }
search.query(boolQuery);
}
@@ -195,7 +205,9 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO {
Query.term(
EndpointRelationServerSideMetrics.DEST_ENDPOINT, destEndpointId
));
-
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(EndpointRelationServerSideMetrics.INDEX_NAME)) {
+ boolQuery.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, EndpointRelationServerSideMetrics.INDEX_NAME));
+ }
sourceBuilder.query(boolQuery);
return loadEndpoint(
@@ -214,11 +226,12 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO {
private List<Call.CallDetail> buildProcessRelation(String serviceInstanceId, long startTB, long endTB, DetectPoint detectPoint) throws IOException {
final SearchBuilder sourceBuilder = Search.builder().size(0);
- sourceBuilder.query(Query.bool()
- .must(Query.term(ProcessRelationServerSideMetrics.SERVICE_INSTANCE_ID, serviceInstanceId))
- .must(Query.range(EndpointRelationServerSideMetrics.TIME_BUCKET)
- .gte(startTB)
- .lte(endTB)));
+ final BoolQueryBuilder query = Query.bool()
+ .must(Query.term(ProcessRelationServerSideMetrics.SERVICE_INSTANCE_ID, serviceInstanceId))
+ .must(Query.range(EndpointRelationServerSideMetrics.TIME_BUCKET)
+ .gte(startTB)
+ .lte(endTB));
+ sourceBuilder.query(query);
sourceBuilder.aggregation(
Aggregation
.terms(Metrics.ENTITY_ID).field(Metrics.ENTITY_ID)
@@ -226,9 +239,13 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO {
.collectMode(TermsAggregationBuilder.CollectMode.BREADTH_FIRST)
.size(1000));
- final String index =
- IndexController.LogicIndicesRegister.getPhysicalTableName(detectPoint == DetectPoint.SERVER ?
- ProcessRelationServerSideMetrics.INDEX_NAME : ProcessRelationClientSideMetrics.INDEX_NAME);
+ String indexName = detectPoint == DetectPoint.SERVER ?
+ ProcessRelationServerSideMetrics.INDEX_NAME : ProcessRelationClientSideMetrics.INDEX_NAME;
+
+ final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(indexName);
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(indexName)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, indexName));
+ }
final SearchResponse response = getClient().search(index, sourceBuilder.build());
final List<Call.CallDetail> calls = new ArrayList<>();
@@ -353,7 +370,7 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO {
}
private void setQueryCondition(SearchBuilder search, long startTB, long endTB,
- List<String> serviceIds) {
+ List<String> serviceIds, String indexName) {
final BoolQueryBuilder query =
Query.bool()
.must(Query.range(ServiceRelationServerSideMetrics.TIME_BUCKET)
@@ -363,7 +380,9 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO {
final BoolQueryBuilder serviceIdBoolQuery = Query.bool();
query.must(serviceIdBoolQuery);
-
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(indexName)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, indexName));
+ }
if (serviceIds.size() == 1) {
serviceIdBoolQuery.should(
Query.term(
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAO.java
index 8c8f2f196c..ec3ac6d6a3 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAO.java
@@ -66,6 +66,9 @@ public class ZipkinQueryEsDAO extends EsDAO implements IZipkinQueryDAO {
final String index =
IndexController.LogicIndicesRegister.getPhysicalTableName(ZipkinServiceTraffic.INDEX_NAME);
final BoolQueryBuilder query = Query.bool();
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ZipkinServiceTraffic.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ZipkinServiceTraffic.INDEX_NAME));
+ }
final SearchBuilder search = Search.builder().query(query).size(SCROLLING_BATCH_SIZE);
final SearchParams params = new SearchParams().scroll(SCROLL_CONTEXT_RETENTION);
final List<String> services = new ArrayList<>();
@@ -98,6 +101,9 @@ public class ZipkinQueryEsDAO extends EsDAO implements IZipkinQueryDAO {
String index = IndexController.LogicIndicesRegister.getPhysicalTableName(
ZipkinServiceRelationTraffic.INDEX_NAME);
BoolQueryBuilder query = Query.bool().must(Query.term(ZipkinServiceRelationTraffic.SERVICE_NAME, serviceName));
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ZipkinServiceRelationTraffic.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ZipkinServiceRelationTraffic.INDEX_NAME));
+ }
SearchBuilder search = Search.builder().query(query).size(NAME_QUERY_MAX_SIZE);
SearchResponse response = getClient().search(index, search.build());
List<String> remoteServices = new ArrayList<>();
@@ -114,6 +120,9 @@ public class ZipkinQueryEsDAO extends EsDAO implements IZipkinQueryDAO {
public List<String> getSpanNames(final String serviceName) {
String index = IndexController.LogicIndicesRegister.getPhysicalTableName(ZipkinServiceSpanTraffic.INDEX_NAME);
BoolQueryBuilder query = Query.bool().must(Query.term(ZipkinServiceSpanTraffic.SERVICE_NAME, serviceName));
+ if (IndexController.LogicIndicesRegister.isPhysicalTable(ZipkinServiceSpanTraffic.INDEX_NAME)) {
+ query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ZipkinServiceSpanTraffic.INDEX_NAME));
+ }
SearchBuilder search = Search.builder().query(query).size(NAME_QUERY_MAX_SIZE);
SearchResponse response = getClient().search(index, search.build());
List<String> spanNames = new ArrayList<>();
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java
index 25a0d735d5..43ef3de019 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java
@@ -20,6 +20,8 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base;
import com.google.common.collect.Lists;
import org.apache.skywalking.oap.server.core.analysis.DownSampling;
+import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
+import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.query.enumeration.Step;
import org.apache.skywalking.oap.server.core.storage.model.Model;
import org.apache.skywalking.oap.server.core.storage.model.SQLDatabaseModelExtension;
@@ -39,13 +41,13 @@ public class TimeSeriesUtilsTest {
@Before
public void prepare() {
superDatasetModel = new Model("superDatasetModel", Lists.newArrayList(),
- 0, DownSampling.Second, true, true, "", true, new SQLDatabaseModelExtension()
+ 0, DownSampling.Second, true, true, Record.class, true, new SQLDatabaseModelExtension()
);
normalRecordModel = new Model("normalRecordModel", Lists.newArrayList(),
- 0, DownSampling.Second, true, false, "", true, new SQLDatabaseModelExtension()
+ 0, DownSampling.Second, true, false, Record.class, true, new SQLDatabaseModelExtension()
);
normalMetricsModel = new Model("normalMetricsModel", Lists.newArrayList(),
- 0, DownSampling.Minute, false, false, "", true, new SQLDatabaseModelExtension()
+ 0, DownSampling.Minute, false, false, Metrics.class, true, new SQLDatabaseModelExtension()
);
TimeSeriesUtils.setSUPER_DATASET_DAY_STEP(1);
TimeSeriesUtils.setDAY_STEP(3);
@@ -75,7 +77,7 @@ public class TimeSeriesUtilsTest {
writeIndexName(normalRecordModel, secondTimeBucket)
);
Assert.assertEquals(
- "normalMetricsModel-20200807",
+ "metrics-all-20200807",
writeIndexName(normalMetricsModel, minuteTimeBucket)
);
secondTimeBucket += 1000000;
@@ -89,7 +91,7 @@ public class TimeSeriesUtilsTest {
writeIndexName(normalRecordModel, secondTimeBucket)
);
Assert.assertEquals(
- "normalMetricsModel-20200810",
+ "metrics-all-20200810",
writeIndexName(normalMetricsModel, minuteTimeBucket)
);
}
diff --git a/test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/docker-compose.yml b/test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/docker-compose.yml
index 73f0017c5d..f2408e2d5a 100644
--- a/test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/docker-compose.yml
+++ b/test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/docker-compose.yml
@@ -43,6 +43,7 @@ services:
SW_STORAGE_ES_CLUSTER_NODES: opensearch:9200
SW_ES_USER: admin
SW_ES_PASSWORD: admin
+ SW_STORAGE_ES_LOGIC_SHARDING: "true"
depends_on:
opensearch:
condition: service_healthy
@@ -67,4 +68,4 @@ services:
condition: service_healthy
networks:
- e2e:
\ No newline at end of file
+ e2e:
diff --git a/test/e2e-v2/cases/storage/banyandb/e2e.yaml b/test/e2e-v2/cases/storage/banyandb/e2e.yaml
index 12b86b8efc..833e1957dc 100644
--- a/test/e2e-v2/cases/storage/banyandb/e2e.yaml
+++ b/test/e2e-v2/cases/storage/banyandb/e2e.yaml
@@ -32,8 +32,11 @@ trigger:
action: http
interval: 3s
times: 10
- url: http://${consumer_host}:${consumer_9092}/info
+ url: http://${consumer_host}:${consumer_9092}/users
method: POST
+ body: '{"id":"123","name":"skywalking"}'
+ headers:
+ "Content-Type": "application/json"
verify:
# verify with retry strategy
diff --git a/test/e2e-v2/cases/storage/es/e2e.yaml b/test/e2e-v2/cases/storage/es/e2e.yaml
index 12b86b8efc..833e1957dc 100644
--- a/test/e2e-v2/cases/storage/es/e2e.yaml
+++ b/test/e2e-v2/cases/storage/es/e2e.yaml
@@ -32,8 +32,11 @@ trigger:
action: http
interval: 3s
times: 10
- url: http://${consumer_host}:${consumer_9092}/info
+ url: http://${consumer_host}:${consumer_9092}/users
method: POST
+ body: '{"id":"123","name":"skywalking"}'
+ headers:
+ "Content-Type": "application/json"
verify:
# verify with retry strategy
diff --git a/test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/docker-compose.yml b/test/e2e-v2/cases/storage/es/es-sharding/docker-compose.yml
similarity index 65%
copy from test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/docker-compose.yml
copy to test/e2e-v2/cases/storage/es/es-sharding/docker-compose.yml
index 73f0017c5d..f96e6ae06b 100644
--- a/test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/docker-compose.yml
+++ b/test/e2e-v2/cases/storage/es/es-sharding/docker-compose.yml
@@ -16,8 +16,8 @@
version: '2.1'
services:
- opensearch:
- image: opensearchproject/opensearch:${OPENSEARCH_VERSION}
+ es:
+ image: elastic/elasticsearch:7.15.0
expose:
- 9200
networks:
@@ -25,46 +25,52 @@ services:
environment:
- discovery.type=single-node
- cluster.routing.allocation.disk.threshold_enabled=false
- - plugins.security.ssl.http.enabled=false
+ - xpack.security.enabled=false
healthcheck:
- test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200" ]
+ test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"]
interval: 5s
timeout: 60s
retries: 120
oap:
extends:
- file: ../docker-compose.yml
+ file: ../../../../script/docker-compose/base-compose.yml
service: oap
- networks:
- - e2e
environment:
SW_STORAGE: elasticsearch
- SW_STORAGE_ES_CLUSTER_NODES: opensearch:9200
- SW_ES_USER: admin
- SW_ES_PASSWORD: admin
+ SW_STORAGE_ES_CLUSTER_NODES: es:9200
+ SW_STORAGE_ES_LOGIC_SHARDING: "true"
+ ports:
+ - 12800
depends_on:
- opensearch:
+ es:
condition: service_healthy
- ports:
- - 12800
-
- sqrt:
- extends:
- file: ../docker-compose.yml
- service: sqrt
networks:
- e2e
- rover:
+ provider:
extends:
- file: ../docker-compose.yml
- service: rover
+ file: ../../../../script/docker-compose/base-compose.yml
+ service: provider
+ ports:
+ - 9090
+ depends_on:
+ oap:
+ condition: service_healthy
networks:
- e2e
+
+ consumer:
+ extends:
+ file: ../../../../script/docker-compose/base-compose.yml
+ service: consumer
+ ports:
+ - 9092
depends_on:
oap:
condition: service_healthy
+ provider:
+ condition: service_healthy
networks:
- e2e:
\ No newline at end of file
+ e2e:
diff --git a/test/e2e-v2/cases/storage/h2/e2e.yaml b/test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml
similarity index 86%
copy from test/e2e-v2/cases/storage/h2/e2e.yaml
copy to test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml
index 12b86b8efc..214270be44 100644
--- a/test/e2e-v2/cases/storage/h2/e2e.yaml
+++ b/test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml
@@ -19,7 +19,7 @@ setup:
env: compose
file: docker-compose.yml
timeout: 20m
- init-system-environment: ../../../script/env
+ init-system-environment: ../../../../script/env
steps:
- name: set PATH
command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH
@@ -32,8 +32,11 @@ trigger:
action: http
interval: 3s
times: 10
- url: http://${consumer_host}:${consumer_9092}/info
+ url: http://${consumer_host}:${consumer_9092}/users
method: POST
+ body: '{"id":"123","name":"skywalking"}'
+ headers:
+ "Content-Type": "application/json"
verify:
# verify with retry strategy
@@ -44,4 +47,4 @@ verify:
interval: 10s
cases:
- includes:
- - ../storage-cases.yaml
+ - ../../storage-cases.yaml
diff --git a/test/e2e-v2/cases/storage/expected/dependency-endpoint-consumer.yml b/test/e2e-v2/cases/storage/expected/dependency-endpoint-consumer.yml
new file mode 100644
index 0000000000..8f5a8b26b4
--- /dev/null
+++ b/test/e2e-v2/cases/storage/expected/dependency-endpoint-consumer.yml
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nodes:
+{{- contains .nodes }}
+- id: {{ b64enc "User" }}.0_{{ b64enc "User" }}
+ name: User
+ serviceid: {{ b64enc "User" }}.0
+ servicename: User
+ type: ""
+ isreal: false
+- id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }}
+ name: POST:/users
+ serviceid: {{ b64enc "e2e-service-consumer" }}.1
+ servicename: e2e-service-consumer
+ type: ""
+ isreal: true
+- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/users" }}
+ name: POST:/users
+ serviceid: {{ b64enc "e2e-service-provider" }}.1
+ servicename: e2e-service-provider
+ type: ""
+ isreal: true
+{{- end }}
+calls:
+{{- contains .calls }}
+- source: {{ b64enc "User" }}.0_{{ b64enc "User" }}
+ sourcecomponents: []
+ target: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }}
+ targetcomponents: []
+ id: {{ b64enc "User" }}.0-{{ b64enc "User" }}-{{ b64enc "e2e-service-consumer" }}.1-{{ b64enc "POST:/users" }}
+ detectpoints:
+ - SERVER
+- source: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }}
+ sourcecomponents: []
+ target: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/users" }}
+ targetcomponents: []
+ id: {{ b64enc "e2e-service-consumer" }}.1-{{ b64enc "POST:/users" }}-{{ b64enc "e2e-service-provider" }}.1-{{ b64enc "POST:/users" }}
+ detectpoints:
+ - SERVER
+{{- end }}
diff --git a/test/e2e-v2/cases/storage/expected/dependency-services.yml b/test/e2e-v2/cases/storage/expected/dependency-endpoint-provider.yml
similarity index 59%
copy from test/e2e-v2/cases/storage/expected/dependency-services.yml
copy to test/e2e-v2/cases/storage/expected/dependency-endpoint-provider.yml
index aa6e2fc5fd..3835336b8b 100644
--- a/test/e2e-v2/cases/storage/expected/dependency-services.yml
+++ b/test/e2e-v2/cases/storage/expected/dependency-endpoint-provider.yml
@@ -15,23 +15,26 @@
nodes:
{{- contains .nodes }}
-- id: {{ b64enc "e2e-service-provider"}}.1
- name: e2e-service-provider
- type: Tomcat
+- id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }}
+ name: POST:/users
+ serviceid: {{ b64enc "e2e-service-consumer" }}.1
+ servicename: e2e-service-consumer
+ type: ""
isreal: true
-- id: {{ b64enc "e2e-service-consumer"}}.1
- name: e2e-service-consumer
- type: Tomcat
+- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/users" }}
+ name: POST:/users
+ serviceid: {{ b64enc "e2e-service-provider" }}.1
+ servicename: e2e-service-provider
+ type: ""
isreal: true
{{- end }}
calls:
{{- contains .calls }}
-- source: {{ b64enc "e2e-service-consumer"}}.1
+- source: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }}
sourcecomponents: []
- target: {{ b64enc "e2e-service-provider"}}.1
+ target: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/users" }}
targetcomponents: []
- id: {{ b64enc "e2e-service-consumer"}}.1-{{ b64enc "e2e-service-provider"}}.1
+ id: {{ b64enc "e2e-service-consumer" }}.1-{{ b64enc "POST:/users" }}-{{ b64enc "e2e-service-provider" }}.1-{{ b64enc "POST:/users" }}
detectpoints:
- - CLIENT
- SERVER
{{- end }}
diff --git a/test/e2e-v2/cases/storage/expected/dependency-services.yml b/test/e2e-v2/cases/storage/expected/dependency-instance.yml
similarity index 60%
copy from test/e2e-v2/cases/storage/expected/dependency-services.yml
copy to test/e2e-v2/cases/storage/expected/dependency-instance.yml
index aa6e2fc5fd..d3b9547c60 100644
--- a/test/e2e-v2/cases/storage/expected/dependency-services.yml
+++ b/test/e2e-v2/cases/storage/expected/dependency-instance.yml
@@ -15,22 +15,26 @@
nodes:
{{- contains .nodes }}
-- id: {{ b64enc "e2e-service-provider"}}.1
- name: e2e-service-provider
- type: Tomcat
+- id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "consumer1" }}
+ name: consumer1
+ serviceid: {{ b64enc "e2e-service-consumer" }}.1
+ servicename: e2e-service-consumer
+ type: ""
isreal: true
-- id: {{ b64enc "e2e-service-consumer"}}.1
- name: e2e-service-consumer
- type: Tomcat
+- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }}
+ name: provider1
+ serviceid: {{ b64enc "e2e-service-provider" }}.1
+ servicename: e2e-service-provider
+ type: "Tomcat"
isreal: true
{{- end }}
calls:
{{- contains .calls }}
-- source: {{ b64enc "e2e-service-consumer"}}.1
+- source: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "consumer1" }}
sourcecomponents: []
- target: {{ b64enc "e2e-service-provider"}}.1
+ target: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }}
targetcomponents: []
- id: {{ b64enc "e2e-service-consumer"}}.1-{{ b64enc "e2e-service-provider"}}.1
+ id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "consumer1" }}-{{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }}
detectpoints:
- CLIENT
- SERVER
diff --git a/test/e2e-v2/cases/storage/expected/dependency-services.yml b/test/e2e-v2/cases/storage/expected/dependency-services-consumer.yml
similarity index 82%
copy from test/e2e-v2/cases/storage/expected/dependency-services.yml
copy to test/e2e-v2/cases/storage/expected/dependency-services-consumer.yml
index aa6e2fc5fd..11ba084ec2 100644
--- a/test/e2e-v2/cases/storage/expected/dependency-services.yml
+++ b/test/e2e-v2/cases/storage/expected/dependency-services-consumer.yml
@@ -15,14 +15,18 @@
nodes:
{{- contains .nodes }}
-- id: {{ b64enc "e2e-service-provider"}}.1
- name: e2e-service-provider
- type: Tomcat
- isreal: true
+- id: {{ b64enc "User"}}.0
+ name: User
+ type: USER
+ isreal: false
- id: {{ b64enc "e2e-service-consumer"}}.1
name: e2e-service-consumer
type: Tomcat
isreal: true
+- id: {{ b64enc "e2e-service-provider"}}.1
+ name: e2e-service-provider
+ type: Tomcat
+ isreal: true
{{- end }}
calls:
{{- contains .calls }}
@@ -34,4 +38,11 @@ calls:
detectpoints:
- CLIENT
- SERVER
+- source: {{ b64enc "User" }}.0
+ sourcecomponents: []
+ target: {{ b64enc "e2e-service-consumer"}}.1
+ targetcomponents: []
+ id: {{ b64enc "User" }}.0-{{ b64enc "e2e-service-consumer"}}.1
+ detectpoints:
+ - SERVER
{{- end }}
diff --git a/test/e2e-v2/cases/storage/expected/dependency-services.yml b/test/e2e-v2/cases/storage/expected/dependency-services-provider.yml
similarity index 80%
rename from test/e2e-v2/cases/storage/expected/dependency-services.yml
rename to test/e2e-v2/cases/storage/expected/dependency-services-provider.yml
index aa6e2fc5fd..600dd6dd99 100644
--- a/test/e2e-v2/cases/storage/expected/dependency-services.yml
+++ b/test/e2e-v2/cases/storage/expected/dependency-services-provider.yml
@@ -23,6 +23,10 @@ nodes:
name: e2e-service-consumer
type: Tomcat
isreal: true
+- id: {{ b64enc "localhost:-1" }}.0
+ name: localhost:-1
+ type: H2
+ isreal: false
{{- end }}
calls:
{{- contains .calls }}
@@ -34,4 +38,11 @@ calls:
detectpoints:
- CLIENT
- SERVER
+- source: {{ b64enc "e2e-service-provider" }}.1
+ sourcecomponents: []
+ target: {{ b64enc "localhost:-1"}}.0
+ targetcomponents: []
+ id: {{ b64enc "e2e-service-provider" }}.1-{{ b64enc "localhost:-1"}}.0
+ detectpoints:
+ - CLIENT
{{- end }}
diff --git a/test/e2e-v2/cases/storage/expected/metrics-has-value.yml b/test/e2e-v2/cases/storage/expected/metrics-has-value-percentile.yml
similarity index 58%
copy from test/e2e-v2/cases/storage/expected/metrics-has-value.yml
copy to test/e2e-v2/cases/storage/expected/metrics-has-value-percentile.yml
index 5359e6d315..ef17490250 100644
--- a/test/e2e-v2/cases/storage/expected/metrics-has-value.yml
+++ b/test/e2e-v2/cases/storage/expected/metrics-has-value-percentile.yml
@@ -14,6 +14,34 @@
# limitations under the License.
{{- contains . }}
-- key: {{ notEmpty .key }}
- value: {{ ge .value 1 }}
+- key: 0
+ value:
+ {{- contains .value }}
+ - key: {{ notEmpty .key }}
+ value: {{ ge .value 1 }}
+ {{- end }}
+- key: 1
+ value:
+ {{- contains .value }}
+ - key: {{ notEmpty .key }}
+ value: {{ ge .value 1 }}
+ {{- end }}
+- key: 2
+ value:
+ {{- contains .value }}
+ - key: {{ notEmpty .key }}
+ value: {{ ge .value 1 }}
+ {{- end }}
+- key: 3
+ value:
+ {{- contains .value }}
+ - key: {{ notEmpty .key }}
+ value: {{ ge .value 1 }}
+ {{- end }}
+- key: 4
+ value:
+ {{- contains .value }}
+ - key: {{ notEmpty .key }}
+ value: {{ ge .value 1 }}
+ {{- end }}
{{- end }}
diff --git a/test/e2e-v2/cases/storage/expected/metrics-has-value.yml b/test/e2e-v2/cases/storage/expected/metrics-has-value.yml
index 5359e6d315..d9c49854c9 100644
--- a/test/e2e-v2/cases/storage/expected/metrics-has-value.yml
+++ b/test/e2e-v2/cases/storage/expected/metrics-has-value.yml
@@ -16,4 +16,4 @@
{{- contains . }}
- key: {{ notEmpty .key }}
value: {{ ge .value 1 }}
-{{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/test/e2e-v2/cases/storage/expected/service-endpoint.yml b/test/e2e-v2/cases/storage/expected/service-endpoint-consumer.yml
similarity index 87%
rename from test/e2e-v2/cases/storage/expected/service-endpoint.yml
rename to test/e2e-v2/cases/storage/expected/service-endpoint-consumer.yml
index d2afb4575b..cedf79a14e 100644
--- a/test/e2e-v2/cases/storage/expected/service-endpoint.yml
+++ b/test/e2e-v2/cases/storage/expected/service-endpoint-consumer.yml
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-{{- contains .}}
-- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/info" }}
- name: POST:/info
+{{- contains . }}
+- id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }}
+ name: POST:/users
{{- end}}
diff --git a/test/e2e-v2/cases/storage/expected/metrics-has-value.yml b/test/e2e-v2/cases/storage/expected/service-endpoint-provider.yml
similarity index 88%
copy from test/e2e-v2/cases/storage/expected/metrics-has-value.yml
copy to test/e2e-v2/cases/storage/expected/service-endpoint-provider.yml
index 5359e6d315..0303f51790 100644
--- a/test/e2e-v2/cases/storage/expected/metrics-has-value.yml
+++ b/test/e2e-v2/cases/storage/expected/service-endpoint-provider.yml
@@ -14,6 +14,6 @@
# limitations under the License.
{{- contains . }}
-- key: {{ notEmpty .key }}
- value: {{ ge .value 1 }}
-{{- end }}
+- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/users" }}
+ name: POST:/users
+{{- end}}
\ No newline at end of file
diff --git a/test/e2e-v2/cases/storage/expected/service-instance.yml b/test/e2e-v2/cases/storage/expected/service-instance-consumer.yml
similarity index 85%
copy from test/e2e-v2/cases/storage/expected/service-instance.yml
copy to test/e2e-v2/cases/storage/expected/service-instance-consumer.yml
index 5f03bc18e7..dc2ac11dbd 100644
--- a/test/e2e-v2/cases/storage/expected/service-instance.yml
+++ b/test/e2e-v2/cases/storage/expected/service-instance-consumer.yml
@@ -15,9 +15,9 @@
# specific language governing permissions and limitations
# under the License.
-{{- contains .}}
-- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }}
- name: {{ notEmpty .name }}
+{{- contains . }}
+- id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "consumer1" }}
+ name: consumer1
attributes:
{{- contains .attributes }}
- name: OS Name
@@ -25,7 +25,7 @@
- name: hostname
value: {{ notEmpty .value }}
- name: Process No.
- value: "1"
+ value: {{ notEmpty .value }}
- name: Start Time
value: {{ notEmpty .value }}
- name: JVM Arguments
@@ -36,5 +36,5 @@
value: {{ notEmpty .value }}
{{- end}}
language: JAVA
- instanceuuid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }}
+ instanceuuid: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "consumer1" }}
{{- end}}
diff --git a/test/e2e-v2/cases/storage/expected/service-instance.yml b/test/e2e-v2/cases/storage/expected/service-instance-provider.yml
similarity index 95%
rename from test/e2e-v2/cases/storage/expected/service-instance.yml
rename to test/e2e-v2/cases/storage/expected/service-instance-provider.yml
index 5f03bc18e7..f32cc2b0f1 100644
--- a/test/e2e-v2/cases/storage/expected/service-instance.yml
+++ b/test/e2e-v2/cases/storage/expected/service-instance-provider.yml
@@ -15,9 +15,9 @@
# specific language governing permissions and limitations
# under the License.
-{{- contains .}}
+{{- contains . }}
- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }}
- name: {{ notEmpty .name }}
+ name: provider1
attributes:
{{- contains .attributes }}
- name: OS Name
@@ -25,7 +25,7 @@
- name: hostname
value: {{ notEmpty .value }}
- name: Process No.
- value: "1"
+ value: {{ notEmpty .value }}
- name: Start Time
value: {{ notEmpty .value }}
- name: JVM Arguments
diff --git a/test/e2e-v2/cases/storage/expected/trace-info-detail.yml b/test/e2e-v2/cases/storage/expected/trace-info-detail.yml
deleted file mode 100644
index 95b9bd98a5..0000000000
--- a/test/e2e-v2/cases/storage/expected/trace-info-detail.yml
+++ /dev/null
@@ -1,68 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-spans:
- {{- contains .spans }}
- - traceid: {{ notEmpty .traceid }}
- segmentid: {{ notEmpty .segmentid }}
- spanid: {{ .spanid }}
- parentspanid: {{ .parentspanid }}
- refs:
- {{- if eq .servicecode "e2e-service-provider" }}
- {{- contains .refs }}
- - traceid: {{ notEmpty .traceid }}
- parentsegmentid: {{ notEmpty .parentsegmentid }}
- parentspanid: 1
- type: CROSS_PROCESS
- {{- end }}
- {{- end }}
- {{- if eq .servicecode "e2e-service-consumer" }}
- []
- {{- end }}
- servicecode: {{ notEmpty .servicecode }}
- serviceinstancename: {{ notEmpty .serviceinstancename }}
- starttime: {{ gt .starttime 0 }}
- endtime: {{ gt .endtime 0 }}
- endpointname:
- {{- if eq .type "Exit" }}
- /info
- {{ else }}
- POST:/info
- {{- end }}
- type: {{ notEmpty .type }}
- peer:
- {{- if eq .type "Exit" }}
- provider:9090
- {{ else }}
- ""
- {{- end }}
- component:
- {{- if eq .type "Exit" }}
- SpringRestTemplate
- {{- end }}
- {{- if eq .type "Entry" }}
- Tomcat
- {{- end }}
- iserror: false
- layer: Http
- tags:
- {{- contains .tags }}
- - key: http.method
- value: POST
- - key: url
- value: {{ notEmpty .value }}
- {{- end }}
- logs: []
- {{- end }}
diff --git a/test/e2e-v2/cases/storage/expected/trace-users-detail.yml b/test/e2e-v2/cases/storage/expected/trace-users-detail.yml
index 4dfdd6ec41..0905893c03 100644
--- a/test/e2e-v2/cases/storage/expected/trace-users-detail.yml
+++ b/test/e2e-v2/cases/storage/expected/trace-users-detail.yml
@@ -14,12 +14,64 @@
# limitations under the License.
spans:
-{{- contains .spans }}
+ {{- contains .spans }}
- traceid: {{ .traceid }}
segmentid: {{ .segmentid }}
spanid: {{ .spanid }}
parentspanid: {{ .parentspanid }}
refs: []
+ servicecode: e2e-service-consumer
+ serviceinstancename: consumer1
+ starttime: {{ gt .starttime 0 }}
+ endtime: {{ gt .endtime 0 }}
+ endpointname: POST:/users
+ type: Entry
+ peer: ""
+ component: Tomcat
+ iserror: false
+ layer: Http
+ tags:
+ {{- contains .tags }}
+ - key: http.method
+ value: POST
+ - key: url
+ value: {{ notEmpty .value }}
+ {{- end }}
+ logs: []
+ - traceid: {{ notEmpty .traceid }}
+ segmentid: {{ .segmentid }}
+ spanid: {{ .spanid }}
+ parentspanid: {{ .parentspanid }}
+ refs: []
+ servicecode: e2e-service-consumer
+ serviceinstancename: consumer1
+ starttime: {{ gt .starttime 0 }}
+ endtime: {{ gt .endtime 0 }}
+ endpointname: /users
+ type: Exit
+ peer: provider:9090
+ component: SpringRestTemplate
+ iserror: false
+ layer: Http
+ tags:
+ {{- contains .tags }}
+ - key: http.method
+ value: POST
+ - key: url
+ value: {{ notEmpty .value }}
+ {{- end }}
+ logs: []
+ - traceid: {{ notEmpty .traceid }}
+ segmentid: {{ .segmentid }}
+ spanid: {{ .spanid }}
+ parentspanid: {{ .parentspanid }}
+ refs:
+ {{- contains .refs }}
+ - traceid: {{ notEmpty .traceid }}
+ parentsegmentid: {{ .parentsegmentid }}
+ parentspanid: 1
+ type: CROSS_PROCESS
+ {{- end }}
servicecode: e2e-service-provider
serviceinstancename: provider1
starttime: {{ gt .starttime 0 }}
diff --git a/test/e2e-v2/cases/storage/expected/traces-list.yml b/test/e2e-v2/cases/storage/expected/traces-list.yml
index 9ff69011e8..4d5a1e5fec 100644
--- a/test/e2e-v2/cases/storage/expected/traces-list.yml
+++ b/test/e2e-v2/cases/storage/expected/traces-list.yml
@@ -17,12 +17,10 @@ traces:
{{- contains .traces }}
- segmentid: {{ notEmpty .segmentid }}
endpointnames:
- {{- contains .endpointnames }}
- - POST:/info
- {{- end }}
+ - POST:/users
duration: {{ ge .duration 0 }}
start: "{{ notEmpty .start}}"
iserror: false
traceids:
- - {{ (index .traceids 0) }}
+ - {{ index .traceids 0 }}
{{- end }}
diff --git a/test/e2e-v2/cases/storage/h2/e2e.yaml b/test/e2e-v2/cases/storage/h2/e2e.yaml
index 12b86b8efc..833e1957dc 100644
--- a/test/e2e-v2/cases/storage/h2/e2e.yaml
+++ b/test/e2e-v2/cases/storage/h2/e2e.yaml
@@ -32,8 +32,11 @@ trigger:
action: http
interval: 3s
times: 10
- url: http://${consumer_host}:${consumer_9092}/info
+ url: http://${consumer_host}:${consumer_9092}/users
method: POST
+ body: '{"id":"123","name":"skywalking"}'
+ headers:
+ "Content-Type": "application/json"
verify:
# verify with retry strategy
diff --git a/test/e2e-v2/cases/storage/mysql/e2e.yaml b/test/e2e-v2/cases/storage/mysql/e2e.yaml
index 12b86b8efc..833e1957dc 100644
--- a/test/e2e-v2/cases/storage/mysql/e2e.yaml
+++ b/test/e2e-v2/cases/storage/mysql/e2e.yaml
@@ -32,8 +32,11 @@ trigger:
action: http
interval: 3s
times: 10
- url: http://${consumer_host}:${consumer_9092}/info
+ url: http://${consumer_host}:${consumer_9092}/users
method: POST
+ body: '{"id":"123","name":"skywalking"}'
+ headers:
+ "Content-Type": "application/json"
verify:
# verify with retry strategy
diff --git a/test/e2e-v2/cases/storage/opensearch/e2e.yaml b/test/e2e-v2/cases/storage/opensearch/e2e.yaml
index 12b86b8efc..833e1957dc 100644
--- a/test/e2e-v2/cases/storage/opensearch/e2e.yaml
+++ b/test/e2e-v2/cases/storage/opensearch/e2e.yaml
@@ -32,8 +32,11 @@ trigger:
action: http
interval: 3s
times: 10
- url: http://${consumer_host}:${consumer_9092}/info
+ url: http://${consumer_host}:${consumer_9092}/users
method: POST
+ body: '{"id":"123","name":"skywalking"}'
+ headers:
+ "Content-Type": "application/json"
verify:
# verify with retry strategy
diff --git a/test/e2e-v2/cases/storage/postgres/e2e.yaml b/test/e2e-v2/cases/storage/postgres/e2e.yaml
index 12b86b8efc..833e1957dc 100644
--- a/test/e2e-v2/cases/storage/postgres/e2e.yaml
+++ b/test/e2e-v2/cases/storage/postgres/e2e.yaml
@@ -32,8 +32,11 @@ trigger:
action: http
interval: 3s
times: 10
- url: http://${consumer_host}:${consumer_9092}/info
+ url: http://${consumer_host}:${consumer_9092}/users
method: POST
+ body: '{"id":"123","name":"skywalking"}'
+ headers:
+ "Content-Type": "application/json"
verify:
# verify with retry strategy
diff --git a/test/e2e-v2/cases/storage/storage-cases.yaml b/test/e2e-v2/cases/storage/storage-cases.yaml
index ec30bf5573..b1ac0b299f 100644
--- a/test/e2e-v2/cases/storage/storage-cases.yaml
+++ b/test/e2e-v2/cases/storage/storage-cases.yaml
@@ -20,40 +20,123 @@ cases:
# service list
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service layer GENERAL
expected: expected/service.yml
- # service metrics
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name service_sla --service-name=e2e-service-provider | yq e 'to_entries' -
- expected: expected/metrics-has-value.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics top --name service_sla 5
- expected: expected/metrics-top-service-sla.yml
- # service endpoint
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=info --service-name=e2e-service-provider
- expected: expected/service-endpoint.yml
- # service endpoint metrics
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name endpoint_cpm --endpoint-name=POST:/info --service-name=e2e-service-provider |yq e 'to_entries' -
- expected: expected/metrics-has-value.yml
- # dependency service
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name="e2e-service-provider"
- expected: expected/dependency-services.yml
# service instance list
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider
- expected: expected/service-instance.yml
- # service instance jvm metrics
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name instance_jvm_thread_live_count --instance-name=provider1 --service-name=e2e-service-provider | yq e 'to_entries' -
- expected: expected/metrics-has-value.yml
-
+ expected: expected/service-instance-provider.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-consumer
+ expected: expected/service-instance-consumer.yml
+ # service endpoint
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider
+ expected: expected/service-endpoint-provider.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-consumer
+ expected: expected/service-endpoint-consumer.yml
+ # dependency service
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name=e2e-service-provider
+ expected: expected/dependency-services-provider.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name=e2e-service-consumer
+ expected: expected/dependency-services-consumer.yml
+ # dependency instance
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency instance --service-name=e2e-service-consumer --dest-service-name=e2e-service-provider
+ expected: expected/dependency-instance.yml
+ # dependency endpoint
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency endpoint --service-name=e2e-service-provider --endpoint-name=POST:/users
+ expected: expected/dependency-endpoint-provider.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency endpoint --service-name=e2e-service-consumer --endpoint-name=POST:/users
+ expected: expected/dependency-endpoint-consumer.yml
# trace segment list
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls --tags http.method=POST
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls
expected: expected/traces-list.yml
# negative tags search: relationship should be logical AND instead of logical OR
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls --tags http.method=POST,http.status_code=200
expected: expected/empty-traces-list.yml
- # native tracing: trace detail
+ # trace detail
- query: |
swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace $( \
swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \
- | yq e '.traces | select(.[].endpointnames[0]=="POST:/info") | .[0].traceids[0]' - \
+ | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' -
)
- expected: expected/trace-info-detail.yml
+ expected: expected/trace-users-detail.yml
+ # service metrics
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics top --name service_sla 5
+ expected: expected/metrics-top-service-sla.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_sla --service-name=e2e-service-provider |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_cpm --service-name=e2e-service-provider |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_resp_time --service-name=e2e-service-provider |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_apdex --service-name=e2e-service-provider |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_sla --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_cpm --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_resp_time --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_apdex --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ # service instance metrics
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_instance_resp_time --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_instance_cpm --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_instance_sla --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ # service instance JVM metrics
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_heap --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_heap_max --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_noheap --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_live_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_daemon_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_peak_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_runnable_state_thread_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_class_loaded_class_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_class_total_loaded_class_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_heap --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_heap_max --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_noheap --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_live_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_daemon_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_peak_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_runnable_state_thread_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_class_loaded_class_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_class_total_loaded_class_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ # service endpoint metrics
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_cpm --endpoint-name=POST:/users --service-name=e2e-service-provider | yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_resp_time --endpoint-name=POST:/users --service-name=e2e-service-provider | yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_sla --endpoint-name=POST:/users --service-name=e2e-service-provider | yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_cpm --endpoint-name=POST:/users --service-name=e2e-service-consumer | yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_resp_time --endpoint-name=POST:/users --service-name=e2e-service-consumer | yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_sla --endpoint-name=POST:/users --service-name=e2e-service-consumer | yq e 'to_entries' -
+ expected: expected/metrics-has-value.yml
+ # service endpoint metrics percentile
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics multiple-linear --name=endpoint_percentile --endpoint-name=POST:/users --service-name=e2e-service-consumer |yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+ expected: expected/metrics-has-value-percentile.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics multiple-linear --name=endpoint_percentile --endpoint-name=POST:/users --service-name=e2e-service-consumer |yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+ expected: expected/metrics-has-value-percentile.yml
# native event: event list
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql event list
diff --git a/test/e2e-v2/cases/storage/tidb/e2e.yaml b/test/e2e-v2/cases/storage/tidb/e2e.yaml
index 12b86b8efc..833e1957dc 100644
--- a/test/e2e-v2/cases/storage/tidb/e2e.yaml
+++ b/test/e2e-v2/cases/storage/tidb/e2e.yaml
@@ -32,8 +32,11 @@ trigger:
action: http
interval: 3s
times: 10
- url: http://${consumer_host}:${consumer_9092}/info
+ url: http://${consumer_host}:${consumer_9092}/users
method: POST
+ body: '{"id":"123","name":"skywalking"}'
+ headers:
+ "Content-Type": "application/json"
verify:
# verify with retry strategy