You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by li...@apache.org on 2025/02/20 08:42:07 UTC

(skywalking) branch master updated: Add SkyPredictor project to E2E Testing alarm with baseline (#13047)

This is an automated email from the ASF dual-hosted git repository.

liuhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git


The following commit(s) were added to refs/heads/master by this push:
     new c09d9394a7 Add SkyPredictor project to E2E Testing alarm with baseline (#13047)
c09d9394a7 is described below

commit c09d9394a770f9af761f6a60f11c79d9108fb75d
Author: mrproliu <74...@qq.com>
AuthorDate: Thu Feb 20 16:42:03 2025 +0800

    Add SkyPredictor project to E2E Testing alarm with baseline (#13047)
---
 .github/workflows/skywalking.yaml                  |  7 +++
 .../ai-pipeline/src/main/proto/baseline.proto      |  2 +-
 test/e2e-v2/cases/alarm/alarm-settings.yml         |  9 ----
 .../e2e-v2/cases/alarm/banyandb/docker-compose.yml | 18 -------
 test/e2e-v2/cases/alarm/es/docker-compose.yml      | 18 -------
 .../cases/alarm/es/es-sharding/docker-compose.yml  | 18 -------
 .../cases/alarm/expected/metrics-has-value.yml     | 30 +++++++++++
 .../expected/silence-after-graphql-critical.yml    | 56 --------------------
 .../expected/silence-before-graphql-critical.yml   | 56 --------------------
 test/e2e-v2/cases/alarm/mysql/docker-compose.yml   | 17 -------
 .../e2e-v2/cases/alarm/postgres/docker-compose.yml | 18 -------
 test/e2e-v2/cases/baseline/alarm-cases.yaml        | 28 ++++++++++
 test/e2e-v2/cases/baseline/alarm-settings.yml      | 23 +++++++++
 .../banyandb/docker-compose.yml                    | 42 ++++++++++++---
 test/e2e-v2/cases/baseline/banyandb/e2e.yaml       | 47 +++++++++++++++++
 .../{alarm => baseline}/es/docker-compose.yml      | 43 ++++++++++++----
 test/e2e-v2/cases/baseline/es/e2e.yaml             | 47 +++++++++++++++++
 .../es/es-sharding/docker-compose.yml              | 44 ++++++++++++----
 test/e2e-v2/cases/baseline/es/es-sharding/e2e.yaml | 47 +++++++++++++++++
 .../expected/critical.yml}                         | 59 +++-------------------
 .../cases/baseline/expected/metrics-has-value.yml  | 30 +++++++++++
 .../src/main/proto/baseline.proto                  |  2 +-
 .../ServiceMeshMetricSenderController.java         | 37 ++++++++++++++
 test/e2e-v2/script/docker-compose/base-compose.yml | 14 +++++
 test/e2e-v2/script/env                             |  1 +
 25 files changed, 424 insertions(+), 289 deletions(-)

diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml
index 4461c5e8d6..156fd8501b 100644
--- a/.github/workflows/skywalking.yaml
+++ b/.github/workflows/skywalking.yaml
@@ -404,6 +404,13 @@ jobs:
           - name: Alarm BanyanDB
             config: test/e2e-v2/cases/alarm/banyandb/e2e.yaml
 
+          - name: Baseline-driven Alarm ES
+            config: test/e2e-v2/cases/baseline/es/e2e.yaml
+          - name: Baseline-driven Alarm ES Sharding
+            config: test/e2e-v2/cases/baseline/es/es-sharding/e2e.yaml
+          - name: Baseline-driven Alarm BanyanDB
+            config: test/e2e-v2/cases/baseline/banyandb/e2e.yaml
+
           - name: TTL ES 7.16.3
             config: test/e2e-v2/cases/ttl/es/e2e.yaml
             env: ES_VERSION=7.16.3
diff --git a/oap-server/ai-pipeline/src/main/proto/baseline.proto b/oap-server/ai-pipeline/src/main/proto/baseline.proto
index dafbd8cfbb..41fe11f216 100644
--- a/oap-server/ai-pipeline/src/main/proto/baseline.proto
+++ b/oap-server/ai-pipeline/src/main/proto/baseline.proto
@@ -17,7 +17,7 @@
  */
 
 syntax = "proto3";
-package skywalking.baseline.v3;
+package skywalking.v3;
 
 import "google/protobuf/empty.proto";
 
diff --git a/test/e2e-v2/cases/alarm/alarm-settings.yml b/test/e2e-v2/cases/alarm/alarm-settings.yml
index de072b729f..04ddbc61cd 100755
--- a/test/e2e-v2/cases/alarm/alarm-settings.yml
+++ b/test/e2e-v2/cases/alarm/alarm-settings.yml
@@ -43,15 +43,6 @@ rules:
     tags:
       level: CRITICAL
       receivers: zhangsan
-  baseline_rule:
-    expression: sum(service_resp_time > baseline(service_resp_time,upper)) >= 1
-    period: 10
-    message: Response time of service {name} is more than baseline in 1 minutes of last 10 minutes.
-    tags:
-      level: CRITICAL
-      receivers: zhangsan
-    hooks:
-      - webhook.none
 
 hooks:
   webhook:
diff --git a/test/e2e-v2/cases/alarm/banyandb/docker-compose.yml b/test/e2e-v2/cases/alarm/banyandb/docker-compose.yml
index 9c42ee12e1..a08b02cad0 100644
--- a/test/e2e-v2/cases/alarm/banyandb/docker-compose.yml
+++ b/test/e2e-v2/cases/alarm/banyandb/docker-compose.yml
@@ -30,14 +30,11 @@ services:
     environment:
       SW_STORAGE: banyandb
       SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
-      SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
     ports:
       - 12800
     depends_on:
       banyandb:
         condition: service_healthy
-      baseline-server:
-        condition: service_healthy
     volumes:
       - ../alarm-settings.yml:/skywalking/config/alarm-settings.yml
 
@@ -51,20 +48,5 @@ services:
       oap:
         condition: service_healthy
 
-  baseline-server:
-    image: "eclipse-temurin:11-jre"
-    volumes:
-      - ../../../java-test-service/e2e-mock-baseline-server/target/e2e-mock-baseline-server-2.0.0.jar:/e2e-mock-baseline-server-2.0.0.jar
-    command: [ "java", "-jar", "/e2e-mock-baseline-server-2.0.0.jar" ]
-    networks:
-      - e2e
-    ports:
-      - 18080
-    healthcheck:
-      test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/18080"]
-      interval: 5s
-      timeout: 60s
-      retries: 120
-
 networks:
   e2e:
diff --git a/test/e2e-v2/cases/alarm/es/docker-compose.yml b/test/e2e-v2/cases/alarm/es/docker-compose.yml
index 6c066a40a5..4e5862e90d 100644
--- a/test/e2e-v2/cases/alarm/es/docker-compose.yml
+++ b/test/e2e-v2/cases/alarm/es/docker-compose.yml
@@ -37,14 +37,11 @@ services:
     environment:
       SW_STORAGE: elasticsearch
       SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
-      SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
     ports:
       - 12800
     depends_on:
       es:
         condition: service_healthy
-      baseline-server:
-        condition: service_healthy
     volumes:
       - ../alarm-settings.yml:/skywalking/config/alarm-settings.yml
 
@@ -58,20 +55,5 @@ services:
       oap:
         condition: service_healthy
 
-  baseline-server:
-    image: "eclipse-temurin:11-jre"
-    volumes:
-      - ../../../java-test-service/e2e-mock-baseline-server/target/e2e-mock-baseline-server-2.0.0.jar:/e2e-mock-baseline-server-2.0.0.jar
-    command: [ "java", "-jar", "/e2e-mock-baseline-server-2.0.0.jar" ]
-    networks:
-      - e2e
-    ports:
-      - 18080
-    healthcheck:
-      test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/18080"]
-      interval: 5s
-      timeout: 60s
-      retries: 120
-
 networks:
   e2e:
diff --git a/test/e2e-v2/cases/alarm/es/es-sharding/docker-compose.yml b/test/e2e-v2/cases/alarm/es/es-sharding/docker-compose.yml
index a8cef85fd9..c009c55292 100644
--- a/test/e2e-v2/cases/alarm/es/es-sharding/docker-compose.yml
+++ b/test/e2e-v2/cases/alarm/es/es-sharding/docker-compose.yml
@@ -38,14 +38,11 @@ services:
       SW_STORAGE: elasticsearch
       SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
       SW_STORAGE_ES_LOGIC_SHARDING: "true"
-      SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
     ports:
       - 12800
     depends_on:
       es:
         condition: service_healthy
-      baseline-server:
-        condition: service_healthy
     volumes:
       - ../../alarm-settings.yml:/skywalking/config/alarm-settings.yml
 
@@ -59,20 +56,5 @@ services:
       oap:
         condition: service_healthy
 
-  baseline-server:
-    image: "eclipse-temurin:11-jre"
-    volumes:
-      - ../../../../java-test-service/e2e-mock-baseline-server/target/e2e-mock-baseline-server-2.0.0.jar:/e2e-mock-baseline-server-2.0.0.jar
-    command: [ "java", "-jar", "/e2e-mock-baseline-server-2.0.0.jar" ]
-    networks:
-      - e2e
-    ports:
-      - 18080
-    healthcheck:
-      test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/18080"]
-      interval: 5s
-      timeout: 60s
-      retries: 120
-
 networks:
   e2e:
diff --git a/test/e2e-v2/cases/alarm/expected/metrics-has-value.yml b/test/e2e-v2/cases/alarm/expected/metrics-has-value.yml
new file mode 100644
index 0000000000..37fb746100
--- /dev/null
+++ b/test/e2e-v2/cases/alarm/expected/metrics-has-value.yml
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+debuggingtrace: null
+type: TIME_SERIES_VALUES
+results:
+  {{- contains .results }}
+  - metric:
+      labels: []
+    values:
+      {{- contains .values }}
+      - id: {{ notEmpty .id }}
+        value: {{ notEmpty .value }}
+        owner: null
+        traceid: null
+      {{- end}}
+  {{- end}}
+error: null
diff --git a/test/e2e-v2/cases/alarm/expected/silence-after-graphql-critical.yml b/test/e2e-v2/cases/alarm/expected/silence-after-graphql-critical.yml
index 6fb600f2fa..0631c162b8 100644
--- a/test/e2e-v2/cases/alarm/expected/silence-after-graphql-critical.yml
+++ b/test/e2e-v2/cases/alarm/expected/silence-after-graphql-critical.yml
@@ -58,60 +58,4 @@ msgs:
           {{- end }}
         {{- end }}
       {{- end }}
-  - starttime: {{ gt .starttime 0 }}
-    scope: Service
-    id: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
-    name: e2e-service-provider
-    message: Response time of service e2e-service-provider is more than baseline in 1 minutes of last 10 minutes.
-    tags:
-      - key: level
-        value: CRITICAL
-      - key: receivers
-        value: zhangsan
-    events:
-      {{- contains .events }}
-      - uuid: {{ notEmpty .uuid }}
-        source:
-          service: e2e-service-provider
-          serviceinstance: ""
-          endpoint: ""
-        name: Alarm
-        type: ""
-        message: {{ notEmpty .message }}
-        parameters: []
-        starttime: {{ gt .starttime 0 }}
-        endtime: {{ gt .endtime 0 }}
-        layer: GENERAL
-      {{- end }}
-    snapshot:
-      expression: sum(service_resp_time > baseline(service_resp_time,upper)) >= 1
-      metrics:
-      {{- contains .snapshot.metrics }}
-      - name: service_resp_time
-        results:
-        {{- contains .results }}
-        - metric:
-            labels: []
-          values:
-          {{- contains .values }}
-          - id: {{ notEmpty .id }}
-            owner: null
-            value: {{ .value }}
-            traceid: null
-          {{- end }}
-        {{- end }}
-      - name: baseline(service_resp_time,upper)
-        results:
-        {{- contains .results }}
-        - metric:
-            labels: []
-          values:
-          {{- contains .values }}
-          - id: {{ notEmpty .id }}
-            owner: null
-            value: 10
-            traceid: null
-          {{- end }}
-        {{- end }}
-      {{- end }}
   {{- end }}
diff --git a/test/e2e-v2/cases/alarm/expected/silence-before-graphql-critical.yml b/test/e2e-v2/cases/alarm/expected/silence-before-graphql-critical.yml
index 6fb600f2fa..0631c162b8 100644
--- a/test/e2e-v2/cases/alarm/expected/silence-before-graphql-critical.yml
+++ b/test/e2e-v2/cases/alarm/expected/silence-before-graphql-critical.yml
@@ -58,60 +58,4 @@ msgs:
           {{- end }}
         {{- end }}
       {{- end }}
-  - starttime: {{ gt .starttime 0 }}
-    scope: Service
-    id: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
-    name: e2e-service-provider
-    message: Response time of service e2e-service-provider is more than baseline in 1 minutes of last 10 minutes.
-    tags:
-      - key: level
-        value: CRITICAL
-      - key: receivers
-        value: zhangsan
-    events:
-      {{- contains .events }}
-      - uuid: {{ notEmpty .uuid }}
-        source:
-          service: e2e-service-provider
-          serviceinstance: ""
-          endpoint: ""
-        name: Alarm
-        type: ""
-        message: {{ notEmpty .message }}
-        parameters: []
-        starttime: {{ gt .starttime 0 }}
-        endtime: {{ gt .endtime 0 }}
-        layer: GENERAL
-      {{- end }}
-    snapshot:
-      expression: sum(service_resp_time > baseline(service_resp_time,upper)) >= 1
-      metrics:
-      {{- contains .snapshot.metrics }}
-      - name: service_resp_time
-        results:
-        {{- contains .results }}
-        - metric:
-            labels: []
-          values:
-          {{- contains .values }}
-          - id: {{ notEmpty .id }}
-            owner: null
-            value: {{ .value }}
-            traceid: null
-          {{- end }}
-        {{- end }}
-      - name: baseline(service_resp_time,upper)
-        results:
-        {{- contains .results }}
-        - metric:
-            labels: []
-          values:
-          {{- contains .values }}
-          - id: {{ notEmpty .id }}
-            owner: null
-            value: 10
-            traceid: null
-          {{- end }}
-        {{- end }}
-      {{- end }}
   {{- end }}
diff --git a/test/e2e-v2/cases/alarm/mysql/docker-compose.yml b/test/e2e-v2/cases/alarm/mysql/docker-compose.yml
index ce79c178f2..0de65f5e04 100644
--- a/test/e2e-v2/cases/alarm/mysql/docker-compose.yml
+++ b/test/e2e-v2/cases/alarm/mysql/docker-compose.yml
@@ -39,15 +39,12 @@ services:
     environment:
       SW_STORAGE: mysql
       SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
-      SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
     entrypoint: ['sh', '-c', '/download-mysql.sh /skywalking/oap-libs && /skywalking/docker-entrypoint.sh']
     ports:
       - 12800
     depends_on:
       mysql:
         condition: service_healthy
-      baseline-server:
-          condition: service_healthy
     volumes:
       - ../alarm-settings.yml:/skywalking/config/alarm-settings.yml
 
@@ -61,19 +58,5 @@ services:
       oap:
         condition: service_healthy
 
-  baseline-server:
-    image: "eclipse-temurin:11-jre"
-    volumes:
-      - ../../../java-test-service/e2e-mock-baseline-server/target/e2e-mock-baseline-server-2.0.0.jar:/e2e-mock-baseline-server-2.0.0.jar
-    command: [ "java", "-jar", "/e2e-mock-baseline-server-2.0.0.jar" ]
-    networks:
-      - e2e
-    ports:
-      - 18080
-    healthcheck:
-      test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/18080"]
-      interval: 5s
-      timeout: 60s
-      retries: 120
 networks:
   e2e:
diff --git a/test/e2e-v2/cases/alarm/postgres/docker-compose.yml b/test/e2e-v2/cases/alarm/postgres/docker-compose.yml
index 1dc4ae7775..11c5f30cc1 100644
--- a/test/e2e-v2/cases/alarm/postgres/docker-compose.yml
+++ b/test/e2e-v2/cases/alarm/postgres/docker-compose.yml
@@ -39,14 +39,11 @@ services:
       SW_STORAGE: postgresql
       SW_JDBC_URL: "jdbc:postgresql://postgres:5432/skywalking"
       SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
-      SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
     ports:
       - 12800
     depends_on:
       postgres:
         condition: service_healthy
-      baseline-server:
-        condition: service_healthy
     volumes:
       - ../alarm-settings.yml:/skywalking/config/alarm-settings.yml
 
@@ -60,20 +57,5 @@ services:
       oap:
         condition: service_healthy
 
-  baseline-server:
-    image: "eclipse-temurin:11-jre"
-    volumes:
-      - ../../../java-test-service/e2e-mock-baseline-server/target/e2e-mock-baseline-server-2.0.0.jar:/e2e-mock-baseline-server-2.0.0.jar
-    command: [ "java", "-jar", "/e2e-mock-baseline-server-2.0.0.jar" ]
-    networks:
-      - e2e
-    ports:
-      - 18080
-    healthcheck:
-      test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/18080"]
-      interval: 5s
-      timeout: 60s
-      retries: 120
-
 networks:
   e2e:
diff --git a/test/e2e-v2/cases/baseline/alarm-cases.yaml b/test/e2e-v2/cases/baseline/alarm-cases.yaml
new file mode 100644
index 0000000000..6785d37eec
--- /dev/null
+++ b/test/e2e-v2/cases/baseline/alarm-cases.yaml
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  cases:
+    - query: |
+        curl -s -XPOST http://${sender_host}:${sender_9093}/sendMetrics4Predict/7 > /dev/null;
+        sleep 10;
+        swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=service_resp_time --service-name=e2e-test-dest-service
+      expected: expected/metrics-has-value.yml
+    # before silence alarm list level=CRITICAL,receivers=zhangsan
+    - query: |
+        sleep 10;
+        curl -s -XPOST http://${sender_host}:${sender_9093}/sendMetrics4Predict/1 > /dev/null; # still trigger the service mesh traffic for keep it have traffic
+        sleep 10;
+        swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql alarm ls --tags level=CRITICAL,receivers=zhangsan
+      expected: expected/critical.yml
diff --git a/test/e2e-v2/cases/baseline/alarm-settings.yml b/test/e2e-v2/cases/baseline/alarm-settings.yml
new file mode 100755
index 0000000000..cd8a7ec89b
--- /dev/null
+++ b/test/e2e-v2/cases/baseline/alarm-settings.yml
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+rules:
+  baseline_rule:
+    expression: sum(service_resp_time > baseline(service_resp_time,lower)) >= 1
+    period: 10
+    message: Response time of service {name} is more than baseline in 1 minutes of last 10 minutes.
+    tags:
+      level: CRITICAL
+      receivers: zhangsan
diff --git a/test/e2e-v2/cases/alarm/banyandb/docker-compose.yml b/test/e2e-v2/cases/baseline/banyandb/docker-compose.yml
similarity index 64%
copy from test/e2e-v2/cases/alarm/banyandb/docker-compose.yml
copy to test/e2e-v2/cases/baseline/banyandb/docker-compose.yml
index 9c42ee12e1..719c046ecc 100644
--- a/test/e2e-v2/cases/alarm/banyandb/docker-compose.yml
+++ b/test/e2e-v2/cases/baseline/banyandb/docker-compose.yml
@@ -30,14 +30,12 @@ services:
     environment:
       SW_STORAGE: banyandb
       SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
-      SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
+      SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-predictor
     ports:
       - 12800
     depends_on:
       banyandb:
         condition: service_healthy
-      baseline-server:
-        condition: service_healthy
     volumes:
       - ../alarm-settings.yml:/skywalking/config/alarm-settings.yml
 
@@ -51,20 +49,48 @@ services:
       oap:
         condition: service_healthy
 
-  baseline-server:
-    image: "eclipse-temurin:11-jre"
-    volumes:
-      - ../../../java-test-service/e2e-mock-baseline-server/target/e2e-mock-baseline-server-2.0.0.jar:/e2e-mock-baseline-server-2.0.0.jar
-    command: [ "java", "-jar", "/e2e-mock-baseline-server-2.0.0.jar" ]
+  baseline-predictor:
+    extends:
+      file: ../../../script/docker-compose/base-compose.yml
+      service: baseline-predictor
     networks:
       - e2e
     ports:
       - 18080
+    environment:
+      BASELINE_FETCH_METRICS: service_resp_time,service_percentile
+      BASELINE_FETCH_CRON: "* * * * *"
+      BASELINE_FETCH_SERVER_ENDPOINT: http://oap:12800
+      BASELINE_FETCH_SERVER_LAYERS: MESH
     healthcheck:
       test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/18080"]
       interval: 5s
       timeout: 60s
       retries: 120
+    depends_on:
+      oap:
+        condition: service_healthy
+
+  sender:
+    image: "eclipse-temurin:8-jre"
+    volumes:
+      - ./../../../java-test-service/e2e-mock-sender/target/e2e-mock-sender-2.0.0.jar:/e2e-mock-sender-2.0.0.jar
+    command: [ "java", "-jar", "/e2e-mock-sender-2.0.0.jar" ]
+    environment:
+      OAP_HOST: oap
+      OAP_GRPC_PORT: 11800
+    networks:
+      - e2e
+    ports:
+      - 9093
+    healthcheck:
+      test: [ "CMD", "sh", "-c", "nc -nz 127.0.0.1 9093" ]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    depends_on:
+      oap:
+        condition: service_healthy
 
 networks:
   e2e:
diff --git a/test/e2e-v2/cases/baseline/banyandb/e2e.yaml b/test/e2e-v2/cases/baseline/banyandb/e2e.yaml
new file mode 100644
index 0000000000..4dcb90bf0f
--- /dev/null
+++ b/test/e2e-v2/cases/baseline/banyandb/e2e.yaml
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is used to show how to write configuration files and can be used to test.
+
+setup:
+  env: compose
+  file: docker-compose.yml
+  timeout: 20m
+  init-system-environment: ../../../script/env
+  steps:
+    - name: set PATH
+      command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH
+    - name: install yq
+      command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq
+    - name: install swctl
+      command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl
+
+trigger:
+  action: http
+  interval: 3s
+  times: 30
+  url: http://${provider_host}:${provider_9090}/users
+  method: POST
+  body: '{"id":"123","name":"skywalking"}'
+  headers:
+    "Content-Type": "application/json"
+
+verify:
+  retry:
+    count: 20
+    interval: 3s
+  cases:
+    - includes:
+        - ../alarm-cases.yaml
diff --git a/test/e2e-v2/cases/alarm/es/docker-compose.yml b/test/e2e-v2/cases/baseline/es/docker-compose.yml
similarity index 63%
copy from test/e2e-v2/cases/alarm/es/docker-compose.yml
copy to test/e2e-v2/cases/baseline/es/docker-compose.yml
index 6c066a40a5..444c43e2c2 100644
--- a/test/e2e-v2/cases/alarm/es/docker-compose.yml
+++ b/test/e2e-v2/cases/baseline/es/docker-compose.yml
@@ -37,14 +37,12 @@ services:
     environment:
       SW_STORAGE: elasticsearch
       SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
-      SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
+      SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-predictor
     ports:
       - 12800
     depends_on:
       es:
         condition: service_healthy
-      baseline-server:
-        condition: service_healthy
     volumes:
       - ../alarm-settings.yml:/skywalking/config/alarm-settings.yml
 
@@ -58,20 +56,47 @@ services:
       oap:
         condition: service_healthy
 
-  baseline-server:
-    image: "eclipse-temurin:11-jre"
-    volumes:
-      - ../../../java-test-service/e2e-mock-baseline-server/target/e2e-mock-baseline-server-2.0.0.jar:/e2e-mock-baseline-server-2.0.0.jar
-    command: [ "java", "-jar", "/e2e-mock-baseline-server-2.0.0.jar" ]
+  baseline-predictor:
+    extends:
+      file: ../../../script/docker-compose/base-compose.yml
+      service: baseline-predictor
     networks:
       - e2e
     ports:
       - 18080
+    environment:
+      BASELINE_FETCH_METRICS: service_resp_time,service_percentile
+      BASELINE_FETCH_CRON: "* * * * *"
+      BASELINE_FETCH_SERVER_ENDPOINT: http://oap:12800
+      BASELINE_FETCH_SERVER_LAYERS: MESH
     healthcheck:
-      test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/18080"]
+      test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/18080" ]
       interval: 5s
       timeout: 60s
       retries: 120
+    depends_on:
+      oap:
+        condition: service_healthy
 
+  sender:
+    image: "eclipse-temurin:8-jre"
+    volumes:
+      - ./../../../java-test-service/e2e-mock-sender/target/e2e-mock-sender-2.0.0.jar:/e2e-mock-sender-2.0.0.jar
+    command: [ "java", "-jar", "/e2e-mock-sender-2.0.0.jar" ]
+    environment:
+      OAP_HOST: oap
+      OAP_GRPC_PORT: 11800
+    networks:
+      - e2e
+    ports:
+      - 9093
+    healthcheck:
+      test: [ "CMD", "sh", "-c", "nc -nz 127.0.0.1 9093" ]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    depends_on:
+      oap:
+        condition: service_healthy
 networks:
   e2e:
diff --git a/test/e2e-v2/cases/baseline/es/e2e.yaml b/test/e2e-v2/cases/baseline/es/e2e.yaml
new file mode 100644
index 0000000000..4dcb90bf0f
--- /dev/null
+++ b/test/e2e-v2/cases/baseline/es/e2e.yaml
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is used to show how to write configuration files and can be used to test.
+
+setup:
+  env: compose
+  file: docker-compose.yml
+  timeout: 20m
+  init-system-environment: ../../../script/env
+  steps:
+    - name: set PATH
+      command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH
+    - name: install yq
+      command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq
+    - name: install swctl
+      command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl
+
+trigger:
+  action: http
+  interval: 3s
+  times: 30
+  url: http://${provider_host}:${provider_9090}/users
+  method: POST
+  body: '{"id":"123","name":"skywalking"}'
+  headers:
+    "Content-Type": "application/json"
+
+verify:
+  retry:
+    count: 20
+    interval: 3s
+  cases:
+    - includes:
+        - ../alarm-cases.yaml
diff --git a/test/e2e-v2/cases/alarm/es/es-sharding/docker-compose.yml b/test/e2e-v2/cases/baseline/es/es-sharding/docker-compose.yml
similarity index 64%
copy from test/e2e-v2/cases/alarm/es/es-sharding/docker-compose.yml
copy to test/e2e-v2/cases/baseline/es/es-sharding/docker-compose.yml
index a8cef85fd9..553bd81479 100644
--- a/test/e2e-v2/cases/alarm/es/es-sharding/docker-compose.yml
+++ b/test/e2e-v2/cases/baseline/es/es-sharding/docker-compose.yml
@@ -38,14 +38,12 @@ services:
       SW_STORAGE: elasticsearch
       SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
       SW_STORAGE_ES_LOGIC_SHARDING: "true"
-      SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
+      SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-predictor
     ports:
       - 12800
     depends_on:
       es:
         condition: service_healthy
-      baseline-server:
-        condition: service_healthy
     volumes:
       - ../../alarm-settings.yml:/skywalking/config/alarm-settings.yml
 
@@ -59,20 +57,48 @@ services:
       oap:
         condition: service_healthy
 
-  baseline-server:
-    image: "eclipse-temurin:11-jre"
-    volumes:
-      - ../../../../java-test-service/e2e-mock-baseline-server/target/e2e-mock-baseline-server-2.0.0.jar:/e2e-mock-baseline-server-2.0.0.jar
-    command: [ "java", "-jar", "/e2e-mock-baseline-server-2.0.0.jar" ]
+  baseline-predictor:
+    extends:
+      file: ../../../../script/docker-compose/base-compose.yml
+      service: baseline-predictor
     networks:
       - e2e
     ports:
       - 18080
+    environment:
+      BASELINE_FETCH_METRICS: service_resp_time,service_percentile
+      BASELINE_FETCH_CRON: "* * * * *"
+      BASELINE_FETCH_SERVER_ENDPOINT: http://oap:12800
+      BASELINE_FETCH_SERVER_LAYERS: MESH
     healthcheck:
-      test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/18080"]
+      test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/18080" ]
       interval: 5s
       timeout: 60s
       retries: 120
+    depends_on:
+      oap:
+        condition: service_healthy
+
+  sender:
+    image: "eclipse-temurin:8-jre"
+    volumes:
+      - ./../../../../java-test-service/e2e-mock-sender/target/e2e-mock-sender-2.0.0.jar:/e2e-mock-sender-2.0.0.jar
+    command: [ "java", "-jar", "/e2e-mock-sender-2.0.0.jar" ]
+    environment:
+      OAP_HOST: oap
+      OAP_GRPC_PORT: 11800
+    networks:
+      - e2e
+    ports:
+      - 9093
+    healthcheck:
+      test: [ "CMD", "sh", "-c", "nc -nz 127.0.0.1 9093" ]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    depends_on:
+      oap:
+        condition: service_healthy
 
 networks:
   e2e:
diff --git a/test/e2e-v2/cases/baseline/es/es-sharding/e2e.yaml b/test/e2e-v2/cases/baseline/es/es-sharding/e2e.yaml
new file mode 100644
index 0000000000..a1fa0e36b9
--- /dev/null
+++ b/test/e2e-v2/cases/baseline/es/es-sharding/e2e.yaml
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is used to show how to write configuration files and can be used to test.
+
+setup:
+  env: compose
+  file: docker-compose.yml
+  timeout: 20m
+  init-system-environment: ../../../../script/env
+  steps:
+    - name: set PATH
+      command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH
+    - name: install yq
+      command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq
+    - name: install swctl
+      command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl
+
+trigger:
+  action: http
+  interval: 3s
+  times: 30
+  url: http://${provider_host}:${provider_9090}/users
+  method: POST
+  body: '{"id":"123","name":"skywalking"}'
+  headers:
+    "Content-Type": "application/json"
+
+verify:
+  retry:
+    count: 20
+    interval: 3s
+  cases:
+    - includes:
+        - ../../alarm-cases.yaml
diff --git a/test/e2e-v2/cases/alarm/expected/silence-after-graphql-critical.yml b/test/e2e-v2/cases/baseline/expected/critical.yml
similarity index 57%
copy from test/e2e-v2/cases/alarm/expected/silence-after-graphql-critical.yml
copy to test/e2e-v2/cases/baseline/expected/critical.yml
index 6fb600f2fa..77d5de6a4a 100644
--- a/test/e2e-v2/cases/alarm/expected/silence-after-graphql-critical.yml
+++ b/test/e2e-v2/cases/baseline/expected/critical.yml
@@ -17,52 +17,9 @@ msgs:
   {{- contains .msgs }}
   - starttime: {{ gt .starttime 0 }}
     scope: Service
-    id: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
-    name: e2e-service-provider
-    message: Service e2e-service-provider response time is more than 10ms and sla is more than 1%.
-    tags:
-      - key: level
-        value: CRITICAL
-      - key: receivers
-        value: zhangsan
-    events:
-      {{- contains .events }}
-      - uuid: {{ notEmpty .uuid}}
-        source:
-          service: e2e-service-provider
-          serviceinstance: ""
-          endpoint: ""
-        name: Alarm
-        type: ""
-        message: {{ notEmpty .message}}
-        parameters: []
-        starttime: {{ gt .starttime 0 }}
-        endtime: {{ gt .endtime 0 }}
-        layer: GENERAL
-      {{- end }}
-    snapshot:
-      expression: sum((service_resp_time > 10) && (service_sla > 100)) >= 1
-      metrics:
-      {{- contains .snapshot.metrics }}
-      - name: service_resp_time
-        results:
-        {{- contains .results }}
-        - metric:
-            labels: []
-          values:
-          {{- contains .values }}
-          - id: {{ notEmpty .id }}
-            owner: null
-            value: {{ .value }}
-            traceid: null
-          {{- end }}
-        {{- end }}
-      {{- end }}
-  - starttime: {{ gt .starttime 0 }}
-    scope: Service
-    id: ZTJlLXNlcnZpY2UtcHJvdmlkZXI=.1
-    name: e2e-service-provider
-    message: Response time of service e2e-service-provider is more than baseline in 1 minutes of last 10 minutes.
+    id: ZTJlLXRlc3QtZGVzdC1zZXJ2aWNl.1
+    name: e2e-test-dest-service
+    message: Response time of service e2e-test-dest-service is more than baseline in 1 minutes of last 10 minutes.
     tags:
       - key: level
         value: CRITICAL
@@ -72,7 +29,7 @@ msgs:
       {{- contains .events }}
       - uuid: {{ notEmpty .uuid }}
         source:
-          service: e2e-service-provider
+          service: e2e-test-dest-service
           serviceinstance: ""
           endpoint: ""
         name: Alarm
@@ -81,10 +38,10 @@ msgs:
         parameters: []
         starttime: {{ gt .starttime 0 }}
         endtime: {{ gt .endtime 0 }}
-        layer: GENERAL
+        layer: MESH
       {{- end }}
     snapshot:
-      expression: sum(service_resp_time > baseline(service_resp_time,upper)) >= 1
+      expression: sum(service_resp_time > baseline(service_resp_time,lower)) >= 1
       metrics:
       {{- contains .snapshot.metrics }}
       - name: service_resp_time
@@ -100,7 +57,7 @@ msgs:
             traceid: null
           {{- end }}
         {{- end }}
-      - name: baseline(service_resp_time,upper)
+      - name: baseline(service_resp_time,lower)
         results:
         {{- contains .results }}
         - metric:
@@ -109,7 +66,7 @@ msgs:
           {{- contains .values }}
           - id: {{ notEmpty .id }}
             owner: null
-            value: 10
+            value: "1999"
             traceid: null
           {{- end }}
         {{- end }}
diff --git a/test/e2e-v2/cases/baseline/expected/metrics-has-value.yml b/test/e2e-v2/cases/baseline/expected/metrics-has-value.yml
new file mode 100644
index 0000000000..37fb746100
--- /dev/null
+++ b/test/e2e-v2/cases/baseline/expected/metrics-has-value.yml
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+debuggingtrace: null
+type: TIME_SERIES_VALUES
+results:
+  {{- contains .results }}
+  - metric:
+      labels: []
+    values:
+      {{- contains .values }}
+      - id: {{ notEmpty .id }}
+        value: {{ notEmpty .value }}
+        owner: null
+        traceid: null
+      {{- end}}
+  {{- end}}
+error: null
diff --git a/test/e2e-v2/java-test-service/e2e-mock-baseline-server/src/main/proto/baseline.proto b/test/e2e-v2/java-test-service/e2e-mock-baseline-server/src/main/proto/baseline.proto
index dafbd8cfbb..41fe11f216 100644
--- a/test/e2e-v2/java-test-service/e2e-mock-baseline-server/src/main/proto/baseline.proto
+++ b/test/e2e-v2/java-test-service/e2e-mock-baseline-server/src/main/proto/baseline.proto
@@ -17,7 +17,7 @@
  */
 
 syntax = "proto3";
-package skywalking.baseline.v3;
+package skywalking.v3;
 
 import "google/protobuf/empty.proto";
 
diff --git a/test/e2e-v2/java-test-service/e2e-mock-sender/src/main/java/org/apache/skywalking/e2e/controller/ServiceMeshMetricSenderController.java b/test/e2e-v2/java-test-service/e2e-mock-sender/src/main/java/org/apache/skywalking/e2e/controller/ServiceMeshMetricSenderController.java
index 1b094c40c8..4c8a196370 100644
--- a/test/e2e-v2/java-test-service/e2e-mock-sender/src/main/java/org/apache/skywalking/e2e/controller/ServiceMeshMetricSenderController.java
+++ b/test/e2e-v2/java-test-service/e2e-mock-sender/src/main/java/org/apache/skywalking/e2e/controller/ServiceMeshMetricSenderController.java
@@ -88,6 +88,43 @@ public class ServiceMeshMetricSenderController {
         return "Metrics send success!";
     }
 
+    @PostMapping("/sendMetrics4Predict/{days}")
+    public String sendMetrics4Predict(@PathVariable("days") int days) throws Exception {
+        final HTTPServiceMeshMetric.Builder builder =
+            HTTPServiceMeshMetric
+                .newBuilder()
+                .setSourceServiceName("e2e-test-source-service")
+                .setSourceServiceInstance("e2e-test-source-service-instance")
+                .setDestServiceName("e2e-test-dest-service")
+                .setDestServiceInstance("e2e-test-dest-service-instance")
+                .setEndpoint("e2e/test")
+                .setLatency(2000)
+                .setResponseCode(200)
+                .setStatus(SUCCESS)
+                .setProtocol(Protocol.HTTP)
+                .setDetectPoint(DetectPoint.server);
+
+        final LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC);
+        LocalDateTime startTime = now.minusDays(days + 1);
+
+        while (!startTime.isAfter(now)) {
+            startTime = startTime.plusMinutes(1);
+            final LocalDateTime endTime = startTime.plusMinutes(1);
+            sendMetrics(ServiceMeshMetrics
+                .newBuilder()
+                .setHttpMetrics(
+                    HTTPServiceMeshMetrics
+                        .newBuilder()
+                        .addMetrics(
+                            builder
+                                .setStartTime(startTime.toEpochSecond(ZoneOffset.UTC) * 1000)
+                                .setEndTime(endTime.toEpochSecond(ZoneOffset.UTC) * 1000)))
+                .build());
+        }
+
+        return "Metrics send success!";
+    }
+
     void sendMetrics(final ServiceMeshMetrics metrics) throws InterruptedException {
         final CountDownLatch latch = new CountDownLatch(1);
 
diff --git a/test/e2e-v2/script/docker-compose/base-compose.yml b/test/e2e-v2/script/docker-compose/base-compose.yml
index 92598c2807..4a30f442ba 100644
--- a/test/e2e-v2/script/docker-compose/base-compose.yml
+++ b/test/e2e-v2/script/docker-compose/base-compose.yml
@@ -114,5 +114,19 @@ services:
       timeout: 60s
       retries: 120
 
+  baseline-predictor:
+    image: "ghcr.io/skyapm/skypredictor:${SW_PREDICTOR_COMMIT}"
+    networks:
+      - e2e
+    expose:
+      - 18080
+    environment:
+      BASELINE_FETCH_SERVER_ENDPOINT: http://oap:12800
+    healthcheck:
+      test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/18080" ]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+
 networks:
   e2e:
diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env
index 81d693d399..8d0ce74a0f 100644
--- a/test/e2e-v2/script/env
+++ b/test/e2e-v2/script/env
@@ -25,5 +25,6 @@ SW_KUBERNETES_COMMIT_SHA=6fe5e6f0d3b7686c6be0457733e825ee68cb9b35
 SW_ROVER_COMMIT=40d03c14d638339bf4aa6ec7376d421323de865a
 SW_BANYANDB_COMMIT=f484391b33674fd5b6c6c1903056803818f5393a
 SW_AGENT_PHP_COMMIT=3192c553002707d344bd6774cfab5bc61f67a1d3
+SW_PREDICTOR_COMMIT=9d6da24001a63f435d8a702b122499788f2b318b
 
 SW_CTL_COMMIT=67cbc89dd7b214d5791321a7ca992f940cb586ba