You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2020/10/16 00:45:24 UTC
[skywalking] branch master updated: test: add a tool to select a
suitable group for a new plugin test and remove the outdated group size of
plugin tests (#5676)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new d113f58 test: add a tool to select a suitable group for a new plugin test and remove the outdated group size of plugin tests (#5676)
d113f58 is described below
commit d113f58645f7f02508da75f648e829dbd83e7d82
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Fri Oct 16 08:44:55 2020 +0800
test: add a tool to select a suitable group for a new plugin test and remove the outdated group size of plugin tests (#5676)
---
.github/workflows/plugins-test.0.yaml | 56 ++++++++++++++++----------------
.github/workflows/plugins-test.1.yaml | 38 +++++++++++-----------
.github/workflows/plugins-test.2.yaml | 50 ++++++++++++++---------------
.github/workflows/plugins-test.3.yaml | 56 ++++++++++++++++----------------
docs/en/guides/Plugin-test.md | 7 ++--
tools/select-group.py | 60 +++++++++++++++++++++++++++++++++++
6 files changed, 164 insertions(+), 103 deletions(-)
diff --git a/.github/workflows/plugins-test.0.yaml b/.github/workflows/plugins-test.0.yaml
index 25d07a9..ef14557 100644
--- a/.github/workflows/plugins-test.0.yaml
+++ b/.github/workflows/plugins-test.0.yaml
@@ -28,32 +28,32 @@ jobs:
fail-fast: true
matrix:
case:
- - { name: 'activemq-scenario', title: 'ActiveMQ 5.10.0-5.15.4 (22)' }
- - { name: 'apm-toolkit-trace-scenario', title: 'apm-toolkit-trace (1)' }
- - { name: 'armeria-0.96minus-scenario', title: 'Armeria 0.63.0-0.95.0 (20)' }
- - { name: 'armeria-0.96plus-scenario', title: 'Armeria 0.96.0-0.97.0 (2)' }
- - { name: 'avro-scenario', title: 'Avro 1.7.0-1.8.2 (10)' }
- - { name: 'canal-scenario', title: 'Canal 1.0.24-1.1.2 (5)' }
- - { name: 'cassandra-java-driver-3.x-scenario', title: 'Cassandra 3.7.0-3.7.2 (3)' }
- - { name: 'customize-scenario', title: 'customize (1)' }
- - { name: 'dubbo-2.5.x-scenario', title: 'Dubbo 2.5.x-2.6.x (10)' }
- - { name: 'dubbo-2.7.x-scenario', title: 'Dubbo 2.7.x (4)' }
- - { name: 'ehcache-2.x-scenario', title: 'Ehcache 2.8.x-2.10.x (19)' }
- - { name: 'elasticsearch-5.x-scenario', title: 'ElasticSearch 5.x (3)' }
- - { name: 'elasticsearch-6.x-scenario', title: 'ElasticSearch 6.7.1-6.8.4 (7)' }
- - { name: 'elasticsearch-7.x-scenario', title: 'ElasticSearch 7.0.0-7.5.2 (14)' }
- - { name: 'exception-checker-spring-scenario', title: 'SpringBoot exception checker 1.0.0 (1)' }
- - { name: 'exception-checker-tomcat-scenario', title: 'Tomcat exception checker 1.0.0 (1)' }
- - { name: 'feign-scenario', title: 'Feign 9.0.0-9.5.1 (8)' }
- - { name: 'finagle-17.10.x-scenario', title: 'Finagle 17.10.0-20.1.0(19)' }
- - { name: 'finagle-6.25.x-scenario', title: 'Finagle 6.25.0-6.43.0 (18)' }
- - { name: 'finagle-6.44.x-scenario', title: 'Finagle 6.44.0-7.1.0 (4)' }
- - { name: 'gateway-2.1.x-scenario', title: 'Spring-Cloud-Gateway 2.1.x (7)' }
- - { name: 'gateway-2.0.x-scenario', title: 'Spring-Cloud-Gateway 2.0.x (3)' }
- - { name: 'grpc-scenario', title: 'gRPC 1.6.0-1.25.0 (22)' }
- - { name: 'gson-scenario', title: 'Gson (7)' }
- - { name: 'elasticjob-3.x-scenario', title: 'elasticjob-3.x-scenario (1)' }
- - { name: 'springmvc-reactive-scenario', title: 'springmvc-reactive-scenario (7)' }
+ - activemq-scenario
+ - apm-toolkit-trace-scenario
+ - armeria-0.96minus-scenario
+ - armeria-0.96plus-scenario
+ - avro-scenario
+ - canal-scenario
+ - cassandra-java-driver-3.x-scenario
+ - customize-scenario
+ - dubbo-2.5.x-scenario
+ - dubbo-2.7.x-scenario
+ - ehcache-2.x-scenario
+ - elasticsearch-5.x-scenario
+ - elasticsearch-6.x-scenario
+ - elasticsearch-7.x-scenario
+ - exception-checker-spring-scenario
+ - exception-checker-tomcat-scenario
+ - feign-scenario
+ - finagle-17.10.x-scenario
+ - finagle-6.25.x-scenario
+ - finagle-6.44.x-scenario
+ - gateway-2.1.x-scenario
+ - gateway-2.0.x-scenario
+ - grpc-scenario
+ - gson-scenario
+ - elasticjob-3.x-scenario
+ - springmvc-reactive-scenario
steps:
- uses: actions/checkout@v2
with:
@@ -72,7 +72,7 @@ jobs:
run: ./mvnw --batch-mode clean package -DskipTests -Pagent
- name: Build Docker image
run: ./mvnw --batch-mode -f test/plugin/pom.xml clean package -DskipTests
- - name: ${{ matrix.case.title }}
- run: bash test/plugin/run.sh ${{ matrix.case.name }}
+ - name: ${{ matrix.case }}
+ run: bash test/plugin/run.sh ${{ matrix.case }}
- name: Report Coverage
run: bash -x tools/coverage/report.sh
diff --git a/.github/workflows/plugins-test.1.yaml b/.github/workflows/plugins-test.1.yaml
index e6989af..f525360 100644
--- a/.github/workflows/plugins-test.1.yaml
+++ b/.github/workflows/plugins-test.1.yaml
@@ -28,23 +28,23 @@ jobs:
fail-fast: true
matrix:
case:
- - { name: 'h2-scenario', title: 'H2 (24)' }
- - { name: 'httpasyncclient-scenario', title: 'HttpAsyncClient 4.0-4.1.3 (7)' }
- - { name: 'httpclient-3.x-scenario', title: 'HttpClient 2.0-3.1 (5)' }
- - { name: 'httpclient-4.3.x-scenario', title: 'HttpClient 4.3.x-4.5.x (14)' }
- - { name: 'hystrix-scenario', title: 'Hystrix 1.4.20-1.5.12 (20)' }
- - { name: 'influxdb-scenario', title: 'InfluxDB Java 2.5-2.17 (12)' }
- - { name: 'jdk-http-scenario', title: 'JDK http (1)' }
- - { name: 'jdk-threading-scenario', title: 'JDK Threading (1)' }
- - { name: 'jedis-scenario', title: 'Jedis 2.4.0-2.9.0 (18)' }
- - { name: 'jetty-scenario', title: 'Jetty 9.x (63)' }
- - { name: 'kafka-scenario', title: 'Kafka 0.11.0.0-2.3.0 (16)' }
- - { name: 'kotlin-coroutine-scenario', title: 'Kotlin Coroutine 1.0.1-1.3.3 (4)' }
- - { name: 'lettuce-scenario', title: 'Lettuce 5.x (17)' }
- - { name: 'mongodb-3.x-scenario', title: 'Mongodb 3.4.0-3.12.7 (31)' }
- - { name: 'mongodb-4.x-scenario', title: 'Mongodb 4.0.0-4.1.0 (7)' }
- - { name: 'netty-socketio-scenario', title: 'Netty-SocketIO 1.x (4)' }
- - { name: 'postgresql-above9.4.1207-scenario', title: 'PostgreSQL 9.4.1207+ (62)' }
+ - h2-scenario
+ - httpasyncclient-scenario
+ - httpclient-3.x-scenario
+ - httpclient-4.3.x-scenario
+ - hystrix-scenario
+ - influxdb-scenario
+ - jdk-http-scenario
+ - jdk-threading-scenario
+ - jedis-scenario
+ - jetty-scenario
+ - kafka-scenario
+ - kotlin-coroutine-scenario
+ - lettuce-scenario
+ - mongodb-3.x-scenario
+ - mongodb-4.x-scenario
+ - netty-socketio-scenario
+ - postgresql-above9.4.1207-scenario
steps:
- uses: actions/checkout@v2
with:
@@ -63,7 +63,7 @@ jobs:
run: ./mvnw --batch-mode clean package -DskipTests -Pagent
- name: Build Docker image
run: ./mvnw --batch-mode -f test/plugin/pom.xml clean package -DskipTests
- - name: ${{ matrix.case.title }}
- run: bash test/plugin/run.sh ${{ matrix.case.name }}
+ - name: ${{ matrix.case }}
+ run: bash test/plugin/run.sh ${{ matrix.case }}
- name: Report Coverage
run: bash -x tools/coverage/report.sh
diff --git a/.github/workflows/plugins-test.2.yaml b/.github/workflows/plugins-test.2.yaml
index 019782c..35fde5d 100644
--- a/.github/workflows/plugins-test.2.yaml
+++ b/.github/workflows/plugins-test.2.yaml
@@ -28,29 +28,29 @@ jobs:
fail-fast: true
matrix:
case:
- - { name: 'okhttp-scenario', title: 'OKHttp 3.0.x-3.14.x (34)' }
- - { name: 'play-scenario', title: 'Play! Framework (12)' }
- - { name: 'postgresql-scenario', title: 'PostgreSQL 9.2.x-9.4.x (36)' }
- - { name: 'pulsar-scenario', title: 'Pulsar 2.2.0-2.4.1 (7)' }
- - { name: 'rabbitmq-scenario', title: 'RabbitMQ (12)' }
- - { name: 'redisson-scenario', title: 'Redisson 3.x (37)' }
- - { name: 'resttemplate-4.x-scenario', title: 'RestTemplate 4.0.0.RELEASE-4.3.26.RELEASE (57)' }
- - { name: 'servicecomb-0.x-scenario', title: 'ServiceComb 0.x (5)' }
- - { name: 'servicecomb-1.x-scenario', title: 'ServiceComb 1.x (5)' }
- - { name: 'shardingsphere-3.x-scenario', title: 'ShardingSphere 3.0.0 (1)' }
- - { name: 'shardingsphere-4.x-RC1-RC2-scenario', title: 'ShardingSphere 4.0.0-RC1-4.0.0-RC2 (2)' }
- - { name: 'shardingsphere-4.x-RC3-scenario', title: 'ShardingSphere 4.0.0-RC3 (1)' }
- - { name: 'shardingsphere-4.0.x-scenario', title: 'ShardingSphere 4.0.0-4.0.1 (2)' }
- - { name: 'shardingsphere-4.x-scenario', title: 'ShardingSphere 4.1.0-4.1.1 (2)' }
- - { name: 'sofarpc-scenario', title: 'SofaRPC 5.4.0-5.6.2 (23)' }
- - { name: 'solrj-7.x-scenario', title: 'SolrJ 7.x (12)' }
- - { name: 'spring-3.0.x-scenario', title: 'Spring 3.0.x (8)' }
- - { name: 'spring-cloud-feign-1.1.x-scenario', title: 'SpringCloud Feign 1.1.x (8)' }
- - { name: 'spring-cloud-feign-1.2.x-scenario', title: 'SpringCloud Feign 1.2.x-1.4.x (23)' }
- - { name: 'spring-cloud-feign-2.x-scenario', title: 'SpringCloud Feign 2.x (14)' }
- - { name: 'spring-tx-scenario', title: 'Spring-tx 4.x+ (10)' }
- - { name: 'struts2.3-scenario', title: 'Struts2.3 (35)' }
- - { name: 'struts2.5-scenario', title: 'Struts2.5 (14)' }
+ - okhttp-scenario
+ - play-scenario
+ - postgresql-scenario
+ - pulsar-scenario
+ - rabbitmq-scenario
+ - redisson-scenario
+ - resttemplate-4.x-scenario
+ - servicecomb-0.x-scenario
+ - servicecomb-1.x-scenario
+ - shardingsphere-3.x-scenario
+ - shardingsphere-4.x-RC1-RC2-scenario
+ - shardingsphere-4.x-RC3-scenario
+ - shardingsphere-4.0.x-scenario
+ - shardingsphere-4.x-scenario
+ - sofarpc-scenario
+ - solrj-7.x-scenario
+ - spring-3.0.x-scenario
+ - spring-cloud-feign-1.1.x-scenario
+ - spring-cloud-feign-1.2.x-scenario
+ - spring-cloud-feign-2.x-scenario
+ - spring-tx-scenario
+ - struts2.3-scenario
+ - struts2.5-scenario
steps:
- uses: actions/checkout@v2
with:
@@ -69,7 +69,7 @@ jobs:
run: ./mvnw --batch-mode clean package -DskipTests -Pagent
- name: Build Docker image
run: ./mvnw --batch-mode -f test/plugin/pom.xml clean package -DskipTests
- - name: ${{ matrix.case.title }}
- run: bash test/plugin/run.sh ${{ matrix.case.name }}
+ - name: ${{ matrix.case }}
+ run: bash test/plugin/run.sh ${{ matrix.case }}
- name: Report Coverage
run: bash -x tools/coverage/report.sh
diff --git a/.github/workflows/plugins-test.3.yaml b/.github/workflows/plugins-test.3.yaml
index baa1a0b..d4c45f9 100644
--- a/.github/workflows/plugins-test.3.yaml
+++ b/.github/workflows/plugins-test.3.yaml
@@ -28,32 +28,32 @@ jobs:
fail-fast: true
matrix:
case:
- - { name: 'mysql-scenario', title: 'MySQL 5.1.2-8.0.15 (30)' }
- - { name: 'undertow-scenario', title: 'Undertow 1.3.0-2.0.27 (23)' }
- - { name: 'webflux-scenario', title: 'Spring-WebFlux 2.x (7)' }
- - { name: 'zookeeper-scenario', title: 'Zookeeper 3.4.x (14)' }
- - { name: 'spring-3.1.x-scenario', title: 'Spring 3.1.x-4.0.x (25)' }
- - { name: 'spring-4.1.x-scenario', title: 'Spring 4.1.x-4.2.x (20)' }
- - { name: 'spring-4.3.x-scenario', title: 'Spring 4.3.x-5.2.x (54)' }
- - { name: 'spring-async-scenario', title: 'Spring Async 4.3.x-5.1.x (35)' }
- - { name: 'vertx-eventbus-3.x-scenario', title: 'Vert.x EventBus 3.2.0-3.9.1 (28)' }
- - { name: 'vertx-web-3.54minus-scenario', title: 'Vert.x Web 3.0.0-3.5.4 (16)' }
- - { name: 'vertx-web-3.6plus-scenario', title: 'Vert.x Web 3.6.0-3.9.1 (14)' }
- - { name: 'mariadb-scenario', title: 'Mariadb 2.x (8)' }
- - { name: 'quasar-scenario', title: 'quasar 0.7.x (5)' }
- - { name: 'baidu-brpc-scenario', title: 'baidu-brpc 2.3.7-2.5.3 (12)' }
- - { name: 'retransform-class-scenario', title: 'Retransform class (1)' }
- - { name: 'retransform-class-tomcat-scenario', title: 'Retransform class (1)' }
- - { name: 'graphql-8.x-scenario', title: 'graphql-8.x 8.0 (1)' }
- - { name: 'graphql-9.x-scenario', title: 'graphql-9.x 9.0-11.0 (3)' }
- - { name: 'graphql-12.x-scenario', title: 'graphql-12.x 12.0-15.0 (4)' }
- - { name: 'hbase-scenario', title: 'hbase-scenario (5)' }
- - { name: 'spring-kafka-2.2.x-scenario', title: 'Spring-Kafka 2.2.x (7)' }
- - { name: 'spring-kafka-2.3.x-scenario', title: 'Spring-Kafka 2.3.x (7)' }
- - { name: 'spring-scheduled-scenario', title: 'Spring Scheduled 3.1.x-5.2.x (9)' }
- - { name: 'elasticjob-2.x-scenario', title: 'elasticjob-2.x-scenario (2)' }
- - { name: 'quartz-scheduler-2.x-scenario', title: 'quartz-scheduler-2.x-scenario (5)' }
- - { name: 'xxl-job-2.x-scenario', title: 'xxl-job-2.x-scenario (1)' }
+ - mysql-scenario
+ - undertow-scenario
+ - webflux-scenario
+ - zookeeper-scenario
+ - spring-3.1.x-scenario
+ - spring-4.1.x-scenario
+ - spring-4.3.x-scenario
+ - spring-async-scenario
+ - vertx-eventbus-3.x-scenario
+ - vertx-web-3.54minus-scenario
+ - vertx-web-3.6plus-scenario
+ - mariadb-scenario
+ - quasar-scenario
+ - baidu-brpc-scenario
+ - retransform-class-scenario
+ - retransform-class-tomcat-scenario
+ - graphql-8.x-scenario
+ - graphql-9.x-scenario
+ - graphql-12.x-scenario
+ - hbase-scenario
+ - spring-kafka-2.2.x-scenario
+ - spring-kafka-2.3.x-scenario
+ - spring-scheduled-scenario
+ - elasticjob-2.x-scenario
+ - quartz-scheduler-2.x-scenario
+ - xxl-job-2.x-scenario
steps:
- uses: actions/checkout@v2
with:
@@ -72,8 +72,8 @@ jobs:
run: ./mvnw --batch-mode clean package -DskipTests -Pagent
- name: Build Docker image
run: ./mvnw --batch-mode -f test/plugin/pom.xml clean package -DskipTests
- - name: ${{ matrix.case.title }}
- run: bash test/plugin/run.sh ${{ matrix.case.name }}
+ - name: ${{ matrix.case }}
+ run: bash test/plugin/run.sh ${{ matrix.case }}
- name: Report Coverage
run: bash -x tools/coverage/report.sh
diff --git a/docs/en/guides/Plugin-test.md b/docs/en/guides/Plugin-test.md
index 166c6e9..e690eb5 100644
--- a/docs/en/guides/Plugin-test.md
+++ b/docs/en/guides/Plugin-test.md
@@ -618,11 +618,12 @@ rather than recompiling it every time.
Use `${SKYWALKING_HOME}/test/plugin/run.sh -h` to know more command options.
-If the local test passed, then you could add it to `.github/workflows/plugins-test.<n>.yaml` file, which will drive the tests running on the Github Actions of official SkyWalking repository.
+If the local test passed, then you could add it to `.github/workflows/plugins-test.<n>.yaml` file, which will drive the tests running on the GitHub Actions of official SkyWalking repository.
Based on your plugin's name, please add the test case into file `.github/workflows/plugins-test.<n>.yaml`, by alphabetical orders.
-Every test case is a Github Actions Job. Please use the `<scenario name> + <version range> + (<supported version count>)` as the Job `title`, and the scenario directory as the Job `name`,
+Every test case is a GitHub Actions Job. Please use the scenario directory name as the case `name`,
mostly you'll just need to decide which file (`plugins-test.<n>.yaml`) to add your test case, and simply put one line (as follows) in it, take the existed cases as examples.
+You can run `python3 tools/select-group.py` to see which file contains the least cases and add your cases into it, in order to balance the running time of each group.
If a test case required to run in JDK 14 environment, please add you test case into file `plugins-jdk14-test.<n>.yaml`.
@@ -637,6 +638,6 @@ jobs:
matrix:
case:
# ...
- - { name: '<your case name>', title: '<PluginName, i.e. Spring> (<Supported Version Count, i.e 12>)' } # <<== insert one line by alphabetical orders
+ - <your scenario test directory name>
# ...
```
diff --git a/tools/select-group.py b/tools/select-group.py
new file mode 100644
index 0000000..d2aad2e
--- /dev/null
+++ b/tools/select-group.py
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+from os import path
+from os.path import dirname
+
+import yaml
+
+
+def main():
+ root_dir = path.realpath(path.join(dirname(__file__), '..'))
+
+ group_count = {}
+ for root, sub_dirs, files in os.walk(root_dir):
+ for filename in files:
+ if re.match('plugins-test\\.\\d+\\.ya?ml', filename):
+ count_plugin_tests(filename, group_count, root_dir)
+
+ group_count = sorted(group_count.items(), key=lambda x: x[1])
+ for gc in group_count:
+ print(gc[0], gc[1])
+
+
+def count_plugin_tests(filename, group_count, root_dir):
+ yaml_file = path.join(root_dir, '.github/workflows/', filename)
+ content = yaml.safe_load(open(yaml_file))
+ group_count[yaml_file] = 0
+ for job in content['jobs'].values():
+ if 'strategy' in job and 'matrix' in job['strategy'] and 'case' in job['strategy']['matrix']:
+ cases = job['strategy']['matrix']['case']
+ for case in cases:
+ count_scenario_cases(case, group_count, root_dir, yaml_file)
+
+
+def count_scenario_cases(case, group_count, root_dir, yaml_file):
+ version_filename = path.join(root_dir, 'test/plugin/scenarios', case, 'support-version.list')
+ with open(version_filename) as version_file:
+ for line in version_file.readlines():
+ if not line.startswith('#') and len(line.strip()) > 0:
+ group_count[yaml_file] += 1
+
+
+if __name__ == '__main__':
+ main()