You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2022/11/29 03:13:08 UTC
[skywalking] branch master updated: Add OpenSearch 1.3.6 and 2.4.0 to test matrix (#10040)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new a7210f6633 Add OpenSearch 1.3.6 and 2.4.0 to test matrix (#10040)
a7210f6633 is described below
commit a7210f66338d4c9938829f620cd78e6534f3c059
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Tue Nov 29 11:13:01 2022 +0800
Add OpenSearch 1.3.6 and 2.4.0 to test matrix (#10040)
---
.github/workflows/skywalking.yaml | 49 +++++++++++++---------
docs/en/setup/backend/backend-storage.md | 21 ++++++----
.../elasticsearch/ElasticSearchVersion.java | 2 +-
...asticSearchTest.java => TCITElasticSearch.java} | 12 +++++-
...ntegrationTest.java => TCITShardingSphere.java} | 4 +-
pom.xml | 7 ++++
6 files changed, 64 insertions(+), 31 deletions(-)
diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml
index 001aa30bae..60145b497d 100644
--- a/.github/workflows/skywalking.yaml
+++ b/.github/workflows/skywalking.yaml
@@ -29,7 +29,7 @@ env:
SW_AGENT_JDK_VERSION: 8
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 # Cache restore timeout
-
+
jobs:
license-header:
if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule')
@@ -271,17 +271,22 @@ jobs:
./mvnw clean verify -q -B -Dcheckstyle.skip -DskipUTs -DskipITs=false || \
./mvnw clean verify -q -B -Dcheckstyle.skip -DskipUTs -DskipITs=false
- it-sharding-proxy:
+ # Time consuming integration tests
+ tc-integration-test:
if: |
always() &&
((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true')
- name: IT Sharding-proxy
+ name: Time Consuming Integration Tests
needs: [sanity-check, changes]
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
matrix:
- java-version: [8, 11]
+ test:
+ - name: ShardingSphere
+ class: org.apache.skywalking.oap.server.storage.plugin.jdbc.shardingsphere.TCITShardingSphere
+ - name: ElasticSearch / OpenSearch
+ class: org.apache.skywalking.library.elasticsearch.TCITElasticSearch
steps:
- uses: actions/checkout@v3
with:
@@ -294,12 +299,12 @@ jobs:
restore-keys: ${{ runner.os }}-maven-
- uses: actions/setup-java@v3
with:
- java-version: ${{ matrix.java-version }}
+ java-version: 8
distribution: adopt
- - name: IT Sharding-proxy
+ - name: ${{ matrix.test.name }}
run: |
- ./mvnw clean test -Dtest=org.apache.skywalking.oap.server.storage.plugin.jdbc.shardingsphere.ShardingIntegrationTest -DfailIfNoTests=false || \
- ./mvnw clean test -Dtest=org.apache.skywalking.oap.server.storage.plugin.jdbc.shardingsphere.ShardingIntegrationTest -DfailIfNoTests=false
+ ./mvnw clean test -Dtest=${{ matrix.test.class }} -DfailIfNoTests=false || \
+ ./mvnw clean test -Dtest=${{ matrix.test.class }} -DfailIfNoTests=false
e2e-test:
if: |
@@ -370,12 +375,12 @@ jobs:
- name: Storage OpenSearch 1.1.0
config: test/e2e-v2/cases/storage/opensearch/e2e.yaml
env: OPENSEARCH_VERSION=1.1.0
- - name: Storage OpenSearch 1.2.0
+ - name: Storage OpenSearch 1.3.6
config: test/e2e-v2/cases/storage/opensearch/e2e.yaml
- env: OPENSEARCH_VERSION=1.2.0
- - name: Storage OpenSearch 2.3.0
+ env: OPENSEARCH_VERSION=1.3.6
+ - name: Storage OpenSearch 2.4.0
config: test/e2e-v2/cases/storage/opensearch/e2e.yaml
- env: OPENSEARCH_VERSION=2.3.0
+ env: OPENSEARCH_VERSION=2.4.0
- name: Storage ES Sharding
config: test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml
@@ -480,9 +485,12 @@ jobs:
- name: Trace Profiling OpenSearch 1.1.0
config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml
env: OPENSEARCH_VERSION=1.1.0
- - name: Trace Profiling OpenSearch 1.2.0
+ - name: Trace Profiling OpenSearch 1.3.6
+ config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml
+ env: OPENSEARCH_VERSION=1.3.6
+ - name: Trace Profiling OpenSearch 2.4.0
config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml
- env: OPENSEARCH_VERSION=1.2.0
+ env: OPENSEARCH_VERSION=2.4.0
- name: eBPF Profiling On CPU BanyanDB
config: test/e2e-v2/cases/profiling/ebpf/oncpu/banyandb/e2e.yaml
@@ -501,9 +509,12 @@ jobs:
- name: eBPF Profiling On CPU OpenSearch 1.1.0
config: test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/e2e.yaml
env: OPENSEARCH_VERSION=1.1.0
- - name: eBPF Profiling On CPU OpenSearch 1.2.0
+ - name: eBPF Profiling On CPU OpenSearch 1.3.6
+ config: test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/e2e.yaml
+ env: OPENSEARCH_VERSION=1.3.6
+ - name: eBPF Profiling On CPU OpenSearch 2.4.0
config: test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/e2e.yaml
- env: OPENSEARCH_VERSION=1.2.0
+ env: OPENSEARCH_VERSION=2.4.0
- name: eBPF Profiling Off CPU
config: test/e2e-v2/cases/profiling/ebpf/offcpu/e2e.yaml
- name: eBPF Profiling Network
@@ -747,7 +758,7 @@ jobs:
- dependency-license
- unit-test
- integration-test
- - it-sharding-proxy
+ - tc-integration-test
- e2e-test
- e2e-test-istio
- e2e-test-java-versions
@@ -766,7 +777,7 @@ jobs:
depLicenseResults=${{ needs.dependency-license.result }}
unitResults=${{ needs.unit-test.result }};
integrationResults=${{ needs.integration-test.result }};
- itShardingResults=${{ needs.it-sharding-proxy.result }};
+ timeConsumingITResults=${{ needs.tc-integration-test.result }};
e2eResults=${{ needs.e2e-test.result }};
e2eIstioResults=${{ needs.e2e-test-istio.result }};
e2eJavaVersionResults=${{ needs.e2e-test-java-versions.result }};
@@ -777,6 +788,6 @@ jobs:
[[ ${e2eResults} == 'success' ]] || [[ ${execute} != 'true' && ${e2eResults} == 'skipped' ]] || exit -5;
[[ ${e2eIstioResults} == 'success' ]] || [[ ${execute} != 'true' && ${e2eIstioResults} == 'skipped' ]] || exit -6;
[[ ${e2eJavaVersionResults} == 'success' ]] || [[ ${execute} != 'true' && ${e2eJavaVersionResults} == 'skipped' ]] || exit -7;
- [[ ${itShardingResults} == 'success' ]] || [[ ${execute} != 'true' && ${itShardingResults} == 'skipped' ]] || exit -8;
+ [[ ${timeConsumingITResults} == 'success' ]] || [[ ${execute} != 'true' && ${timeConsumingITResults} == 'skipped' ]] || exit -8;
exit 0;
diff --git a/docs/en/setup/backend/backend-storage.md b/docs/en/setup/backend/backend-storage.md
index d6d83f0d55..edc65625d7 100644
--- a/docs/en/setup/backend/backend-storage.md
+++ b/docs/en/setup/backend/backend-storage.md
@@ -41,6 +41,11 @@ OpenSearch is a fork from ElasticSearch 7.11 but licensed in Apache 2.0.
OpenSearch storage shares the same configurations as ElasticSearch.
In order to activate OpenSearch as storage, set the storage provider to **elasticsearch**.
+We support and tested the following versions of OpenSearch:
+
+- 1.1.0, 1.3.6
+- 2.4.0
+
## ElasticSearch
**NOTE:** Elastic announced through their blog that Elasticsearch will be moving over to a Server Side Public
@@ -161,19 +166,19 @@ The following settings control the number of shards and replicas for new and exi
storage:
elasticsearch:
# ......
- indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1}
- indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:1}
+ indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1}
+ indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:1}
specificIndexSettings: ${SW_STORAGE_ES_SPECIFIC_INDEX_SETTINGS:""}
- superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5}
+ superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5}
superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0}
```
The following table shows the relationship between those config items and Elasticsearch `index number_of_shards/number_of_replicas`.
-And also you can [specify the settings for each index individually.](#specify-settings-for-each-elasticsearch-index-individually)
+And also you can [specify the settings for each index individually.](#specify-settings-for-each-elasticsearch-index-individually)
-| index | number_of_shards | number_of_replicas |
+| index | number_of_shards | number_of_replicas |
|--------------------------------------|------------------|----------------------|
-| sw_ui_template | indexShardsNumber | indexReplicasNumber |
-| sw_metrics-all-`${day-format}` | indexShardsNumber | indexReplicasNumber |
+| sw_ui_template | indexShardsNumber | indexReplicasNumber |
+| sw_metrics-all-`${day-format}` | indexShardsNumber | indexReplicasNumber |
| sw_log-`${day-format}` | indexShardsNumber * superDatasetIndexShardsFactor | superDatasetIndexReplicasNumber |
| sw_segment-`${day-format}` | indexShardsNumber * superDatasetIndexShardsFactor | superDatasetIndexReplicasNumber |
| sw_browser_error_log-`${day-format}` | indexShardsNumber * superDatasetIndexShardsFactor | superDatasetIndexReplicasNumber |
@@ -200,7 +205,7 @@ Supported settings:
- number_of_shards
- number_of_replicas
-**NOTE:** These settings have the highest priority and will override the existing
+**NOTE:** These settings have the highest priority and will override the existing
generic settings mentioned in [index settings doc](#index-settings).
The settings are in `JSON` format. The index name here is logic entity name, which should exclude the `${SW_NAMESPACE}` which is `sw` by default, e.g.
diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchVersion.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchVersion.java
index 6607961f6b..97cd695fa3 100644
--- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchVersion.java
+++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchVersion.java
@@ -43,7 +43,7 @@ public final class ElasticSearchVersion {
this.minor = minor;
if (distribution.equalsIgnoreCase("OpenSearch")) {
- requestFactory = new V78RequestFactory(this);
+ requestFactory = new V81RequestFactory(this);
codec = V78Codec.INSTANCE;
return;
}
diff --git a/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ITElasticSearchTest.java b/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/TCITElasticSearch.java
similarity index 96%
rename from oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ITElasticSearchTest.java
rename to oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/TCITElasticSearch.java
index d171afbb8c..1babf84ef8 100644
--- a/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ITElasticSearchTest.java
+++ b/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/TCITElasticSearch.java
@@ -55,7 +55,7 @@ import static org.junit.Assert.assertTrue;
@RequiredArgsConstructor
@RunWith(Parameterized.class)
-public class ITElasticSearchTest {
+public class TCITElasticSearch {
@Parameterized.Parameters(name = "version: {0}")
public static Collection<Object[]> es() {
@@ -104,6 +104,16 @@ public class ITElasticSearchTest {
"docker.elastic.co/elasticsearch/elasticsearch-oss"))
.withEnv("plugins.security.disabled", "true")
.withStartupTimeout(java.time.Duration.ofMinutes(5))
+ },
+ {
+ "OpenSearch 2.4.0",
+ new ElasticsearchContainer(
+ DockerImageName.parse("opensearchproject/opensearch")
+ .withTag("2.4.0")
+ .asCompatibleSubstituteFor(
+ "docker.elastic.co/elasticsearch/elasticsearch-oss"))
+ .withEnv("plugins.security.disabled", "true")
+ .withStartupTimeout(java.time.Duration.ofMinutes(5))
}
});
}
diff --git a/oap-server/server-storage-plugin/storage-shardingsphere-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/shardingsphere/ShardingIntegrationTest.java b/oap-server/server-storage-plugin/storage-shardingsphere-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/shardingsphere/TCITShardingSphere.java
similarity index 99%
rename from oap-server/server-storage-plugin/storage-shardingsphere-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/shardingsphere/ShardingIntegrationTest.java
rename to oap-server/server-storage-plugin/storage-shardingsphere-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/shardingsphere/TCITShardingSphere.java
index c9776758ec..f70ad53ede 100644
--- a/oap-server/server-storage-plugin/storage-shardingsphere-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/shardingsphere/ShardingIntegrationTest.java
+++ b/oap-server/server-storage-plugin/storage-shardingsphere-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/shardingsphere/TCITShardingSphere.java
@@ -116,7 +116,7 @@ import static org.mockito.Mockito.when;
"org.w3c.*"
})
@PrepareForTest({DefaultScopeDefine.class})
-public class ShardingIntegrationTest {
+public class TCITShardingSphere {
@BeforeClass
public static void setup() {
PowerMockito.mockStatic(DefaultScopeDefine.class);
@@ -172,7 +172,7 @@ public class ShardingIntegrationTest {
}
private void startEnv(String dockerComposeName, int dsServicePort) {
- environment = new DockerComposeContainer<>(new File(ShardingIntegrationTest.class
+ environment = new DockerComposeContainer<>(new File(TCITShardingSphere.class
.getClassLoader()
.getResource(dockerComposeName).getPath()))
.withExposedService("sharding-proxy", 3307,
diff --git a/pom.xml b/pom.xml
index ab76862160..00dc8a3099 100755
--- a/pom.xml
+++ b/pom.xml
@@ -310,6 +310,13 @@
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<skip>${skipITs}</skip>
+ <!--
+ Exclude time-consuming tests and run those tests separately in skywalking.yaml.
+ TCIT means Time Consuming Integration Test.
+ -->
+ <excludes>
+ <exclude>**/TCIT*.java</exclude>
+ </excludes>
</configuration>
<executions>
<execution>