You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2022/02/21 00:12:02 UTC
[skywalking] branch master updated: Support ElasticSearch 8 and add it into E2E tests (#8564)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 68e40eb Support ElasticSearch 8 and add it into E2E tests (#8564)
68e40eb is described below
commit 68e40eb9fc0a891a55fe927d49b267d0fd80000c
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Mon Feb 21 08:11:27 2022 +0800
Support ElasticSearch 8 and add it into E2E tests (#8564)
---
.github/workflows/e2e.log.yaml | 2 ++
.github/workflows/e2e.storages.yaml | 2 ++
.github/workflows/e2e.ttl.yaml | 2 ++
CHANGES.md | 2 ++
docs/en/setup/backend/backend-storage.md | 6 +++---
.../apache/skywalking/oap/server/core/alarm/AlarmRecord.java | 2 +-
.../skywalking/library/elasticsearch/ElasticSearchVersion.java | 5 +++++
.../storage/plugin/elasticsearch/base/StorageEsInstaller.java | 10 +++++-----
test/e2e-v2/cases/log/es/docker-compose.yml | 1 +
test/e2e-v2/cases/log/fluent-bit/docker-compose.yml | 1 +
test/e2e-v2/cases/storage/es/docker-compose.yml | 1 +
test/e2e-v2/cases/ttl/es/docker-compose.yml | 1 +
12 files changed, 26 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/e2e.log.yaml b/.github/workflows/e2e.log.yaml
index b42896d..55ccc27 100644
--- a/.github/workflows/e2e.log.yaml
+++ b/.github/workflows/e2e.log.yaml
@@ -58,6 +58,8 @@ jobs:
config-file: log/es/e2e.yaml
- es-version: 7.15.0
config-file: log/es/e2e.yaml
+ - es-version: 8.0.0
+ config-file: log/es/e2e.yaml
- es-version: 6.3.2
config-file: log/fluent-bit/e2e.yaml
diff --git a/.github/workflows/e2e.storages.yaml b/.github/workflows/e2e.storages.yaml
index 7b3d018..b35a092 100644
--- a/.github/workflows/e2e.storages.yaml
+++ b/.github/workflows/e2e.storages.yaml
@@ -61,6 +61,8 @@ jobs:
config-file: storage/es/e2e.yaml
- es-version: 7.15.0
config-file: storage/es/e2e.yaml
+ - es-version: 8.0.0
+ config-file: storage/es/e2e.yaml
steps:
- uses: actions/checkout@v2
diff --git a/.github/workflows/e2e.ttl.yaml b/.github/workflows/e2e.ttl.yaml
index 4457fb0..38bc321 100644
--- a/.github/workflows/e2e.ttl.yaml
+++ b/.github/workflows/e2e.ttl.yaml
@@ -49,6 +49,8 @@ jobs:
config-file: ttl/es/e2e.yaml
- es-version: 7.15.0
config-file: ttl/es/e2e.yaml
+ - es-version: 8.0.0
+ config-file: ttl/es/e2e.yaml
steps:
- uses: actions/checkout@v2
with:
diff --git a/CHANGES.md b/CHANGES.md
index 96911ef..ff1d8f2 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -61,6 +61,8 @@ Release Notes.
* Support all metrics from MAL engine in alarm core, including Prometheus, OC receiver, meter receiver.
* Allow updating non-metrics templates when structure changed.
* Set default connection timeout of ElasticSearch to 3000 milliseconds.
+* Support ElasticSearch 8 and add it into E2E tests.
+* Disable indexing for field `alarm_record.tags_raw_data` of binary type in ElasticSearch storage.
#### UI
diff --git a/docs/en/setup/backend/backend-storage.md b/docs/en/setup/backend/backend-storage.md
index 44ace76..2e82b4e 100644
--- a/docs/en/setup/backend/backend-storage.md
+++ b/docs/en/setup/backend/backend-storage.md
@@ -10,7 +10,7 @@ storage:
Natively supported storage:
- H2
- OpenSearch
-- ElasticSearch 6, 7
+- ElasticSearch 6, 7, 8
- MySQL
- TiDB
- InfluxDB
@@ -50,8 +50,8 @@ Since 8.8.0, SkyWalking rebuilds the ElasticSearch client on top of ElasticSearc
correct request formats according to the server side version, hence you don't need to download different binaries
and don't need to configure different storage selector for different ElasticSearch server side version anymore.
-For now, SkyWalking supports ElasticSearch 6.x, ElasticSearch 7.x, and OpenSearch 1.x, their configurations are as
-follows:
+For now, SkyWalking supports ElasticSearch 6.x, ElasticSearch 7.x, ElasticSearch 8.x, and OpenSearch 1.x, their
+configurations are as follows:
```yaml
storage:
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
index b6dbc1e..5804115 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
@@ -77,7 +77,7 @@ public class AlarmRecord extends Record {
private String ruleName;
@Column(columnName = TAGS)
private List<String> tagsInString;
- @Column(columnName = TAGS_RAW_DATA)
+ @Column(columnName = TAGS_RAW_DATA, storageOnly = true)
private byte[] tagsRawData;
@Setter
@Getter
diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchVersion.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchVersion.java
index 5e78d34..4f51773 100644
--- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchVersion.java
+++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchVersion.java
@@ -63,6 +63,11 @@ public final class ElasticSearchVersion {
}
return;
}
+ if (major == 8) {
+ requestFactory = new V78RequestFactory(this);
+ codec = V78Codec.INSTANCE;
+ return;
+ }
}
throw new UnsupportedOperationException("Unsupported version: " + this);
}
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
index 478e5ea..a6a87e9 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
@@ -200,12 +200,12 @@ public class StorageEsInstaller extends ModelInstaller {
protected Mappings createMapping(Model model) {
Map<String, Object> properties = new HashMap<>();
for (ModelColumn columnDefine : model.getColumns()) {
+ final String type = columnTypeEsMapping.transform(columnDefine.getType(), columnDefine.getGenericType());
if (columnDefine.isMatchQuery()) {
String matchCName = MatchCNameBuilder.INSTANCE.build(columnDefine.getColumnName().getName());
Map<String, Object> originalColumn = new HashMap<>();
- originalColumn.put(
- "type", columnTypeEsMapping.transform(columnDefine.getType(), columnDefine.getGenericType()));
+ originalColumn.put("type", type);
originalColumn.put("copy_to", matchCName);
properties.put(columnDefine.getColumnName().getName(), originalColumn);
@@ -215,9 +215,9 @@ public class StorageEsInstaller extends ModelInstaller {
properties.put(matchCName, matchColumn);
} else {
Map<String, Object> column = new HashMap<>();
- column.put(
- "type", columnTypeEsMapping.transform(columnDefine.getType(), columnDefine.getGenericType()));
- if (columnDefine.isStorageOnly()) {
+ column.put("type", type);
+ // no index parameter is allowed for binary type, since ES 8.0
+ if (columnDefine.isStorageOnly() && !"binary".equals(type)) {
column.put("index", false);
}
properties.put(columnDefine.getColumnName().getName(), column);
diff --git a/test/e2e-v2/cases/log/es/docker-compose.yml b/test/e2e-v2/cases/log/es/docker-compose.yml
index 46edc68..f7bfbfd 100644
--- a/test/e2e-v2/cases/log/es/docker-compose.yml
+++ b/test/e2e-v2/cases/log/es/docker-compose.yml
@@ -24,6 +24,7 @@ services:
- e2e
environment:
- discovery.type=single-node
+ - xpack.security.enabled=false
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"]
interval: 5s
diff --git a/test/e2e-v2/cases/log/fluent-bit/docker-compose.yml b/test/e2e-v2/cases/log/fluent-bit/docker-compose.yml
index 2a3e4a2..448b407 100644
--- a/test/e2e-v2/cases/log/fluent-bit/docker-compose.yml
+++ b/test/e2e-v2/cases/log/fluent-bit/docker-compose.yml
@@ -24,6 +24,7 @@ services:
- e2e
environment:
- discovery.type=single-node
+ - xpack.security.enabled=false
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"]
interval: 5s
diff --git a/test/e2e-v2/cases/storage/es/docker-compose.yml b/test/e2e-v2/cases/storage/es/docker-compose.yml
index d1881bd..221d427 100644
--- a/test/e2e-v2/cases/storage/es/docker-compose.yml
+++ b/test/e2e-v2/cases/storage/es/docker-compose.yml
@@ -25,6 +25,7 @@ services:
environment:
- discovery.type=single-node
- cluster.routing.allocation.disk.threshold_enabled=false
+ - xpack.security.enabled=false
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"]
interval: 5s
diff --git a/test/e2e-v2/cases/ttl/es/docker-compose.yml b/test/e2e-v2/cases/ttl/es/docker-compose.yml
index 426a4b1..8cd921a 100644
--- a/test/e2e-v2/cases/ttl/es/docker-compose.yml
+++ b/test/e2e-v2/cases/ttl/es/docker-compose.yml
@@ -25,6 +25,7 @@ services:
environment:
- discovery.type=single-node
- cluster.routing.allocation.disk.threshold_enabled=false
+ - xpack.security.enabled=false
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"]
interval: 5s