You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2021/08/10 05:22:30 UTC
[skywalking] 01/01: Fix potential issues in ES7.14 and add log
cases to ES7.14
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch es7.14
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit ee54f5e4992e0ae6b10e4b1ce6e7b13f81555e5b
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Tue Aug 10 12:35:45 2021 +0800
Fix potential issues in ES7.14 and add log cases to ES7.14
---
.github/workflows/e2e.log.yaml | 5 +-
.../oap/server/core/alarm/AlarmRecord.java | 6 ++-
.../analysis/manual/endpoint/EndpointTraffic.java | 6 ++-
.../analysis/manual/log/AbstractLogRecord.java | 9 +++-
.../analysis/manual/segment/SegmentRecord.java | 3 +-
.../analysis/manual/service/ServiceTraffic.java | 6 ++-
.../manual/errorlog/BrowserErrorLogRecord.java | 7 ++-
.../storage/plugin/zipkin/ZipkinSpanRecord.java | 7 ++-
.../e2e-test/docker/log/docker-compose.es7.14.yml | 56 ++++++++++++++++++++++
9 files changed, 96 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/e2e.log.yaml b/.github/workflows/e2e.log.yaml
index c8a4d9a..1a756a3 100644
--- a/.github/workflows/e2e.log.yaml
+++ b/.github/workflows/e2e.log.yaml
@@ -23,6 +23,9 @@ on:
- '!**.md'
schedule:
- cron: '0 18 * * *'
+ push:
+ branches:
+ - es7.14
env:
SW_AGENT_JDK_VERSION: 8
@@ -39,7 +42,7 @@ jobs:
timeout-minutes: 90
strategy:
matrix:
- storage: ['h2', 'mysql', 'es6', 'es7', 'influxdb']
+ storage: ['h2', 'mysql', 'es6', 'es7', 'es7.14', 'influxdb']
env:
SW_STORAGE: ${{ matrix.storage }}
steps:
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
index 8fc26a3..e752450 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
@@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.core.alarm;
+import com.google.common.base.Strings;
import lombok.Getter;
import lombok.Setter;
import org.apache.skywalking.apm.util.StringUtil;
@@ -37,6 +38,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static java.util.Objects.nonNull;
import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ALARM;
@Getter
@@ -92,7 +94,9 @@ public class AlarmRecord extends Record {
map.put(NAME, storageData.getName());
map.put(ID0, storageData.getId0());
map.put(ID1, storageData.getId1());
- map.put(ALARM_MESSAGE, storageData.getAlarmMessage());
+ if (nonNull(storageData.getAlarmMessage())) {
+ map.put(ALARM_MESSAGE, storageData.getAlarmMessage());
+ }
map.put(START_TIME, storageData.getStartTime());
map.put(TIME_BUCKET, storageData.getTimeBucket());
map.put(RULE_NAME, storageData.getRuleName());
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
index e302904..90690ee 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
@@ -35,6 +35,8 @@ import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.StorageHashMapBuilder;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import static java.util.Objects.nonNull;
+
@Stream(name = EndpointTraffic.INDEX_NAME, scopeId = DefaultScopeDefine.ENDPOINT,
builder = EndpointTraffic.Builder.class, processor = MetricsStreamProcessor.class)
@MetricsExtension(supportDownSampling = false, supportUpdate = false)
@@ -101,7 +103,9 @@ public class EndpointTraffic extends Metrics {
public Map<String, Object> entity2Storage(EndpointTraffic storageData) {
Map<String, Object> map = new HashMap<>();
map.put(SERVICE_ID, storageData.getServiceId());
- map.put(NAME, storageData.getName());
+ if (nonNull(storageData.getName())) {
+ map.put(NAME, storageData.getName());
+ }
map.put(TIME_BUCKET, storageData.getTimeBucket());
return map;
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java
index 8988d00..b913971 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java
@@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.core.analysis.manual.log;
+import com.google.common.base.Strings;
import java.util.Base64;
import java.util.List;
import java.util.Map;
@@ -33,6 +34,8 @@ import org.apache.skywalking.oap.server.core.storage.StorageHashMapBuilder;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.library.util.CollectionUtils;
+import static java.util.Objects.nonNull;
+
public abstract class AbstractLogRecord extends Record {
public static final String SERVICE_ID = "service_id";
@@ -121,13 +124,15 @@ public abstract class AbstractLogRecord extends Record {
map.put(SERVICE_ID, record.getServiceId());
map.put(SERVICE_INSTANCE_ID, record.getServiceInstanceId());
map.put(ENDPOINT_ID, record.getEndpointId());
- map.put(ENDPOINT_NAME, record.getEndpointName());
+ if (nonNull(record.getEndpointName())) {
+ map.put(ENDPOINT_NAME, record.getEndpointName());
+ }
map.put(TRACE_ID, record.getTraceId());
map.put(TRACE_SEGMENT_ID, record.getTraceSegmentId());
map.put(SPAN_ID, record.getSpanId());
map.put(TIME_BUCKET, record.getTimeBucket());
map.put(CONTENT_TYPE, record.getContentType());
- map.put(CONTENT, record.getContent());
+ map.put(CONTENT, Strings.nullToEmpty(record.getContent()));
map.put(TIMESTAMP, record.getTimestamp());
if (CollectionUtils.isEmpty(record.getTagsRawData())) {
map.put(TAGS_RAW_DATA, Const.EMPTY_STRING);
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
index 7af4419..a13a0a6 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
@@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.core.analysis.manual.segment;
+import com.google.common.base.Strings;
import java.util.Base64;
import java.util.HashMap;
import java.util.List;
@@ -119,7 +120,7 @@ public class SegmentRecord extends Record {
map.put(TRACE_ID, storageData.getTraceId());
map.put(SERVICE_ID, storageData.getServiceId());
map.put(SERVICE_INSTANCE_ID, storageData.getServiceInstanceId());
- map.put(ENDPOINT_NAME, storageData.getEndpointName());
+ map.put(ENDPOINT_NAME, Strings.nullToEmpty(storageData.getEndpointName()));
map.put(ENDPOINT_ID, storageData.getEndpointId());
map.put(START_TIME, storageData.getStartTime());
map.put(LATENCY, storageData.getLatency());
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java
index d5ab808..35e1a9e 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java
@@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.core.analysis.manual.service;
+import com.google.common.base.Strings;
import java.util.HashMap;
import java.util.Map;
import lombok.EqualsAndHashCode;
@@ -35,6 +36,7 @@ import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.StorageHashMapBuilder;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import static java.util.Objects.nonNull;
import static org.apache.skywalking.oap.server.core.Const.DOUBLE_COLONS_SPLIT;
@Stream(name = ServiceTraffic.INDEX_NAME, scopeId = DefaultScopeDefine.SERVICE,
@@ -115,7 +117,9 @@ public class ServiceTraffic extends Metrics {
}
}
Map<String, Object> map = new HashMap<>();
- map.put(NAME, serviceName);
+ if (nonNull(serviceName)) {
+ map.put(NAME, serviceName);
+ }
map.put(NODE_TYPE, storageData.getNodeType().value());
map.put(GROUP, storageData.getGroup());
return map;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java
index 7cc404b..d182ec9 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java
@@ -17,6 +17,7 @@
package org.apache.skywalking.oap.server.core.browser.manual.errorlog;
+import com.google.common.base.Strings;
import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
@@ -33,6 +34,8 @@ import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset;
import org.apache.skywalking.oap.server.library.util.CollectionUtils;
+import static java.util.Objects.nonNull;
+
@SuperDataset
@Stream(name = BrowserErrorLogRecord.INDEX_NAME, scopeId = DefaultScopeDefine.BROWSER_ERROR_LOG, builder = BrowserErrorLogRecord.Builder.class, processor = RecordStreamProcessor.class)
public class BrowserErrorLogRecord extends Record {
@@ -119,7 +122,9 @@ public class BrowserErrorLogRecord extends Record {
map.put(SERVICE_ID, storageData.getServiceId());
map.put(SERVICE_VERSION_ID, storageData.getServiceVersionId());
map.put(PAGE_PATH_ID, storageData.getPagePathId());
- map.put(PAGE_PATH, storageData.getPagePath());
+ if (nonNull(storageData.getPagePath())) {
+ map.put(PAGE_PATH, storageData.getPagePath());
+ }
map.put(TIMESTAMP, storageData.getTimestamp());
map.put(TIME_BUCKET, storageData.getTimeBucket());
map.put(ERROR_CATEGORY, storageData.getErrorCategory());
diff --git a/oap-server/server-storage-plugin/storage-zipkin-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java b/oap-server/server-storage-plugin/storage-zipkin-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java
index 434b3eb..58e0197 100644
--- a/oap-server/server-storage-plugin/storage-zipkin-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java
+++ b/oap-server/server-storage-plugin/storage-zipkin-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java
@@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.storage.plugin.zipkin;
+import com.google.common.base.Strings;
import java.util.Base64;
import java.util.HashMap;
import java.util.List;
@@ -35,6 +36,8 @@ import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset;
import org.apache.skywalking.oap.server.library.util.CollectionUtils;
+import static java.util.Objects.nonNull;
+
@SuperDataset
@Stream(name = ZipkinSpanRecord.INDEX_NAME, scopeId = DefaultScopeDefine.ZIPKIN_SPAN, builder = ZipkinSpanRecord.Builder.class, processor = RecordStreamProcessor.class)
public class ZipkinSpanRecord extends Record {
@@ -120,7 +123,9 @@ public class ZipkinSpanRecord extends Record {
map.put(SPAN_ID, storageData.getSpanId());
map.put(SERVICE_ID, storageData.getServiceId());
map.put(SERVICE_INSTANCE_ID, storageData.getServiceInstanceId());
- map.put(ENDPOINT_NAME, storageData.getEndpointName());
+ if (nonNull(storageData.getEndpointName())) {
+ map.put(ENDPOINT_NAME, storageData.getEndpointName());
+ }
map.put(ENDPOINT_ID, storageData.getEndpointId());
map.put(START_TIME, storageData.getStartTime());
map.put(END_TIME, storageData.getEndTime());
diff --git a/test/e2e/e2e-test/docker/log/docker-compose.es7.14.yml b/test/e2e/e2e-test/docker/log/docker-compose.es7.14.yml
new file mode 100644
index 0000000..2a7cadc
--- /dev/null
+++ b/test/e2e/e2e-test/docker/log/docker-compose.es7.14.yml
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '2.1'
+
+services:
+ es:
+ image: elastic/elasticsearch:7.14.0
+ expose:
+ - 9200
+ networks:
+ - e2e
+ environment:
+ - discovery.type=single-node
+ healthcheck:
+ test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"]
+ interval: 5s
+ timeout: 60s
+ retries: 120
+
+ oap:
+ extends:
+ file: ../base-compose.yml
+ service: oap-es7
+ environment:
+ SW_STORAGE: elasticsearch7
+ SW_LOG_LAL_FILES: test
+ SW_LOG_MAL_FILES: test
+ volumes:
+ - ./lal.yaml:/skywalking/config/lal/test.yaml
+ - ./log-mal.yaml:/skywalking/config/log-mal-rules/test.yaml
+ depends_on:
+ es:
+ condition: service_healthy
+
+ provider:
+ extends:
+ file: ../base-compose.yml
+ service: provider
+ depends_on:
+ oap:
+ condition: service_healthy
+networks:
+ e2e: