You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2021/06/20 09:13:40 UTC
[skywalking] 01/01: Add FluentBit integration E2E test and doc
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch log/fluentbit
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 6736bb46ba9b98d2331cc7d45b459c73411bcb9c
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Sun Jun 20 17:13:14 2021 +0800
Add FluentBit integration E2E test and doc
---
.github/workflows/e2e.log.yaml | 27 ++++++++
docs/en/setup/backend/log-analyzer.md | 10 +--
.../docker/log/docker-compose.fluentbit.yml | 59 +++++++++++++++++
.../docker/log/fluent-bit/fluent-bit-parser.conf | 21 +++++++
.../docker/log/fluent-bit/fluent-bit-script.lua | 21 +++++++
.../e2e-test/docker/log/fluent-bit/fluent-bit.conf | 40 ++++++++++++
.../apache/skywalking/e2e/log/FluentBitE2E.java | 73 ++++++++++++++++++++++
.../java/org/apache/skywalking/e2e/log/LogE2E.java | 34 +++-------
test/e2e/pom.xml | 1 +
9 files changed, 257 insertions(+), 29 deletions(-)
diff --git a/.github/workflows/e2e.log.yaml b/.github/workflows/e2e.log.yaml
index 88c8212..04ddf7b 100644
--- a/.github/workflows/e2e.log.yaml
+++ b/.github/workflows/e2e.log.yaml
@@ -23,6 +23,9 @@ on:
- '!**.md'
schedule:
- cron: '0 18 * * *'
+ push:
+ branches:
+ - log/fluentbit
env:
SW_AGENT_JDK_VERSION: 8
@@ -60,3 +63,27 @@ jobs:
uses: ./.github/actions/e2e-test
with:
test_class: org.apache.skywalking.e2e.log.LogE2E
+
+ LogFluentBit:
+ if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule')
+ name: Log FluentBit
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ submodules: true
+ - name: Set Skip Env Var
+ uses: ./.github/actions/skip
+ - name: Cache local Maven repository
+ if: env.SKIP_CI != 'true'
+ uses: actions/cache@v2
+ with:
+ path: ~/.m2/repository
+ key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-maven-
+ - name: Run E2E Test
+ if: env.SKIP_CI != 'true'
+ uses: ./.github/actions/e2e-test
+ with:
+ test_class: org.apache.skywalking.e2e.log.LogFluentBitE2E
diff --git a/docs/en/setup/backend/log-analyzer.md b/docs/en/setup/backend/log-analyzer.md
index 68f8a28..b323136 100644
--- a/docs/en/setup/backend/log-analyzer.md
+++ b/docs/en/setup/backend/log-analyzer.md
@@ -5,14 +5,16 @@ There are various ways to collect logs from application.
### Log files collector
-You can use [Filebeat](https://www.elastic.co/cn/beats/filebeat) 、[Fluentd](https://fluentd.org) to
-collect file logs including to use Kafka MQ to transport [native-json](../../protocols/Log-Data-Protocol.md#Native-Json-Protocol)
-format logs. When use this, need to open [kafka-fetcher](backend-fetcher.md#kafka-fetcher)
-and enable configs `enableNativeJsonLog`.
+You can use [Filebeat](https://www.elastic.co/cn/beats/filebeat), [Fluentd](https://fluentd.org)
+and [FluentBit](http://fluentbit.io) to collect logs, and then transport the logs to SkyWalking OAP through Kafka or
+HTTP protocol, with the formats [Kafka JSON](../../protocols/Log-Data-Protocol.md#native-kafka-protocol)
+or [HTTP JSON array](../../protocols/Log-Data-Protocol.md#http-api). When using Kafka protocol, you need to
+open [kafka-fetcher](backend-fetcher.md#kafka-fetcher) and enable configs `enableNativeJsonLog`.
Collector config examples:
- [filebeat.yml](../../../../test/e2e/e2e-test/docker/kafka/filebeat.yml)
- [fluentd.conf](../../../../test/e2e/e2e-test/docker/kafka/fluentd.conf)
+- [fluent-bit.conf](../../../../test/e2e/e2e-test/docker/log/fluent-bit)
### Java agent's toolkits
Java agent provides toolkit for
diff --git a/test/e2e/e2e-test/docker/log/docker-compose.fluentbit.yml b/test/e2e/e2e-test/docker/log/docker-compose.fluentbit.yml
new file mode 100644
index 0000000..f829d82
--- /dev/null
+++ b/test/e2e/e2e-test/docker/log/docker-compose.fluentbit.yml
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '2.1'
+
+services:
+ oap:
+ extends:
+ file: ../base-compose.yml
+ service: oap
+ environment:
+ SW_STORAGE: h2
+ SW_LOG_LAL_FILES: test
+ SW_LOG_MAL_FILES: test
+ volumes:
+ - ./lal.yaml:/skywalking/config/lal/test.yaml
+ - ./log-mal.yaml:/skywalking/config/log-mal-rules/test.yaml
+ networks:
+ - e2e
+
+ provider:
+ extends:
+ file: ../base-compose.yml
+ service: provider
+ volumes:
+ - /tmp/skywalking-logs/:/tmp/skywalking-logs/
+ networks:
+ - e2e
+ depends_on:
+ oap:
+ condition: service_healthy
+
+ fluentbit:
+ image: fluent/fluent-bit:1.7
+ volumes:
+ - /tmp/skywalking-logs:/tmp/skywalking-logs
+ - ./fluent-bit:/fluent-bit/etc
+ networks:
+ - e2e
+ depends_on:
+ oap:
+ condition: service_healthy
+ provider:
+ condition: service_healthy
+
+networks:
+ e2e:
diff --git a/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-parser.conf b/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-parser.conf
new file mode 100644
index 0000000..be298d6
--- /dev/null
+++ b/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-parser.conf
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[PARSER]
+ Name my-log-format
+ Format regex
+ Regex ^\[SW_CTX: ?\[(?<service>[^,]+),(?<serviceInstance>[^,]+),(?<traceId>[^,]+),(?<traceSegmentId>[^,]+),(?<spanId>[^\]]+)\]\] \[(?<level>.+?)\] (?<logtime>[^\]]*) \[(?<thread>[^ ]*)\] (?<logger>[^\]]*):(?<line>[^\]]*) - (?<body>[^\]]*)$
+ Time_Key time
+ Time_Format %d/%b/%Y:%H:%M:%S %z
diff --git a/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-script.lua b/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-script.lua
new file mode 100644
index 0000000..2cbb386
--- /dev/null
+++ b/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit-script.lua
@@ -0,0 +1,21 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+function rewrite_body(tag, timestamp, record)
+ record["body"] = {text={text=record["body"]}}
+ return 1, timestamp, record
+end
diff --git a/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit.conf b/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit.conf
new file mode 100644
index 0000000..28d9554
--- /dev/null
+++ b/test/e2e/e2e-test/docker/log/fluent-bit/fluent-bit.conf
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[SERVICE]
+ Flush 5
+ Daemon Off
+ Log_Level warn
+ Parsers_File fluent-bit-parser.conf
+[INPUT]
+ Name tail
+ Path /tmp/skywalking-logs/*/e2e-service-provider.log
+ Parser my-log-format
+[FILTER]
+ Name lua
+ Match *
+ Script fluent-bit-script.lua
+ Call rewrite_body
+[OUTPUT]
+ Name stdout
+ Match *
+ Format json
+[OUTPUT]
+ Name http
+ Match *
+ Host oap
+ Port 12800
+ URI /v3/logs
+ Format json
diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/FluentBitE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/FluentBitE2E.java
new file mode 100644
index 0000000..3df2287
--- /dev/null
+++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/FluentBitE2E.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.e2e.log;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.e2e.annotation.ContainerHostAndPort;
+import org.apache.skywalking.e2e.annotation.DockerCompose;
+import org.apache.skywalking.e2e.base.SkyWalkingE2E;
+import org.apache.skywalking.e2e.common.HostAndPort;
+import org.apache.skywalking.e2e.retryable.RetryableTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.testcontainers.containers.DockerComposeContainer;
+
+@Slf4j
+@SkyWalkingE2E
+public class FluentBitE2E extends LogE2E {
+ @SuppressWarnings("unused")
+ @DockerCompose({
+ "docker/log/docker-compose.fluentbit.yml"
+ })
+ protected DockerComposeContainer<?> justForSideEffects;
+
+ @SuppressWarnings("unused")
+ @ContainerHostAndPort(name = "oap", port = 12800)
+ protected HostAndPort swWebappHostPort;
+
+ @SuppressWarnings("unused")
+ @ContainerHostAndPort(name = "provider", port = 9090)
+ protected HostAndPort providerHostPort;
+
+ @Override
+ @BeforeAll
+ public void setUp() throws Exception {
+ queryClient(swWebappHostPort);
+ trafficController(providerHostPort, "/file/logs/trigger");
+ }
+
+ @Override
+ @RetryableTest
+ public void verifyService() throws Exception {
+ super.verifyService();
+ }
+
+ @RetryableTest
+ public void verifyFluentBitLog4jLog() throws Exception {
+ verify("log4j fileLogger");
+ }
+
+ @RetryableTest
+ public void verifyFluentBitLog4j2Log() throws Exception {
+ verify("log4j2 fileLogger");
+ }
+
+ @RetryableTest
+ public void verifyFluentBitLogbackLog() throws Exception {
+ verify("logback fileLogger");
+ }
+}
diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java
index 0ed77ab..2ca0b24 100644
--- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java
+++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java
@@ -51,11 +51,11 @@ public class LogE2E extends SkyWalkingTestAdapter {
@DockerCompose({
"docker/log/docker-compose.${SW_STORAGE}.yml"
})
- private DockerComposeContainer<?> justForSideEffects;
+ protected DockerComposeContainer<?> justForSideEffects;
@SuppressWarnings("unused")
@ContainerHostAndPort(name = "oap", port = 12800)
- private HostAndPort swWebappHostPort;
+ protected HostAndPort swWebappHostPort;
@SuppressWarnings("unused")
@ContainerHostAndPort(name = "provider", port = 9090)
@@ -92,42 +92,26 @@ public class LogE2E extends SkyWalkingTestAdapter {
@RetryableTest
public void verifyLog4jLog() throws Exception {
- LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")
- .start(startTime)
- .end(Times.now())
- .addTag("level", "INFO");
- if (graphql.supportQueryLogsByKeywords()) {
- logsQuery.keywordsOfContent("log4j message");
- }
- final List<Log> logs = graphql.logs(logsQuery);
- LOGGER.info("logs: {}", logs);
-
- load("expected/log/logs.yml").as(LogsMatcher.class).verifyLoosely(logs);
+ verify("log4j message");
}
@RetryableTest
public void verifyLog4j2Log() throws Exception {
- LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")
- .start(startTime)
- .end(Times.now())
- .addTag("level", "INFO");
- if (graphql.supportQueryLogsByKeywords()) {
- logsQuery.keywordsOfContent("log4j2 message");
- }
- final List<Log> logs = graphql.logs(logsQuery);
- LOGGER.info("logs: {}", logs);
-
- load("expected/log/logs.yml").as(LogsMatcher.class).verifyLoosely(logs);
+ verify("log4j2 message");
}
@RetryableTest
public void verifyLogbackLog() throws Exception {
+ verify("logback message");
+ }
+
+ protected void verify(String keyword) throws Exception {
LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")
.start(startTime)
.end(Times.now())
.addTag("level", "INFO");
if (graphql.supportQueryLogsByKeywords()) {
- logsQuery.keywordsOfContent("logback message");
+ logsQuery.keywordsOfContent("keyword");
}
final List<Log> logs = graphql.logs(logsQuery);
LOGGER.info("logs: {}", logs);
diff --git a/test/e2e/pom.xml b/test/e2e/pom.xml
index 7245949..4d40e60 100644
--- a/test/e2e/pom.xml
+++ b/test/e2e/pom.xml
@@ -50,6 +50,7 @@
(if you build from command line, just use ./mvnw -Dsw.version=x.y.z),
but make sure not to check it into the code base / git
-->
+ 8.6.0
</sw.version>
<java.version>1.8</java.version>