You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2021/02/04 12:44:56 UTC
[skywalking] branch als/save-error-logs created (now 4d42173)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a change to branch als/save-error-logs
in repository https://gitbox.apache.org/repos/asf/skywalking.git.
at 4d42173 Save Envoy http access logs when error occurs
This branch includes the following new commits:
new 4d42173 Save Envoy http access logs when error occurs
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[skywalking] 01/01: Save Envoy http access logs when error occurs
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch als/save-error-logs
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 4d421739df1537d3e710ae48bdbbe6cb494961f4
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Thu Feb 4 20:37:11 2021 +0800
Save Envoy http access logs when error occurs
---
CHANGES.md | 1 +
.../oap/server/library/util/ProtoBufJsonUtils.java | 11 ++-
.../envoy-metrics-receiver-plugin/pom.xml | 5 ++
.../envoy/AccessLogServiceGRPCHandler.java | 15 +++-
.../server/receiver/envoy/ErrorLogsAnalyzer.java | 97 ++++++++++++++++++++++
5 files changed, 124 insertions(+), 5 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index e7fbef0..f0d9be6 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -15,6 +15,7 @@ Release Notes.
#### OAP-Backend
* Allow user-defined `JAVA_OPTS` in the startup script.
* Metrics combination API supports abandoning results.
+* Save Envoy http access logs when error occurs.
#### UI
diff --git a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ProtoBufJsonUtils.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ProtoBufJsonUtils.java
index 2f35d18..0bb837f 100644
--- a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ProtoBufJsonUtils.java
+++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ProtoBufJsonUtils.java
@@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.library.util;
+import com.google.protobuf.BytesValue;
import com.google.protobuf.Message;
import com.google.protobuf.util.JsonFormat;
import java.io.IOException;
@@ -25,7 +26,15 @@ import java.io.IOException;
public class ProtoBufJsonUtils {
public static String toJSON(Message sourceMessage) throws IOException {
- return JsonFormat.printer().print(sourceMessage);
+ return JsonFormat.printer()
+ .usingTypeRegistry(
+ JsonFormat
+ .TypeRegistry
+ .newBuilder()
+ .add(BytesValue.getDescriptor())
+ .build()
+ )
+ .print(sourceMessage);
}
/**
diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/pom.xml
index e430882..43cfb53 100644
--- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/pom.xml
+++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/pom.xml
@@ -41,6 +41,11 @@
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
+ <artifactId>log-analyzer</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.skywalking</groupId>
<artifactId>skywalking-mesh-receiver-plugin</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/AccessLogServiceGRPCHandler.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/AccessLogServiceGRPCHandler.java
index 4d47072..7645b72 100644
--- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/AccessLogServiceGRPCHandler.java
+++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/AccessLogServiceGRPCHandler.java
@@ -30,7 +30,6 @@ import org.apache.skywalking.aop.server.receiver.mesh.TelemetryDataDispatcher;
import org.apache.skywalking.apm.network.servicemesh.v3.ServiceMeshMetric;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
-import org.apache.skywalking.oap.server.library.util.CollectionUtils;
import org.apache.skywalking.oap.server.receiver.envoy.als.ALSHTTPAnalysis;
import org.apache.skywalking.oap.server.receiver.envoy.als.Role;
import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
@@ -41,6 +40,8 @@ import org.apache.skywalking.oap.server.telemetry.api.MetricsTag;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.skywalking.oap.server.library.util.CollectionUtils.isNotEmpty;
+
public class AccessLogServiceGRPCHandler extends AccessLogServiceGrpc.AccessLogServiceImplBase {
private static final Logger LOGGER = LoggerFactory.getLogger(AccessLogServiceGRPCHandler.class);
private final List<ALSHTTPAnalysis> envoyHTTPAnalysisList;
@@ -48,6 +49,7 @@ public class AccessLogServiceGRPCHandler extends AccessLogServiceGrpc.AccessLogS
private final CounterMetrics counter;
private final HistogramMetrics histogram;
private final CounterMetrics sourceDispatcherCounter;
+ private final ErrorLogsAnalyzer errorLogsAnalyzer;
public AccessLogServiceGRPCHandler(ModuleManager manager,
EnvoyMetricReceiverConfig config) throws ModuleStartException {
@@ -64,6 +66,8 @@ public class AccessLogServiceGRPCHandler extends AccessLogServiceGrpc.AccessLogS
LOGGER.debug("envoy HTTP analysis: " + envoyHTTPAnalysisList);
+ errorLogsAnalyzer = new ErrorLogsAnalyzer(manager);
+
MetricsCreator metricCreator = manager.find(TelemetryModule.NAME).provider().getService(MetricsCreator.class);
counter = metricCreator.createCounter(
"envoy_als_in_count", "The count of envoy ALS metric received", MetricsTag.EMPTY_KEY,
@@ -117,15 +121,18 @@ public class AccessLogServiceGRPCHandler extends AccessLogServiceGrpc.AccessLogS
List<ServiceMeshMetric.Builder> sourceResult = new ArrayList<>();
for (final HTTPAccessLogEntry log : logs.getLogEntryList()) {
+ List<ServiceMeshMetric.Builder> result = null;
for (ALSHTTPAnalysis analysis : envoyHTTPAnalysisList) {
- final List<ServiceMeshMetric.Builder> result =
- analysis.analysis(identifier, log, role);
- if (CollectionUtils.isNotEmpty(result)) {
+ result = analysis.analysis(identifier, log, role);
+ if (isNotEmpty(result)) {
// Once the analysis has results, don't need to continue analysis in lower priority analyzers.
sourceResult.addAll(result);
break;
}
}
+ if (isNotEmpty(result)) {
+ errorLogsAnalyzer.analyze(result, log);
+ }
}
sourceDispatcherCounter.inc(sourceResult.size());
diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/ErrorLogsAnalyzer.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/ErrorLogsAnalyzer.java
new file mode 100644
index 0000000..dac9e3a
--- /dev/null
+++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/ErrorLogsAnalyzer.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.receiver.envoy;
+
+import com.google.protobuf.TextFormat;
+import io.envoyproxy.envoy.data.accesslog.v3.HTTPAccessLogEntry;
+import java.io.IOException;
+import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.apm.network.common.v3.DetectPoint;
+import org.apache.skywalking.apm.network.logging.v3.JSONLog;
+import org.apache.skywalking.apm.network.logging.v3.LogData;
+import org.apache.skywalking.apm.network.logging.v3.LogDataBody;
+import org.apache.skywalking.apm.network.servicemesh.v3.ServiceMeshMetric;
+import org.apache.skywalking.oap.log.analyzer.module.LogAnalyzerModule;
+import org.apache.skywalking.oap.log.analyzer.provider.log.ILogAnalyzerService;
+import org.apache.skywalking.oap.server.library.module.ModuleManager;
+
+import static org.apache.skywalking.apm.util.StringUtil.isNotBlank;
+import static org.apache.skywalking.oap.server.library.util.ProtoBufJsonUtils.toJSON;
+
+/**
+ * {@code ErrorLogsAnalyzer} analyzes the error logs and persists them to the log system.
+ */
+@Slf4j
+public class ErrorLogsAnalyzer {
+ private final ILogAnalyzerService logAnalyzerService;
+
+ public ErrorLogsAnalyzer(final ModuleManager manager) {
+ logAnalyzerService = manager.find(LogAnalyzerModule.NAME)
+ .provider()
+ .getService(ILogAnalyzerService.class);
+ }
+
+ public void analyze(final List<ServiceMeshMetric.Builder> result,
+ final HTTPAccessLogEntry logEntry) {
+ result.stream()
+ .filter(this::hasError)
+ .findFirst()
+ .ifPresent(metrics -> {
+ try {
+ final LogData logData = convertToLogData(logEntry, metrics);
+ logAnalyzerService.doAnalysis(logData);
+ } catch (IOException e) {
+ log.error(
+ "Failed to parse error log entry to log data: {}",
+ TextFormat.shortDebugString(logEntry),
+ e
+ );
+ }
+ });
+ }
+
+ public LogData convertToLogData(final HTTPAccessLogEntry logEntry,
+ final ServiceMeshMetric.Builder metrics) throws IOException {
+ final boolean isServerSide = metrics.getDetectPoint() == DetectPoint.server;
+ final String svc = isServerSide ? metrics.getDestServiceName() : metrics.getSourceServiceName();
+ final String svcInst = isServerSide ? metrics.getDestServiceInstance() : metrics.getSourceServiceInstance();
+
+ return LogData
+ .newBuilder()
+ .setService(svc)
+ .setServiceInstance(svcInst)
+ .setEndpoint(metrics.getEndpoint())
+ .setTimestamp(metrics.getEndTime())
+ .setBody(
+ LogDataBody
+ .newBuilder()
+ .setJson(
+ JSONLog
+ .newBuilder()
+ .setJson(toJSON(logEntry))
+ )
+ )
+ .build();
+ }
+
+ private boolean hasError(final ServiceMeshMetric.Builder it) {
+ return isNotBlank(it.getInternalErrorCode());
+ }
+}