You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2021/02/04 12:44:57 UTC
[skywalking] 01/01: Save Envoy http access logs when error occurs
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch als/save-error-logs
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 4d421739df1537d3e710ae48bdbbe6cb494961f4
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Thu Feb 4 20:37:11 2021 +0800
Save Envoy http access logs when error occurs
---
CHANGES.md | 1 +
.../oap/server/library/util/ProtoBufJsonUtils.java | 11 ++-
.../envoy-metrics-receiver-plugin/pom.xml | 5 ++
.../envoy/AccessLogServiceGRPCHandler.java | 15 +++-
.../server/receiver/envoy/ErrorLogsAnalyzer.java | 97 ++++++++++++++++++++++
5 files changed, 124 insertions(+), 5 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index e7fbef0..f0d9be6 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -15,6 +15,7 @@ Release Notes.
#### OAP-Backend
* Allow user-defined `JAVA_OPTS` in the startup script.
* Metrics combination API supports abandoning results.
+* Save Envoy http access logs when error occurs.
#### UI
diff --git a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ProtoBufJsonUtils.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ProtoBufJsonUtils.java
index 2f35d18..0bb837f 100644
--- a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ProtoBufJsonUtils.java
+++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ProtoBufJsonUtils.java
@@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.library.util;
+import com.google.protobuf.BytesValue;
import com.google.protobuf.Message;
import com.google.protobuf.util.JsonFormat;
import java.io.IOException;
@@ -25,7 +26,15 @@ import java.io.IOException;
public class ProtoBufJsonUtils {
public static String toJSON(Message sourceMessage) throws IOException {
- return JsonFormat.printer().print(sourceMessage);
+ return JsonFormat.printer()
+ .usingTypeRegistry(
+ JsonFormat
+ .TypeRegistry
+ .newBuilder()
+ .add(BytesValue.getDescriptor())
+ .build()
+ )
+ .print(sourceMessage);
}
/**
diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/pom.xml
index e430882..43cfb53 100644
--- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/pom.xml
+++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/pom.xml
@@ -41,6 +41,11 @@
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
+ <artifactId>log-analyzer</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.skywalking</groupId>
<artifactId>skywalking-mesh-receiver-plugin</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/AccessLogServiceGRPCHandler.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/AccessLogServiceGRPCHandler.java
index 4d47072..7645b72 100644
--- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/AccessLogServiceGRPCHandler.java
+++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/AccessLogServiceGRPCHandler.java
@@ -30,7 +30,6 @@ import org.apache.skywalking.aop.server.receiver.mesh.TelemetryDataDispatcher;
import org.apache.skywalking.apm.network.servicemesh.v3.ServiceMeshMetric;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
-import org.apache.skywalking.oap.server.library.util.CollectionUtils;
import org.apache.skywalking.oap.server.receiver.envoy.als.ALSHTTPAnalysis;
import org.apache.skywalking.oap.server.receiver.envoy.als.Role;
import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
@@ -41,6 +40,8 @@ import org.apache.skywalking.oap.server.telemetry.api.MetricsTag;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.skywalking.oap.server.library.util.CollectionUtils.isNotEmpty;
+
public class AccessLogServiceGRPCHandler extends AccessLogServiceGrpc.AccessLogServiceImplBase {
private static final Logger LOGGER = LoggerFactory.getLogger(AccessLogServiceGRPCHandler.class);
private final List<ALSHTTPAnalysis> envoyHTTPAnalysisList;
@@ -48,6 +49,7 @@ public class AccessLogServiceGRPCHandler extends AccessLogServiceGrpc.AccessLogS
private final CounterMetrics counter;
private final HistogramMetrics histogram;
private final CounterMetrics sourceDispatcherCounter;
+ private final ErrorLogsAnalyzer errorLogsAnalyzer;
public AccessLogServiceGRPCHandler(ModuleManager manager,
EnvoyMetricReceiverConfig config) throws ModuleStartException {
@@ -64,6 +66,8 @@ public class AccessLogServiceGRPCHandler extends AccessLogServiceGrpc.AccessLogS
LOGGER.debug("envoy HTTP analysis: " + envoyHTTPAnalysisList);
+ errorLogsAnalyzer = new ErrorLogsAnalyzer(manager);
+
MetricsCreator metricCreator = manager.find(TelemetryModule.NAME).provider().getService(MetricsCreator.class);
counter = metricCreator.createCounter(
"envoy_als_in_count", "The count of envoy ALS metric received", MetricsTag.EMPTY_KEY,
@@ -117,15 +121,18 @@ public class AccessLogServiceGRPCHandler extends AccessLogServiceGrpc.AccessLogS
List<ServiceMeshMetric.Builder> sourceResult = new ArrayList<>();
for (final HTTPAccessLogEntry log : logs.getLogEntryList()) {
+ List<ServiceMeshMetric.Builder> result = null;
for (ALSHTTPAnalysis analysis : envoyHTTPAnalysisList) {
- final List<ServiceMeshMetric.Builder> result =
- analysis.analysis(identifier, log, role);
- if (CollectionUtils.isNotEmpty(result)) {
+ result = analysis.analysis(identifier, log, role);
+ if (isNotEmpty(result)) {
// Once the analysis has results, don't need to continue analysis in lower priority analyzers.
sourceResult.addAll(result);
break;
}
}
+ if (isNotEmpty(result)) {
+ errorLogsAnalyzer.analyze(result, log);
+ }
}
sourceDispatcherCounter.inc(sourceResult.size());
diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/ErrorLogsAnalyzer.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/ErrorLogsAnalyzer.java
new file mode 100644
index 0000000..dac9e3a
--- /dev/null
+++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/ErrorLogsAnalyzer.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.receiver.envoy;
+
+import com.google.protobuf.TextFormat;
+import io.envoyproxy.envoy.data.accesslog.v3.HTTPAccessLogEntry;
+import java.io.IOException;
+import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.apm.network.common.v3.DetectPoint;
+import org.apache.skywalking.apm.network.logging.v3.JSONLog;
+import org.apache.skywalking.apm.network.logging.v3.LogData;
+import org.apache.skywalking.apm.network.logging.v3.LogDataBody;
+import org.apache.skywalking.apm.network.servicemesh.v3.ServiceMeshMetric;
+import org.apache.skywalking.oap.log.analyzer.module.LogAnalyzerModule;
+import org.apache.skywalking.oap.log.analyzer.provider.log.ILogAnalyzerService;
+import org.apache.skywalking.oap.server.library.module.ModuleManager;
+
+import static org.apache.skywalking.apm.util.StringUtil.isNotBlank;
+import static org.apache.skywalking.oap.server.library.util.ProtoBufJsonUtils.toJSON;
+
+/**
+ * {@code ErrorLogsAnalyzer} analyzes the error logs and persists them to the log system.
+ */
+@Slf4j
+public class ErrorLogsAnalyzer {
+ private final ILogAnalyzerService logAnalyzerService;
+
+ public ErrorLogsAnalyzer(final ModuleManager manager) {
+ logAnalyzerService = manager.find(LogAnalyzerModule.NAME)
+ .provider()
+ .getService(ILogAnalyzerService.class);
+ }
+
+ public void analyze(final List<ServiceMeshMetric.Builder> result,
+ final HTTPAccessLogEntry logEntry) {
+ result.stream()
+ .filter(this::hasError)
+ .findFirst()
+ .ifPresent(metrics -> {
+ try {
+ final LogData logData = convertToLogData(logEntry, metrics);
+ logAnalyzerService.doAnalysis(logData);
+ } catch (IOException e) {
+ log.error(
+ "Failed to parse error log entry to log data: {}",
+ TextFormat.shortDebugString(logEntry),
+ e
+ );
+ }
+ });
+ }
+
+ public LogData convertToLogData(final HTTPAccessLogEntry logEntry,
+ final ServiceMeshMetric.Builder metrics) throws IOException {
+ final boolean isServerSide = metrics.getDetectPoint() == DetectPoint.server;
+ final String svc = isServerSide ? metrics.getDestServiceName() : metrics.getSourceServiceName();
+ final String svcInst = isServerSide ? metrics.getDestServiceInstance() : metrics.getSourceServiceInstance();
+
+ return LogData
+ .newBuilder()
+ .setService(svc)
+ .setServiceInstance(svcInst)
+ .setEndpoint(metrics.getEndpoint())
+ .setTimestamp(metrics.getEndTime())
+ .setBody(
+ LogDataBody
+ .newBuilder()
+ .setJson(
+ JSONLog
+ .newBuilder()
+ .setJson(toJSON(logEntry))
+ )
+ )
+ .build();
+ }
+
+ private boolean hasError(final ServiceMeshMetric.Builder it) {
+ return isNotBlank(it.getInternalErrorCode());
+ }
+}