You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2023/04/15 10:55:00 UTC
[skywalking] branch master updated: Support no-proxy mode for aws-firehose receiver (#10684)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 8c6bc1b4c5 Support no-proxy mode for aws-firehose receiver (#10684)
8c6bc1b4c5 is described below
commit 8c6bc1b4c5df7aeef5fd1fd90672d63cd3239678
Author: pg.yang <pg...@hotmail.com>
AuthorDate: Sat Apr 15 18:54:47 2023 +0800
Support no-proxy mode for aws-firehose receiver (#10684)
* upgrade armeria to 1.23.1
* Support no-proxy mode for aws-firehose receiver
---
.licenserc.yaml | 6 ++-
dist-material/release-docs/LICENSE | 51 +++++++++++-----------
docs/en/changes/changes.md | 2 +
docs/en/setup/backend/aws-firehose-receiver.md | 4 +-
docs/en/setup/backend/configuration-vocabulary.md | 3 ++
oap-server-bom/pom.xml | 2 +-
.../oap/server/library/server/http/HTTPServer.java | 14 ++++++
.../library/server/http/HTTPServerConfig.java | 2 +
.../firehose/AWSFirehoseReceiverModuleConfig.java | 3 ++
.../AWSFirehoseReceiverModuleProvider.java | 11 ++++-
.../src/main/resources/application.yml | 3 ++
11 files changed, 72 insertions(+), 29 deletions(-)
diff --git a/.licenserc.yaml b/.licenserc.yaml
index 918eda48fa..f91763696f 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -106,7 +106,7 @@ dependency:
version: 2.13.4
license: Apache-2.0
- name: com.fasterxml.jackson.datatype:jackson-datatype-jsr310
- version: 2.14.1
+ version: 2.14.2
license: Apache-2.0
- name: com.graphql-java:graphql-java-extended-scalars
version: 18.1
@@ -129,3 +129,7 @@ dependency:
- name: build.buf.protoc-gen-validate:protoc-gen-validate
version: 0.6.13
license: Apache-2.0
+ - name: com.aayushatharva.brotli4j:service
+ version: 1.11.0
+ license: Apache-2.0
+
diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE
index 4022dd0fbd..712305f0fa 100644
--- a/dist-material/release-docs/LICENSE
+++ b/dist-material/release-docs/LICENSE
@@ -210,18 +210,19 @@ The following components are provided under the Apache-2.0 License. See project
The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/pgv-java-stub/0.6.13 Apache-2.0
https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/protoc-gen-validate/0.6.13 Apache-2.0
- https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/brotli4j/1.8.0 Apache-2.0
+ https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/brotli4j/1.11.0 Apache-2.0
+ https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/service/1.11.0 Apache-2.0
https://mvnrepository.com/artifact/com.alibaba.nacos/nacos-api/1.4.2 Apache-2.0
https://mvnrepository.com/artifact/com.alibaba.nacos/nacos-client/1.4.2 Apache-2.0
https://mvnrepository.com/artifact/com.alibaba.nacos/nacos-common/1.4.2 Apache-2.0
https://mvnrepository.com/artifact/com.ctrip.framework.apollo/apollo-client/1.8.0 Apache-2.0
https://mvnrepository.com/artifact/com.ctrip.framework.apollo/apollo-core/1.8.0 Apache-2.0
- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.14.1 Apache-2.0
- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core/2.14.1 Apache-2.0
- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind/2.14.1 Apache-2.0
+ https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.14.2 Apache-2.0
+ https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core/2.14.2 Apache-2.0
+ https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind/2.14.2 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-guava/2.12.0 Apache-2.0
- https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.14.1 Apache-2.0
- https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.14.1 Apache-2.0
+ https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.14.2 Apache-2.0
+ https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.14.2 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-kotlin/2.13.4 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml/classmate/1.5.1 Apache-2.0
https://mvnrepository.com/artifact/com.google.api.grpc/proto-google-common-protos/2.9.0 Apache-2.0
@@ -235,10 +236,10 @@ The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/com.google.inject/guice/4.1.0 Apache-2.0
https://mvnrepository.com/artifact/com.google.j2objc/j2objc-annotations/1.3 Apache-2.0
https://mvnrepository.com/artifact/com.graphql-java/java-dataloader/3.2.0 Apache-2.0
- https://mvnrepository.com/artifact/com.linecorp.armeria/armeria/1.21.0 Apache-2.0
- https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-graphql/1.21.0 Apache-2.0
- https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-graphql-protocol/1.21.0 Apache-2.0
- https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-protobuf/1.21.0 Apache-2.0
+ https://mvnrepository.com/artifact/com.linecorp.armeria/armeria/1.23.1 Apache-2.0
+ https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-graphql/1.23.1 Apache-2.0
+ https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-graphql-protocol/1.23.1 Apache-2.0
+ https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-protobuf/1.23.1 Apache-2.0
https://mvnrepository.com/artifact/com.orbitz.consul/consul-client/1.5.3 Apache-2.0
https://mvnrepository.com/artifact/com.squareup.okhttp3/logging-interceptor/4.10.0 Apache-2.0
https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp/4.9.0 Apache-2.0
@@ -268,32 +269,32 @@ The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/io.kubernetes/client-java/18.0.0 Apache-2.0
https://mvnrepository.com/artifact/io.kubernetes/client-java-api/18.0.0 Apache-2.0
https://mvnrepository.com/artifact/io.kubernetes/client-java-proto/18.0.0 Apache-2.0
- https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.10.2 Apache-2.0
- https://mvnrepository.com/artifact/io.micrometer/micrometer-core/1.10.2 Apache-2.0
- https://mvnrepository.com/artifact/io.micrometer/micrometer-observation/1.10.2 Apache-2.0
+ https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.10.5 Apache-2.0
+ https://mvnrepository.com/artifact/io.micrometer/micrometer-core/1.10.5 Apache-2.0
+ https://mvnrepository.com/artifact/io.micrometer/micrometer-observation/1.10.5 Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-buffer/4.1.86.Final Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-codec/4.1.86.Final Apache-2.0
- https://mvnrepository.com/artifact/io.netty/netty-codec-dns/4.1.86.Final Apache-2.0
- https://mvnrepository.com/artifact/io.netty/netty-codec-haproxy/4.1.86.Final Apache-2.0
+ https://mvnrepository.com/artifact/io.netty/netty-codec-dns/4.1.91.Final Apache-2.0
+ https://mvnrepository.com/artifact/io.netty/netty-codec-haproxy/4.1.91.Final Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-codec-http/4.1.86.Final Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-codec-http2/4.1.86.Final Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-codec-socks/4.1.86.Final Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-common/4.1.86.Final Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-handler/4.1.86.Final Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-handler-proxy/4.1.86.Final Apache-2.0
- https://mvnrepository.com/artifact/io.netty/netty-resolver/4.1.86.Final Apache-2.0
- https://mvnrepository.com/artifact/io.netty/netty-resolver-dns/4.1.86.Final Apache-2.0
- https://mvnrepository.com/artifact/io.netty/netty-resolver-dns-classes-macos/4.1.86.Final Apache-2.0
- https://mvnrepository.com/artifact/io.netty/netty-resolver-dns-native-macos/4.1.86.Final Apache-2.0
+ https://mvnrepository.com/artifact/io.netty/netty-resolver/4.1.91.Final Apache-2.0
+ https://mvnrepository.com/artifact/io.netty/netty-resolver-dns/4.1.91.Final Apache-2.0
+ https://mvnrepository.com/artifact/io.netty/netty-resolver-dns-classes-macos/4.1.91.Final Apache-2.0
+ https://mvnrepository.com/artifact/io.netty/netty-resolver-dns-native-macos/4.1.91.Final Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-tcnative-boringssl-static/2.0.52.Final Apache-2.0
- https://mvnrepository.com/artifact/io.netty/netty-tcnative-boringssl-static/2.0.54.Final Apache-2.0
- https://mvnrepository.com/artifact/io.netty/netty-tcnative-classes/2.0.54.Final Apache-2.0
- https://mvnrepository.com/artifact/io.netty/netty-transport/4.1.86.Final Apache-2.0
- https://mvnrepository.com/artifact/io.netty/netty-transport-classes-epoll/4.1.86.Final Apache-2.0
+ https://mvnrepository.com/artifact/io.netty/netty-tcnative-boringssl-static/2.0.59.Final Apache-2.0
+ https://mvnrepository.com/artifact/io.netty/netty-tcnative-classes/2.0.59.Final Apache-2.0
+ https://mvnrepository.com/artifact/io.netty/netty-transport/4.1.91.Final Apache-2.0
+ https://mvnrepository.com/artifact/io.netty/netty-transport-classes-epoll/4.1.91.Final Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-transport-native-epoll/4.1.45.Final Apache-2.0
- https://mvnrepository.com/artifact/io.netty/netty-transport-native-epoll/4.1.86.Final Apache-2.0
+ https://mvnrepository.com/artifact/io.netty/netty-transport-native-epoll/4.1.91.Final Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-transport-native-unix-common/4.1.77.Final Apache-2.0
- https://mvnrepository.com/artifact/io.netty/netty-transport-native-unix-common/4.1.86.Final Apache-2.0
+ https://mvnrepository.com/artifact/io.netty/netty-transport-native-unix-common/4.1.91.Final Apache-2.0
https://mvnrepository.com/artifact/io.perfmark/perfmark-api/0.25.0 Apache-2.0
https://mvnrepository.com/artifact/io.prometheus/simpleclient/0.6.0 Apache-2.0
https://mvnrepository.com/artifact/io.prometheus/simpleclient_common/0.6.0 Apache-2.0
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 4049ab19e7..b1b56ec41d 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -32,6 +32,8 @@
* Bump up Kubernetes Java client.
* Support Redis Monitoring.
* Add component ID for amqp, amqp-producer and amqp-consumer.
+* Support no-proxy mode for aws-firehose receiver
+* Bump up armeria to 1.23.1
#### UI
* Revert: cpm5d function. This feature is cancelled from backend.
diff --git a/docs/en/setup/backend/aws-firehose-receiver.md b/docs/en/setup/backend/aws-firehose-receiver.md
index 23155e4f5f..e39efb7fde 100644
--- a/docs/en/setup/backend/aws-firehose-receiver.md
+++ b/docs/en/setup/backend/aws-firehose-receiver.md
@@ -32,5 +32,7 @@ The following blogs demonstrate complete setup process for AWS S3 and API Gatewa
## Notice
1. Only OpenTelemetry format is supported (refer to [Metric streams output formats](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-formats.html))
-2. A proxy(e.g. Nginx, Envoy) is required in front of OAP's Firehose receiver to accept HTTPS requests from AWS Firehose through port `443` (refer to [Amazon Kinesis Data Firehose Delivery Stream HTTP Endpoint Delivery Specifications](https://docs.aws.amazon.com/firehose/latest/dev/httpdeliveryrequestresponse.html).
+2. According to HTTPS requirement by AWS Firehose(refer to [Amazon Kinesis Data Firehose Delivery Stream HTTP Endpoint Delivery Specifications](https://docs.aws.amazon.com/firehose/latest/dev/httpdeliveryrequestresponse.html), users have two options
+ - A proxy(e.g. Nginx, Envoy) is required in front of OAP's Firehose receiver to accept HTTPS requests from AWS Firehose through port `443`. (Recommended based on the general security policy)
+ - Set `aws-firehose/enableTLS=true` with suitable cert/key files through `aws-firehose/tlsKeyPath` and `aws-firehose/tlsCertChainPath` at OAP side to accept requests from firehose directly.
3. AWS Firehose receiver support setting accessKey for Kinesis Data Firehose, please refer to [configuration vocabulary](./configuration-vocabulary.md)
diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md
index d5c3bc4b00..9584ae482d 100644
--- a/docs/en/setup/backend/configuration-vocabulary.md
+++ b/docs/en/setup/backend/configuration-vocabulary.md
@@ -313,6 +313,9 @@ The Configuration Vocabulary lists all available configurations provided by `app
| - | - | acceptQueueSize | Maximum allowed number of open connections [...]
| - | - | maxRequestHeaderSize | Maximum length of all headers in an HTTP/1 response [...]
| - | - | firehoseAccessKey | The AccessKey of AWS firhose [...]
+| - | - | enableTLS | Indicate if enable HTTPS for the server [...]
+| - | - | tlsKeyPath | TLS key path [...]
+| - | - | tlsCertChainPath | TLS certificate chain path [...]
## Note
diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml
index 370246fc8e..d2611c2e22 100644
--- a/oap-server-bom/pom.xml
+++ b/oap-server-bom/pom.xml
@@ -68,7 +68,7 @@
<postgresql.version>42.4.1</postgresql.version>
<jetcd.version>0.5.3</jetcd.version>
<testcontainers.version>1.17.6</testcontainers.version>
- <armeria.version>1.21.0</armeria.version>
+ <armeria.version>1.23.1</armeria.version>
<awaitility.version>3.0.0</awaitility.version>
<httpcore.version>4.4.13</httpcore.version>
<commons-compress.version>1.21</commons-compress.version>
diff --git a/oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/http/HTTPServer.java b/oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/http/HTTPServer.java
index 67de3c2ca0..787d942a36 100644
--- a/oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/http/HTTPServer.java
+++ b/oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/http/HTTPServer.java
@@ -95,6 +95,10 @@ public class HTTPServer implements Server {
sb.maxNumConnections(config.getAcceptQueueSize());
}
+ if (config.isAcceptProxyRequest()) {
+ sb.absoluteUriTransformer(this::transformAbsoluteURI);
+ }
+
log.info("Server root context path: {}", contextPath);
}
@@ -119,4 +123,14 @@ public class HTTPServer implements Server {
public void start() {
sb.build().start().join();
}
+
+ private String transformAbsoluteURI(final String uri) {
+ if (uri.startsWith("https://")) {
+ return uri.substring(uri.indexOf("/", 8));
+ }
+ if (uri.startsWith("http://")) {
+ return uri.substring(uri.indexOf("/", 7));
+ }
+ return uri;
+ }
}
diff --git a/oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/http/HTTPServerConfig.java b/oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/http/HTTPServerConfig.java
index 9d9daa1356..2354815388 100644
--- a/oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/http/HTTPServerConfig.java
+++ b/oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/http/HTTPServerConfig.java
@@ -45,4 +45,6 @@ public class HTTPServerConfig {
private String tlsKeyPath;
private String tlsCertChainPath;
+
+ private boolean acceptProxyRequest;
}
diff --git a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleConfig.java b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleConfig.java
index 4ebedebb68..76cb60aa17 100644
--- a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleConfig.java
+++ b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleConfig.java
@@ -30,4 +30,7 @@ public class AWSFirehoseReceiverModuleConfig extends ModuleConfig {
private int acceptQueueSize = 0;
private int maxRequestHeaderSize = 8192;
private String firehoseAccessKey;
+ private boolean enableTLS = false;
+ private String tlsKeyPath;
+ private String tlsCertChainPath;
}
diff --git a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleProvider.java b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleProvider.java
index 8f84a4fb0f..1ca73ae1b3 100644
--- a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleProvider.java
+++ b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleProvider.java
@@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.receiver.aws.firehose;
import com.linecorp.armeria.common.HttpMethod;
import java.util.Collections;
+import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.library.module.ModuleDefine;
import org.apache.skywalking.oap.server.library.module.ModuleProvider;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
@@ -29,6 +30,7 @@ import org.apache.skywalking.oap.server.library.server.http.HTTPServerConfig;
import org.apache.skywalking.oap.server.receiver.otel.OtelMetricReceiverModule;
import org.apache.skywalking.oap.server.receiver.otel.otlp.OpenTelemetryMetricRequestProcessor;
+@Slf4j
public class AWSFirehoseReceiverModuleProvider extends ModuleProvider {
public static final String NAME = "default";
@@ -67,9 +69,16 @@ public class AWSFirehoseReceiverModuleProvider extends ModuleProvider {
.contextPath(moduleConfig.getContextPath())
.maxThreads(moduleConfig.getMaxThreads())
.idleTimeOut(moduleConfig.getIdleTimeOut())
- .acceptQueueSize(moduleConfig.getAcceptQueueSize())
+ .acceptQueueSize(
+ moduleConfig.getAcceptQueueSize())
.maxRequestHeaderSize(
moduleConfig.getMaxRequestHeaderSize())
+ //set acceptProxyRequest same with enableTLS
+ .acceptProxyRequest(
+ moduleConfig.isEnableTLS())
+ .enableTLS(moduleConfig.isEnableTLS())
+ .tlsKeyPath(moduleConfig.getTlsKeyPath())
+ .tlsCertChainPath(moduleConfig.getTlsCertChainPath())
.build();
httpServer = new HTTPServer(httpServerConfig);
httpServer.initialize();
diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml
index 9aa6c2bcc6..774fd2c4d5 100644
--- a/oap-server/server-starter/src/main/resources/application.yml
+++ b/oap-server/server-starter/src/main/resources/application.yml
@@ -547,3 +547,6 @@ aws-firehose:
acceptQueueSize: ${SW_RECEIVER_AWS_FIREHOSE_HTTP_ACCEPT_QUEUE_SIZE:0}
maxRequestHeaderSize: ${SW_RECEIVER_AWS_FIREHOSE_HTTP_MAX_REQUEST_HEADER_SIZE:8192}
firehoseAccessKey: ${SW_RECEIVER_AWS_FIREHOSE_ACCESS_KEY:}
+ enableTLS: ${SW_RECEIVER_AWS_FIREHOSE_HTTP_ENABLE_TLS:false}
+ tlsKeyPath: ${SW_RECEIVER_AWS_FIREHOSE_HTTP_TLS_KEY_PATH:}
+ tlsCertChainPath: ${SW_RECEIVER_AWS_FIREHOSE_HTTP_TLS_CERT_CHAIN_PATH:}