You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by yp...@apache.org on 2023/03/04 16:36:25 UTC
[skywalking] branch master updated: Polish aws-firehose-receiver to support setting accessKey (#10484)
This is an automated email from the ASF dual-hosted git repository.
ypg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 4274b6ec5e Polish aws-firehose-receiver to support setting accessKey (#10484)
4274b6ec5e is described below
commit 4274b6ec5e9a20d1ccf19deff28d2876942b50cb
Author: pg.yang <pg...@hotmail.com>
AuthorDate: Sun Mar 5 00:36:11 2023 +0800
Polish aws-firehose-receiver to support setting accessKey (#10484)
---
docs/en/setup/backend/aws-firehose-receiver.md | 1 +
docs/en/setup/backend/configuration-vocabulary.md | 1 +
.../aws/firehose/AWSFirehoseReceiverModuleConfig.java | 1 +
.../aws/firehose/AWSFirehoseReceiverModuleProvider.java | 2 +-
.../receiver/aws/firehose/FirehoseHTTPHandler.java | 17 ++++++++++++++++-
.../server-starter/src/main/resources/application.yml | 1 +
6 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/docs/en/setup/backend/aws-firehose-receiver.md b/docs/en/setup/backend/aws-firehose-receiver.md
index 6f3b783d81..f667c74cda 100644
--- a/docs/en/setup/backend/aws-firehose-receiver.md
+++ b/docs/en/setup/backend/aws-firehose-receiver.md
@@ -26,3 +26,4 @@ CloudWatch metrics with S3 --> CloudWatch Metric Stream (OpenTelemetry formart)
1. Only OpenTelemetry format is supported (refer to [Metric streams output formats](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-formats.html))
2. A proxy(e.g. Nginx, Envoy) is required in front of OAP's Firehose receiver to accept HTTPS requests from AWS Firehose through port `443` (refer to [Amazon Kinesis Data Firehose Delivery Stream HTTP Endpoint Delivery Specifications](https://docs.aws.amazon.com/firehose/latest/dev/httpdeliveryrequestresponse.html).
+3. AWS Firehose receiver support setting accessKey for Kinesis Data Firehose, please refer to [configuration vocabulary](./configuration-vocabulary.md)
diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md
index 8ab8681e6d..68608ce79c 100644
--- a/docs/en/setup/backend/configuration-vocabulary.md
+++ b/docs/en/setup/backend/configuration-vocabulary.md
@@ -319,6 +319,7 @@ The Configuration Vocabulary lists all available configurations provided by `app
| - | - | idleTimeOut | Idle timeout of a connection for keep-alive. [...]
| - | - | acceptQueueSize | Maximum allowed number of open connections [...]
| - | - | maxRequestHeaderSize | Maximum length of all headers in an HTTP/1 response [...]
+| - | - | firehoseAccessKey | The AccessKey of AWS firhose [...]
## Note
diff --git a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleConfig.java b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleConfig.java
index 5955f319d2..4ebedebb68 100644
--- a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleConfig.java
+++ b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleConfig.java
@@ -29,4 +29,5 @@ public class AWSFirehoseReceiverModuleConfig extends ModuleConfig {
private long idleTimeOut = 30000;
private int acceptQueueSize = 0;
private int maxRequestHeaderSize = 8192;
+ private String firehoseAccessKey;
}
diff --git a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleProvider.java b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleProvider.java
index 36dd7e40dd..8f84a4fb0f 100644
--- a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleProvider.java
+++ b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/AWSFirehoseReceiverModuleProvider.java
@@ -82,7 +82,7 @@ public class AWSFirehoseReceiverModuleProvider extends ModuleProvider {
.getService(
OpenTelemetryMetricRequestProcessor.class);
httpServer.addHandler(
- new FirehoseHTTPHandler(processor),
+ new FirehoseHTTPHandler(processor, moduleConfig.getFirehoseAccessKey()),
Collections.singletonList(HttpMethod.POST)
);
}
diff --git a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/FirehoseHTTPHandler.java b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/FirehoseHTTPHandler.java
index 9a11018a94..e03f9dc0c1 100644
--- a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/FirehoseHTTPHandler.java
+++ b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/FirehoseHTTPHandler.java
@@ -21,6 +21,8 @@ import com.google.protobuf.InvalidProtocolBufferException;
import com.linecorp.armeria.common.HttpResponse;
import com.linecorp.armeria.common.HttpStatus;
import com.linecorp.armeria.server.annotation.ConsumesJson;
+import com.linecorp.armeria.server.annotation.Default;
+import com.linecorp.armeria.server.annotation.Header;
import com.linecorp.armeria.server.annotation.Post;
import com.linecorp.armeria.server.annotation.ProducesJson;
import io.opentelemetry.proto.collector.metrics.firehose.v0_7.ExportMetricsServiceRequest;
@@ -28,17 +30,30 @@ import java.io.ByteArrayInputStream;
import java.util.Base64;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.oap.server.library.util.StringUtil;
import org.apache.skywalking.oap.server.receiver.otel.otlp.OpenTelemetryMetricRequestProcessor;
@Slf4j
@AllArgsConstructor
public class FirehoseHTTPHandler {
private final OpenTelemetryMetricRequestProcessor openTelemetryMetricRequestProcessor;
+ private final String firehoseAccessKey;
@Post("/aws/firehose/metrics")
@ConsumesJson
@ProducesJson
- public HttpResponse collectMetrics(final FirehoseReq firehoseReq) {
+ public HttpResponse collectMetrics(final FirehoseReq firehoseReq,
+ @Default @Header(value = "X-Amz-Firehose-Access-Key") String accessKey) {
+
+ if (StringUtil.isNotBlank(firehoseAccessKey) && !firehoseAccessKey.equals(accessKey)) {
+ return HttpResponse.ofJson(
+ HttpStatus.UNAUTHORIZED,
+ new FirehoseRes(firehoseReq.getRequestId(), System.currentTimeMillis(),
+ "AccessKey incorrect, please check your config"
+ )
+ );
+ }
+
try {
for (RequestData record : firehoseReq.getRecords()) {
final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(
diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml
index c79cffeb21..c69f7d92b9 100644
--- a/oap-server/server-starter/src/main/resources/application.yml
+++ b/oap-server/server-starter/src/main/resources/application.yml
@@ -567,3 +567,4 @@ aws-firehose:
idleTimeOut: ${SW_RECEIVER_AWS_FIREHOSE_HTTP_IDLE_TIME_OUT:30000}
acceptQueueSize: ${SW_RECEIVER_AWS_FIREHOSE_HTTP_ACCEPT_QUEUE_SIZE:0}
maxRequestHeaderSize: ${SW_RECEIVER_AWS_FIREHOSE_HTTP_MAX_REQUEST_HEADER_SIZE:8192}
+ firehoseAccessKey: ${SW_RECEIVER_AWS_FIREHOSE_ACCESS_KEY:}