You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ac...@apache.org on 2022/11/24 12:18:19 UTC
[camel-kamelets] 01/02: Added an it-tests for AWS S3 CDC
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch cdc-ex
in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
commit 3909b00a08d8eb29c4f3bfd89c0458d9d426c712
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Thu Nov 24 12:24:57 2022 +0100
Added an it-tests for AWS S3 CDC
Signed-off-by: Andrea Cosentino <an...@gmail.com>
---
it-tests/aws/aws-s3-cdc/source/aws-keys.properties | 2 +
.../aws-s3-cdc/source/aws-s3-cdc-log-it-test.sh | 45 +++++++
it-tests/aws/aws-s3-cdc/source/aws-s3-cdc-log.yaml | 29 +++++
it-tests/aws/aws-s3-cdc/source/example-file.txt | 1 +
it-tests/aws/aws-s3-cdc/source/terraform/main.tf | 130 +++++++++++++++++++++
5 files changed, 207 insertions(+)
diff --git a/it-tests/aws/aws-s3-cdc/source/aws-keys.properties b/it-tests/aws/aws-s3-cdc/source/aws-keys.properties
new file mode 100644
index 00000000..56709101
--- /dev/null
+++ b/it-tests/aws/aws-s3-cdc/source/aws-keys.properties
@@ -0,0 +1,2 @@
+camel.kamelet.aws-s3-cdc-source.accessKey = <access_key>
+camel.kamelet.aws-s3-cdc-source.secretKey = <secret_key>
diff --git a/it-tests/aws/aws-s3-cdc/source/aws-s3-cdc-log-it-test.sh b/it-tests/aws/aws-s3-cdc/source/aws-s3-cdc-log-it-test.sh
new file mode 100755
index 00000000..8daa81c6
--- /dev/null
+++ b/it-tests/aws/aws-s3-cdc/source/aws-s3-cdc-log-it-test.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+if [ $# -ne 1 ]; then
+ echo $0: usage: aws-s3-cdc-log-it-test.sh camel-version
+ exit 1
+fi
+
+camel_version=$1
+
+cd terraform/
+terraform init
+terraform apply -auto-approve
+cd ../
+
+jbang run -Dcamel.jbang.version=$camel_version camel@apache/camel run --local-kamelet-dir=../../../../kamelets/ --properties=aws-keys.properties aws-s3-cdc-log.yaml &
+
+sleep 5
+
+aws s3api put-object --bucket s3-camel-test-123 --key example-file-uploaded-1.txt --body example-file.txt --region eu-west-1
+aws s3api put-object --bucket s3-camel-test-123 --key example-file-uploaded-2.txt --body example-file.txt --region eu-west-1
+aws s3api put-object --bucket s3-camel-test-123 --key example-file-uploaded-3.txt --body example-file.txt --region eu-west-1
+aws s3api put-object --bucket s3-camel-test-123 --key example-file-uploaded-4.txt --body example-file.txt --region eu-west-1
+aws s3api put-object --bucket s3-camel-test-123 --key example-file-uploaded-5.txt --body example-file.txt --region eu-west-1
+
+sleep 20
+
+variable=`jbang run -Dcamel.jbang.version=$camel_version camel@apache/camel get | tail -n +2`
+success=`echo $variable | cut -d' ' -f11`
+fail=`echo $variable | cut -d' ' -f12`
+if [[ $success == 5 && $fail == 0 ]]
+then
+ mkdir -p ../../../tests/
+ echo "Test Successful" > ../../../tests/aws-s3-cdc-log-it-test.result;
+else
+ mkdir -p ../../../tests/
+ echo "Test failed" > ../../../tests/aws-s3-cdc-log-it-test.result;
+fi
+
+jbang run -Dcamel.jbang.version=$camel_version camel@apache/camel stop aws-s3-cdc-log
+
+cd terraform/
+terraform destroy -auto-approve
+cd ../
+
+cat ../../../tests/aws-s3-cdc-log-it-test.result
diff --git a/it-tests/aws/aws-s3-cdc/source/aws-s3-cdc-log.yaml b/it-tests/aws/aws-s3-cdc/source/aws-s3-cdc-log.yaml
new file mode 100644
index 00000000..7ed4ff01
--- /dev/null
+++ b/it-tests/aws/aws-s3-cdc/source/aws-s3-cdc-log.yaml
@@ -0,0 +1,29 @@
+## ---------------------------------------------------------------------------
+## Licensed to the Apache Software Foundation (ASF) under one or more
+## contributor license agreements. See the NOTICE file distributed with
+## this work for additional information regarding copyright ownership.
+## The ASF licenses this file to You under the Apache License, Version 2.0
+## (the "License"); you may not use this file except in compliance with
+## the License. You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+## ---------------------------------------------------------------------------
+
+- route:
+ from:
+ uri: "kamelet:aws-s3-cdc-source"
+ parameters:
+ queueNameOrArn: "sqs-camel-test-123"
+ region: "eu-west-1"
+ getObject: true
+ steps:
+ - to:
+ uri: "kamelet:log-sink"
+ parameters:
+ showStreams: true
diff --git a/it-tests/aws/aws-s3-cdc/source/example-file.txt b/it-tests/aws/aws-s3-cdc/source/example-file.txt
new file mode 100644
index 00000000..9daeafb9
--- /dev/null
+++ b/it-tests/aws/aws-s3-cdc/source/example-file.txt
@@ -0,0 +1 @@
+test
diff --git a/it-tests/aws/aws-s3-cdc/source/terraform/main.tf b/it-tests/aws/aws-s3-cdc/source/terraform/main.tf
new file mode 100644
index 00000000..252c981d
--- /dev/null
+++ b/it-tests/aws/aws-s3-cdc/source/terraform/main.tf
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "~> 3.27"
+ }
+ }
+
+ required_version = ">= 0.14.9"
+}
+
+provider "aws" {
+ profile = "default"
+ region = "eu-west-1"
+}
+
+variable "s3_bucket_name" {
+ type = string
+ default = "s3-camel-test-123"
+}
+
+variable "sqs_queue_name" {
+ type = string
+ default = "sqs-camel-test-123"
+}
+
+
+data "aws_caller_identity" "current" {}
+
+# Create a new S3 bucket
+resource "aws_s3_bucket" "MyS3Bucket" {
+ bucket = var.s3_bucket_name
+ force_destroy = true
+}
+
+# Send notifications to EventBridge for all events in the bucket
+resource "aws_s3_bucket_notification" "MyS3BucketNotification" {
+ bucket = aws_s3_bucket.MyS3Bucket.id
+ eventbridge = true
+}
+
+# Create an EventBridge rule
+resource "aws_cloudwatch_event_rule" "MyEventRule" {
+ description = "Object create events on bucket s3://${aws_s3_bucket.MyS3Bucket.id}"
+ event_pattern = <<EOF
+{
+ "source": [
+ "aws.s3"
+ ],
+ "detail": {
+ "bucket": {
+ "name": ["${aws_s3_bucket.MyS3Bucket.id}"]
+ }
+ }
+}
+EOF
+}
+
+# Set the SNS topic as a target of the EventBridge rule
+resource "aws_cloudwatch_event_target" "MyEventRuleTarget" {
+ rule = aws_cloudwatch_event_rule.MyEventRule.name
+ arn = aws_sqs_queue.sqs-queue.arn
+}
+
+# Create a new SQS queue
+resource "aws_sqs_queue" "sqs-queue" {
+ name = var.sqs_queue_name
+}
+
+# Allow EventBridge to publish to the SQS queue
+resource "aws_sqs_queue_policy" "MySQSQueuePolicy" {
+ queue_url = aws_sqs_queue.sqs-queue.id
+ policy = <<POLICY
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Sid": "AWSEventsPermission",
+ "Effect": "Allow",
+ "Principal": {
+ "Service": "events.amazonaws.com"
+ },
+ "Action": "sqs:SendMessage",
+ "Resource": "${aws_sqs_queue.sqs-queue.arn}",
+ "Condition": {
+ "ArnEquals": {
+ "aws:SourceArn": "${aws_cloudwatch_event_rule.MyEventRule.arn}"
+ }
+ }
+ }
+ ]
+}
+POLICY
+}
+
+# Display the EventBridge rule, S3 bucket and SQS queue
+output "EventBridge-Rule-Name" {
+ value = aws_cloudwatch_event_rule.MyEventRule.name
+ description = "The EventBridge Rule Name"
+}
+output "S3-Bucket" {
+ value = aws_s3_bucket.MyS3Bucket.id
+ description = "The S3 Bucket"
+}
+output "SQS-Queue-Name" {
+ value = aws_sqs_queue.sqs-queue.name
+ description = "The SQS Queue Name"
+}
+output "SQS-Queue-ARN" {
+ value = aws_sqs_queue.sqs-queue.arn
+ description = "The SQS Queue Arn"
+}
+