You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ac...@apache.org on 2022/11/02 12:14:41 UTC
[camel-kamelets] 01/05: Create an AWS S3 CDC Source Kamelet
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch 1133
in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
commit 4d4187bee3bb30a6651abb587bfdad6929afdbcb
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Wed Nov 2 12:16:20 2022 +0100
Create an AWS S3 CDC Source Kamelet
Signed-off-by: Andrea Cosentino <an...@gmail.com>
---
kamelets/aws-s3-cdc-source.kamelet.yaml | 217 ++++++++++++++++++++++++++++++++
1 file changed, 217 insertions(+)
diff --git a/kamelets/aws-s3-cdc-source.kamelet.yaml b/kamelets/aws-s3-cdc-source.kamelet.yaml
new file mode 100644
index 00000000..31792f1f
--- /dev/null
+++ b/kamelets/aws-s3-cdc-source.kamelet.yaml
@@ -0,0 +1,217 @@
+apiVersion: camel.apache.org/v1alpha1
+kind: Kamelet
+metadata:
+ name: aws-s3-cdc-source
+ annotations:
+ camel.apache.org/kamelet.support.level: Stable
+ camel.apache.org/catalog.version: "main-SNAPSHOT"
+ camel.apache.org/kamelet.icon: >-
+ data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgeG1sbnM6ZGM9Imh0dHA6Ly9wdXJsLm9yZy9kYy9lbGVtZW50cy8xLjEvIgogICB4bWxuczpjYz0iaHR0cDovL2NyZWF0aXZlY29tbW9ucy5vcmcvbnMjIgogICB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiCiAgIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCiAgIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIKICAgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFRE [...]
+ camel.apache.org/provider: Apache Software Foundation
+ camel.apache.org/kamelet.group: AWS S3 CDC
+ camel.apache.org/keda.type: aws-s3-cdc-queue
+ labels:
+ camel.apache.org/kamelet.type: source
+spec:
+ definition:
+ title: AWS S3 CDC Source
+ description: >-
+ Receive data from AWS SQS subscribed to Eventbridge Bus reporting events related to an S3 bucket or multiple buckets.
+
+ Access Key/Secret Key are the basic method for authenticating to the AWS
+ SQS Service.
+
+ To use this Kamelet you'll need to set up Eventbridge on your bucket and subscribe Eventbridge bus to an SQS Queue.
+
+ For doing this you'll need to enable Evenbridge notification on your bucket and creating a rule on Eventbridge console related to all the events on S3 bucket and pointing to the SQS Queue specified as parameter in this Kamelet.
+ required:
+ - accessKey
+ - secretKey
+ - queueNameOrArn
+ - region
+ type: object
+ properties:
+ queueNameOrArn:
+ title: Queue Name
+ description: The SQS Queue Name or ARN
+ type: string
+ deleteAfterRead:
+ title: Auto-delete Messages
+ description: Delete messages after consuming them
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: true
+ accessKey:
+ title: Access Key
+ description: The access key obtained from AWS.
+ type: string
+ format: password
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:password'
+ - 'urn:camel:group:credentials'
+ - 'urn:keda:authentication:awsAccessKeyID'
+ - 'urn:keda:required'
+ secretKey:
+ title: Secret Key
+ description: The secret key obtained from AWS.
+ type: string
+ format: password
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:password'
+ - 'urn:camel:group:credentials'
+ - 'urn:keda:authentication:awsSecretAccessKey'
+ - 'urn:keda:required'
+ region:
+ title: AWS Region
+ description: The AWS region to access.
+ type: string
+ x-descriptors:
+ - 'urn:keda:metadata:awsRegion'
+ - 'urn:keda:required'
+ enum:
+ - ap-south-1
+ - eu-south-1
+ - us-gov-east-1
+ - me-central-1
+ - ca-central-1
+ - eu-central-1
+ - us-iso-west-1
+ - us-west-1
+ - us-west-2
+ - af-south-1
+ - eu-north-1
+ - eu-west-3
+ - eu-west-2
+ - eu-west-1
+ - ap-northeast-3
+ - ap-northeast-2
+ - ap-northeast-1
+ - me-south-1
+ - sa-east-1
+ - ap-east-1
+ - cn-north-1
+ - us-gov-west-1
+ - ap-southeast-1
+ - ap-southeast-2
+ - us-iso-east-1
+ - ap-southeast-3
+ - us-east-1
+ - us-east-2
+ - cn-northwest-1
+ - us-isob-east-1
+ - aws-global
+ - aws-cn-global
+ - aws-us-gov-global
+ - aws-iso-global
+ - aws-iso-b-global
+ autoCreateQueue:
+ title: Autocreate Queue
+ description: Setting the autocreation of the SQS queue.
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: false
+ amazonAWSHost:
+ title: AWS Host
+ description: The hostname of the Amazon AWS cloud.
+ type: string
+ default: amazonaws.com
+ protocol:
+ title: Protocol
+ description: The underlying protocol used to communicate with SQS
+ type: string
+ example: http or https
+ default: https
+ queueURL:
+ title: Queue URL
+ description: The full SQS Queue URL (required if using KEDA)
+ type: string
+ x-descriptors:
+ - 'urn:keda:metadata:queueURL'
+ - 'urn:keda:required'
+ uriEndpointOverride:
+ title: Overwrite Endpoint URI
+ description: >-
+ The overriding endpoint URI. To use this option, you must also select
+ the `overrideEndpoint` option.
+ type: string
+ overrideEndpoint:
+ title: Endpoint Overwrite
+ description: >-
+ Select this option to override the endpoint URI. To use this option,
+ you must also provide a URI for the `uriEndpointOverride` option.
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: false
+ delay:
+ title: Delay
+ description: The number of milliseconds before the next poll of the selected stream
+ type: integer
+ default: 500
+ greedy:
+ title: Greedy Scheduler
+ description: >-
+ If greedy is enabled, then the polling will happen immediately again,
+ if the previous run polled 1 or more messages.
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: false
+ getObject:
+ title: Greedy Object in Bucket
+ description: >-
+ If getObject is enabled, then the file created in the bucket will be
+ get and returned as body, if not only the event will returned as body.
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: false
+ dependencies:
+ - 'camel:core'
+ - 'camel:aws2-sqs'
+ - 'camel:kamelet'
+ - 'camel:jackson'
+ template:
+ from:
+ uri: 'aws2-sqs:{{queueNameOrArn}}'
+ parameters:
+ autoCreateQueue: '{{autoCreateQueue}}'
+ secretKey: '{{?secretKey}}'
+ accessKey: '{{?accessKey}}'
+ region: '{{region}}'
+ deleteAfterRead: '{{deleteAfterRead}}'
+ amazonAWSHost: '{{?amazonAWSHost}}'
+ protocol: '{{?protocol}}'
+ uriEndpointOverride: '{{?uriEndpointOverride}}'
+ overrideEndpoint: '{{overrideEndpoint}}'
+ delay: '{{delay}}'
+ greedy: '{{greedy}}'
+ steps:
+ - choice:
+ precondition: true
+ when:
+ - simple: '${properties:getObject:true}'
+ steps:
+ - unmarshal:
+ json:
+ library: Jackson
+ unmarshalType: com.fasterxml.jackson.databind.JsonNode
+ - set-property:
+ name: s3-event-name
+ jsonpath: $.detail.reason
+ - choice:
+ when:
+ - simple: '${exchangeProperty.s3-event-name} == "PutObject"'
+ steps:
+ - set-property:
+ name: aws-s3-name
+ jsonpath: $.detail.object.key
+ - set-property:
+ name: aws-s3-bucket
+ jsonpath: $.detail.bucket.name
+ - toD: >-
+ aws2-s3:${exchangeProperty.aws-s3-bucket}?accessKey={{accessKey}}&secretKey={{secretKey}}®ion={{region}}&operation=getObject&keyName=${exchangeProperty.aws-s3-name}
+ - to: 'kamelet:sink'
+