You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ac...@apache.org on 2022/11/02 12:14:40 UTC
[camel-kamelets] branch 1133 created (now e9421b8b)
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a change to branch 1133
in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
at e9421b8b Create an AWS S3 CDC Source Kamelet
This branch includes the following new commits:
new 4d4187be Create an AWS S3 CDC Source Kamelet
new ea742601 Create an AWS S3 CDC Source Kamelet
new 1918a2ea Create an AWS S3 CDC Source Kamelet
new 2371a741 Create an AWS S3 CDC Source Kamelet
new e9421b8b Create an AWS S3 CDC Source Kamelet
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[camel-kamelets] 05/05: Create an AWS S3 CDC Source Kamelet
Posted by ac...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch 1133
in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
commit e9421b8bea852340e7d543ab1ed435c25cc3c688
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Wed Nov 2 13:01:36 2022 +0100
Create an AWS S3 CDC Source Kamelet
Signed-off-by: Andrea Cosentino <an...@gmail.com>
---
.../src/main/resources/kamelets/aws-s3-cdc-source.kamelet.yaml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/library/camel-kamelets/src/main/resources/kamelets/aws-s3-cdc-source.kamelet.yaml b/library/camel-kamelets/src/main/resources/kamelets/aws-s3-cdc-source.kamelet.yaml
index 31792f1f..e2795608 100644
--- a/library/camel-kamelets/src/main/resources/kamelets/aws-s3-cdc-source.kamelet.yaml
+++ b/library/camel-kamelets/src/main/resources/kamelets/aws-s3-cdc-source.kamelet.yaml
@@ -171,6 +171,8 @@ spec:
dependencies:
- 'camel:core'
- 'camel:aws2-sqs'
+ - 'camel:aws2-s3'
+ - 'camel:jsonpath'
- 'camel:kamelet'
- 'camel:jackson'
template:
[camel-kamelets] 03/05: Create an AWS S3 CDC Source Kamelet
Posted by ac...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch 1133
in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
commit 1918a2ea30f3fb1cb87e89319f349ece146df836
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Wed Nov 2 12:48:01 2022 +0100
Create an AWS S3 CDC Source Kamelet
Signed-off-by: Andrea Cosentino <an...@gmail.com>
---
script/validator/validator.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/script/validator/validator.go b/script/validator/validator.go
index 30d82161..33ad60c5 100644
--- a/script/validator/validator.go
+++ b/script/validator/validator.go
@@ -410,7 +410,7 @@ func listKamelets(dir string) []KameletInfo {
func verifyUsedParams(kamelets []KameletInfo) (errors []error) {
for _, k := range kamelets {
- if (k.FileName != "../../kamelets/azure-storage-blob-source.kamelet.yaml") {
+ if (k.FileName != "../../kamelets/azure-storage-blob-source.kamelet.yaml" && k.FileName != "../../kamelets/aws-s3-cdc-source.kamelet.yaml") {
used := getUsedParams(k.Kamelet)
declared := getDeclaredParams(k.Kamelet)
for p := range used {
[camel-kamelets] 04/05: Create an AWS S3 CDC Source Kamelet
Posted by ac...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch 1133
in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
commit 2371a741c604f7f2cca41bf6ce8729192374a365
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Wed Nov 2 13:01:02 2022 +0100
Create an AWS S3 CDC Source Kamelet
Signed-off-by: Andrea Cosentino <an...@gmail.com>
---
kamelets/aws-s3-cdc-source.kamelet.yaml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/kamelets/aws-s3-cdc-source.kamelet.yaml b/kamelets/aws-s3-cdc-source.kamelet.yaml
index 31792f1f..e2795608 100644
--- a/kamelets/aws-s3-cdc-source.kamelet.yaml
+++ b/kamelets/aws-s3-cdc-source.kamelet.yaml
@@ -171,6 +171,8 @@ spec:
dependencies:
- 'camel:core'
- 'camel:aws2-sqs'
+ - 'camel:aws2-s3'
+ - 'camel:jsonpath'
- 'camel:kamelet'
- 'camel:jackson'
template:
[camel-kamelets] 02/05: Create an AWS S3 CDC Source Kamelet
Posted by ac...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch 1133
in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
commit ea742601fe1de47a0098fb52dbf5d72738be809a
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Wed Nov 2 12:45:33 2022 +0100
Create an AWS S3 CDC Source Kamelet
Signed-off-by: Andrea Cosentino <an...@gmail.com>
---
.../kamelets/aws-s3-cdc-source.kamelet.yaml | 217 +++++++++++++++++++++
1 file changed, 217 insertions(+)
diff --git a/library/camel-kamelets/src/main/resources/kamelets/aws-s3-cdc-source.kamelet.yaml b/library/camel-kamelets/src/main/resources/kamelets/aws-s3-cdc-source.kamelet.yaml
new file mode 100644
index 00000000..31792f1f
--- /dev/null
+++ b/library/camel-kamelets/src/main/resources/kamelets/aws-s3-cdc-source.kamelet.yaml
@@ -0,0 +1,217 @@
+apiVersion: camel.apache.org/v1alpha1
+kind: Kamelet
+metadata:
+ name: aws-s3-cdc-source
+ annotations:
+ camel.apache.org/kamelet.support.level: Stable
+ camel.apache.org/catalog.version: "main-SNAPSHOT"
+ camel.apache.org/kamelet.icon: >-
+ data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgeG1sbnM6ZGM9Imh0dHA6Ly9wdXJsLm9yZy9kYy9lbGVtZW50cy8xLjEvIgogICB4bWxuczpjYz0iaHR0cDovL2NyZWF0aXZlY29tbW9ucy5vcmcvbnMjIgogICB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiCiAgIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCiAgIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIKICAgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFRE [...]
+ camel.apache.org/provider: Apache Software Foundation
+ camel.apache.org/kamelet.group: AWS S3 CDC
+ camel.apache.org/keda.type: aws-s3-cdc-queue
+ labels:
+ camel.apache.org/kamelet.type: source
+spec:
+ definition:
+ title: AWS S3 CDC Source
+ description: >-
+ Receive data from AWS SQS subscribed to Eventbridge Bus reporting events related to an S3 bucket or multiple buckets.
+
+ Access Key/Secret Key are the basic method for authenticating to the AWS
+ SQS Service.
+
+ To use this Kamelet you'll need to set up Eventbridge on your bucket and subscribe Eventbridge bus to an SQS Queue.
+
+ For doing this you'll need to enable Evenbridge notification on your bucket and creating a rule on Eventbridge console related to all the events on S3 bucket and pointing to the SQS Queue specified as parameter in this Kamelet.
+ required:
+ - accessKey
+ - secretKey
+ - queueNameOrArn
+ - region
+ type: object
+ properties:
+ queueNameOrArn:
+ title: Queue Name
+ description: The SQS Queue Name or ARN
+ type: string
+ deleteAfterRead:
+ title: Auto-delete Messages
+ description: Delete messages after consuming them
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: true
+ accessKey:
+ title: Access Key
+ description: The access key obtained from AWS.
+ type: string
+ format: password
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:password'
+ - 'urn:camel:group:credentials'
+ - 'urn:keda:authentication:awsAccessKeyID'
+ - 'urn:keda:required'
+ secretKey:
+ title: Secret Key
+ description: The secret key obtained from AWS.
+ type: string
+ format: password
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:password'
+ - 'urn:camel:group:credentials'
+ - 'urn:keda:authentication:awsSecretAccessKey'
+ - 'urn:keda:required'
+ region:
+ title: AWS Region
+ description: The AWS region to access.
+ type: string
+ x-descriptors:
+ - 'urn:keda:metadata:awsRegion'
+ - 'urn:keda:required'
+ enum:
+ - ap-south-1
+ - eu-south-1
+ - us-gov-east-1
+ - me-central-1
+ - ca-central-1
+ - eu-central-1
+ - us-iso-west-1
+ - us-west-1
+ - us-west-2
+ - af-south-1
+ - eu-north-1
+ - eu-west-3
+ - eu-west-2
+ - eu-west-1
+ - ap-northeast-3
+ - ap-northeast-2
+ - ap-northeast-1
+ - me-south-1
+ - sa-east-1
+ - ap-east-1
+ - cn-north-1
+ - us-gov-west-1
+ - ap-southeast-1
+ - ap-southeast-2
+ - us-iso-east-1
+ - ap-southeast-3
+ - us-east-1
+ - us-east-2
+ - cn-northwest-1
+ - us-isob-east-1
+ - aws-global
+ - aws-cn-global
+ - aws-us-gov-global
+ - aws-iso-global
+ - aws-iso-b-global
+ autoCreateQueue:
+ title: Autocreate Queue
+ description: Setting the autocreation of the SQS queue.
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: false
+ amazonAWSHost:
+ title: AWS Host
+ description: The hostname of the Amazon AWS cloud.
+ type: string
+ default: amazonaws.com
+ protocol:
+ title: Protocol
+ description: The underlying protocol used to communicate with SQS
+ type: string
+ example: http or https
+ default: https
+ queueURL:
+ title: Queue URL
+ description: The full SQS Queue URL (required if using KEDA)
+ type: string
+ x-descriptors:
+ - 'urn:keda:metadata:queueURL'
+ - 'urn:keda:required'
+ uriEndpointOverride:
+ title: Overwrite Endpoint URI
+ description: >-
+ The overriding endpoint URI. To use this option, you must also select
+ the `overrideEndpoint` option.
+ type: string
+ overrideEndpoint:
+ title: Endpoint Overwrite
+ description: >-
+ Select this option to override the endpoint URI. To use this option,
+ you must also provide a URI for the `uriEndpointOverride` option.
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: false
+ delay:
+ title: Delay
+ description: The number of milliseconds before the next poll of the selected stream
+ type: integer
+ default: 500
+ greedy:
+ title: Greedy Scheduler
+ description: >-
+ If greedy is enabled, then the polling will happen immediately again,
+ if the previous run polled 1 or more messages.
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: false
+ getObject:
+ title: Greedy Object in Bucket
+ description: >-
+ If getObject is enabled, then the file created in the bucket will be
+ get and returned as body, if not only the event will returned as body.
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: false
+ dependencies:
+ - 'camel:core'
+ - 'camel:aws2-sqs'
+ - 'camel:kamelet'
+ - 'camel:jackson'
+ template:
+ from:
+ uri: 'aws2-sqs:{{queueNameOrArn}}'
+ parameters:
+ autoCreateQueue: '{{autoCreateQueue}}'
+ secretKey: '{{?secretKey}}'
+ accessKey: '{{?accessKey}}'
+ region: '{{region}}'
+ deleteAfterRead: '{{deleteAfterRead}}'
+ amazonAWSHost: '{{?amazonAWSHost}}'
+ protocol: '{{?protocol}}'
+ uriEndpointOverride: '{{?uriEndpointOverride}}'
+ overrideEndpoint: '{{overrideEndpoint}}'
+ delay: '{{delay}}'
+ greedy: '{{greedy}}'
+ steps:
+ - choice:
+ precondition: true
+ when:
+ - simple: '${properties:getObject:true}'
+ steps:
+ - unmarshal:
+ json:
+ library: Jackson
+ unmarshalType: com.fasterxml.jackson.databind.JsonNode
+ - set-property:
+ name: s3-event-name
+ jsonpath: $.detail.reason
+ - choice:
+ when:
+ - simple: '${exchangeProperty.s3-event-name} == "PutObject"'
+ steps:
+ - set-property:
+ name: aws-s3-name
+ jsonpath: $.detail.object.key
+ - set-property:
+ name: aws-s3-bucket
+ jsonpath: $.detail.bucket.name
+ - toD: >-
+ aws2-s3:${exchangeProperty.aws-s3-bucket}?accessKey={{accessKey}}&secretKey={{secretKey}}®ion={{region}}&operation=getObject&keyName=${exchangeProperty.aws-s3-name}
+ - to: 'kamelet:sink'
+
[camel-kamelets] 01/05: Create an AWS S3 CDC Source Kamelet
Posted by ac...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch 1133
in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
commit 4d4187bee3bb30a6651abb587bfdad6929afdbcb
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Wed Nov 2 12:16:20 2022 +0100
Create an AWS S3 CDC Source Kamelet
Signed-off-by: Andrea Cosentino <an...@gmail.com>
---
kamelets/aws-s3-cdc-source.kamelet.yaml | 217 ++++++++++++++++++++++++++++++++
1 file changed, 217 insertions(+)
diff --git a/kamelets/aws-s3-cdc-source.kamelet.yaml b/kamelets/aws-s3-cdc-source.kamelet.yaml
new file mode 100644
index 00000000..31792f1f
--- /dev/null
+++ b/kamelets/aws-s3-cdc-source.kamelet.yaml
@@ -0,0 +1,217 @@
+apiVersion: camel.apache.org/v1alpha1
+kind: Kamelet
+metadata:
+ name: aws-s3-cdc-source
+ annotations:
+ camel.apache.org/kamelet.support.level: Stable
+ camel.apache.org/catalog.version: "main-SNAPSHOT"
+ camel.apache.org/kamelet.icon: >-
+ data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgeG1sbnM6ZGM9Imh0dHA6Ly9wdXJsLm9yZy9kYy9lbGVtZW50cy8xLjEvIgogICB4bWxuczpjYz0iaHR0cDovL2NyZWF0aXZlY29tbW9ucy5vcmcvbnMjIgogICB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiCiAgIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCiAgIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIKICAgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFRE [...]
+ camel.apache.org/provider: Apache Software Foundation
+ camel.apache.org/kamelet.group: AWS S3 CDC
+ camel.apache.org/keda.type: aws-s3-cdc-queue
+ labels:
+ camel.apache.org/kamelet.type: source
+spec:
+ definition:
+ title: AWS S3 CDC Source
+ description: >-
+ Receive data from AWS SQS subscribed to Eventbridge Bus reporting events related to an S3 bucket or multiple buckets.
+
+ Access Key/Secret Key are the basic method for authenticating to the AWS
+ SQS Service.
+
+ To use this Kamelet you'll need to set up Eventbridge on your bucket and subscribe Eventbridge bus to an SQS Queue.
+
+ For doing this you'll need to enable Evenbridge notification on your bucket and creating a rule on Eventbridge console related to all the events on S3 bucket and pointing to the SQS Queue specified as parameter in this Kamelet.
+ required:
+ - accessKey
+ - secretKey
+ - queueNameOrArn
+ - region
+ type: object
+ properties:
+ queueNameOrArn:
+ title: Queue Name
+ description: The SQS Queue Name or ARN
+ type: string
+ deleteAfterRead:
+ title: Auto-delete Messages
+ description: Delete messages after consuming them
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: true
+ accessKey:
+ title: Access Key
+ description: The access key obtained from AWS.
+ type: string
+ format: password
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:password'
+ - 'urn:camel:group:credentials'
+ - 'urn:keda:authentication:awsAccessKeyID'
+ - 'urn:keda:required'
+ secretKey:
+ title: Secret Key
+ description: The secret key obtained from AWS.
+ type: string
+ format: password
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:password'
+ - 'urn:camel:group:credentials'
+ - 'urn:keda:authentication:awsSecretAccessKey'
+ - 'urn:keda:required'
+ region:
+ title: AWS Region
+ description: The AWS region to access.
+ type: string
+ x-descriptors:
+ - 'urn:keda:metadata:awsRegion'
+ - 'urn:keda:required'
+ enum:
+ - ap-south-1
+ - eu-south-1
+ - us-gov-east-1
+ - me-central-1
+ - ca-central-1
+ - eu-central-1
+ - us-iso-west-1
+ - us-west-1
+ - us-west-2
+ - af-south-1
+ - eu-north-1
+ - eu-west-3
+ - eu-west-2
+ - eu-west-1
+ - ap-northeast-3
+ - ap-northeast-2
+ - ap-northeast-1
+ - me-south-1
+ - sa-east-1
+ - ap-east-1
+ - cn-north-1
+ - us-gov-west-1
+ - ap-southeast-1
+ - ap-southeast-2
+ - us-iso-east-1
+ - ap-southeast-3
+ - us-east-1
+ - us-east-2
+ - cn-northwest-1
+ - us-isob-east-1
+ - aws-global
+ - aws-cn-global
+ - aws-us-gov-global
+ - aws-iso-global
+ - aws-iso-b-global
+ autoCreateQueue:
+ title: Autocreate Queue
+ description: Setting the autocreation of the SQS queue.
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: false
+ amazonAWSHost:
+ title: AWS Host
+ description: The hostname of the Amazon AWS cloud.
+ type: string
+ default: amazonaws.com
+ protocol:
+ title: Protocol
+ description: The underlying protocol used to communicate with SQS
+ type: string
+ example: http or https
+ default: https
+ queueURL:
+ title: Queue URL
+ description: The full SQS Queue URL (required if using KEDA)
+ type: string
+ x-descriptors:
+ - 'urn:keda:metadata:queueURL'
+ - 'urn:keda:required'
+ uriEndpointOverride:
+ title: Overwrite Endpoint URI
+ description: >-
+ The overriding endpoint URI. To use this option, you must also select
+ the `overrideEndpoint` option.
+ type: string
+ overrideEndpoint:
+ title: Endpoint Overwrite
+ description: >-
+ Select this option to override the endpoint URI. To use this option,
+ you must also provide a URI for the `uriEndpointOverride` option.
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: false
+ delay:
+ title: Delay
+ description: The number of milliseconds before the next poll of the selected stream
+ type: integer
+ default: 500
+ greedy:
+ title: Greedy Scheduler
+ description: >-
+ If greedy is enabled, then the polling will happen immediately again,
+ if the previous run polled 1 or more messages.
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: false
+ getObject:
+ title: Greedy Object in Bucket
+ description: >-
+ If getObject is enabled, then the file created in the bucket will be
+ get and returned as body, if not only the event will returned as body.
+ type: boolean
+ x-descriptors:
+ - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+ default: false
+ dependencies:
+ - 'camel:core'
+ - 'camel:aws2-sqs'
+ - 'camel:kamelet'
+ - 'camel:jackson'
+ template:
+ from:
+ uri: 'aws2-sqs:{{queueNameOrArn}}'
+ parameters:
+ autoCreateQueue: '{{autoCreateQueue}}'
+ secretKey: '{{?secretKey}}'
+ accessKey: '{{?accessKey}}'
+ region: '{{region}}'
+ deleteAfterRead: '{{deleteAfterRead}}'
+ amazonAWSHost: '{{?amazonAWSHost}}'
+ protocol: '{{?protocol}}'
+ uriEndpointOverride: '{{?uriEndpointOverride}}'
+ overrideEndpoint: '{{overrideEndpoint}}'
+ delay: '{{delay}}'
+ greedy: '{{greedy}}'
+ steps:
+ - choice:
+ precondition: true
+ when:
+ - simple: '${properties:getObject:true}'
+ steps:
+ - unmarshal:
+ json:
+ library: Jackson
+ unmarshalType: com.fasterxml.jackson.databind.JsonNode
+ - set-property:
+ name: s3-event-name
+ jsonpath: $.detail.reason
+ - choice:
+ when:
+ - simple: '${exchangeProperty.s3-event-name} == "PutObject"'
+ steps:
+ - set-property:
+ name: aws-s3-name
+ jsonpath: $.detail.object.key
+ - set-property:
+ name: aws-s3-bucket
+ jsonpath: $.detail.bucket.name
+ - toD: >-
+ aws2-s3:${exchangeProperty.aws-s3-bucket}?accessKey={{accessKey}}&secretKey={{secretKey}}®ion={{region}}&operation=getObject&keyName=${exchangeProperty.aws-s3-name}
+ - to: 'kamelet:sink'
+