You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ac...@apache.org on 2022/12/02 08:08:55 UTC
[camel-kamelets] 23/28: Improve CloudEvents output produced by AWS S3 source
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
commit c64f31b0f6ebb7667aca034a7d67954356cde5ff
Author: Christoph Deppisch <cd...@redhat.com>
AuthorDate: Wed Nov 30 13:02:55 2022 +0100
Improve CloudEvents output produced by AWS S3 source
- Align with CloudEvents spec in creating proper event type and source values
- Enable Knative YAKS tests
---
.github/actions/install-knative/action.yml | 26 ++++
.github/actions/install-knative/install-knative.sh | 142 +++++++++++++++++++++
.github/workflows/yaks-tests.yaml | 7 +-
.../aws2/s3/AWS2S3CloudEventOutputType.java | 4 +-
.../aws2/s3/AWS2S3CloudEventOutputTypeTest.java | 4 +-
test/aws-s3/aws-s3-cloudevents.feature | 8 +-
test/aws-s3/aws-s3-knative-binding.feature | 16 ++-
test/aws-s3/aws-s3-knative.feature | 5 +
test/aws-s3/yaks-config.yaml | 2 +-
test/utils/knative-channel-to-log.yaml | 34 +++++
10 files changed, 230 insertions(+), 18 deletions(-)
diff --git a/.github/actions/install-knative/action.yml b/.github/actions/install-knative/action.yml
new file mode 100644
index 00000000..24dd36f9
--- /dev/null
+++ b/.github/actions/install-knative/action.yml
@@ -0,0 +1,26 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+name: install-knative
+description: 'Install Knative serving and eventing'
+runs:
+ using: "composite"
+ steps:
+ - name: Install Knative
+ shell: bash
+ run: |
+ ./.github/actions/install-knative/install-knative.sh
diff --git a/.github/actions/install-knative/install-knative.sh b/.github/actions/install-knative/install-knative.sh
new file mode 100755
index 00000000..8434afc9
--- /dev/null
+++ b/.github/actions/install-knative/install-knative.sh
@@ -0,0 +1,142 @@
+#!/bin/bash
+
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+####
+#
+# Install the knative setup
+#
+####
+
+set -e
+
+# Prerequisites
+sudo wget https://github.com/mikefarah/yq/releases/download/v4.26.1/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
+
+set +e
+
+export SERVING_VERSION=knative-v1.6.0
+export EVENTING_VERSION=knative-v1.6.0
+export KOURIER_VERSION=knative-v1.6.0
+
+apply() {
+ local file="${1:-}"
+ if [ -z "${file}" ]; then
+ echo "Error: Cannot apply. No file."
+ exit 1
+ fi
+
+ kubectl apply --filename ${file}
+ if [ $? != 0 ]; then
+ sleep 5
+ echo "Re-applying ${file} ..."
+ kubectl apply --filename ${file}
+ if [ $? != 0 ]; then
+ echo "Error: Application of resource failed."
+ exit 1
+ fi
+ fi
+}
+
+SERVING_CRDS="https://github.com/knative/serving/releases/download/${SERVING_VERSION}/serving-crds.yaml"
+SERVING_CORE="https://github.com/knative/serving/releases/download/${SERVING_VERSION}/serving-core.yaml"
+KOURIER="https://github.com/knative-sandbox/net-kourier/releases/download/${KOURIER_VERSION}/kourier.yaml"
+EVENTING_CRDS="https://github.com/knative/eventing/releases/download/${EVENTING_VERSION}/eventing-crds.yaml"
+EVENTING_CORE="https://github.com/knative/eventing/releases/download/${EVENTING_VERSION}/eventing-core.yaml"
+IN_MEMORY_CHANNEL="https://github.com/knative/eventing/releases/download/${EVENTING_VERSION}/in-memory-channel.yaml"
+CHANNEL_BROKER="https://github.com/knative/eventing/releases/download/${EVENTING_VERSION}/mt-channel-broker.yaml"
+
+# Serving
+apply "${SERVING_CRDS}"
+
+YAML=$(mktemp serving-core-XXX.yaml)
+curl -L -s ${SERVING_CORE} | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - > ${YAML}
+if [ -s ${YAML} ]; then
+ apply ${YAML}
+ echo "Waiting for pods to be ready in knative-serving (dependency for kourier)"
+ kubectl wait --for=condition=Ready pod --all -n knative-serving --timeout=60s
+else
+ echo "Error: Failed to correctly download ${SERVING_CORE}"
+ exit 1
+fi
+
+# Kourier
+apply "${KOURIER}"
+
+sleep 5
+
+kubectl patch configmap/config-network \
+ --namespace knative-serving \
+ --type merge \
+ --patch '{"data":{"ingress.class":"kourier.ingress.networking.knative.dev"}}'
+if [ $? != 0 ]; then
+ echo "Error: Failed to patch configmap"
+ exit 1
+fi
+
+# Eventing
+apply "${EVENTING_CRDS}"
+
+YAML=$(mktemp eventing-XXX.yaml)
+curl -L -s ${EVENTING_CORE} | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - > ${YAML}
+if [ -s ${YAML} ]; then
+ apply ${YAML}
+else
+ echo "Error: Failed to correctly download ${SERVING_CORE}"
+ exit 1
+fi
+
+# Eventing channels
+YAML=$(mktemp in-memory-XXX.yaml)
+curl -L -s ${IN_MEMORY_CHANNEL} | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - > ${YAML}
+if [ -s ${YAML} ]; then
+ apply ${YAML}
+else
+ echo "Error: Failed to correctly download ${SERVING_CORE}"
+ exit 1
+fi
+
+# Eventing broker
+YAML=$(mktemp channel-broker-XXX.yaml)
+curl -L -s ${CHANNEL_BROKER} | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - > ${YAML}
+if [ -s ${YAML} ]; then
+ apply ${YAML}
+else
+ echo "Error: Failed to correctly download ${SERVING_CORE}"
+ exit 1
+fi
+
+# Eventing sugar controller configuration
+echo "Patching Knative eventing configuration"
+kubectl patch configmap/config-sugar \
+ -n knative-eventing \
+ --type merge \
+ -p '{"data":{"namespace-selector":"{\"matchExpressions\":[{\"key\":\"eventing.knative.dev/injection\",\"operator\":\"In\",\"values\":[\"enabled\"]}]}"}}'
+
+kubectl patch configmap/config-sugar \
+ -n knative-eventing \
+ --type merge \
+ -p '{"data":{"trigger-selector":"{\"matchExpressions\":[{\"key\":\"eventing.knative.dev/injection\",\"operator\":\"In\",\"values\":[\"enabled\"]}]}"}}'
+
+# Wait for installation completed
+echo "Waiting for all pods to be ready in kourier-system"
+kubectl wait --for=condition=Ready pod --all -n kourier-system --timeout=60s
+echo "Waiting for all pods to be ready in knative-serving"
+kubectl wait --for=condition=Ready pod --all -n knative-serving --timeout=60s
+echo "Waiting for all pods to be ready in knative-eventing"
+kubectl wait --for=condition=Ready pod --all -n knative-eventing --timeout=60s
diff --git a/.github/workflows/yaks-tests.yaml b/.github/workflows/yaks-tests.yaml
index 4acd7c8a..73dcec77 100644
--- a/.github/workflows/yaks-tests.yaml
+++ b/.github/workflows/yaks-tests.yaml
@@ -91,6 +91,8 @@ jobs:
kubectl version
kubectl cluster-info
kubectl describe nodes
+ - name: Install Knative
+ uses: ./.github/actions/install-knative
- name: Install Camel K
run: |
# Configure install options
@@ -110,10 +112,7 @@ jobs:
echo "Running tests"
yaks run test/aws-ddb-sink $YAKS_RUN_OPTIONS
- yaks run test/aws-s3/aws-s3-uri-binding.feature $YAKS_RUN_OPTIONS
- yaks run test/aws-s3/aws-s3-source-property-conf.feature $YAKS_RUN_OPTIONS
- yaks run test/aws-s3/aws-s3-source-secret-conf.feature $YAKS_RUN_OPTIONS
- yaks run test/aws-s3/aws-s3-source-uri-conf.feature $YAKS_RUN_OPTIONS
+ yaks run test/aws-s3 $YAKS_RUN_OPTIONS
yaks run test/extract-field-action $YAKS_RUN_OPTIONS
yaks run test/insert-field-action $YAKS_RUN_OPTIONS
diff --git a/library/camel-kamelets-utils/src/main/java/org/apache/camel/kamelets/utils/format/converter/aws2/s3/AWS2S3CloudEventOutputType.java b/library/camel-kamelets-utils/src/main/java/org/apache/camel/kamelets/utils/format/converter/aws2/s3/AWS2S3CloudEventOutputType.java
index d1906f24..4bc87192 100644
--- a/library/camel-kamelets-utils/src/main/java/org/apache/camel/kamelets/utils/format/converter/aws2/s3/AWS2S3CloudEventOutputType.java
+++ b/library/camel-kamelets-utils/src/main/java/org/apache/camel/kamelets/utils/format/converter/aws2/s3/AWS2S3CloudEventOutputType.java
@@ -44,8 +44,8 @@ public class AWS2S3CloudEventOutputType implements DataTypeConverter {
public void convert(Exchange exchange) {
final Map<String, Object> headers = exchange.getMessage().getHeaders();
- headers.put(CAMEL_CLOUD_EVENT_TYPE, "kamelet.aws.s3.source");
- headers.put(CAMEL_CLOUD_EVENT_SOURCE, exchange.getMessage().getHeader(AWS2S3Constants.BUCKET_NAME, String.class));
+ headers.put(CAMEL_CLOUD_EVENT_TYPE, "org.apache.camel.event.aws.s3.getObject");
+ headers.put(CAMEL_CLOUD_EVENT_SOURCE, "aws.s3.bucket." + exchange.getMessage().getHeader(AWS2S3Constants.BUCKET_NAME, String.class));
headers.put(CAMEL_CLOUD_EVENT_SUBJECT, exchange.getMessage().getHeader(AWS2S3Constants.KEY, String.class));
headers.put(CAMEL_CLOUD_EVENT_TIME, getEventTime(exchange));
}
diff --git a/library/camel-kamelets-utils/src/test/java/org/apache/camel/kamelets/utils/format/converter/aws2/s3/AWS2S3CloudEventOutputTypeTest.java b/library/camel-kamelets-utils/src/test/java/org/apache/camel/kamelets/utils/format/converter/aws2/s3/AWS2S3CloudEventOutputTypeTest.java
index f2d41606..084f4c16 100644
--- a/library/camel-kamelets-utils/src/test/java/org/apache/camel/kamelets/utils/format/converter/aws2/s3/AWS2S3CloudEventOutputTypeTest.java
+++ b/library/camel-kamelets-utils/src/test/java/org/apache/camel/kamelets/utils/format/converter/aws2/s3/AWS2S3CloudEventOutputTypeTest.java
@@ -52,9 +52,9 @@ class AWS2S3CloudEventOutputTypeTest {
Assertions.assertTrue(exchange.getMessage().hasHeaders());
Assertions.assertTrue(exchange.getMessage().getHeaders().containsKey(AWS2S3Constants.KEY));
- assertEquals("kamelet.aws.s3.source", exchange.getMessage().getHeader(AWS2S3CloudEventOutputType.CAMEL_CLOUD_EVENT_TYPE));
+ assertEquals("org.apache.camel.event.aws.s3.getObject", exchange.getMessage().getHeader(AWS2S3CloudEventOutputType.CAMEL_CLOUD_EVENT_TYPE));
assertEquals("test1.txt", exchange.getMessage().getHeader(AWS2S3CloudEventOutputType.CAMEL_CLOUD_EVENT_SUBJECT));
- assertEquals("myBucket", exchange.getMessage().getHeader(AWS2S3CloudEventOutputType.CAMEL_CLOUD_EVENT_SOURCE));
+ assertEquals("aws.s3.bucket.myBucket", exchange.getMessage().getHeader(AWS2S3CloudEventOutputType.CAMEL_CLOUD_EVENT_SOURCE));
}
@Test
diff --git a/test/aws-s3/aws-s3-cloudevents.feature b/test/aws-s3/aws-s3-cloudevents.feature
index 1e2f7d1e..52ac84c5 100644
--- a/test/aws-s3/aws-s3-cloudevents.feature
+++ b/test/aws-s3/aws-s3-cloudevents.feature
@@ -20,6 +20,10 @@ Feature: AWS S3 Kamelet - cloud events data type
Given New global Camel context
Given load to Camel registry amazonS3Client.groovy
+ Scenario: Create Knative broker
+ Given create Knative broker default
+ And Knative broker default is running
+
Scenario: Create AWS-S3 Kamelet to Knative binding
Given variable loginfo is "Installed features"
When load KameletBinding aws-s3-to-knative.yaml
@@ -35,8 +39,8 @@ Feature: AWS S3 Kamelet - cloud events data type
Given send Camel exchange to("aws2-s3://${aws.s3.bucketNameOrArn}?amazonS3Client=#amazonS3Client") with body: ${aws.s3.message}
Then expect Knative event data: ${aws.s3.message}
And verify Knative event
- | type | kamelet.aws.s3.source |
- | source | ${aws.s3.bucketNameOrArn} |
+ | type | org.apache.camel.event.aws.s3.getObject |
+ | source | aws.s3.bucket.${aws.s3.bucketNameOrArn} |
| subject | ${aws.s3.key} |
| id | @ignore@ |
diff --git a/test/aws-s3/aws-s3-knative-binding.feature b/test/aws-s3/aws-s3-knative-binding.feature
index c143bbee..e94ab715 100644
--- a/test/aws-s3/aws-s3-knative-binding.feature
+++ b/test/aws-s3/aws-s3-knative-binding.feature
@@ -17,31 +17,33 @@ Feature: AWS S3 Kamelet - binding to Knative
Given New global Camel context
Given load to Camel registry amazonS3Client.groovy
- Scenario: Create Knative broker
+ Scenario: Create Knative broker and channel
Given create Knative broker default
And Knative broker default is running
+ And create Knative channel messages
Scenario: Create AWS-S3 Kamelet to InMemoryChannel binding
Given variable loginfo is "Installed features"
Given load KameletBinding aws-s3-to-knative.yaml
- Given load KameletBinding knative-to-log.yaml
+ Given load KameletBinding knative-channel-to-log.yaml
Then KameletBinding aws-s3-to-knative should be available
- And KameletBinding knative-to-log should be available
+ And KameletBinding knative-channel-to-log should be available
And Camel K integration aws-s3-to-knative is running
- And Camel K integration knative-to-log is running
+ And Camel K integration knative-channel-to-log is running
And Camel K integration aws-s3-to-knative should print ${loginfo}
- And Camel K integration knative-to-log should print ${loginfo}
+ And Camel K integration knative-channel-to-log should print ${loginfo}
Then sleep 10000 ms
Scenario: Verify Kamelet source
Given Camel exchange message header CamelAwsS3Key="${aws.s3.key}"
Given send Camel exchange to("aws2-s3://${aws.s3.bucketNameOrArn}?amazonS3Client=#amazonS3Client") with body: ${aws.s3.message}
- Then Camel K integration knative-to-log should print ${aws.s3.message}
+ Then Camel K integration knative-channel-to-log should print ${aws.s3.message}
Scenario: Remove resources
Given delete KameletBinding aws-s3-to-knative
- Given delete KameletBinding knative-to-log
+ Given delete KameletBinding knative-channel-to-log
Given delete Knative broker default
+ Given delete Knative channel messages
Scenario: Stop container
Given stop LocalStack container
diff --git a/test/aws-s3/aws-s3-knative.feature b/test/aws-s3/aws-s3-knative.feature
index 148ec1d6..fe080fa2 100644
--- a/test/aws-s3/aws-s3-knative.feature
+++ b/test/aws-s3/aws-s3-knative.feature
@@ -20,6 +20,10 @@ Feature: AWS S3 Kamelet - Knative binding
Given New global Camel context
Given load to Camel registry amazonS3Client.groovy
+ Scenario: Create Knative broker
+ Given create Knative broker default
+ And Knative broker default is running
+
Scenario: Create AWS-S3 Kamelet to Knative binding
Given variable loginfo is "Installed features"
When load KameletBinding aws-s3-to-knative.yaml
@@ -42,6 +46,7 @@ Feature: AWS S3 Kamelet - Knative binding
Scenario: Remove Camel K resources
Given delete KameletBinding aws-s3-to-knative
Given delete Kubernetes service event-consumer-service
+ Given delete Knative broker default
Scenario: Stop container
Given stop LocalStack container
diff --git a/test/aws-s3/yaks-config.yaml b/test/aws-s3/yaks-config.yaml
index 0d70ba75..4ef910c4 100644
--- a/test/aws-s3/yaks-config.yaml
+++ b/test/aws-s3/yaks-config.yaml
@@ -43,7 +43,7 @@ config:
- aws-s3-to-log-secret-based.groovy
- aws-s3-uri-binding.yaml
- aws-s3-to-knative.yaml
- - ../utils/knative-to-log.yaml
+ - ../utils/knative-channel-to-log.yaml
cucumber:
tags:
- "not @ignored"
diff --git a/test/utils/knative-channel-to-log.yaml b/test/utils/knative-channel-to-log.yaml
new file mode 100644
index 00000000..4fc551c6
--- /dev/null
+++ b/test/utils/knative-channel-to-log.yaml
@@ -0,0 +1,34 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+apiVersion: camel.apache.org/v1alpha1
+kind: KameletBinding
+metadata:
+ name: knative-channel-to-log
+spec:
+ source:
+ ref:
+ kind: InMemoryChannel
+ apiVersion: messaging.knative.dev/v1
+ name: messages
+ sink:
+ ref:
+ kind: Kamelet
+ apiVersion: camel.apache.org/v1alpha1
+ name: log-sink
+ properties:
+ showHeaders: true