You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ac...@apache.org on 2024/06/27 09:20:49 UTC
(camel-examples) 01/01: CAMEL-20914 - Camel-AWS-Kinesis: KCL Consumers add an example
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch CAMEL-20914
in repository https://gitbox.apache.org/repos/asf/camel-examples.git
commit d4c29d39dc51bc59a57119342e07abc82bf0db49
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Thu Jun 27 11:20:21 2024 +0200
CAMEL-20914 - Camel-AWS-Kinesis: KCL Consumers add an example
Signed-off-by: Andrea Cosentino <an...@gmail.com>
---
README.adoc | 4 +-
aws/main-endpointdsl-aws-kcl-kinesis/README.adoc | 43 ++++++++
aws/main-endpointdsl-aws-kcl-kinesis/pom.xml | 112 +++++++++++++++++++++
.../org/apache/camel/example/MyApplication.java | 36 +++++++
.../org/apache/camel/example/MyRouteBuilder.java | 62 ++++++++++++
.../src/main/resources/application.properties | 23 +++++
.../src/main/resources/log4j2.properties | 23 +++++
aws/pom.xml | 1 +
.../org/apache/camel/example/MyBeanConfigurer.java | 2 +
9 files changed, 305 insertions(+), 1 deletion(-)
diff --git a/README.adoc b/README.adoc
index b20a235d..46525016 100644
--- a/README.adoc
+++ b/README.adoc
@@ -28,7 +28,7 @@ readme's instructions.
== Examples
// examples: START
-Number of Examples: 76 (0 deprecated)
+Number of Examples: 77 (0 deprecated)
[width="100%",cols="4,2,4",options="header"]
|===
@@ -90,6 +90,8 @@ Number of Examples: 76 (0 deprecated)
| link:main-endpointdsl/README.adoc[Main Endpointdsl] (main-endpointdsl) | Beginner | An example for showing standalone Camel with Endpoint DSL
+| link:aws/main-endpointdsl-aws-kcl-kinesis/README.adoc[Main Endpointdsl Aws Kcl Kinesis] (main-endpointdsl-aws-kcl-kinesis) | Beginner | An example for showing standalone Camel with Endpoint DSL and AWS2-Kinesis
+
| link:aws/main-endpointdsl-aws2-s3/README.adoc[Main Endpointdsl Aws2 S3] (main-endpointdsl-aws2-s3) | Beginner | An example for showing standalone Camel with Endpoint DSL and AWS2-S3
| link:aws/main-endpointdsl-aws2-s3-kafka/README.adoc[Main Endpointdsl Aws2 S3 Kafka] (main-endpointdsl-aws2-s3-kafka) | Beginner | An example for showing standalone Camel with Endpoint DSL and AWS2-S3 and Kafka
diff --git a/aws/main-endpointdsl-aws-kcl-kinesis/README.adoc b/aws/main-endpointdsl-aws-kcl-kinesis/README.adoc
new file mode 100644
index 00000000..36ceb405
--- /dev/null
+++ b/aws/main-endpointdsl-aws-kcl-kinesis/README.adoc
@@ -0,0 +1,43 @@
+== Camel Example Main Endpoint DSL with AWS2 Kinesis component
+
+This example shows how to use the endpoint DSL in your Camel routes
+to define endpoints using type safe fluent builders, which are Java methods
+that are compiled.
+
+The example will poll a Kinesis Data Stream by using KCL consumers.
+
+You'll need to create the stream from your AWS console or your AWS CLI.
+
+Notice how you can configure Camel in the `application.properties` file.
+
+This example will use the AWS default credentials Provider: https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html
+Set your credentials accordingly.
+Don't forget to add the stream name.
+
+=== Build
+
+First compile the example by executing:
+
+[source,sh]
+----
+$ mvn compile
+----
+
+=== How to run
+
+You can run this example using
+
+[source,sh]
+----
+$ mvn camel:run
+----
+
+=== Help and contributions
+
+If you hit any problem using Camel or have some feedback, then please
+https://camel.apache.org/community/support/[let us know].
+
+We also love contributors, so
+https://camel.apache.org/community/contributing/[get involved] :-)
+
+The Camel riders!
diff --git a/aws/main-endpointdsl-aws-kcl-kinesis/pom.xml b/aws/main-endpointdsl-aws-kcl-kinesis/pom.xml
new file mode 100644
index 00000000..27a785e2
--- /dev/null
+++ b/aws/main-endpointdsl-aws-kcl-kinesis/pom.xml
@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.camel.example</groupId>
+ <artifactId>camel-examples-aws-parent</artifactId>
+ <version>4.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>camel-example-main-endpointdsl-aws2-kinesis-kcl</artifactId>
+ <packaging>jar</packaging>
+ <name>Camel :: Example :: Main :: Endpoint DSL :: AWS2 KCL Kinesis</name>
+ <description>An example for showing standalone Camel with Endpoint DSL and AWS2-Kinesis</description>
+
+ <properties>
+ <category>Beginner</category>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <!-- Add Camel BOM -->
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-bom</artifactId>
+ <version>${camel.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-main</artifactId>
+ </dependency>
+ <!-- we use the endpoint-dsl -->
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-endpointdsl</artifactId>
+ </dependency>
+ <!-- we use these 2 camel components in this example -->
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-bean</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-aws2-kinesis</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-timer</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-file</artifactId>
+ </dependency>
+
+ <!-- logging -->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${log4j2-version}</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j2-impl</artifactId>
+ <version>${log4j2-version}</version>
+ <scope>runtime</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!-- to run the application -->
+ <plugin>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-maven-plugin</artifactId>
+ <version>${camel.version}</version>
+ <configuration>
+ <mainClass>org.apache.camel.example.MyApplication</mainClass>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
diff --git a/aws/main-endpointdsl-aws-kcl-kinesis/src/main/java/org/apache/camel/example/MyApplication.java b/aws/main-endpointdsl-aws-kcl-kinesis/src/main/java/org/apache/camel/example/MyApplication.java
new file mode 100644
index 00000000..31197dc0
--- /dev/null
+++ b/aws/main-endpointdsl-aws-kcl-kinesis/src/main/java/org/apache/camel/example/MyApplication.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.example;
+
+import org.apache.camel.main.Main;
+
+/**
+ * Main class that boot the Camel application
+ */
+public final class MyApplication {
+
+ private MyApplication() {
+ }
+
+ public static void main(String[] args) throws Exception {
+ // use Camels Main class
+ Main main = new Main(MyApplication.class);
+ // now keep the application running until the JVM is terminated (ctrl + c or sigterm)
+ main.run(args);
+ }
+
+}
diff --git a/aws/main-endpointdsl-aws-kcl-kinesis/src/main/java/org/apache/camel/example/MyRouteBuilder.java b/aws/main-endpointdsl-aws-kcl-kinesis/src/main/java/org/apache/camel/example/MyRouteBuilder.java
new file mode 100644
index 00000000..22257112
--- /dev/null
+++ b/aws/main-endpointdsl-aws-kcl-kinesis/src/main/java/org/apache/camel/example/MyRouteBuilder.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.example;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.camel.builder.endpoint.EndpointRouteBuilder;
+import org.apache.camel.component.aws2.kinesis.Kinesis2Constants;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * To use the endpoint DSL then we must extend EndpointRouteBuilder instead of RouteBuilder
+ */
+public class MyRouteBuilder extends EndpointRouteBuilder {
+
+ @Override
+ public void configure() throws Exception {
+
+ from(aws2Kinesis("{{streamName}}").useDefaultCredentialsProvider(true).advanced().useKclConsumers(true).asyncClient(true))
+ .log("The content is ${body} from ${headers.CamelAwsKinesisShardId}").process(new Processor() {
+ @Override
+ public void process(Exchange exchange) throws Exception {
+ ByteBuffer buffer = exchange.getMessage().getBody(ByteBuffer.class);
+ buffer.flip();
+ CharBuffer charBuffer = StandardCharsets.UTF_8.decode(buffer);
+ exchange.getMessage().setBody(charBuffer.toString());
+ }
+ }).to(file("./target/kcl-files/")).startupOrder(1);
+
+ from(timer("kcl-ingest-1").repeatCount(5))
+ .setBody(constant("Camel KCL Test Partition 1"))
+ .setHeader(Kinesis2Constants.PARTITION_KEY, constant("partition-1"))
+ .to(aws2Kinesis("{{streamName}}").useDefaultCredentialsProvider(true)).startupOrder(2);
+
+ from(timer("kcl-ingest-2").repeatCount(5))
+ .setBody(constant("Camel KCL Test Partition 2"))
+ .setHeader(Kinesis2Constants.PARTITION_KEY, constant("partition-2"))
+ .to(aws2Kinesis("{{streamName}}").useDefaultCredentialsProvider(true)).startupOrder(3);
+
+ from(timer("kcl-ingest-3").repeatCount(5))
+ .setBody(constant("Camel KCL Test Partition 3"))
+ .setHeader(Kinesis2Constants.PARTITION_KEY, constant("partition-3"))
+ .to(aws2Kinesis("{{streamName}}").useDefaultCredentialsProvider(true)).startupOrder(4);
+ }
+}
diff --git a/aws/main-endpointdsl-aws-kcl-kinesis/src/main/resources/application.properties b/aws/main-endpointdsl-aws-kcl-kinesis/src/main/resources/application.properties
new file mode 100644
index 00000000..6d4401ea
--- /dev/null
+++ b/aws/main-endpointdsl-aws-kcl-kinesis/src/main/resources/application.properties
@@ -0,0 +1,23 @@
+## ---------------------------------------------------------------------------
+## Licensed to the Apache Software Foundation (ASF) under one or more
+## contributor license agreements. See the NOTICE file distributed with
+## this work for additional information regarding copyright ownership.
+## The ASF licenses this file to You under the Apache License, Version 2.0
+## (the "License"); you may not use this file except in compliance with
+## the License. You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+## ---------------------------------------------------------------------------
+
+# here you can configure options on camel main
+# https://camel.apache.org/components/next/others/main.html
+camel.main.name = AWS2-KCL-Kinesis
+
+# properties used in the route
+streamName=kcl-stream
diff --git a/aws/main-endpointdsl-aws-kcl-kinesis/src/main/resources/log4j2.properties b/aws/main-endpointdsl-aws-kcl-kinesis/src/main/resources/log4j2.properties
new file mode 100644
index 00000000..d9f0508a
--- /dev/null
+++ b/aws/main-endpointdsl-aws-kcl-kinesis/src/main/resources/log4j2.properties
@@ -0,0 +1,23 @@
+## ---------------------------------------------------------------------------
+## Licensed to the Apache Software Foundation (ASF) under one or more
+## contributor license agreements. See the NOTICE file distributed with
+## this work for additional information regarding copyright ownership.
+## The ASF licenses this file to You under the Apache License, Version 2.0
+## (the "License"); you may not use this file except in compliance with
+## the License. You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+## ---------------------------------------------------------------------------
+
+appender.out.type = Console
+appender.out.name = out
+appender.out.layout.type = PatternLayout
+appender.out.layout.pattern = [%30.30t] %-30.30c{1} %-5p %m%n
+rootLogger.level = INFO
+rootLogger.appenderRef.out.ref = out
diff --git a/aws/pom.xml b/aws/pom.xml
index a5428605..0df489ac 100644
--- a/aws/pom.xml
+++ b/aws/pom.xml
@@ -40,6 +40,7 @@
<module>main-endpointdsl-aws2</module>
<module>main-endpointdsl-aws2-s3</module>
<module>main-endpointdsl-aws2-s3-kafka</module>
+ <module>main-endpointdsl-aws-kcl-kinesis</module>
<module>main-endpointdsl-kafka-aws2-s3</module>
<module>main-endpointdsl-kafka-aws2-s3-restarting-policy</module>
</modules>
diff --git a/main-tiny/src/generated/java/org/apache/camel/example/MyBeanConfigurer.java b/main-tiny/src/generated/java/org/apache/camel/example/MyBeanConfigurer.java
index d6cdd510..3818ba2e 100644
--- a/main-tiny/src/generated/java/org/apache/camel/example/MyBeanConfigurer.java
+++ b/main-tiny/src/generated/java/org/apache/camel/example/MyBeanConfigurer.java
@@ -1,6 +1,7 @@
/* Generated by camel build tools - do NOT edit this file! */
package org.apache.camel.example;
+import javax.annotation.processing.Generated;
import java.util.Map;
import org.apache.camel.CamelContext;
@@ -14,6 +15,7 @@ import org.apache.camel.example.MyBean;
/**
* Generated by camel build tools - do NOT edit this file!
*/
+@Generated("org.apache.camel.maven.packaging.GenerateConfigurerMojo")
@SuppressWarnings("unchecked")
public class MyBeanConfigurer extends org.apache.camel.support.component.PropertyConfigurerSupport implements GeneratedPropertyConfigurer, PropertyConfigurerGetter {