You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by df...@apache.org on 2021/04/20 18:16:53 UTC

[camel-examples] 17/35: Added an AWS2-S3 Example of restarting policy usage with streaming upload

This is an automated email from the ASF dual-hosted git repository.

dfoulks pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-examples.git

commit a2c030dcb66974cbdcbdc2046ed92e0527bc9550
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Thu Apr 8 15:05:09 2021 +0200

    Added an AWS2-S3 Example of restarting policy usage with streaming upload
---
 .../README.adoc                                    |  69 ++++++++++++++
 .../data/burst.sh                                  |   5 +
 .../data/msg.txt                                   |   2 +
 .../pom.xml                                        | 106 +++++++++++++++++++++
 .../org/apache/camel/example/MyApplication.java    |  38 ++++++++
 .../org/apache/camel/example/MyRouteBuilder.java   |  33 +++++++
 .../src/main/resources/application.properties      |  29 ++++++
 .../src/main/resources/logback.xml                 |  30 ++++++
 examples/aws/pom.xml                               |   1 +
 9 files changed, 313 insertions(+)

diff --git a/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/README.adoc b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/README.adoc
new file mode 100644
index 0000000..b793dd1
--- /dev/null
+++ b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/README.adoc
@@ -0,0 +1,69 @@
+== Camel Example Main Endpoint DSL with AWS2 S3 component to Kafka with restarting policy
+
+This example shows how to use the endpoint DSL in your Camel routes
+to define endpoints using type safe fluent builders, which are Java methods
+that are compiled and it will show the AWS2-S3 stream mode.
+
+The example will poll one kafka topic s3.topic.1 and upload batch of 25 messages as single file into an s3 bucket (mycamel-1).
+
+On your bucket you'll see:
+
+s3.topic.1/partition_<partition-number>/s3.topic.1.txt
+s3.topic.1/partition_<partition-number>/s3.topic.1-1.txt
+
+and so on
+
+At the end you should have a total of 20 files.
+
+Notice how you can configure Camel in the `application.properties` file.
+
+Don't forget to add your AWS Credentials and the bucket name (already created ahead of time) and point to the correct topic.
+You'll need also a running kafka broker.
+You'll need to have kafkacat installed.
+
+This example supposed the s3.topic.1 has 1 partition only.
+
+But this should work with multiple partitions too.
+
+=== How to run
+
+You can run this example using
+
+[source,sh]
+----
+$ mvn compile
+----
+
+[source,sh]
+----
+$ mvn camel:run
+----
+
+Now run
+
+[source,sh]
+----
+$ data/burst.sh s3.topic.1 250 0 msg.txt
+----
+
+Stop the route with CTRL + C.
+
+At this point you should see an s3.topic.1/partition_0 folder, with 10 files.
+
+Restart the route and run 
+
+----
+$ data/burst.sh s3.topic.1 250 0 msg.txt
+----
+
+Now in the same s3.topic.1/partition_0 folder, you should see 20 files correctly numbered.
+
+=== Help and contributions
+
+If you hit any problem using Camel or have some feedback, then please
+https://camel.apache.org/support.html[let us know].
+
+We also love contributors, so
+https://camel.apache.org/contributing.html[get involved] :-)
+
+The Camel riders!
diff --git a/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/data/burst.sh b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/data/burst.sh
new file mode 100755
index 0000000..faab584
--- /dev/null
+++ b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/data/burst.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+for (( c=1; c<=$2; c++ ))
+do  
+  kafkacat -P -b localhost:9092 -t $1 -p $3 $4
+done
diff --git a/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/data/msg.txt b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/data/msg.txt
new file mode 100644
index 0000000..9f24223
--- /dev/null
+++ b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/data/msg.txt
@@ -0,0 +1,2 @@
+Hello
+
diff --git a/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/pom.xml b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/pom.xml
new file mode 100644
index 0000000..e3e6d23
--- /dev/null
+++ b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/pom.xml
@@ -0,0 +1,106 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.apache.camel.example</groupId>
+        <artifactId>camel-examples-aws-parent</artifactId>
+        <version>3.10.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>camel-example-main-endpointdsl-kafka-aws2-s3-restarting-policy</artifactId>
+    <packaging>jar</packaging>
+    <name>Camel :: Example :: Main :: Endpoint DSL :: Kafka AWS2 S3 Restarting Policy</name>
+    <description>An example for showing standalone Camel with Endpoint DSL and Kafka and AWS2 S3</description>
+
+    <properties>
+        <category>Beginner</category>
+    </properties>
+
+    <dependencyManagement>
+        <dependencies>
+            <!-- Add Camel BOM -->
+            <dependency>
+                <groupId>org.apache.camel</groupId>
+                <artifactId>camel-bom</artifactId>
+                <version>${camel.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.apache.camel</groupId>
+            <artifactId>camel-main</artifactId>
+        </dependency>
+        <!-- we use the endpoint-dsl -->
+        <dependency>
+            <groupId>org.apache.camel</groupId>
+            <artifactId>camel-endpointdsl</artifactId>
+        </dependency>
+        <!-- we use these 2 camel components in this example -->
+        <dependency>
+            <groupId>org.apache.camel</groupId>
+            <artifactId>camel-bean</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.camel</groupId>
+            <artifactId>camel-aws2-s3</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.camel</groupId>
+            <artifactId>camel-kafka</artifactId>
+        </dependency>
+
+        <!-- logging -->
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-core</artifactId>
+            <version>${logback-version}</version>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>${logback-version}</version>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <!-- to run the application -->
+            <plugin>
+                <groupId>org.apache.camel</groupId>
+                <artifactId>camel-maven-plugin</artifactId>
+                <version>${camel.version}</version>
+                <configuration>
+                    <mainClass>org.apache.camel.example.MyApplication</mainClass>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
diff --git a/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/src/main/java/org/apache/camel/example/MyApplication.java b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/src/main/java/org/apache/camel/example/MyApplication.java
new file mode 100644
index 0000000..496b37d
--- /dev/null
+++ b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/src/main/java/org/apache/camel/example/MyApplication.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.example;
+
+import org.apache.camel.main.Main;
+
+/**
+ * Main class that boot the Camel application
+ */
+public final class MyApplication {
+
+    private MyApplication() {
+    }
+
+    public static void main(String[] args) throws Exception {
+        // use Camels Main class
+        Main main = new Main();
+        // and add the routes (you can specify multiple classes)
+        main.configure().addRoutesBuilder(MyRouteBuilder.class);
+        // now keep the application running until the JVM is terminated (ctrl + c or sigterm)
+        main.run(args);
+    }
+
+}
diff --git a/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/src/main/java/org/apache/camel/example/MyRouteBuilder.java b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/src/main/java/org/apache/camel/example/MyRouteBuilder.java
new file mode 100644
index 0000000..29f739e
--- /dev/null
+++ b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/src/main/java/org/apache/camel/example/MyRouteBuilder.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.example;
+
+import org.apache.camel.builder.endpoint.EndpointRouteBuilder;
+import org.apache.camel.builder.endpoint.dsl.AWS2S3EndpointBuilderFactory;
+import software.amazon.awssdk.regions.Region;
+
+public class MyRouteBuilder extends EndpointRouteBuilder {
+
+    @Override
+    public void configure() throws Exception {
+
+        
+        from(kafka("{{kafkaTopic1}}").brokers("{{kafkaBrokers}}"))
+              .log("Kafka Message is: ${body}")
+              .toD(aws2S3("{{bucketName}}").streamingUploadMode(true).restartingPolicy(AWS2S3EndpointBuilderFactory.AWSS3RestartingPolicyEnum.lastPart).batchMessageNumber(25).namingStrategy(AWS2S3EndpointBuilderFactory.AWSS3NamingStrategyEnum.progressive).keyName("{{kafkaTopic1}}/partition_${headers.kafka.PARTITION}/{{kafkaTopic1}}.txt"));
+    }
+}
diff --git a/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/src/main/resources/application.properties b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/src/main/resources/application.properties
new file mode 100644
index 0000000..9a68e03
--- /dev/null
+++ b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/src/main/resources/application.properties
@@ -0,0 +1,29 @@
+## ---------------------------------------------------------------------------
+## Licensed to the Apache Software Foundation (ASF) under one or more
+## contributor license agreements.  See the NOTICE file distributed with
+## this work for additional information regarding copyright ownership.
+## The ASF licenses this file to You under the Apache License, Version 2.0
+## (the "License"); you may not use this file except in compliance with
+## the License.  You may obtain a copy of the License at
+##
+##      http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+## ---------------------------------------------------------------------------
+
+# to configure camel main
+# here you can configure options on camel main (see MainConfigurationProperties class)
+camel.main.name = Kafka-to-AWS2-S3-Stream
+
+# properties used in the route
+camel.component.aws2-s3.accessKey=xxxxx
+camel.component.aws2-s3.secretKey=yyyyy
+camel.component.aws2-s3.region=region
+bucketName=camel-1
+
+kafkaTopic1=s3.topic.1
+kafkaBrokers=localhost:9092
diff --git a/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/src/main/resources/logback.xml b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/src/main/resources/logback.xml
new file mode 100644
index 0000000..a798d0b
--- /dev/null
+++ b/examples/aws/main-endpointdsl-kafka-aws2-s3-restarting-policy/src/main/resources/logback.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<configuration>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="STDOUT" />
+    </root>
+</configuration>
diff --git a/examples/aws/pom.xml b/examples/aws/pom.xml
index a820e68..9d9397c 100644
--- a/examples/aws/pom.xml
+++ b/examples/aws/pom.xml
@@ -42,6 +42,7 @@
         <module>main-endpointdsl-aws2-s3</module>
         <module>main-endpointdsl-aws2-s3-kafka</module>
         <module>main-endpointdsl-kafka-aws2-s3</module>
+        <module>main-endpointdsl-kafka-aws2-s3-restarting-policy</module>
     </modules>
 
 </project>