You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lg...@apache.org on 2019/02/11 12:47:20 UTC

[beam] branch master updated: [BEAM-6567] Added Kinesis support to SyntheticDataPublisher

This is an automated email from the ASF dual-hosted git repository.

lgajowy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new cacbc6c  [BEAM-6567] Added Kinesis support to SyntheticDataPublisher
     new 9b24032  Merge pull request #7803: [BEAM-6567] Added Kinesis support to SyntheticDataPublisher
cacbc6c is described below

commit cacbc6c0ccbe8c0403f9b3f4cb895077eb4ad685
Author: Michal Walenia <mi...@polidea.com>
AuthorDate: Thu Jan 31 13:47:51 2019 +0100

    [BEAM-6567] Added Kinesis support to SyntheticDataPublisher
    
    Signed-off-by: Michal Walenia <mi...@polidea.com>
---
 sdks/java/testing/load-tests/build.gradle          |  1 +
 .../beam/sdk/loadtests/SyntheticDataPublisher.java | 73 ++++++++++++++++++++--
 2 files changed, 70 insertions(+), 4 deletions(-)

diff --git a/sdks/java/testing/load-tests/build.gradle b/sdks/java/testing/load-tests/build.gradle
index a2f5d71..70b0d6d 100644
--- a/sdks/java/testing/load-tests/build.gradle
+++ b/sdks/java/testing/load-tests/build.gradle
@@ -63,6 +63,7 @@ dependencies {
   shadow project(path: ":beam-sdks-java-test-utils", configuration: "shadow")
   shadow project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow")
   shadow project(path: ":beam-sdks-java-io-kafka", configuration: "shadow")
+  shadow project(path: ":beam-sdks-java-io-kinesis", configuration: "shadow")
 
   gradleRun project(path: project.path, configuration: "shadow")
   gradleRun project(path: runnerDependency, configuration: "shadow")
diff --git a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/SyntheticDataPublisher.java b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/SyntheticDataPublisher.java
index 4929833..e35054f 100644
--- a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/SyntheticDataPublisher.java
+++ b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/SyntheticDataPublisher.java
@@ -20,6 +20,7 @@ package org.apache.beam.sdk.loadtests;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.apache.beam.sdk.util.CoderUtils.encodeToByteArray;
 
+import com.amazonaws.regions.Regions;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -32,6 +33,7 @@ import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage;
 import org.apache.beam.sdk.io.kafka.KafkaIO;
+import org.apache.beam.sdk.io.kinesis.KinesisIO;
 import org.apache.beam.sdk.io.synthetic.SyntheticBoundedSource;
 import org.apache.beam.sdk.io.synthetic.SyntheticOptions;
 import org.apache.beam.sdk.io.synthetic.SyntheticSourceOptions;
@@ -47,7 +49,8 @@ import org.apache.beam.sdk.values.PCollection;
 import org.apache.kafka.common.serialization.StringSerializer;
 
 /**
- * Pipeline that generates synthetic data and publishes it in PubSub topic.
+ * Pipeline that generates synthetic data and publishes it in a PubSub or Kafka topic or in a
+ * Kinesis stream.
  *
  * <p>To run it manually, use the following command:
  *
@@ -56,12 +59,17 @@ import org.apache.kafka.common.serialization.StringSerializer;
  *    --pubSubTopic=TOPIC_NAME
  *    --kafkaBootstrapServerAddress=SERVER_ADDRESS
  *    --kafkaTopic=KAFKA_TOPIC_NAME
- *    --sourceOptions={"numRecords":1000,...}'
+ *    --sourceOptions={"numRecords":1000,...}
+ *    --kinesisStreamName=KINESIS_STREAM_NAME
+ *    --kinesisPartitionKey=KINESIS_PARTITION_KEY
+ *    --kinesisAwsKey=KINESIS_AWS_KEY
+ *    --kinesisAwsSecret=KINESIS_AWS_SECRET
+ *    --kinesisAwsRegion=KINESIS_AWS_REGION'
  *    -PloadTest.mainClass="org.apache.beam.sdk.loadtests.SyntheticDataPublisher"
  *  </pre>
  *
- * <p>If parameters related to a specific sink are provided (Kafka or PubSub), the pipeline writes
- * to the sink. Writing to both sinks is also acceptable.
+ * <p>If parameters related to a specific sink are provided (Kafka, PubSub or Kinesis), the pipeline
+ * writes to the sink. Writing to more than one sink is also acceptable.
  */
 public class SyntheticDataPublisher {
 
@@ -93,6 +101,31 @@ public class SyntheticDataPublisher {
     String getKafkaTopic();
 
     void setKafkaTopic(String topic);
+
+    @Description("Kinesis partition key")
+    String getKinesisPartitionKey();
+
+    void setKinesisPartitionKey(String partitionKey);
+
+    @Description("Kinesis partition name")
+    String getKinesisStreamName();
+
+    void setKinesisStreamName(String kinesisStreamName);
+
+    @Description("AWS secret for Kinesis")
+    String getKinesisAwsSecret();
+
+    void setKinesisAwsSecret(String kinesisAwsSecret);
+
+    @Description("AWS key for Kinesis")
+    String getKinesisAwsKey();
+
+    void setKinesisAwsKey(String kinesisAwsKey);
+
+    @Description("AWS region for Kinesis")
+    String getKinesisAwsRegion();
+
+    void setKinesisAwsRegion(String kinesisAwsRegion);
   }
 
   public static void main(String[] args) throws IOException {
@@ -111,9 +144,20 @@ public class SyntheticDataPublisher {
     if (options.getPubSubTopic() != null) {
       writeToPubSub(syntheticData);
     }
+    if (allKinesisOptionsConfigured()) {
+      writeToKinesis(syntheticData);
+    }
     pipeline.run().waitUntilFinish();
   }
 
+  private static boolean allKinesisOptionsConfigured() {
+    return options.getKinesisAwsKey() != null
+        && options.getKinesisAwsRegion() != null
+        && options.getKinesisAwsSecret() != null
+        && options.getKinesisPartitionKey() != null
+        && options.getKinesisStreamName() != null;
+  }
+
   private static void writeToPubSub(PCollection<KV<byte[], byte[]>> collection) {
     collection
         .apply("Map to PubSub messages", MapElements.via(new MapBytesToPubSubMessage()))
@@ -132,6 +176,20 @@ public class SyntheticDataPublisher {
                 .values());
   }
 
+  private static void writeToKinesis(PCollection<KV<byte[], byte[]>> collection) {
+    collection
+        .apply("Map to byte array for Kinesis", MapElements.via(new MapKVToByteArray()))
+        .apply(
+            "Write to Kinesis",
+            KinesisIO.write()
+                .withStreamName(options.getKinesisStreamName())
+                .withPartitionKey(options.getKinesisPartitionKey())
+                .withAWSClientsProvider(
+                    options.getKinesisAwsKey(),
+                    options.getKinesisAwsSecret(),
+                    Regions.fromName(options.getKinesisAwsRegion())));
+  }
+
   private static class MapKVToString extends SimpleFunction<KV<byte[], byte[]>, String> {
     @Override
     public String apply(KV<byte[], byte[]> input) {
@@ -140,6 +198,13 @@ public class SyntheticDataPublisher {
     }
   }
 
+  private static class MapKVToByteArray extends SimpleFunction<KV<byte[], byte[]>, byte[]> {
+    @Override
+    public byte[] apply(KV<byte[], byte[]> input) {
+      return encodeInputElement(input);
+    }
+  }
+
   private static class MapBytesToPubSubMessage
       extends SimpleFunction<KV<byte[], byte[]>, PubsubMessage> {
     @Override