You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lg...@apache.org on 2019/02/11 12:47:20 UTC
[beam] branch master updated: [BEAM-6567] Added Kinesis support to
SyntheticDataPublisher
This is an automated email from the ASF dual-hosted git repository.
lgajowy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new cacbc6c [BEAM-6567] Added Kinesis support to SyntheticDataPublisher
new 9b24032 Merge pull request #7803: [BEAM-6567] Added Kinesis support to SyntheticDataPublisher
cacbc6c is described below
commit cacbc6c0ccbe8c0403f9b3f4cb895077eb4ad685
Author: Michal Walenia <mi...@polidea.com>
AuthorDate: Thu Jan 31 13:47:51 2019 +0100
[BEAM-6567] Added Kinesis support to SyntheticDataPublisher
Signed-off-by: Michal Walenia <mi...@polidea.com>
---
sdks/java/testing/load-tests/build.gradle | 1 +
.../beam/sdk/loadtests/SyntheticDataPublisher.java | 73 ++++++++++++++++++++--
2 files changed, 70 insertions(+), 4 deletions(-)
diff --git a/sdks/java/testing/load-tests/build.gradle b/sdks/java/testing/load-tests/build.gradle
index a2f5d71..70b0d6d 100644
--- a/sdks/java/testing/load-tests/build.gradle
+++ b/sdks/java/testing/load-tests/build.gradle
@@ -63,6 +63,7 @@ dependencies {
shadow project(path: ":beam-sdks-java-test-utils", configuration: "shadow")
shadow project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow")
shadow project(path: ":beam-sdks-java-io-kafka", configuration: "shadow")
+ shadow project(path: ":beam-sdks-java-io-kinesis", configuration: "shadow")
gradleRun project(path: project.path, configuration: "shadow")
gradleRun project(path: runnerDependency, configuration: "shadow")
diff --git a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/SyntheticDataPublisher.java b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/SyntheticDataPublisher.java
index 4929833..e35054f 100644
--- a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/SyntheticDataPublisher.java
+++ b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/SyntheticDataPublisher.java
@@ -20,6 +20,7 @@ package org.apache.beam.sdk.loadtests;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.beam.sdk.util.CoderUtils.encodeToByteArray;
+import com.amazonaws.regions.Regions;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
@@ -32,6 +33,7 @@ import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage;
import org.apache.beam.sdk.io.kafka.KafkaIO;
+import org.apache.beam.sdk.io.kinesis.KinesisIO;
import org.apache.beam.sdk.io.synthetic.SyntheticBoundedSource;
import org.apache.beam.sdk.io.synthetic.SyntheticOptions;
import org.apache.beam.sdk.io.synthetic.SyntheticSourceOptions;
@@ -47,7 +49,8 @@ import org.apache.beam.sdk.values.PCollection;
import org.apache.kafka.common.serialization.StringSerializer;
/**
- * Pipeline that generates synthetic data and publishes it in PubSub topic.
+ * Pipeline that generates synthetic data and publishes it in a PubSub or Kafka topic or in a
+ * Kinesis stream.
*
* <p>To run it manually, use the following command:
*
@@ -56,12 +59,17 @@ import org.apache.kafka.common.serialization.StringSerializer;
* --pubSubTopic=TOPIC_NAME
* --kafkaBootstrapServerAddress=SERVER_ADDRESS
* --kafkaTopic=KAFKA_TOPIC_NAME
- * --sourceOptions={"numRecords":1000,...}'
+ * --sourceOptions={"numRecords":1000,...}
+ * --kinesisStreamName=KINESIS_STREAM_NAME
+ * --kinesisPartitionKey=KINESIS_PARTITION_KEY
+ * --kinesisAwsKey=KINESIS_AWS_KEY
+ * --kinesisAwsSecret=KINESIS_AWS_SECRET
+ * --kinesisAwsRegion=KINESIS_AWS_REGION'
* -PloadTest.mainClass="org.apache.beam.sdk.loadtests.SyntheticDataPublisher"
* </pre>
*
- * <p>If parameters related to a specific sink are provided (Kafka or PubSub), the pipeline writes
- * to the sink. Writing to both sinks is also acceptable.
+ * <p>If parameters related to a specific sink are provided (Kafka, PubSub or Kinesis), the pipeline
+ * writes to the sink. Writing to more than one sink is also acceptable.
*/
public class SyntheticDataPublisher {
@@ -93,6 +101,31 @@ public class SyntheticDataPublisher {
String getKafkaTopic();
void setKafkaTopic(String topic);
+
+ @Description("Kinesis partition key")
+ String getKinesisPartitionKey();
+
+ void setKinesisPartitionKey(String partitionKey);
+
+ @Description("Kinesis partition name")
+ String getKinesisStreamName();
+
+ void setKinesisStreamName(String kinesisStreamName);
+
+ @Description("AWS secret for Kinesis")
+ String getKinesisAwsSecret();
+
+ void setKinesisAwsSecret(String kinesisAwsSecret);
+
+ @Description("AWS key for Kinesis")
+ String getKinesisAwsKey();
+
+ void setKinesisAwsKey(String kinesisAwsKey);
+
+ @Description("AWS region for Kinesis")
+ String getKinesisAwsRegion();
+
+ void setKinesisAwsRegion(String kinesisAwsRegion);
}
public static void main(String[] args) throws IOException {
@@ -111,9 +144,20 @@ public class SyntheticDataPublisher {
if (options.getPubSubTopic() != null) {
writeToPubSub(syntheticData);
}
+ if (allKinesisOptionsConfigured()) {
+ writeToKinesis(syntheticData);
+ }
pipeline.run().waitUntilFinish();
}
+ private static boolean allKinesisOptionsConfigured() {
+ return options.getKinesisAwsKey() != null
+ && options.getKinesisAwsRegion() != null
+ && options.getKinesisAwsSecret() != null
+ && options.getKinesisPartitionKey() != null
+ && options.getKinesisStreamName() != null;
+ }
+
private static void writeToPubSub(PCollection<KV<byte[], byte[]>> collection) {
collection
.apply("Map to PubSub messages", MapElements.via(new MapBytesToPubSubMessage()))
@@ -132,6 +176,20 @@ public class SyntheticDataPublisher {
.values());
}
+ private static void writeToKinesis(PCollection<KV<byte[], byte[]>> collection) {
+ collection
+ .apply("Map to byte array for Kinesis", MapElements.via(new MapKVToByteArray()))
+ .apply(
+ "Write to Kinesis",
+ KinesisIO.write()
+ .withStreamName(options.getKinesisStreamName())
+ .withPartitionKey(options.getKinesisPartitionKey())
+ .withAWSClientsProvider(
+ options.getKinesisAwsKey(),
+ options.getKinesisAwsSecret(),
+ Regions.fromName(options.getKinesisAwsRegion())));
+ }
+
private static class MapKVToString extends SimpleFunction<KV<byte[], byte[]>, String> {
@Override
public String apply(KV<byte[], byte[]> input) {
@@ -140,6 +198,13 @@ public class SyntheticDataPublisher {
}
}
+ private static class MapKVToByteArray extends SimpleFunction<KV<byte[], byte[]>, byte[]> {
+ @Override
+ public byte[] apply(KV<byte[], byte[]> input) {
+ return encodeInputElement(input);
+ }
+ }
+
private static class MapBytesToPubSubMessage
extends SimpleFunction<KV<byte[], byte[]>, PubsubMessage> {
@Override