You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by pa...@apache.org on 2022/12/08 21:02:55 UTC
[beam] branch master updated: Implement Kafka Write Schema Transform (#24495)
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 00bf2d42eac Implement Kafka Write Schema Transform (#24495)
00bf2d42eac is described below
commit 00bf2d42eac4f3007642c9a9fe6968054267600b
Author: Pablo Estrada <pa...@users.noreply.github.com>
AuthorDate: Thu Dec 8 13:02:49 2022 -0800
Implement Kafka Write Schema Transform (#24495)
* Implement Kafka Write Schema Transform
* rebased and fixdup
* fixups
* fix json+avro tests
* fix test
---
.../io/kafka/KafkaReadSchemaTransformProvider.java | 38 ++++-
.../kafka/KafkaWriteSchemaTransformProvider.java | 166 +++++++++++++++++++++
.../org/apache/beam/sdk/io/kafka/KafkaIOIT.java | 144 ++++++++++++++++++
.../KafkaReadSchemaTransformProviderTest.java | 2 +-
4 files changed, 344 insertions(+), 6 deletions(-)
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProvider.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProvider.java
index 5c84922db20..89c8d986ac1 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProvider.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProvider.java
@@ -37,15 +37,30 @@ import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionRowTuple;
import org.apache.beam.sdk.values.Row;
import org.apache.beam.sdk.values.TypeDescriptors;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.joda.time.Duration;
@AutoService(SchemaTransformProvider.class)
public class KafkaReadSchemaTransformProvider
extends TypedSchemaTransformProvider<KafkaReadSchemaTransformConfiguration> {
+ final Boolean isTest;
+ final Integer testTimeoutSecs;
+
+ public KafkaReadSchemaTransformProvider() {
+ this(false, 0);
+ }
+
+ @VisibleForTesting
+ KafkaReadSchemaTransformProvider(Boolean isTest, Integer testTimeoutSecs) {
+ this.isTest = isTest;
+ this.testTimeoutSecs = testTimeoutSecs;
+ }
+
@Override
protected Class<KafkaReadSchemaTransformConfiguration> configurationClass() {
return KafkaReadSchemaTransformConfiguration.class;
@@ -53,7 +68,7 @@ public class KafkaReadSchemaTransformProvider
@Override
protected SchemaTransform from(KafkaReadSchemaTransformConfiguration configuration) {
- return new KafkaReadSchemaTransform(configuration);
+ return new KafkaReadSchemaTransform(configuration, isTest, testTimeoutSecs);
}
@Override
@@ -68,15 +83,22 @@ public class KafkaReadSchemaTransformProvider
@Override
public List<String> outputCollectionNames() {
- return Lists.newArrayList("OUTPUT");
+ return Lists.newArrayList("output");
}
private static class KafkaReadSchemaTransform implements SchemaTransform {
private final KafkaReadSchemaTransformConfiguration configuration;
+ private final Boolean isTest;
+ private final Integer testTimeoutSeconds;
- KafkaReadSchemaTransform(KafkaReadSchemaTransformConfiguration configuration) {
+ KafkaReadSchemaTransform(
+ KafkaReadSchemaTransformConfiguration configuration,
+ Boolean isTest,
+ Integer testTimeoutSeconds) {
configuration.validate();
this.configuration = configuration;
+ this.isTest = isTest;
+ this.testTimeoutSeconds = testTimeoutSeconds;
}
@Override
@@ -116,9 +138,12 @@ public class KafkaReadSchemaTransformProvider
autoOffsetReset))
.withTopic(configuration.getTopic())
.withBootstrapServers(configuration.getBootstrapServers());
+ if (isTest) {
+ kafkaRead = kafkaRead.withMaxReadTime(Duration.standardSeconds(testTimeoutSeconds));
+ }
return PCollectionRowTuple.of(
- "OUTPUT",
+ "output",
input
.getPipeline()
.apply(kafkaRead.withoutMetadata())
@@ -157,6 +182,9 @@ public class KafkaReadSchemaTransformProvider
.withValueDeserializer(
ConfluentSchemaRegistryDeserializerProvider.of(
confluentSchemaRegUrl, confluentSchemaRegSubject));
+ if (isTest) {
+ kafkaRead = kafkaRead.withMaxReadTime(Duration.standardSeconds(testTimeoutSeconds));
+ }
PCollection<GenericRecord> kafkaValues =
input.getPipeline().apply(kafkaRead.withoutMetadata()).apply(Values.create());
@@ -164,7 +192,7 @@ public class KafkaReadSchemaTransformProvider
assert kafkaValues.getCoder().getClass() == AvroCoder.class;
AvroCoder<GenericRecord> coder = (AvroCoder<GenericRecord>) kafkaValues.getCoder();
kafkaValues = kafkaValues.setCoder(AvroUtils.schemaCoder(coder.getSchema()));
- return PCollectionRowTuple.of("OUTPUT", kafkaValues.apply(Convert.toRows()));
+ return PCollectionRowTuple.of("output", kafkaValues.apply(Convert.toRows()));
}
};
}
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriteSchemaTransformProvider.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriteSchemaTransformProvider.java
new file mode 100644
index 00000000000..45f9dbff599
--- /dev/null
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriteSchemaTransformProvider.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.kafka;
+
+import com.google.auto.value.AutoValue;
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.schemas.AutoValueSchema;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.annotations.DefaultSchema;
+import org.apache.beam.sdk.schemas.transforms.SchemaTransform;
+import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider;
+import org.apache.beam.sdk.schemas.utils.AvroUtils;
+import org.apache.beam.sdk.schemas.utils.JsonUtils;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollectionRowTuple;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.checkerframework.checker.initialization.qual.Initialized;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.UnknownKeyFor;
+
+public class KafkaWriteSchemaTransformProvider
+ extends TypedSchemaTransformProvider<
+ KafkaWriteSchemaTransformProvider.KafkaWriteSchemaTransformConfiguration> {
+
+ public static final Set<String> SUPPORTED_FORMATS = Sets.newHashSet("JSON", "AVRO");
+
+ @Override
+ protected @UnknownKeyFor @NonNull @Initialized Class<KafkaWriteSchemaTransformConfiguration>
+ configurationClass() {
+ return KafkaWriteSchemaTransformConfiguration.class;
+ }
+
+ @Override
+ protected @UnknownKeyFor @NonNull @Initialized SchemaTransform from(
+ KafkaWriteSchemaTransformConfiguration configuration) {
+ if (!SUPPORTED_FORMATS.contains(configuration.getFormat())) {
+ throw new IllegalArgumentException(
+ "Format "
+ + configuration.getFormat()
+ + " is not supported. "
+ + "Supported formats are: "
+ + String.join(", ", SUPPORTED_FORMATS));
+ }
+ return new KafkaWriteSchemaTransform(configuration);
+ }
+
+ static final class KafkaWriteSchemaTransform implements SchemaTransform, Serializable {
+ final KafkaWriteSchemaTransformConfiguration configuration;
+
+ KafkaWriteSchemaTransform(KafkaWriteSchemaTransformConfiguration configuration) {
+ this.configuration = configuration;
+ }
+
+ @Override
+ public @UnknownKeyFor @NonNull @Initialized PTransform<
+ @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple,
+ @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple>
+ buildTransform() {
+ return new PTransform<PCollectionRowTuple, PCollectionRowTuple>() {
+ @Override
+ public PCollectionRowTuple expand(PCollectionRowTuple input) {
+ Schema inputSchema = input.get("input").getSchema();
+ final SerializableFunction<Row, byte[]> toBytesFn =
+ configuration.getFormat().equals("JSON")
+ ? JsonUtils.getRowToJsonBytesFunction(inputSchema)
+ : AvroUtils.getRowToAvroBytesFunction(inputSchema);
+
+ final Map<String, String> configOverrides = configuration.getProducerConfigUpdates();
+ input
+ .get("input")
+ .apply(
+ "Map Rows to Kafka Messages",
+ MapElements.via(
+ new SimpleFunction<Row, KV<byte[], byte[]>>(
+ row -> KV.of(new byte[1], toBytesFn.apply(row))) {}))
+ .apply(
+ KafkaIO.<byte[], byte[]>write()
+ .withTopic(configuration.getTopic())
+ .withBootstrapServers(configuration.getBootstrapServers())
+ .withProducerConfigUpdates(
+ configOverrides == null
+ ? new HashMap<>()
+ : new HashMap<String, Object>(configOverrides))
+ .withKeySerializer(ByteArraySerializer.class)
+ .withValueSerializer(ByteArraySerializer.class));
+ return PCollectionRowTuple.empty(input.getPipeline());
+ }
+ };
+ }
+ }
+
+ @Override
+ public @UnknownKeyFor @NonNull @Initialized String identifier() {
+ return "beam:schematransform:org.apache.beam:kafka_write:v1";
+ }
+
+ @Override
+ public @UnknownKeyFor @NonNull @Initialized List<@UnknownKeyFor @NonNull @Initialized String>
+ inputCollectionNames() {
+ return Collections.singletonList("input");
+ }
+
+ @Override
+ public @UnknownKeyFor @NonNull @Initialized List<@UnknownKeyFor @NonNull @Initialized String>
+ outputCollectionNames() {
+ return Collections.emptyList();
+ }
+
+ @AutoValue
+ @DefaultSchema(AutoValueSchema.class)
+ public abstract static class KafkaWriteSchemaTransformConfiguration implements Serializable {
+ public abstract String getFormat();
+
+ public abstract String getTopic();
+
+ public abstract String getBootstrapServers();
+
+ @Nullable
+ public abstract Map<String, String> getProducerConfigUpdates();
+
+ public static Builder builder() {
+ return new AutoValue_KafkaWriteSchemaTransformProvider_KafkaWriteSchemaTransformConfiguration
+ .Builder();
+ }
+
+ @AutoValue.Builder
+ public abstract static class Builder {
+ public abstract Builder setFormat(String format);
+
+ public abstract Builder setTopic(String topic);
+
+ public abstract Builder setBootstrapServers(String bootstrapServers);
+
+ public abstract Builder setProducerConfigUpdates(Map<String, String> producerConfigUpdates);
+
+ public abstract KafkaWriteSchemaTransformConfiguration build();
+ }
+ }
+}
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java
index d2a441d8e58..497496db1b3 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java
@@ -29,12 +29,16 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.function.BiFunction;
+import java.util.stream.Collectors;
+import java.util.stream.LongStream;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.NullableCoder;
+import org.apache.beam.sdk.io.GenerateSequence;
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.io.common.HashingFn;
import org.apache.beam.sdk.io.common.IOITHelper;
@@ -49,6 +53,9 @@ import org.apache.beam.sdk.options.ExperimentalOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.StreamingOptions;
import org.apache.beam.sdk.options.Validation;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.utils.AvroUtils;
+import org.apache.beam.sdk.schemas.utils.JsonUtils;
import org.apache.beam.sdk.testing.ExpectedLogs;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
@@ -72,9 +79,12 @@ import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionRowTuple;
import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.TypeDescriptors;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists;
import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.admin.NewPartitions;
import org.apache.kafka.clients.admin.NewTopic;
@@ -502,6 +512,140 @@ public class KafkaIOIT {
readElementMetric(readResult, NAMESPACE, READ_ELEMENT_METRIC_NAME));
}
+ public static final Schema KAFKA_TOPIC_SCHEMA =
+ Schema.builder()
+ .addStringField("name")
+ .addInt64Field("userId")
+ .addInt64Field("age")
+ .addBooleanField("ageIsEven")
+ .addDoubleField("temperature")
+ .addArrayField("childrenNames", Schema.FieldType.STRING)
+ .build();
+
+ public static final String SCHEMA_IN_JSON =
+ "{\n"
+ + " \"type\": \"object\",\n"
+ + " \"properties\": {\n"
+ + " \"name\": {\n"
+ + " \"type\": \"string\"\n"
+ + " },\n"
+ + " \"userId\": {\n"
+ + " \"type\": \"integer\"\n"
+ + " },\n"
+ + " \"age\": {\n"
+ + " \"type\": \"integer\"\n"
+ + " },\n"
+ + " \"ageIsEven\": {\n"
+ + " \"type\": \"boolean\"\n"
+ + " },\n"
+ + " \"temperature\": {\n"
+ + " \"type\": \"number\"\n"
+ + " },\n"
+ + " \"childrenNames\": {\n"
+ + " \"type\": \"array\",\n"
+ + " \"items\": {\n"
+ + " \"type\": \"string\"\n"
+ + " }\n"
+ + " }\n"
+ + " }\n"
+ + "}";
+
+ private static final int FIVE_MINUTES_IN_MS = 5 * 60 * 1000;
+
+ @Test(timeout = FIVE_MINUTES_IN_MS)
+ public void testKafkaViaSchemaTransformJson() {
+ runReadWriteKafkaViaSchemaTransforms(
+ "JSON", SCHEMA_IN_JSON, JsonUtils.beamSchemaFromJsonSchema(SCHEMA_IN_JSON));
+ }
+
+ @Test(timeout = FIVE_MINUTES_IN_MS)
+ public void testKafkaViaSchemaTransformAvro() {
+ runReadWriteKafkaViaSchemaTransforms(
+ "AVRO", AvroUtils.toAvroSchema(KAFKA_TOPIC_SCHEMA).toString(), KAFKA_TOPIC_SCHEMA);
+ }
+
+ public void runReadWriteKafkaViaSchemaTransforms(
+ String format, String schemaDefinition, Schema beamSchema) {
+ String topicName = options.getKafkaTopic() + "-schema-transform" + UUID.randomUUID();
+ PCollectionRowTuple.of(
+ "input",
+ writePipeline
+ .apply("Generate records", GenerateSequence.from(0).to(1000))
+ .apply(
+ "Transform to Beam Rows",
+ MapElements.into(TypeDescriptors.rows())
+ .via(
+ numb ->
+ Row.withSchema(beamSchema)
+ .withFieldValue("name", numb.toString())
+ .withFieldValue(
+ "userId", Long.valueOf(numb.hashCode())) // User ID
+ .withFieldValue("age", Long.valueOf(numb.intValue())) // Age
+ .withFieldValue("ageIsEven", numb % 2 == 0) // ageIsEven
+ .withFieldValue("temperature", new Random(numb).nextDouble())
+ .withFieldValue(
+ "childrenNames",
+ Lists.newArrayList(
+ Long.toString(numb + 1),
+ Long.toString(numb + 2))) // childrenNames
+ .build()))
+ .setRowSchema(beamSchema))
+ .apply(
+ "Write to Kafka",
+ new KafkaWriteSchemaTransformProvider()
+ .from(
+ KafkaWriteSchemaTransformProvider.KafkaWriteSchemaTransformConfiguration
+ .builder()
+ .setTopic(topicName)
+ .setBootstrapServers(options.getKafkaBootstrapServerAddresses())
+ .setFormat(format)
+ .build())
+ .buildTransform());
+
+ PAssert.that(
+ PCollectionRowTuple.empty(readPipeline)
+ .apply(
+ "Read from unbounded Kafka",
+ // A timeout of 30s for local, container-based tests, and 2 minutes for
+ // real-kafka tests.
+ new KafkaReadSchemaTransformProvider(
+ true, options.isWithTestcontainers() ? 30 : 120)
+ .from(
+ KafkaReadSchemaTransformConfiguration.builder()
+ .setDataFormat(format)
+ .setAutoOffsetResetConfig("earliest")
+ .setSchema(schemaDefinition)
+ .setTopic(topicName)
+ .setBootstrapServers(options.getKafkaBootstrapServerAddresses())
+ .build())
+ .buildTransform())
+ .get("output"))
+ .containsInAnyOrder(
+ LongStream.range(0L, 1000L)
+ .<Row>mapToObj(
+ numb ->
+ Row.withSchema(beamSchema)
+ .withFieldValue("name", Long.toString(numb)) // Name
+ .withFieldValue("userId", Long.valueOf(Long.hashCode(numb))) // User ID
+ .withFieldValue("age", Long.valueOf(numb)) // Age
+ .withFieldValue("ageIsEven", numb % 2 == 0) // ageIsEven
+ .withFieldValue("temperature", new Random(numb).nextDouble())
+ .withFieldValue(
+ "childrenNames",
+ Lists.newArrayList(
+ Long.toString(numb + 1),
+ Long.toString(numb + 2))) // childrenNames
+ .build())
+ .collect(Collectors.toList()));
+
+ PipelineResult writeResult = writePipeline.run();
+ writeResult.waitUntilFinish();
+
+ PipelineResult readResult = readPipeline.run();
+ readResult.waitUntilFinish(Duration.standardSeconds(options.getReadTimeout()));
+ assertEquals(PipelineResult.State.DONE, readResult.getState());
+ }
+
private static class DelayedCheckStopReadingFn
implements SerializableFunction<TopicPartition, Boolean> {
int checkCount = 0;
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java
index 0ac42c38462..591f05ed6aa 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java
@@ -88,7 +88,7 @@ public class KafkaReadSchemaTransformProviderTest {
.filter(provider -> provider.getClass() == KafkaReadSchemaTransformProvider.class)
.collect(Collectors.toList());
SchemaTransformProvider kafkaProvider = providers.get(0);
- assertEquals(kafkaProvider.outputCollectionNames(), Lists.newArrayList("OUTPUT"));
+ assertEquals(kafkaProvider.outputCollectionNames(), Lists.newArrayList("output"));
assertEquals(kafkaProvider.inputCollectionNames(), Lists.newArrayList());
assertEquals(