You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by pa...@apache.org on 2022/12/08 21:02:55 UTC

[beam] branch master updated: Implement Kafka Write Schema Transform (#24495)

This is an automated email from the ASF dual-hosted git repository.

pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 00bf2d42eac Implement Kafka Write Schema Transform (#24495)
00bf2d42eac is described below

commit 00bf2d42eac4f3007642c9a9fe6968054267600b
Author: Pablo Estrada <pa...@users.noreply.github.com>
AuthorDate: Thu Dec 8 13:02:49 2022 -0800

    Implement Kafka Write Schema Transform (#24495)
    
    * Implement Kafka Write Schema Transform
    
    * rebased and fixdup
    
    * fixups
    
    * fix json+avro tests
    
    * fix test
---
 .../io/kafka/KafkaReadSchemaTransformProvider.java |  38 ++++-
 .../kafka/KafkaWriteSchemaTransformProvider.java   | 166 +++++++++++++++++++++
 .../org/apache/beam/sdk/io/kafka/KafkaIOIT.java    | 144 ++++++++++++++++++
 .../KafkaReadSchemaTransformProviderTest.java      |   2 +-
 4 files changed, 344 insertions(+), 6 deletions(-)

diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProvider.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProvider.java
index 5c84922db20..89c8d986ac1 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProvider.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProvider.java
@@ -37,15 +37,30 @@ import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionRowTuple;
 import org.apache.beam.sdk.values.Row;
 import org.apache.beam.sdk.values.TypeDescriptors;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.joda.time.Duration;
 
 @AutoService(SchemaTransformProvider.class)
 public class KafkaReadSchemaTransformProvider
     extends TypedSchemaTransformProvider<KafkaReadSchemaTransformConfiguration> {
 
+  final Boolean isTest;
+  final Integer testTimeoutSecs;
+
+  public KafkaReadSchemaTransformProvider() {
+    this(false, 0);
+  }
+
+  @VisibleForTesting
+  KafkaReadSchemaTransformProvider(Boolean isTest, Integer testTimeoutSecs) {
+    this.isTest = isTest;
+    this.testTimeoutSecs = testTimeoutSecs;
+  }
+
   @Override
   protected Class<KafkaReadSchemaTransformConfiguration> configurationClass() {
     return KafkaReadSchemaTransformConfiguration.class;
@@ -53,7 +68,7 @@ public class KafkaReadSchemaTransformProvider
 
   @Override
   protected SchemaTransform from(KafkaReadSchemaTransformConfiguration configuration) {
-    return new KafkaReadSchemaTransform(configuration);
+    return new KafkaReadSchemaTransform(configuration, isTest, testTimeoutSecs);
   }
 
   @Override
@@ -68,15 +83,22 @@ public class KafkaReadSchemaTransformProvider
 
   @Override
   public List<String> outputCollectionNames() {
-    return Lists.newArrayList("OUTPUT");
+    return Lists.newArrayList("output");
   }
 
   private static class KafkaReadSchemaTransform implements SchemaTransform {
     private final KafkaReadSchemaTransformConfiguration configuration;
+    private final Boolean isTest;
+    private final Integer testTimeoutSeconds;
 
-    KafkaReadSchemaTransform(KafkaReadSchemaTransformConfiguration configuration) {
+    KafkaReadSchemaTransform(
+        KafkaReadSchemaTransformConfiguration configuration,
+        Boolean isTest,
+        Integer testTimeoutSeconds) {
       configuration.validate();
       this.configuration = configuration;
+      this.isTest = isTest;
+      this.testTimeoutSeconds = testTimeoutSeconds;
     }
 
     @Override
@@ -116,9 +138,12 @@ public class KafkaReadSchemaTransformProvider
                             autoOffsetReset))
                     .withTopic(configuration.getTopic())
                     .withBootstrapServers(configuration.getBootstrapServers());
+            if (isTest) {
+              kafkaRead = kafkaRead.withMaxReadTime(Duration.standardSeconds(testTimeoutSeconds));
+            }
 
             return PCollectionRowTuple.of(
-                "OUTPUT",
+                "output",
                 input
                     .getPipeline()
                     .apply(kafkaRead.withoutMetadata())
@@ -157,6 +182,9 @@ public class KafkaReadSchemaTransformProvider
                     .withValueDeserializer(
                         ConfluentSchemaRegistryDeserializerProvider.of(
                             confluentSchemaRegUrl, confluentSchemaRegSubject));
+            if (isTest) {
+              kafkaRead = kafkaRead.withMaxReadTime(Duration.standardSeconds(testTimeoutSeconds));
+            }
 
             PCollection<GenericRecord> kafkaValues =
                 input.getPipeline().apply(kafkaRead.withoutMetadata()).apply(Values.create());
@@ -164,7 +192,7 @@ public class KafkaReadSchemaTransformProvider
             assert kafkaValues.getCoder().getClass() == AvroCoder.class;
             AvroCoder<GenericRecord> coder = (AvroCoder<GenericRecord>) kafkaValues.getCoder();
             kafkaValues = kafkaValues.setCoder(AvroUtils.schemaCoder(coder.getSchema()));
-            return PCollectionRowTuple.of("OUTPUT", kafkaValues.apply(Convert.toRows()));
+            return PCollectionRowTuple.of("output", kafkaValues.apply(Convert.toRows()));
           }
         };
       }
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriteSchemaTransformProvider.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriteSchemaTransformProvider.java
new file mode 100644
index 00000000000..45f9dbff599
--- /dev/null
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriteSchemaTransformProvider.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.kafka;
+
+import com.google.auto.value.AutoValue;
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.schemas.AutoValueSchema;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.annotations.DefaultSchema;
+import org.apache.beam.sdk.schemas.transforms.SchemaTransform;
+import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider;
+import org.apache.beam.sdk.schemas.utils.AvroUtils;
+import org.apache.beam.sdk.schemas.utils.JsonUtils;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollectionRowTuple;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.checkerframework.checker.initialization.qual.Initialized;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.UnknownKeyFor;
+
+public class KafkaWriteSchemaTransformProvider
+    extends TypedSchemaTransformProvider<
+        KafkaWriteSchemaTransformProvider.KafkaWriteSchemaTransformConfiguration> {
+
+  public static final Set<String> SUPPORTED_FORMATS = Sets.newHashSet("JSON", "AVRO");
+
+  @Override
+  protected @UnknownKeyFor @NonNull @Initialized Class<KafkaWriteSchemaTransformConfiguration>
+      configurationClass() {
+    return KafkaWriteSchemaTransformConfiguration.class;
+  }
+
+  @Override
+  protected @UnknownKeyFor @NonNull @Initialized SchemaTransform from(
+      KafkaWriteSchemaTransformConfiguration configuration) {
+    if (!SUPPORTED_FORMATS.contains(configuration.getFormat())) {
+      throw new IllegalArgumentException(
+          "Format "
+              + configuration.getFormat()
+              + " is not supported. "
+              + "Supported formats are: "
+              + String.join(", ", SUPPORTED_FORMATS));
+    }
+    return new KafkaWriteSchemaTransform(configuration);
+  }
+
+  static final class KafkaWriteSchemaTransform implements SchemaTransform, Serializable {
+    final KafkaWriteSchemaTransformConfiguration configuration;
+
+    KafkaWriteSchemaTransform(KafkaWriteSchemaTransformConfiguration configuration) {
+      this.configuration = configuration;
+    }
+
+    @Override
+    public @UnknownKeyFor @NonNull @Initialized PTransform<
+            @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple,
+            @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple>
+        buildTransform() {
+      return new PTransform<PCollectionRowTuple, PCollectionRowTuple>() {
+        @Override
+        public PCollectionRowTuple expand(PCollectionRowTuple input) {
+          Schema inputSchema = input.get("input").getSchema();
+          final SerializableFunction<Row, byte[]> toBytesFn =
+              configuration.getFormat().equals("JSON")
+                  ? JsonUtils.getRowToJsonBytesFunction(inputSchema)
+                  : AvroUtils.getRowToAvroBytesFunction(inputSchema);
+
+          final Map<String, String> configOverrides = configuration.getProducerConfigUpdates();
+          input
+              .get("input")
+              .apply(
+                  "Map Rows to Kafka Messages",
+                  MapElements.via(
+                      new SimpleFunction<Row, KV<byte[], byte[]>>(
+                          row -> KV.of(new byte[1], toBytesFn.apply(row))) {}))
+              .apply(
+                  KafkaIO.<byte[], byte[]>write()
+                      .withTopic(configuration.getTopic())
+                      .withBootstrapServers(configuration.getBootstrapServers())
+                      .withProducerConfigUpdates(
+                          configOverrides == null
+                              ? new HashMap<>()
+                              : new HashMap<String, Object>(configOverrides))
+                      .withKeySerializer(ByteArraySerializer.class)
+                      .withValueSerializer(ByteArraySerializer.class));
+          return PCollectionRowTuple.empty(input.getPipeline());
+        }
+      };
+    }
+  }
+
+  @Override
+  public @UnknownKeyFor @NonNull @Initialized String identifier() {
+    return "beam:schematransform:org.apache.beam:kafka_write:v1";
+  }
+
+  @Override
+  public @UnknownKeyFor @NonNull @Initialized List<@UnknownKeyFor @NonNull @Initialized String>
+      inputCollectionNames() {
+    return Collections.singletonList("input");
+  }
+
+  @Override
+  public @UnknownKeyFor @NonNull @Initialized List<@UnknownKeyFor @NonNull @Initialized String>
+      outputCollectionNames() {
+    return Collections.emptyList();
+  }
+
+  @AutoValue
+  @DefaultSchema(AutoValueSchema.class)
+  public abstract static class KafkaWriteSchemaTransformConfiguration implements Serializable {
+    public abstract String getFormat();
+
+    public abstract String getTopic();
+
+    public abstract String getBootstrapServers();
+
+    @Nullable
+    public abstract Map<String, String> getProducerConfigUpdates();
+
+    public static Builder builder() {
+      return new AutoValue_KafkaWriteSchemaTransformProvider_KafkaWriteSchemaTransformConfiguration
+          .Builder();
+    }
+
+    @AutoValue.Builder
+    public abstract static class Builder {
+      public abstract Builder setFormat(String format);
+
+      public abstract Builder setTopic(String topic);
+
+      public abstract Builder setBootstrapServers(String bootstrapServers);
+
+      public abstract Builder setProducerConfigUpdates(Map<String, String> producerConfigUpdates);
+
+      public abstract KafkaWriteSchemaTransformConfiguration build();
+    }
+  }
+}
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java
index d2a441d8e58..497496db1b3 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java
@@ -29,12 +29,16 @@ import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 import java.util.UUID;
 import java.util.function.BiFunction;
+import java.util.stream.Collectors;
+import java.util.stream.LongStream;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.NullableCoder;
+import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.common.HashingFn;
 import org.apache.beam.sdk.io.common.IOITHelper;
@@ -49,6 +53,9 @@ import org.apache.beam.sdk.options.ExperimentalOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.StreamingOptions;
 import org.apache.beam.sdk.options.Validation;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.utils.AvroUtils;
+import org.apache.beam.sdk.schemas.utils.JsonUtils;
 import org.apache.beam.sdk.testing.ExpectedLogs;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
@@ -72,9 +79,12 @@ import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionRowTuple;
 import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.TypeDescriptors;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists;
 import org.apache.kafka.clients.admin.AdminClient;
 import org.apache.kafka.clients.admin.NewPartitions;
 import org.apache.kafka.clients.admin.NewTopic;
@@ -502,6 +512,140 @@ public class KafkaIOIT {
         readElementMetric(readResult, NAMESPACE, READ_ELEMENT_METRIC_NAME));
   }
 
+  public static final Schema KAFKA_TOPIC_SCHEMA =
+      Schema.builder()
+          .addStringField("name")
+          .addInt64Field("userId")
+          .addInt64Field("age")
+          .addBooleanField("ageIsEven")
+          .addDoubleField("temperature")
+          .addArrayField("childrenNames", Schema.FieldType.STRING)
+          .build();
+
+  public static final String SCHEMA_IN_JSON =
+      "{\n"
+          + "  \"type\": \"object\",\n"
+          + "  \"properties\": {\n"
+          + "    \"name\": {\n"
+          + "      \"type\": \"string\"\n"
+          + "    },\n"
+          + "    \"userId\": {\n"
+          + "      \"type\": \"integer\"\n"
+          + "    },\n"
+          + "    \"age\": {\n"
+          + "      \"type\": \"integer\"\n"
+          + "    },\n"
+          + "    \"ageIsEven\": {\n"
+          + "      \"type\": \"boolean\"\n"
+          + "    },\n"
+          + "    \"temperature\": {\n"
+          + "      \"type\": \"number\"\n"
+          + "    },\n"
+          + "    \"childrenNames\": {\n"
+          + "      \"type\": \"array\",\n"
+          + "      \"items\": {\n"
+          + "        \"type\": \"string\"\n"
+          + "      }\n"
+          + "    }\n"
+          + "  }\n"
+          + "}";
+
+  private static final int FIVE_MINUTES_IN_MS = 5 * 60 * 1000;
+
+  @Test(timeout = FIVE_MINUTES_IN_MS)
+  public void testKafkaViaSchemaTransformJson() {
+    runReadWriteKafkaViaSchemaTransforms(
+        "JSON", SCHEMA_IN_JSON, JsonUtils.beamSchemaFromJsonSchema(SCHEMA_IN_JSON));
+  }
+
+  @Test(timeout = FIVE_MINUTES_IN_MS)
+  public void testKafkaViaSchemaTransformAvro() {
+    runReadWriteKafkaViaSchemaTransforms(
+        "AVRO", AvroUtils.toAvroSchema(KAFKA_TOPIC_SCHEMA).toString(), KAFKA_TOPIC_SCHEMA);
+  }
+
+  public void runReadWriteKafkaViaSchemaTransforms(
+      String format, String schemaDefinition, Schema beamSchema) {
+    String topicName = options.getKafkaTopic() + "-schema-transform" + UUID.randomUUID();
+    PCollectionRowTuple.of(
+            "input",
+            writePipeline
+                .apply("Generate records", GenerateSequence.from(0).to(1000))
+                .apply(
+                    "Transform to Beam Rows",
+                    MapElements.into(TypeDescriptors.rows())
+                        .via(
+                            numb ->
+                                Row.withSchema(beamSchema)
+                                    .withFieldValue("name", numb.toString())
+                                    .withFieldValue(
+                                        "userId", Long.valueOf(numb.hashCode())) // User ID
+                                    .withFieldValue("age", Long.valueOf(numb.intValue())) // Age
+                                    .withFieldValue("ageIsEven", numb % 2 == 0) // ageIsEven
+                                    .withFieldValue("temperature", new Random(numb).nextDouble())
+                                    .withFieldValue(
+                                        "childrenNames",
+                                        Lists.newArrayList(
+                                            Long.toString(numb + 1),
+                                            Long.toString(numb + 2))) // childrenNames
+                                    .build()))
+                .setRowSchema(beamSchema))
+        .apply(
+            "Write to Kafka",
+            new KafkaWriteSchemaTransformProvider()
+                .from(
+                    KafkaWriteSchemaTransformProvider.KafkaWriteSchemaTransformConfiguration
+                        .builder()
+                        .setTopic(topicName)
+                        .setBootstrapServers(options.getKafkaBootstrapServerAddresses())
+                        .setFormat(format)
+                        .build())
+                .buildTransform());
+
+    PAssert.that(
+            PCollectionRowTuple.empty(readPipeline)
+                .apply(
+                    "Read from unbounded Kafka",
+                    // A timeout of 30s for local, container-based tests, and 2 minutes for
+                    // real-kafka tests.
+                    new KafkaReadSchemaTransformProvider(
+                            true, options.isWithTestcontainers() ? 30 : 120)
+                        .from(
+                            KafkaReadSchemaTransformConfiguration.builder()
+                                .setDataFormat(format)
+                                .setAutoOffsetResetConfig("earliest")
+                                .setSchema(schemaDefinition)
+                                .setTopic(topicName)
+                                .setBootstrapServers(options.getKafkaBootstrapServerAddresses())
+                                .build())
+                        .buildTransform())
+                .get("output"))
+        .containsInAnyOrder(
+            LongStream.range(0L, 1000L)
+                .<Row>mapToObj(
+                    numb ->
+                        Row.withSchema(beamSchema)
+                            .withFieldValue("name", Long.toString(numb)) // Name
+                            .withFieldValue("userId", Long.valueOf(Long.hashCode(numb))) // User ID
+                            .withFieldValue("age", Long.valueOf(numb)) // Age
+                            .withFieldValue("ageIsEven", numb % 2 == 0) // ageIsEven
+                            .withFieldValue("temperature", new Random(numb).nextDouble())
+                            .withFieldValue(
+                                "childrenNames",
+                                Lists.newArrayList(
+                                    Long.toString(numb + 1),
+                                    Long.toString(numb + 2))) // childrenNames
+                            .build())
+                .collect(Collectors.toList()));
+
+    PipelineResult writeResult = writePipeline.run();
+    writeResult.waitUntilFinish();
+
+    PipelineResult readResult = readPipeline.run();
+    readResult.waitUntilFinish(Duration.standardSeconds(options.getReadTimeout()));
+    assertEquals(PipelineResult.State.DONE, readResult.getState());
+  }
+
   private static class DelayedCheckStopReadingFn
       implements SerializableFunction<TopicPartition, Boolean> {
     int checkCount = 0;
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java
index 0ac42c38462..591f05ed6aa 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java
@@ -88,7 +88,7 @@ public class KafkaReadSchemaTransformProviderTest {
             .filter(provider -> provider.getClass() == KafkaReadSchemaTransformProvider.class)
             .collect(Collectors.toList());
     SchemaTransformProvider kafkaProvider = providers.get(0);
-    assertEquals(kafkaProvider.outputCollectionNames(), Lists.newArrayList("OUTPUT"));
+    assertEquals(kafkaProvider.outputCollectionNames(), Lists.newArrayList("output"));
     assertEquals(kafkaProvider.inputCollectionNames(), Lists.newArrayList());
 
     assertEquals(