You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by am...@apache.org on 2021/02/10 23:10:49 UTC
[beam] 01/01: Revert "[BEAM-11659] Extract common payload
serialization behavior for Row transforms. (#13825)"
This is an automated email from the ASF dual-hosted git repository.
amaliujia pushed a commit to branch revert-13825-payload_utils
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 0012fc6f2b8d3650ff50a1f7dcc101f5a4c488d1
Author: Rui Wang <am...@gmail.com>
AuthorDate: Wed Feb 10 15:10:08 2021 -0800
Revert "[BEAM-11659] Extract common payload serialization behavior for Row transforms. (#13825)"
This reverts commit e6eba9edb2632bfd132337873adc1f5c9f5bc703.
---
.../src/main/resources/beam/suppressions.xml | 1 -
.../org/apache/beam/sdk/schemas/io/GenericDlq.java | 17 ++-
.../beam/sdk/schemas/io/GenericDlqProvider.java | 6 +-
.../org/apache/beam/sdk/schemas/io/Providers.java | 54 --------
.../io/payloads/AvroPayloadSerializerProvider.java | 42 ------
.../io/payloads/JsonPayloadSerializerProvider.java | 52 -------
.../sdk/schemas/io/payloads/PayloadSerializer.java | 51 -------
.../io/payloads/PayloadSerializerProvider.java | 37 -----
.../schemas/io/payloads/PayloadSerializers.java | 47 -------
.../beam/sdk/schemas/io/payloads/package-info.java | 24 ----
.../org/apache/beam/sdk/util/RowJsonUtils.java | 2 +-
.../io/AvroPayloadSerializerProviderTest.java | 64 ---------
.../io/JsonPayloadSerializerProviderTest.java | 60 --------
.../meta/provider/kafka/BeamKafkaAvroTable.java | 89 ++++++++++++
.../meta/provider/kafka/BeamKafkaJsonTable.java | 107 ++++++++++++++
.../meta/provider/kafka/BeamKafkaProtoTable.java | 119 ++++++++++++++++
.../meta/provider/kafka/BeamKafkaThriftTable.java | 138 ++++++++++++++++++
.../meta/provider/kafka/KafkaTableProvider.java | 86 +++++++++---
.../kafka/PayloadSerializerKafkaTable.java | 69 ---------
.../payloads/ProtoPayloadSerializerProvider.java | 85 ------------
.../payloads/ThriftPayloadSerializerProvider.java | 108 ---------------
.../sql/meta/provider/payloads/package-info.java | 24 ----
.../provider/kafka/BeamKafkaTableAvroTest.java | 12 +-
.../provider/kafka/BeamKafkaTableJsonTest.java | 12 +-
.../provider/kafka/BeamKafkaTableProtoTest.java | 36 ++---
.../provider/kafka/BeamKafkaTableThriftTest.java | 35 ++---
.../provider/kafka/KafkaTableProviderTest.java | 12 +-
.../ProtoPayloadSerializerProviderTest.java | 111 ---------------
.../ThriftPayloadSerializerProviderTest.java | 154 ---------------------
.../beam/sdk/io/gcp/pubsub/PubsubMessageToRow.java | 98 ++++++++++---
.../sdk/io/gcp/pubsub/PubsubSchemaIOProvider.java | 49 +------
.../beam/sdk/io/gcp/pubsub/RowToPubsubMessage.java | 60 ++++++--
.../sdk/io/gcp/pubsub/PubsubMessageToRowTest.java | 40 ++++--
33 files changed, 732 insertions(+), 1169 deletions(-)
diff --git a/sdks/java/build-tools/src/main/resources/beam/suppressions.xml b/sdks/java/build-tools/src/main/resources/beam/suppressions.xml
index 89b7744..ee03dc3 100644
--- a/sdks/java/build-tools/src/main/resources/beam/suppressions.xml
+++ b/sdks/java/build-tools/src/main/resources/beam/suppressions.xml
@@ -98,7 +98,6 @@
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*google.*cloud.*spanner.*FakeBatchTransactionId\.java" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*google.*cloud.*spanner.*FakePartitionFactory\.java" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*extensions.*sql.*datastore.*" />
- <suppress id="ForbidNonVendoredGrpcProtobuf" files=".*extensions.*sql.*ProtoPayloadSerializerProvider.*" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*datacatalog.*DataCatalogTableProvider\.java" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*zetasql.*DateTimeUtils\.java" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*zetasql.*ZetaSqlBeamTranslationUtils\.java" />
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlq.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlq.java
index 03c9912..f059c11 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlq.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlq.java
@@ -19,8 +19,10 @@ package org.apache.beam.sdk.schemas.io;
import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.ServiceLoader;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
import org.apache.beam.sdk.annotations.Internal;
@@ -35,8 +37,19 @@ import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter;
public final class GenericDlq {
private GenericDlq() {}
- private static final Map<String, GenericDlqProvider> PROVIDERS =
- Providers.loadProviders(GenericDlqProvider.class);
+ private static final Map<String, GenericDlqProvider> PROVIDERS = loadDlqProviders();
+
+ private static Map<String, GenericDlqProvider> loadDlqProviders() {
+ Map<String, GenericDlqProvider> providers = new HashMap<>();
+ for (GenericDlqProvider provider : ServiceLoader.load(GenericDlqProvider.class)) {
+ checkArgument(
+ !providers.containsKey(provider.identifier()),
+ "Duplicate providers exist with identifier `%s`.",
+ provider.identifier());
+ providers.put(provider.identifier(), provider);
+ }
+ return providers;
+ }
@SuppressWarnings("dereference.of.nullable")
public static PTransform<PCollection<Failure>, PDone> getDlqTransform(String fullConfig) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlqProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlqProvider.java
index a74a6fd..e42fb37 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlqProvider.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlqProvider.java
@@ -20,7 +20,6 @@ package org.apache.beam.sdk.schemas.io;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
import org.apache.beam.sdk.annotations.Internal;
-import org.apache.beam.sdk.schemas.io.Providers.Identifyable;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PDone;
@@ -28,7 +27,10 @@ import org.apache.beam.sdk.values.PDone;
/** A Provider for generic DLQ transforms that handle deserialization failures. */
@Internal
@Experimental(Kind.SCHEMAS)
-public interface GenericDlqProvider extends Identifyable {
+public interface GenericDlqProvider {
+ /** Returns an id that uniquely represents this Dlq Provider. */
+ String identifier();
+
/** Generate a DLQ output from the provided config value. */
PTransform<PCollection<Failure>, PDone> newDlqTransform(String config);
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/Providers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/Providers.java
deleted file mode 100644
index dc0f758..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/Providers.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.schemas.io;
-
-import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.ServiceLoader;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.annotations.Internal;
-
-/** Helpers for implementing the "Provider" pattern. */
-@Internal
-@Experimental(Kind.SCHEMAS)
-public final class Providers {
- public interface Identifyable {
- /**
- * Returns an id that uniquely represents this among others implementing its derived interface.
- */
- String identifier();
- }
-
- private Providers() {}
-
- public static <T extends Identifyable> Map<String, T> loadProviders(Class<T> klass) {
- Map<String, T> providers = new HashMap<>();
- for (T provider : ServiceLoader.load(klass)) {
- checkArgument(
- !providers.containsKey(provider.identifier()),
- "Duplicate providers exist with identifier `%s` for class %s.",
- provider.identifier(),
- klass);
- providers.put(provider.identifier(), provider);
- }
- return providers;
- }
-}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/AvroPayloadSerializerProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/AvroPayloadSerializerProvider.java
deleted file mode 100644
index ace671f..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/AvroPayloadSerializerProvider.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.schemas.io.payloads;
-
-import com.google.auto.service.AutoService;
-import java.util.Map;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.annotations.Internal;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.utils.AvroUtils;
-
-@Internal
-@Experimental(Kind.SCHEMAS)
-@AutoService(PayloadSerializerProvider.class)
-public class AvroPayloadSerializerProvider implements PayloadSerializerProvider {
- @Override
- public String identifier() {
- return "avro";
- }
-
- @Override
- public PayloadSerializer getSerializer(Schema schema, Map<String, Object> tableParams) {
- return PayloadSerializer.of(
- AvroUtils.getRowToAvroBytesFunction(schema), AvroUtils.getAvroBytesToRowFunction(schema));
- }
-}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/JsonPayloadSerializerProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/JsonPayloadSerializerProvider.java
deleted file mode 100644
index a8a85ed..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/JsonPayloadSerializerProvider.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.schemas.io.payloads;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.auto.service.AutoService;
-import java.util.Map;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.annotations.Internal;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.util.RowJson.RowJsonDeserializer;
-import org.apache.beam.sdk.util.RowJson.RowJsonSerializer;
-import org.apache.beam.sdk.util.RowJsonUtils;
-
-@Internal
-@Experimental(Kind.SCHEMAS)
-@AutoService(PayloadSerializerProvider.class)
-public class JsonPayloadSerializerProvider implements PayloadSerializerProvider {
- @Override
- public String identifier() {
- return "json";
- }
-
- @Override
- public PayloadSerializer getSerializer(Schema schema, Map<String, Object> tableParams) {
- ObjectMapper deserializeMapper =
- RowJsonUtils.newObjectMapperWith(RowJsonDeserializer.forSchema(schema));
- ObjectMapper serializeMapper =
- RowJsonUtils.newObjectMapperWith(RowJsonSerializer.forSchema(schema));
- return PayloadSerializer.of(
- row -> RowJsonUtils.rowToJson(serializeMapper, row).getBytes(UTF_8),
- bytes -> RowJsonUtils.jsonToRow(deserializeMapper, new String(bytes, UTF_8)));
- }
-}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializer.java
deleted file mode 100644
index a902553..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializer.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.schemas.io.payloads;
-
-import java.io.Serializable;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.annotations.Internal;
-import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.values.Row;
-
-@Internal
-@Experimental(Kind.SCHEMAS)
-public interface PayloadSerializer extends Serializable {
- long serialVersionUID = 5645783967169L;
-
- byte[] serialize(Row row);
-
- Row deserialize(byte[] bytes);
-
- static PayloadSerializer of(
- SerializableFunction<Row, byte[]> serializeFn,
- SerializableFunction<byte[], Row> deserializeFn) {
- return new PayloadSerializer() {
- @Override
- public byte[] serialize(Row row) {
- return serializeFn.apply(row);
- }
-
- @Override
- public Row deserialize(byte[] bytes) {
- return deserializeFn.apply(bytes);
- }
- };
- }
-}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializerProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializerProvider.java
deleted file mode 100644
index eeda798..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializerProvider.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.schemas.io.payloads;
-
-import java.util.Map;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.annotations.Internal;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.io.Providers.Identifyable;
-
-@Internal
-@Experimental(Kind.SCHEMAS)
-public interface PayloadSerializerProvider extends Identifyable {
- /**
- * Get a PayloadSerializer.
- *
- * @param schema the schema of the payload
- * @param tableParams parameters passed at table declaration time for configuring the serializer
- */
- PayloadSerializer getSerializer(Schema schema, Map<String, Object> tableParams);
-}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializers.java
deleted file mode 100644
index 0a8199f..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializers.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.schemas.io.payloads;
-
-import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument;
-
-import java.util.Map;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.annotations.Internal;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.io.Providers;
-
-@Internal
-@Experimental(Kind.SCHEMAS)
-public final class PayloadSerializers {
- private PayloadSerializers() {}
-
- private static final Map<String, PayloadSerializerProvider> PROVIDERS =
- Providers.loadProviders(PayloadSerializerProvider.class);
-
- @SuppressWarnings("dereference.of.nullable")
- public static PayloadSerializer getSerializer(
- String id, Schema schema, Map<String, Object> tableParams) {
- PayloadSerializerProvider provider = PROVIDERS.get(id);
- checkArgument(
- provider != null,
- "Invalid config, no serializer provider exists with identifier `%s`.",
- id);
- return provider.getSerializer(schema, tableParams);
- }
-}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/package-info.java
deleted file mode 100644
index 91fd27c..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** Provides abstractions for schema-aware IOs. */
-@DefaultAnnotation(NonNull.class)
-package org.apache.beam.sdk.schemas.io.payloads;
-
-import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
-import org.checkerframework.checker.nullness.qual.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/RowJsonUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/RowJsonUtils.java
index 6538a14..956cdd0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/RowJsonUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/RowJsonUtils.java
@@ -68,7 +68,7 @@ public class RowJsonUtils {
try {
return objectMapper.writeValueAsString(row);
} catch (JsonProcessingException e) {
- throw new IllegalArgumentException("Unable to serialize row: " + row, e);
+ throw new IllegalArgumentException("Unable to serilize row: " + row, e);
}
}
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/AvroPayloadSerializerProviderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/AvroPayloadSerializerProviderTest.java
deleted file mode 100644
index dbe9ae0..0000000
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/AvroPayloadSerializerProviderTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.schemas.io;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.generic.GenericRecordBuilder;
-import org.apache.beam.sdk.coders.AvroCoder;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.io.payloads.AvroPayloadSerializerProvider;
-import org.apache.beam.sdk.schemas.utils.AvroUtils;
-import org.apache.beam.sdk.values.Row;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-@RunWith(JUnit4.class)
-public class AvroPayloadSerializerProviderTest {
- private static final Schema SCHEMA =
- Schema.builder().addInt64Field("abc").addStringField("xyz").build();
- private static final org.apache.avro.Schema AVRO_SCHEMA = AvroUtils.toAvroSchema(SCHEMA);
- private static final AvroCoder<GenericRecord> AVRO_CODER = AvroCoder.of(AVRO_SCHEMA);
- private static final Row DESERIALIZED =
- Row.withSchema(SCHEMA).withFieldValue("abc", 3L).withFieldValue("xyz", "qqq").build();
- private static final GenericRecord SERIALIZED =
- new GenericRecordBuilder(AVRO_SCHEMA).set("abc", 3L).set("xyz", "qqq").build();
-
- private final AvroPayloadSerializerProvider provider = new AvroPayloadSerializerProvider();
-
- @Test
- public void serialize() throws Exception {
- byte[] bytes = provider.getSerializer(SCHEMA, ImmutableMap.of()).serialize(DESERIALIZED);
- GenericRecord record = AVRO_CODER.decode(new ByteArrayInputStream(bytes));
- assertEquals(3L, record.get("abc"));
- assertEquals("qqq", record.get("xyz").toString());
- }
-
- @Test
- public void deserialize() throws Exception {
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- AVRO_CODER.encode(SERIALIZED, os);
- Row row = provider.getSerializer(SCHEMA, ImmutableMap.of()).deserialize(os.toByteArray());
- assertEquals(DESERIALIZED, row);
- }
-}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/JsonPayloadSerializerProviderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/JsonPayloadSerializerProviderTest.java
deleted file mode 100644
index 51cecad..0000000
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/JsonPayloadSerializerProviderTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.schemas.io;
-
-import static org.junit.Assert.assertEquals;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import java.nio.charset.StandardCharsets;
-import java.util.Map;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.io.payloads.JsonPayloadSerializerProvider;
-import org.apache.beam.sdk.values.Row;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-@RunWith(JUnit4.class)
-public class JsonPayloadSerializerProviderTest {
- private static final Schema SCHEMA =
- Schema.builder().addInt64Field("abc").addStringField("xyz").build();
- private static final String SERIALIZED = "{ \"abc\": 3, \"xyz\": \"qqq\" }";
- private static final Row DESERIALIZED =
- Row.withSchema(SCHEMA).withFieldValue("abc", 3L).withFieldValue("xyz", "qqq").build();
-
- private final JsonPayloadSerializerProvider provider = new JsonPayloadSerializerProvider();
-
- @Test
- public void serialize() throws Exception {
- byte[] bytes = provider.getSerializer(SCHEMA, ImmutableMap.of()).serialize(DESERIALIZED);
- ObjectMapper mapper = new ObjectMapper();
- Map<String, Object> result = mapper.readValue(bytes, Map.class);
- assertEquals(3, result.get("abc"));
- assertEquals("qqq", result.get("xyz"));
- }
-
- @Test
- public void deserialize() {
- Row row =
- provider
- .getSerializer(SCHEMA, ImmutableMap.of())
- .deserialize(SERIALIZED.getBytes(StandardCharsets.UTF_8));
- assertEquals(DESERIALIZED, row);
- }
-}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaAvroTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaAvroTable.java
new file mode 100644
index 0000000..4045106
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaAvroTable.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
+
+import java.util.List;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.utils.AvroUtils;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.TypeDescriptor;
+
+public class BeamKafkaAvroTable extends BeamKafkaTable {
+
+ public BeamKafkaAvroTable(Schema beamSchema, String bootstrapServers, List<String> topics) {
+ super(beamSchema, bootstrapServers, topics);
+ }
+
+ @Override
+ protected PTransform<PCollection<KV<byte[], byte[]>>, PCollection<Row>> getPTransformForInput() {
+ return new AvroRecorderDecoder(schema);
+ }
+
+ @Override
+ protected PTransform<PCollection<Row>, PCollection<KV<byte[], byte[]>>> getPTransformForOutput() {
+ return new AvroRecorderEncoder(schema);
+ }
+
+ /** A PTransform to convert {@code KV<byte[], byte[]>} to {@link Row}. */
+ private static class AvroRecorderDecoder
+ extends PTransform<PCollection<KV<byte[], byte[]>>, PCollection<Row>> {
+ private final Schema schema;
+
+ AvroRecorderDecoder(Schema schema) {
+ this.schema = schema;
+ }
+
+ @Override
+ public PCollection<Row> expand(PCollection<KV<byte[], byte[]>> input) {
+ return input
+ .apply(
+ "extractValue", MapElements.into(TypeDescriptor.of(byte[].class)).via(KV::getValue))
+ .apply("decodeAvroRecord", MapElements.via(AvroUtils.getAvroBytesToRowFunction(schema)))
+ .setRowSchema(schema);
+ }
+ }
+
+ /** A PTransform to convert {@link Row} to {@code KV<byte[], byte[]>}. */
+ private static class AvroRecorderEncoder
+ extends PTransform<PCollection<Row>, PCollection<KV<byte[], byte[]>>> {
+ private final Schema schema;
+
+ AvroRecorderEncoder(Schema schema) {
+ this.schema = schema;
+ }
+
+ @Override
+ public PCollection<KV<byte[], byte[]>> expand(PCollection<Row> input) {
+ return input
+ .apply("encodeAvroRecord", MapElements.via(AvroUtils.getRowToAvroBytesFunction(schema)))
+ .apply("mapToKV", MapElements.via(new MakeBytesKVFn()));
+ }
+
+ private static class MakeBytesKVFn extends SimpleFunction<byte[], KV<byte[], byte[]>> {
+ @Override
+ public KV<byte[], byte[]> apply(byte[] bytes) {
+ return KV.of(new byte[] {}, bytes);
+ }
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaJsonTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaJsonTable.java
new file mode 100644
index 0000000..16a5d8e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaJsonTable.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.List;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.util.RowJson;
+import org.apache.beam.sdk.util.RowJsonUtils;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.Row;
+
+public class BeamKafkaJsonTable extends BeamKafkaTable {
+ public BeamKafkaJsonTable(Schema beamSchema, String bootstrapServers, List<String> topics) {
+ super(beamSchema, bootstrapServers, topics);
+ }
+
+ @Override
+ public PTransform<PCollection<KV<byte[], byte[]>>, PCollection<Row>> getPTransformForInput() {
+ ObjectMapper objectMapper =
+ RowJsonUtils.newObjectMapperWith(RowJson.RowJsonDeserializer.forSchema(schema));
+ return new BeamKafkaJsonTable.JsonRecorderDecoder(schema, objectMapper);
+ }
+
+ @Override
+ public PTransform<PCollection<Row>, PCollection<KV<byte[], byte[]>>> getPTransformForOutput() {
+ ObjectMapper objectMapper =
+ RowJsonUtils.newObjectMapperWith(RowJson.RowJsonSerializer.forSchema(schema));
+ return new BeamKafkaJsonTable.JsonRecorderEncoder(objectMapper);
+ }
+
+ /** A PTransform to convert {@code KV<byte[], byte[]>} to {@link Row}. */
+ private static class JsonRecorderDecoder
+ extends PTransform<PCollection<KV<byte[], byte[]>>, PCollection<Row>> {
+ private final Schema schema;
+ private final ObjectMapper objectMapper;
+
+ public JsonRecorderDecoder(Schema schema, ObjectMapper objectMapper) {
+ this.schema = schema;
+ this.objectMapper = objectMapper;
+ }
+
+ @Override
+ public PCollection<Row> expand(PCollection<KV<byte[], byte[]>> input) {
+ return input
+ .apply(
+ "decodeJsonRecord",
+ ParDo.of(
+ new DoFn<KV<byte[], byte[]>, Row>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ String rowInString = new String(c.element().getValue(), UTF_8);
+ Row row = RowJsonUtils.jsonToRow(objectMapper, rowInString);
+ c.output(row);
+ }
+ }))
+ .setRowSchema(schema);
+ }
+ }
+
+ /** A PTransform to convert {@link Row} to {@code KV<byte[], byte[]>}. */
+ private static class JsonRecorderEncoder
+ extends PTransform<PCollection<Row>, PCollection<KV<byte[], byte[]>>> {
+ private final ObjectMapper objectMapper;
+
+ public JsonRecorderEncoder(ObjectMapper objectMapper) {
+ this.objectMapper = objectMapper;
+ }
+
+ @Override
+ public PCollection<KV<byte[], byte[]>> expand(PCollection<Row> input) {
+ return input.apply(
+ "encodeJsonRecord",
+ ParDo.of(
+ new DoFn<Row, KV<byte[], byte[]>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(
+ KV.of(
+ new byte[] {},
+ RowJsonUtils.rowToJson(objectMapper, c.element()).getBytes(UTF_8)));
+ }
+ }));
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaProtoTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaProtoTable.java
new file mode 100644
index 0000000..7dc6134
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaProtoTable.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.protobuf.ProtoMessageSchema;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.TypeDescriptor;
+
+@SuppressWarnings({
+ "nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402)
+})
+public class BeamKafkaProtoTable extends BeamKafkaTable {
+ private final Class<?> protoClass;
+
+ public BeamKafkaProtoTable(
+ Schema messageSchema, String bootstrapServers, List<String> topics, Class<?> protoClass) {
+ super(inferAndVerifySchema(protoClass, messageSchema), bootstrapServers, topics);
+ this.protoClass = protoClass;
+ }
+
+ @Override
+ public PTransform<PCollection<KV<byte[], byte[]>>, PCollection<Row>> getPTransformForInput() {
+ return new ProtoRecorderDecoder(schema, protoClass);
+ }
+
+ @Override
+ public PTransform<PCollection<Row>, PCollection<KV<byte[], byte[]>>> getPTransformForOutput() {
+ return new ProtoRecorderEncoder(protoClass);
+ }
+
+ private static Schema inferAndVerifySchema(Class<?> protoClass, Schema messageSchema) {
+ Schema inferredSchema = new ProtoMessageSchema().schemaFor(TypeDescriptor.of(protoClass));
+ if (!messageSchema.equivalent(inferredSchema)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Given message schema: '%s'%n"
+ + "does not match schema inferred from protobuf class.%n"
+ + "Protobuf class: '%s'%n"
+ + "Inferred schema: '%s'",
+ messageSchema, protoClass.getName(), inferredSchema));
+ }
+ return inferredSchema;
+ }
+
+ /** A PTransform to convert {@code KV<byte[], byte[]>} to {@link Row}. */
+ private static class ProtoRecorderDecoder
+ extends PTransform<PCollection<KV<byte[], byte[]>>, PCollection<Row>> {
+ private final Schema schema;
+ private final Class<?> clazz;
+
+ ProtoRecorderDecoder(Schema schema, Class<?> clazz) {
+ this.schema = schema;
+ this.clazz = clazz;
+ }
+
+ @Override
+ public PCollection<Row> expand(PCollection<KV<byte[], byte[]>> input) {
+ SimpleFunction<byte[], Row> toRowFn = ProtoMessageSchema.getProtoBytesToRowFn(clazz);
+ return input
+ .apply("decodeProtoRecord", MapElements.via(new KvToBytes()))
+ .apply("Map bytes to rows", MapElements.via(toRowFn))
+ .setRowSchema(schema);
+ }
+
+ private static class KvToBytes extends SimpleFunction<KV<byte[], byte[]>, byte[]> {
+ @Override
+ public byte[] apply(KV<byte[], byte[]> kv) {
+ return kv.getValue();
+ }
+ }
+ }
+
+ /** A PTransform to convert {@link Row} to {@code KV<byte[], byte[]>}. */
+ private static class ProtoRecorderEncoder
+ extends PTransform<PCollection<Row>, PCollection<KV<byte[], byte[]>>> {
+ private final Class<?> clazz;
+
+ public ProtoRecorderEncoder(Class<?> clazz) {
+ this.clazz = clazz;
+ }
+
+ @Override
+ public PCollection<KV<byte[], byte[]>> expand(PCollection<Row> input) {
+ SimpleFunction<Row, byte[]> toBytesFn = ProtoMessageSchema.getRowToProtoBytesFn(clazz);
+ return input
+ .apply("Encode proto bytes to row", MapElements.via(toBytesFn))
+ .apply("Bytes to KV", MapElements.via(new BytesToKV()));
+ }
+
+ private static class BytesToKV extends SimpleFunction<byte[], KV<byte[], byte[]>> {
+ @Override
+ public KV<byte[], byte[]> apply(byte[] bytes) {
+ return KV.of(new byte[] {}, bytes);
+ }
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaThriftTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaThriftTable.java
new file mode 100644
index 0000000..5768ace
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaThriftTable.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
+
+import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull;
+
+import java.util.List;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.thrift.ThriftCoder;
+import org.apache.beam.sdk.io.thrift.ThriftSchema;
+import org.apache.beam.sdk.schemas.RowMessages;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.SchemaProvider;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.Values;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
+import org.apache.thrift.TBase;
+import org.apache.thrift.TFieldIdEnum;
+import org.apache.thrift.protocol.TProtocolFactory;
+
+/**
+ * Kafka table that handles thrift data.
+ *
+ * <p>Uses the default thrift schema {@link ThriftSchema#provider() provider} , so it can't handle
+ * {@link ThriftSchema#custom() custom} container typedefs.
+ *
+ * @param <FieldT> thrift field type, i.e. {@code ThriftGeneratedClass._Fields}
+ * @param <T> thrift generated class
+ */
+public class BeamKafkaThriftTable<FieldT extends TFieldIdEnum, T extends TBase<T, FieldT>>
+ extends BeamKafkaTable {
+
+ private static final SchemaProvider schemaProvider = ThriftSchema.provider();
+
+ private final TypeDescriptor<T> typeDescriptor;
+ private final Coder<T> coder;
+
+ public BeamKafkaThriftTable(
+ Schema requiredSchema,
+ String bootstrapServers,
+ List<String> topics,
+ Class<T> thriftClass,
+ TProtocolFactory protocolFactory) {
+ super(thriftSchema(thriftClass, requiredSchema), bootstrapServers, topics);
+ typeDescriptor = TypeDescriptor.of(thriftClass);
+ coder = ThriftCoder.of(thriftClass, protocolFactory);
+ }
+
+ private static Schema thriftSchema(Class<?> thriftClass, Schema requiredSchema) {
+ final TypeDescriptor<?> typeDescriptor = TypeDescriptor.of(thriftClass);
+ final Schema schema =
+ checkArgumentNotNull(BeamKafkaThriftTable.schemaProvider.schemaFor(typeDescriptor));
+ if (!schema.assignableTo(requiredSchema)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Given message schema: '%s'%n"
+ + "does not match schema inferred from thrift class.%n"
+ + "Thrift class: '%s'%n"
+ + "Inferred schema: '%s'",
+ requiredSchema, thriftClass.getName(), schema));
+ }
+ return schema;
+ }
+
+ @Override
+ protected PTransform<PCollection<KV<byte[], byte[]>>, PCollection<Row>> getPTransformForInput() {
+ return new InputTransformer(schema, typeDescriptor, coder);
+ }
+
+ private static class InputTransformer<T extends TBase<?, ?>>
+ extends PTransform<PCollection<KV<byte[], byte[]>>, PCollection<Row>> {
+
+ private final Schema schema;
+ private final SimpleFunction<byte[], Row> toRowFn;
+
+ InputTransformer(Schema schema, TypeDescriptor<T> typeDescriptor, Coder<T> coder) {
+ this.schema = schema;
+ toRowFn = RowMessages.bytesToRowFn(schemaProvider, typeDescriptor, coder);
+ }
+
+ @Override
+ public PCollection<Row> expand(PCollection<KV<byte[], byte[]>> input) {
+ return input
+ .apply("drop-kafka-keys", Values.create())
+ .apply("thrift-to-row", MapElements.via(toRowFn))
+ .setRowSchema(schema);
+ }
+ }
+
+ @Override
+ protected PTransform<PCollection<Row>, PCollection<KV<byte[], byte[]>>> getPTransformForOutput() {
+ return new OutputTransformer(typeDescriptor, coder);
+ }
+
+ private static class OutputTransformer<T extends TBase<?, ?>>
+ extends PTransform<PCollection<Row>, PCollection<KV<byte[], byte[]>>> {
+
+ private static final TypeDescriptor<byte[]> binTypeDescriptor = TypeDescriptor.of(byte[].class);
+ private static final byte[] emptyKey = {};
+
+ private final SimpleFunction<Row, byte[]> fromRowFn;
+
+ OutputTransformer(TypeDescriptor<T> typeDescriptor, Coder<T> coder) {
+ this.fromRowFn = RowMessages.rowToBytesFn(schemaProvider, typeDescriptor, coder);
+ }
+
+ @Override
+ public PCollection<KV<byte[], byte[]>> expand(PCollection<Row> input) {
+ return input
+ .apply("row-to-thrift-bytes", MapElements.via(fromRowFn))
+ .apply(
+ "bytes-to-kvs",
+ MapElements.into(TypeDescriptors.kvs(binTypeDescriptor, binTypeDescriptor))
+ .via(bytes -> KV.of(emptyKey, bytes)));
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProvider.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProvider.java
index a1b06ae..486b63b 100644
--- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProvider.java
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProvider.java
@@ -20,6 +20,7 @@ package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.auto.service.AutoService;
+import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable;
@@ -27,7 +28,8 @@ import org.apache.beam.sdk.extensions.sql.meta.Table;
import org.apache.beam.sdk.extensions.sql.meta.provider.InMemoryMetaTableProvider;
import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers;
+import org.apache.thrift.TBase;
+import org.apache.thrift.protocol.TProtocolFactory;
/**
* Kafka table provider.
@@ -46,6 +48,15 @@ import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers;
*/
@AutoService(TableProvider.class)
public class KafkaTableProvider extends InMemoryMetaTableProvider {
+
+ private enum PayloadFormat {
+ CSV,
+ AVRO,
+ JSON,
+ PROTO,
+ THRIFT
+ }
+
@Override
public BeamSqlTable buildBeamSqlTable(Table table) {
Schema schema = table.getSchema();
@@ -58,22 +69,65 @@ public class KafkaTableProvider extends InMemoryMetaTableProvider {
topics.add(topic.toString());
}
- String payloadFormat =
- properties.containsKey("format") ? properties.getString("format") : "csv";
- /*
- * CSV is handled separately because multiple rows can be produced from a single message, which
- * adds complexity to payload extraction. It remains here and as the default because it is the
- * historical default, but it will not be extended to support attaching extended attributes to
- * rows.
- */
- if (payloadFormat.equals("csv")) {
- return new BeamKafkaCSVTable(schema, bootstrapServers, topics);
+ PayloadFormat payloadFormat =
+ properties.containsKey("format")
+ ? PayloadFormat.valueOf(properties.getString("format").toUpperCase())
+ : PayloadFormat.CSV;
+
+ switch (payloadFormat) {
+ case CSV:
+ return new BeamKafkaCSVTable(schema, bootstrapServers, topics);
+ case AVRO:
+ return new BeamKafkaAvroTable(schema, bootstrapServers, topics);
+ case JSON:
+ return new BeamKafkaJsonTable(schema, bootstrapServers, topics);
+ case PROTO:
+ return protoTable(schema, bootstrapServers, topics, properties);
+ case THRIFT:
+ return thriftTable(schema, bootstrapServers, topics, properties);
+ default:
+ throw new IllegalArgumentException("Unsupported payload format: " + payloadFormat);
+ }
+ }
+
+ private BeamKafkaProtoTable protoTable(
+ Schema schema, String bootstrapServers, List<String> topics, JSONObject properties) {
+ String protoClassName = properties.getString("protoClass");
+ try {
+ Class<?> protoClass = Class.forName(protoClassName);
+ return new BeamKafkaProtoTable(schema, bootstrapServers, topics, protoClass);
+ } catch (ClassNotFoundException e) {
+ throw new IllegalArgumentException("Incorrect proto class provided: " + protoClassName);
+ }
+ }
+
+ private BeamKafkaThriftTable thriftTable(
+ Schema schema, String bootstrapServers, List<String> topics, JSONObject properties) {
+ final String thriftClassName = properties.getString("thriftClass");
+ final String thriftProtocolFactoryClassName =
+ properties.getString("thriftProtocolFactoryClass");
+ try {
+ final Class<TBase> thriftClass = (Class<TBase>) Class.forName(thriftClassName);
+ final TProtocolFactory thriftProtocolFactory;
+ try {
+ final Class<TProtocolFactory> thriftProtocolFactoryClass =
+ (Class<TProtocolFactory>) Class.forName(thriftProtocolFactoryClassName);
+ thriftProtocolFactory = thriftProtocolFactoryClass.getDeclaredConstructor().newInstance();
+ } catch (ClassNotFoundException e) {
+ throw new IllegalArgumentException(
+ "Incorrect thrift protocol factory class provided: " + thriftProtocolFactoryClassName);
+ } catch (InstantiationException
+ | IllegalAccessException
+ | InvocationTargetException
+ | NoSuchMethodException e) {
+ throw new IllegalStateException(
+ "Could not instantiate the thrift protocol factory class", e);
+ }
+ return new BeamKafkaThriftTable<>(
+ schema, bootstrapServers, topics, thriftClass, thriftProtocolFactory);
+ } catch (ClassNotFoundException e) {
+ throw new IllegalArgumentException("Incorrect thrift class provided: " + thriftClassName);
}
- return new PayloadSerializerKafkaTable(
- schema,
- bootstrapServers,
- topics,
- PayloadSerializers.getSerializer(payloadFormat, schema, properties.getInnerMap()));
}
@Override
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/PayloadSerializerKafkaTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/PayloadSerializerKafkaTable.java
deleted file mode 100644
index ab0e814..0000000
--- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/PayloadSerializerKafkaTable.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
-
-import java.util.List;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer;
-import org.apache.beam.sdk.transforms.MapElements;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.Row;
-import org.apache.beam.sdk.values.TypeDescriptor;
-
-public class PayloadSerializerKafkaTable extends BeamKafkaTable {
- private final PayloadSerializer serializer;
-
- PayloadSerializerKafkaTable(
- Schema requiredSchema,
- String bootstrapServers,
- List<String> topics,
- PayloadSerializer serializer) {
- super(requiredSchema, bootstrapServers, topics);
- this.serializer = serializer;
- }
-
- @Override
- protected PTransform<PCollection<KV<byte[], byte[]>>, PCollection<Row>> getPTransformForInput() {
- return new PTransform<PCollection<KV<byte[], byte[]>>, PCollection<Row>>(
- "deserialize-kafka-rows") {
- @Override
- public PCollection<Row> expand(PCollection<KV<byte[], byte[]>> input) {
- return input
- .apply(
- MapElements.into(TypeDescriptor.of(Row.class))
- .via(kv -> serializer.deserialize(kv.getValue())))
- .setRowSchema(getSchema());
- }
- };
- }
-
- @Override
- protected PTransform<PCollection<Row>, PCollection<KV<byte[], byte[]>>> getPTransformForOutput() {
- return new PTransform<PCollection<Row>, PCollection<KV<byte[], byte[]>>>(
- "serialize-kafka-rows") {
- @Override
- public PCollection<KV<byte[], byte[]>> expand(PCollection<Row> input) {
- return input.apply(
- MapElements.into(new TypeDescriptor<KV<byte[], byte[]>>() {})
- .via(row -> KV.of(new byte[] {}, serializer.serialize(row))));
- }
- };
- }
-}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/ProtoPayloadSerializerProvider.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/ProtoPayloadSerializerProvider.java
deleted file mode 100644
index 1b571f8..0000000
--- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/ProtoPayloadSerializerProvider.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.extensions.sql.meta.provider.payloads;
-
-import static org.apache.beam.sdk.schemas.transforms.Cast.castRow;
-import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull;
-
-import com.google.auto.service.AutoService;
-import com.google.protobuf.Message;
-import java.util.Map;
-import javax.annotation.Nonnull;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.annotations.Internal;
-import org.apache.beam.sdk.extensions.protobuf.ProtoMessageSchema;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer;
-import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializerProvider;
-import org.apache.beam.sdk.transforms.SimpleFunction;
-import org.apache.beam.sdk.values.Row;
-import org.apache.beam.sdk.values.TypeDescriptor;
-
-@Internal
-@Experimental(Kind.SCHEMAS)
-@AutoService(PayloadSerializerProvider.class)
-public class ProtoPayloadSerializerProvider implements PayloadSerializerProvider {
- @Override
- public String identifier() {
- return "proto";
- }
-
- private static Class<? extends Message> getClass(Map<String, Object> tableParams) {
- String protoClassName = checkArgumentNotNull(tableParams.get("protoClass")).toString();
- try {
- Class<?> protoClass = Class.forName(protoClassName);
- return protoClass.asSubclass(Message.class);
- } catch (ClassNotFoundException e) {
- throw new IllegalArgumentException("Incorrect proto class provided: " + protoClassName, e);
- }
- }
-
- private static <T extends Message> void inferAndVerifySchema(
- Class<T> protoClass, Schema requiredSchema) {
- @Nonnull
- Schema inferredSchema =
- checkArgumentNotNull(new ProtoMessageSchema().schemaFor(TypeDescriptor.of(protoClass)));
- if (!inferredSchema.assignableTo(requiredSchema)) {
- throw new IllegalArgumentException(
- String.format(
- "Given message schema: '%s'%n"
- + "does not match schema inferred from protobuf class.%n"
- + "Protobuf class: '%s'%n"
- + "Inferred schema: '%s'",
- requiredSchema, protoClass.getName(), inferredSchema));
- }
- }
-
- @Override
- public PayloadSerializer getSerializer(Schema schema, Map<String, Object> tableParams) {
- Class<? extends Message> protoClass = getClass(tableParams);
- inferAndVerifySchema(protoClass, schema);
- SimpleFunction<byte[], Row> toRowFn = ProtoMessageSchema.getProtoBytesToRowFn(protoClass);
- return PayloadSerializer.of(
- ProtoMessageSchema.getRowToProtoBytesFn(protoClass),
- bytes -> {
- Row rawRow = toRowFn.apply(bytes);
- return castRow(rawRow, rawRow.getSchema(), schema);
- });
- }
-}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/ThriftPayloadSerializerProvider.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/ThriftPayloadSerializerProvider.java
deleted file mode 100644
index af50011..0000000
--- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/ThriftPayloadSerializerProvider.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.extensions.sql.meta.provider.payloads;
-
-import static org.apache.beam.sdk.schemas.transforms.Cast.castRow;
-import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull;
-
-import com.google.auto.service.AutoService;
-import java.util.Map;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.annotations.Internal;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.io.thrift.ThriftCoder;
-import org.apache.beam.sdk.io.thrift.ThriftSchema;
-import org.apache.beam.sdk.schemas.RowMessages;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer;
-import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializerProvider;
-import org.apache.beam.sdk.transforms.SimpleFunction;
-import org.apache.beam.sdk.values.Row;
-import org.apache.beam.sdk.values.TypeDescriptor;
-import org.apache.thrift.TBase;
-import org.apache.thrift.protocol.TProtocolFactory;
-
-@Internal
-@Experimental(Kind.SCHEMAS)
-@AutoService(PayloadSerializerProvider.class)
-public class ThriftPayloadSerializerProvider implements PayloadSerializerProvider {
- @Override
- public String identifier() {
- return "thrift";
- }
-
- private static Class<? extends TBase> getMessageClass(Map<String, Object> tableParams) {
- String thriftClassName = checkArgumentNotNull(tableParams.get("thriftClass")).toString();
- try {
- Class<?> thriftClass = Class.forName(thriftClassName);
- return thriftClass.asSubclass(TBase.class);
- } catch (ClassNotFoundException e) {
- throw new IllegalArgumentException("Incorrect thrift class provided: " + thriftClassName, e);
- }
- }
-
- private static TProtocolFactory getProtocolFactory(Map<String, Object> tableParams) {
- String thriftFactoryClassName =
- checkArgumentNotNull(tableParams.get("thriftProtocolFactoryClass")).toString();
- try {
- Class<?> thriftClass = Class.forName(thriftFactoryClassName);
- return thriftClass.asSubclass(TProtocolFactory.class).getDeclaredConstructor().newInstance();
- } catch (ReflectiveOperationException e) {
- throw new IllegalArgumentException(
- "Incorrect thrift protocol factory class provided: " + thriftFactoryClassName, e);
- }
- }
-
- private static void inferAndVerifySchema(Class<?> thriftClass, Schema requiredSchema) {
- TypeDescriptor<?> typeDescriptor = TypeDescriptor.of(thriftClass);
- Schema schema = checkArgumentNotNull(ThriftSchema.provider().schemaFor(typeDescriptor));
- if (!schema.assignableTo(requiredSchema)) {
- throw new IllegalArgumentException(
- String.format(
- "Given message schema: '%s'%n"
- + "does not match schema inferred from thrift class.%n"
- + "Thrift class: '%s'%n"
- + "Inferred schema: '%s'",
- requiredSchema, thriftClass.getName(), schema));
- }
- }
-
- /** A helper needed to fix the type `T` of thriftClass to satisfy RowMessages constraints. */
- private static <T extends TBase> PayloadSerializer getPayloadSerializer(
- Schema schema, TProtocolFactory protocolFactory, Class<T> thriftClass) {
- Coder<T> coder = ThriftCoder.of(thriftClass, protocolFactory);
- TypeDescriptor<T> descriptor = TypeDescriptor.of(thriftClass);
- SimpleFunction<byte[], Row> toRowFn =
- RowMessages.bytesToRowFn(ThriftSchema.provider(), descriptor, coder);
- return PayloadSerializer.of(
- RowMessages.rowToBytesFn(ThriftSchema.provider(), descriptor, coder),
- bytes -> {
- Row rawRow = toRowFn.apply(bytes);
- return castRow(rawRow, rawRow.getSchema(), schema);
- });
- }
-
- @Override
- public PayloadSerializer getSerializer(Schema schema, Map<String, Object> tableParams) {
- Class<? extends TBase> thriftClass = getMessageClass(tableParams);
- TProtocolFactory protocolFactory = getProtocolFactory(tableParams);
- inferAndVerifySchema(thriftClass, schema);
- return getPayloadSerializer(schema, protocolFactory, thriftClass);
- }
-}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/package-info.java
deleted file mode 100644
index 663e2e6..0000000
--- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** Provides abstractions for schema-aware IOs. */
-@DefaultAnnotation(NonNull.class)
-package org.apache.beam.sdk.extensions.sql.meta.provider.payloads;
-
-import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
-import org.checkerframework.checker.nullness.qual.NonNull;
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableAvroTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableAvroTest.java
index a603276..987c64c 100644
--- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableAvroTest.java
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableAvroTest.java
@@ -17,14 +17,12 @@
*/
package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
-import com.alibaba.fastjson.JSON;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.List;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.GenericRecordBuilder;
import org.apache.beam.sdk.coders.AvroCoder;
-import org.apache.beam.sdk.extensions.sql.meta.Table;
import org.apache.beam.sdk.schemas.Schema;
import org.apache.beam.sdk.schemas.utils.AvroUtils;
import org.apache.beam.sdk.values.Row;
@@ -86,14 +84,6 @@ public class BeamKafkaTableAvroTest extends BeamKafkaTableTest {
@Override
protected BeamKafkaTable getBeamKafkaTable() {
- return (BeamKafkaTable)
- (new KafkaTableProvider()
- .buildBeamSqlTable(
- Table.builder()
- .name("kafka")
- .type("kafka")
- .schema(TEST_SCHEMA)
- .properties(JSON.parseObject("{ \"topics\": [], \"format\": \"avro\" }"))
- .build()));
+ return new BeamKafkaAvroTable(TEST_SCHEMA, "", ImmutableList.of());
}
}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableJsonTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableJsonTest.java
index 6421d10..842d551 100644
--- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableJsonTest.java
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableJsonTest.java
@@ -19,9 +19,7 @@ package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
import static java.nio.charset.StandardCharsets.UTF_8;
-import com.alibaba.fastjson.JSON;
import java.util.List;
-import org.apache.beam.sdk.extensions.sql.meta.Table;
import org.apache.beam.sdk.schemas.Schema;
import org.apache.beam.sdk.values.Row;
import org.testcontainers.shaded.com.google.common.collect.ImmutableList;
@@ -59,15 +57,7 @@ public class BeamKafkaTableJsonTest extends BeamKafkaTableTest {
@Override
protected BeamKafkaTable getBeamKafkaTable() {
- return (BeamKafkaTable)
- (new KafkaTableProvider()
- .buildBeamSqlTable(
- Table.builder()
- .name("kafka")
- .type("kafka")
- .schema(TEST_SCHEMA)
- .properties(JSON.parseObject("{ \"topics\": [], \"format\": \"json\" }"))
- .build()));
+ return new BeamKafkaJsonTable(TEST_SCHEMA, "", ImmutableList.of());
}
private String createJson(int i) {
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableProtoTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableProtoTest.java
index 6e685b8..a16c731 100644
--- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableProtoTest.java
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableProtoTest.java
@@ -21,9 +21,7 @@ import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.junit.Assert.assertThrows;
-import com.alibaba.fastjson.JSON;
import java.util.List;
-import org.apache.beam.sdk.extensions.sql.meta.Table;
import org.apache.beam.sdk.schemas.Schema;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.transforms.Create;
@@ -36,6 +34,7 @@ import org.junit.Test;
"nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402)
})
public class BeamKafkaTableProtoTest extends BeamKafkaTableTest {
+
private static final Schema TEST_SCHEMA =
Schema.builder()
.addInt64Field("f_long")
@@ -55,15 +54,17 @@ public class BeamKafkaTableProtoTest extends BeamKafkaTableTest {
.build();
@Test
- public void testWithShuffledSchema() throws Exception {
- BeamKafkaTable kafkaTable = getBeamKafkaTable(SHUFFLED_SCHEMA);
+ public void testWithShuffledSchema() {
+ BeamKafkaTable kafkaTable =
+ new BeamKafkaProtoTable(
+ SHUFFLED_SCHEMA, "", ImmutableList.of(), KafkaMessages.TestMessage.class);
PCollection<Row> result =
pipeline
.apply(Create.of(shuffledRow(1), shuffledRow(2)))
.apply(kafkaTable.getPTransformForOutput())
.apply(kafkaTable.getPTransformForInput());
- PAssert.that(result).containsInAnyOrder(shuffledRow(1), shuffledRow(2));
+ PAssert.that(result).containsInAnyOrder(generateRow(1), generateRow(2));
pipeline.run();
}
@@ -72,32 +73,21 @@ public class BeamKafkaTableProtoTest extends BeamKafkaTableTest {
Schema schema = Schema.builder().addStringField("non_existing_field").build();
IllegalArgumentException e =
- assertThrows(IllegalArgumentException.class, () -> getBeamKafkaTable(schema));
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ new BeamKafkaProtoTable(
+ schema, "", ImmutableList.of(), KafkaMessages.TestMessage.class));
assertThat(
e.getMessage(),
containsString("does not match schema inferred from protobuf class.\nProtobuf class: "));
}
- private static BeamKafkaTable getBeamKafkaTable(Schema schema) {
- return (BeamKafkaTable)
- (new KafkaTableProvider()
- .buildBeamSqlTable(
- Table.builder()
- .name("kafka")
- .type("kafka")
- .schema(schema)
- .properties(
- JSON.parseObject(
- "{ \"topics\": [], \"format\": \"proto\", \"protoClass\": \""
- + KafkaMessages.TestMessage.class.getName()
- + "\" }"))
- .build()));
- }
-
@Override
protected BeamKafkaTable getBeamKafkaTable() {
- return getBeamKafkaTable(TEST_SCHEMA);
+ return new BeamKafkaProtoTable(
+ TEST_SCHEMA, "", ImmutableList.of(), KafkaMessages.TestMessage.class);
}
@Override
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableThriftTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableThriftTest.java
index 3045d45..35bc803 100644
--- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableThriftTest.java
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableThriftTest.java
@@ -21,9 +21,7 @@ import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.junit.Assert.assertThrows;
-import com.alibaba.fastjson.JSON;
import java.util.List;
-import org.apache.beam.sdk.extensions.sql.meta.Table;
import org.apache.beam.sdk.extensions.sql.meta.provider.kafka.thrift.TestThriftMessage;
import org.apache.beam.sdk.schemas.Schema;
import org.apache.beam.sdk.testing.PAssert;
@@ -63,14 +61,16 @@ public class BeamKafkaTableThriftTest extends BeamKafkaTableTest {
@Test
public void testWithShuffledSchema() {
- BeamKafkaTable kafkaTable = getBeamKafkaTable(SHUFFLED_SCHEMA);
+ BeamKafkaTable kafkaTable =
+ new BeamKafkaThriftTable(
+ SHUFFLED_SCHEMA, "", ImmutableList.of(), TestThriftMessage.class, protocolFactory);
PCollection<Row> result =
pipeline
.apply(Create.of(shuffledRow(1), shuffledRow(2)))
.apply(kafkaTable.getPTransformForOutput())
.apply(kafkaTable.getPTransformForInput());
- PAssert.that(result).containsInAnyOrder(shuffledRow(1), shuffledRow(2));
+ PAssert.that(result).containsInAnyOrder(generateRow(1), generateRow(2));
pipeline.run();
}
@@ -79,34 +79,21 @@ public class BeamKafkaTableThriftTest extends BeamKafkaTableTest {
Schema schema = Schema.builder().addStringField("non_existing_field").build();
IllegalArgumentException e =
- assertThrows(IllegalArgumentException.class, () -> getBeamKafkaTable(schema));
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ new BeamKafkaThriftTable(
+ schema, "", ImmutableList.of(), TestThriftMessage.class, protocolFactory));
assertThat(
e.getMessage(),
containsString("does not match schema inferred from thrift class.\nThrift class: "));
}
- private static BeamKafkaTable getBeamKafkaTable(Schema schema) {
- return (BeamKafkaTable)
- (new KafkaTableProvider()
- .buildBeamSqlTable(
- Table.builder()
- .name("kafka")
- .type("kafka")
- .schema(schema)
- .properties(
- JSON.parseObject(
- "{ \"topics\": [], \"format\": \"thrift\", \"thriftClass\": \""
- + TestThriftMessage.class.getName()
- + "\", \"thriftProtocolFactoryClass\": \""
- + TCompactProtocol.Factory.class.getName()
- + "\" }"))
- .build()));
- }
-
@Override
protected BeamKafkaTable getBeamKafkaTable() {
- return getBeamKafkaTable(TEST_SCHEMA);
+ return new BeamKafkaThriftTable(
+ TEST_SCHEMA, "", ImmutableList.of(), TestThriftMessage.class, protocolFactory);
}
@Override
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java
index 6366186..bb5b276 100644
--- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java
@@ -59,9 +59,9 @@ public class KafkaTableProviderTest {
BeamSqlTable sqlTable = provider.buildBeamSqlTable(table);
assertNotNull(sqlTable);
- assertTrue(sqlTable instanceof BeamKafkaTable);
+ assertTrue(sqlTable instanceof BeamKafkaAvroTable);
- BeamKafkaTable csvTable = (BeamKafkaTable) sqlTable;
+ BeamKafkaAvroTable csvTable = (BeamKafkaAvroTable) sqlTable;
assertEquals("localhost:9092", csvTable.getBootstrapServers());
assertEquals(ImmutableList.of("topic1", "topic2"), csvTable.getTopics());
}
@@ -72,9 +72,9 @@ public class KafkaTableProviderTest {
BeamSqlTable sqlTable = provider.buildBeamSqlTable(table);
assertNotNull(sqlTable);
- assertTrue(sqlTable instanceof BeamKafkaTable);
+ assertTrue(sqlTable instanceof BeamKafkaProtoTable);
- BeamKafkaTable protoTable = (BeamKafkaTable) sqlTable;
+ BeamKafkaProtoTable protoTable = (BeamKafkaProtoTable) sqlTable;
assertEquals("localhost:9092", protoTable.getBootstrapServers());
assertEquals(ImmutableList.of("topic1", "topic2"), protoTable.getTopics());
}
@@ -86,9 +86,9 @@ public class KafkaTableProviderTest {
BeamSqlTable sqlTable = provider.buildBeamSqlTable(table);
assertNotNull(sqlTable);
- assertTrue(sqlTable instanceof BeamKafkaTable);
+ assertTrue(sqlTable instanceof BeamKafkaThriftTable);
- BeamKafkaTable thriftTable = (BeamKafkaTable) sqlTable;
+ BeamKafkaThriftTable thriftTable = (BeamKafkaThriftTable) sqlTable;
assertEquals("localhost:9092", thriftTable.getBootstrapServers());
assertEquals(ImmutableList.of("topic1", "topic2"), thriftTable.getTopics());
}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/ProtoPayloadSerializerProviderTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/ProtoPayloadSerializerProviderTest.java
deleted file mode 100644
index f41ea47..0000000
--- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/ProtoPayloadSerializerProviderTest.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.extensions.sql.meta.provider.payloads;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThrows;
-
-import org.apache.beam.sdk.extensions.sql.meta.provider.kafka.KafkaMessages;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.Schema.FieldType;
-import org.apache.beam.sdk.values.Row;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-@RunWith(JUnit4.class)
-public class ProtoPayloadSerializerProviderTest {
- private static final Schema SHUFFLED_SCHEMA =
- Schema.builder()
- .addStringField("f_string")
- .addInt32Field("f_int")
- .addArrayField("f_float_array", Schema.FieldType.FLOAT)
- .addDoubleField("f_double")
- .addInt64Field("f_long")
- .build();
- private static final Row ROW =
- Row.withSchema(SHUFFLED_SCHEMA)
- .withFieldValue("f_string", "string")
- .withFieldValue("f_int", 123)
- .withFieldValue("f_float_array", ImmutableList.of(8.0f))
- .withFieldValue("f_double", 9.0)
- .withFieldValue("f_long", 456L)
- .build();
- private static final KafkaMessages.TestMessage MESSAGE =
- KafkaMessages.TestMessage.newBuilder()
- .setFLong(456)
- .setFInt(123)
- .setFDouble(9.0)
- .setFString("string")
- .addFFloatArray(8.0f)
- .build();
-
- private final ProtoPayloadSerializerProvider provider = new ProtoPayloadSerializerProvider();
-
- @Test
- public void invalidArgs() {
- assertThrows(
- IllegalArgumentException.class,
- () -> provider.getSerializer(SHUFFLED_SCHEMA, ImmutableMap.of()));
- assertThrows(
- IllegalArgumentException.class,
- () -> provider.getSerializer(SHUFFLED_SCHEMA, ImmutableMap.of("protoClass", "")));
- assertThrows(
- ClassCastException.class,
- () ->
- provider.getSerializer(
- SHUFFLED_SCHEMA, ImmutableMap.of("protoClass", ImmutableList.class.getName())));
- assertThrows(
- IllegalArgumentException.class,
- () ->
- provider.getSerializer(
- Schema.builder()
- .addStringField("f_NOTACTUALLYINMESSAGE")
- .addInt32Field("f_int")
- .addArrayField("f_float_array", FieldType.FLOAT)
- .addDoubleField("f_double")
- .addInt64Field("f_long")
- .build(),
- ImmutableMap.of("protoClass", KafkaMessages.TestMessage.class.getName())));
- }
-
- @Test
- public void serialize() throws Exception {
- byte[] bytes =
- provider
- .getSerializer(
- SHUFFLED_SCHEMA,
- ImmutableMap.of("protoClass", KafkaMessages.TestMessage.class.getName()))
- .serialize(ROW);
- KafkaMessages.TestMessage result = KafkaMessages.TestMessage.parseFrom(bytes);
- assertEquals(MESSAGE, result);
- }
-
- @Test
- public void deserialize() {
- Row row =
- provider
- .getSerializer(
- SHUFFLED_SCHEMA,
- ImmutableMap.of("protoClass", KafkaMessages.TestMessage.class.getName()))
- .deserialize(MESSAGE.toByteArray());
- assertEquals(ROW, row);
- }
-}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/ThriftPayloadSerializerProviderTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/ThriftPayloadSerializerProviderTest.java
deleted file mode 100644
index dd79289..0000000
--- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/payloads/ThriftPayloadSerializerProviderTest.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.extensions.sql.meta.provider.payloads;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThrows;
-
-import org.apache.beam.sdk.extensions.sql.meta.provider.kafka.thrift.TestThriftMessage;
-import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializerProvider;
-import org.apache.beam.sdk.values.Row;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
-import org.apache.thrift.TDeserializer;
-import org.apache.thrift.TSerializer;
-import org.apache.thrift.protocol.TCompactProtocol;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-@RunWith(JUnit4.class)
-public class ThriftPayloadSerializerProviderTest {
- private static final Schema SHUFFLED_SCHEMA =
- Schema.builder()
- .addStringField("f_string")
- .addInt32Field("f_int")
- .addArrayField("f_double_array", Schema.FieldType.DOUBLE)
- .addDoubleField("f_double")
- .addInt64Field("f_long")
- .build();
- private static final Row ROW =
- Row.withSchema(SHUFFLED_SCHEMA)
- .withFieldValue("f_string", "string")
- .withFieldValue("f_int", 123)
- .withFieldValue("f_double_array", ImmutableList.of(8.0))
- .withFieldValue("f_double", 9.0)
- .withFieldValue("f_long", 456L)
- .build();
- private static final TestThriftMessage MESSAGE =
- new TestThriftMessage()
- .setFLong(456L)
- .setFInt(123)
- .setFDouble(9.0)
- .setFString("string")
- .setFDoubleArray(ImmutableList.of(8.0));
-
- private final PayloadSerializerProvider provider = new ThriftPayloadSerializerProvider();
-
- @Test
- public void invalidArgs() {
- assertThrows(
- IllegalArgumentException.class,
- () -> provider.getSerializer(SHUFFLED_SCHEMA, ImmutableMap.of()));
- assertThrows(
- IllegalArgumentException.class,
- () ->
- provider.getSerializer(
- SHUFFLED_SCHEMA,
- ImmutableMap.of("thriftClass", "", "thriftProtocolFactoryClass", "")));
- assertThrows(
- IllegalArgumentException.class,
- () ->
- provider.getSerializer(
- SHUFFLED_SCHEMA,
- ImmutableMap.of(
- "thriftClass",
- "",
- "thriftProtocolFactoryClass",
- TCompactProtocol.Factory.class.getName())));
- assertThrows(
- IllegalArgumentException.class,
- () ->
- provider.getSerializer(
- SHUFFLED_SCHEMA,
- ImmutableMap.of(
- "thriftClass",
- TestThriftMessage.class.getName(),
- "thriftProtocolFactoryClass",
- "")));
- assertThrows(
- ClassCastException.class,
- () ->
- provider.getSerializer(
- SHUFFLED_SCHEMA,
- ImmutableMap.of(
- "thriftClass", ImmutableList.class.getName(),
- "thriftProtocolFactoryClass", TCompactProtocol.Factory.class.getName())));
- assertThrows(
- ClassCastException.class,
- () ->
- provider.getSerializer(
- SHUFFLED_SCHEMA,
- ImmutableMap.of(
- "thriftClass", TestThriftMessage.class.getName(),
- "thriftProtocolFactoryClass", ImmutableList.class.getName())));
- assertThrows(
- IllegalArgumentException.class,
- () ->
- provider.getSerializer(
- Schema.builder()
- .addStringField("f_NOTACTUALLYINMESSAGE")
- .addInt32Field("f_int")
- .addArrayField("f_double_array", Schema.FieldType.DOUBLE)
- .addDoubleField("f_double")
- .addInt64Field("f_long")
- .build(),
- ImmutableMap.of(
- "thriftClass", TestThriftMessage.class.getName(),
- "thriftProtocolFactoryClass", TCompactProtocol.Factory.class.getName())));
- }
-
- @Test
- public void serialize() throws Exception {
- byte[] bytes =
- provider
- .getSerializer(
- SHUFFLED_SCHEMA,
- ImmutableMap.of(
- "thriftClass", TestThriftMessage.class.getName(),
- "thriftProtocolFactoryClass", TCompactProtocol.Factory.class.getName()))
- .serialize(ROW);
- TestThriftMessage result = new TestThriftMessage();
- new TDeserializer(new TCompactProtocol.Factory()).deserialize(result, bytes);
- assertEquals(MESSAGE, result);
- }
-
- @Test
- public void deserialize() throws Exception {
- Row row =
- provider
- .getSerializer(
- SHUFFLED_SCHEMA,
- ImmutableMap.of(
- "thriftClass", TestThriftMessage.class.getName(),
- "thriftProtocolFactoryClass", TCompactProtocol.Factory.class.getName()))
- .deserialize(new TSerializer(new TCompactProtocol.Factory()).serialize(MESSAGE));
- assertEquals(ROW, row);
- }
-}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRow.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRow.java
index df25df7..f2d6022 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRow.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRow.java
@@ -17,27 +17,36 @@
*/
package org.apache.beam.sdk.io.gcp.pubsub;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.stream.Collectors.toList;
+import static org.apache.beam.sdk.util.RowJsonUtils.newObjectMapperWith;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.auto.value.AutoValue;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
+import org.apache.avro.AvroRuntimeException;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.gcp.pubsub.PubsubSchemaIOProvider.PayloadFormat;
import org.apache.beam.sdk.schemas.Schema;
import org.apache.beam.sdk.schemas.Schema.TypeName;
-import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer;
+import org.apache.beam.sdk.schemas.utils.AvroUtils;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.util.RowJson.RowJsonDeserializer;
+import org.apache.beam.sdk.util.RowJson.UnsupportedRowJsonException;
+import org.apache.beam.sdk.util.RowJsonUtils;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.Row;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
import org.joda.time.Instant;
/** Read side converter for {@link PubsubMessage} with JSON/AVRO payload. */
@@ -49,8 +58,6 @@ import org.joda.time.Instant;
})
abstract class PubsubMessageToRow extends PTransform<PCollection<PubsubMessage>, PCollectionTuple>
implements Serializable {
- interface SerializerProvider extends SerializableFunction<Schema, PayloadSerializer> {}
-
static final String TIMESTAMP_FIELD = "event_timestamp";
static final String ATTRIBUTES_FIELD = "attributes";
static final String PAYLOAD_FIELD = "payload";
@@ -78,11 +85,10 @@ abstract class PubsubMessageToRow extends PTransform<PCollection<PubsubMessage>,
public abstract boolean useFlatSchema();
- // A provider for a PayloadSerializer given the expected payload schema.
- public abstract SerializerProvider serializerProvider();
+ public abstract PayloadFormat payloadFormat();
public static Builder builder() {
- return new AutoValue_PubsubMessageToRow.Builder();
+ return new AutoValue_PubsubMessageToRow.Builder().payloadFormat(PayloadFormat.JSON);
}
@Override
@@ -92,15 +98,63 @@ abstract class PubsubMessageToRow extends PTransform<PCollection<PubsubMessage>,
ParDo.of(
useFlatSchema()
? new FlatSchemaPubsubMessageToRow(
- messageSchema(), useDlq(), serializerProvider())
+ messageSchema(), useDlq(), payloadFormat())
: new NestedSchemaPubsubMessageToRow(
- messageSchema(), useDlq(), serializerProvider()))
+ messageSchema(), useDlq(), payloadFormat()))
.withOutputTags(
MAIN_TAG, useDlq() ? TupleTagList.of(DLQ_TAG) : TupleTagList.empty()));
rows.get(MAIN_TAG).setRowSchema(messageSchema());
return rows;
}
+ @VisibleForTesting
+ static SimpleFunction<PubsubMessage, Row> getParsePayloadFn(
+ PayloadFormat format, Schema payloadSchema) {
+ switch (format) {
+ case JSON:
+ return new ParseJsonPayloadFn(payloadSchema);
+ case AVRO:
+ return new ParseAvroPayloadFn(payloadSchema);
+ default:
+ throw new IllegalArgumentException("Unsupported payload format given: " + format);
+ }
+ }
+
+ private static class ParseJsonPayloadFn extends SimpleFunction<PubsubMessage, Row> {
+ private final ObjectMapper jsonMapper;
+
+ ParseJsonPayloadFn(Schema payloadSchema) {
+ jsonMapper = newObjectMapperWith(RowJsonDeserializer.forSchema(payloadSchema));
+ }
+
+ @Override
+ public Row apply(PubsubMessage message) {
+ String payloadJson = new String(message.getPayload(), UTF_8);
+ try {
+ return RowJsonUtils.jsonToRow(jsonMapper, payloadJson);
+ } catch (UnsupportedRowJsonException e) {
+ throw new ParseException(e);
+ }
+ }
+ }
+
+ private static class ParseAvroPayloadFn extends SimpleFunction<PubsubMessage, Row> {
+ private final SimpleFunction<byte[], Row> avroBytesToRowFn;
+
+ public ParseAvroPayloadFn(Schema payloadSchema) {
+ avroBytesToRowFn = AvroUtils.getAvroBytesToRowFunction(payloadSchema);
+ }
+
+ @Override
+ public Row apply(PubsubMessage message) {
+ try {
+ return avroBytesToRowFn.apply(message.getPayload());
+ } catch (AvroRuntimeException e) {
+ throw new ParseException(e);
+ }
+ }
+ }
+
/**
* A {@link DoFn} to convert a flat schema{@link PubsubMessage} with JSON/AVRO payload to {@link
* Row}.
@@ -112,10 +166,10 @@ abstract class PubsubMessageToRow extends PTransform<PCollection<PubsubMessage>,
private final boolean useDlq;
- private final PayloadSerializer payloadSerializer;
+ private final SimpleFunction<PubsubMessage, Row> parsePayloadFn;
protected FlatSchemaPubsubMessageToRow(
- Schema messageSchema, boolean useDlq, SerializerProvider serializerProvider) {
+ Schema messageSchema, boolean useDlq, PayloadFormat payloadFormat) {
this.messageSchema = messageSchema;
// Construct flat payload schema.
Schema payloadSchema =
@@ -124,7 +178,7 @@ abstract class PubsubMessageToRow extends PTransform<PCollection<PubsubMessage>,
.filter(f -> !f.getName().equals(TIMESTAMP_FIELD))
.collect(Collectors.toList()));
this.useDlq = useDlq;
- this.payloadSerializer = serializerProvider.apply(payloadSchema);
+ this.parsePayloadFn = getParsePayloadFn(payloadFormat, payloadSchema);
}
/**
@@ -144,17 +198,17 @@ abstract class PubsubMessageToRow extends PTransform<PCollection<PubsubMessage>,
public void processElement(
@Element PubsubMessage element, @Timestamp Instant timestamp, MultiOutputReceiver o) {
try {
- Row payload = payloadSerializer.deserialize(element.getPayload());
+ Row payload = parsePayloadFn.apply(element);
List<Object> values =
messageSchema.getFields().stream()
.map(field -> getValueForFieldFlatSchema(field, timestamp, payload))
.collect(toList());
o.get(MAIN_TAG).output(Row.withSchema(messageSchema).addValues(values).build());
- } catch (Exception e) {
+ } catch (ParseException pe) {
if (useDlq) {
o.get(DLQ_TAG).output(element);
} else {
- throw new RuntimeException(e);
+ throw new RuntimeException(pe);
}
}
}
@@ -171,14 +225,14 @@ abstract class PubsubMessageToRow extends PTransform<PCollection<PubsubMessage>,
private final boolean useDlq;
- private final PayloadSerializer payloadSerializer;
+ private final SimpleFunction<PubsubMessage, Row> parsePayloadFn;
protected NestedSchemaPubsubMessageToRow(
- Schema messageSchema, boolean useDlq, SerializerProvider serializerProvider) {
+ Schema messageSchema, boolean useDlq, PayloadFormat payloadFormat) {
this.messageSchema = messageSchema;
this.useDlq = useDlq;
Schema payloadSchema = messageSchema.getField(PAYLOAD_FIELD).getType().getRowSchema();
- this.payloadSerializer = serializerProvider.apply(payloadSchema);
+ this.parsePayloadFn = getParsePayloadFn(payloadFormat, payloadSchema);
}
/** Get the value for a field int the order they're specified in the nested schema. */
@@ -205,7 +259,7 @@ abstract class PubsubMessageToRow extends PTransform<PCollection<PubsubMessage>,
public void processElement(
@Element PubsubMessage element, @Timestamp Instant timestamp, MultiOutputReceiver o) {
try {
- Row payload = payloadSerializer.deserialize(element.getPayload());
+ Row payload = parsePayloadFn.apply(element);
List<Object> values =
messageSchema.getFields().stream()
.map(
@@ -214,11 +268,11 @@ abstract class PubsubMessageToRow extends PTransform<PCollection<PubsubMessage>,
field, timestamp, element.getAttributeMap(), payload))
.collect(toList());
o.get(MAIN_TAG).output(Row.withSchema(messageSchema).addValues(values).build());
- } catch (Exception e) {
+ } catch (ParseException exception) {
if (useDlq) {
o.get(DLQ_TAG).output(element);
} else {
- throw new RuntimeException(e);
+ throw new RuntimeException(exception);
}
}
}
@@ -232,7 +286,7 @@ abstract class PubsubMessageToRow extends PTransform<PCollection<PubsubMessage>,
public abstract Builder useFlatSchema(boolean useFlatSchema);
- public abstract Builder serializerProvider(SerializerProvider serializerProvider);
+ public abstract Builder payloadFormat(PayloadFormat payloadFormat);
public abstract PubsubMessageToRow build();
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaIOProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaIOProvider.java
index 2cc05f0..fa1c0e2 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaIOProvider.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaIOProvider.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.sdk.io.gcp.pubsub;
-import static java.util.stream.Collectors.toList;
import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.ATTRIBUTES_FIELD;
import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.DLQ_TAG;
import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.MAIN_TAG;
@@ -28,18 +27,14 @@ import static org.apache.beam.sdk.schemas.Schema.TypeName.ROW;
import com.google.auto.service.AutoService;
import com.google.auto.value.AutoValue;
import java.io.Serializable;
-import java.util.List;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.schemas.AutoValueSchema;
import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.Schema.Field;
import org.apache.beam.sdk.schemas.Schema.FieldType;
import org.apache.beam.sdk.schemas.io.InvalidConfigurationException;
import org.apache.beam.sdk.schemas.io.InvalidSchemaException;
import org.apache.beam.sdk.schemas.io.SchemaIO;
import org.apache.beam.sdk.schemas.io.SchemaIOProvider;
-import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer;
-import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
@@ -47,7 +42,6 @@ import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.POutput;
import org.apache.beam.sdk.values.Row;
import org.apache.beam.sdk.values.TypeDescriptor;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
import org.checkerframework.checker.nullness.qual.Nullable;
/**
@@ -124,11 +118,6 @@ public class PubsubSchemaIOProvider implements SchemaIOProvider {
.addNullableField("timestampAttributeKey", FieldType.STRING)
.addNullableField("deadLetterQueue", FieldType.STRING)
.addNullableField("format", FieldType.STRING)
- // For ThriftPayloadSerializerProvider
- .addNullableField("thriftClass", FieldType.STRING)
- .addNullableField("thriftProtocolFactoryClass", FieldType.STRING)
- // For ProtoPayloadSerializerProvider
- .addNullableField("protoClass", FieldType.STRING)
.build();
}
@@ -215,7 +204,7 @@ public class PubsubSchemaIOProvider implements SchemaIOProvider {
.messageSchema(dataSchema)
.useDlq(config.useDeadLetterQueue())
.useFlatSchema(useFlatSchema)
- .serializerProvider(config::serializer)
+ .payloadFormat(config.format())
.build());
rowsWithDlq.get(MAIN_TAG).setRowSchema(dataSchema);
@@ -241,8 +230,7 @@ public class PubsubSchemaIOProvider implements SchemaIOProvider {
return input
.apply(
RowToPubsubMessage.of(
- config.useTimestampAttribute(),
- config.serializer(stripFromTimestampField(dataSchema))))
+ config.useTimestampAttribute(), config.format(), dataSchema))
.apply(createPubsubMessageWrite());
}
};
@@ -288,14 +276,6 @@ public class PubsubSchemaIOProvider implements SchemaIOProvider {
}
}
- private static Schema stripFromTimestampField(Schema schema) {
- List<Field> selectedFields =
- schema.getFields().stream()
- .filter(field -> !TIMESTAMP_FIELD.equals(field.getName()))
- .collect(toList());
- return Schema.of(selectedFields.toArray(new Schema.Field[0]));
- }
-
@AutoValue
abstract static class Config implements Serializable {
@@ -305,14 +285,6 @@ public class PubsubSchemaIOProvider implements SchemaIOProvider {
abstract @Nullable String getFormat();
- // For ThriftPayloadSerializerProvider
- abstract @Nullable String getThriftClass();
-
- abstract @Nullable String getThriftProtocolFactoryClass();
-
- // For ProtoPayloadSerializerProvider
- abstract @Nullable String getProtoClass();
-
boolean useDeadLetterQueue() {
return getDeadLetterQueue() != null;
}
@@ -321,19 +293,10 @@ public class PubsubSchemaIOProvider implements SchemaIOProvider {
return getTimestampAttributeKey() != null;
}
- PayloadSerializer serializer(Schema schema) {
- String format = getFormat() == null ? "json" : getFormat();
- ImmutableMap.Builder<String, Object> params = ImmutableMap.builder();
- if (getThriftClass() != null) {
- params.put("thriftClass", getThriftClass());
- }
- if (getThriftProtocolFactoryClass() != null) {
- params.put("thriftProtocolFactoryClass", getThriftProtocolFactoryClass());
- }
- if (getProtoClass() != null) {
- params.put("protoClass", getProtoClass());
- }
- return PayloadSerializers.getSerializer(format, schema, params.build());
+ PayloadFormat format() {
+ return getFormat() == null
+ ? PayloadFormat.JSON
+ : PayloadFormat.valueOf(getFormat().toUpperCase());
}
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/RowToPubsubMessage.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/RowToPubsubMessage.java
index e59a949..27a8807 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/RowToPubsubMessage.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/RowToPubsubMessage.java
@@ -17,18 +17,24 @@
*/
package org.apache.beam.sdk.io.gcp.pubsub;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.stream.Collectors.toList;
import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.TIMESTAMP_FIELD;
+import static org.apache.beam.sdk.io.gcp.pubsub.PubsubSchemaIOProvider.PayloadFormat;
-import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer;
+import java.util.List;
+import org.apache.beam.sdk.schemas.Schema;
import org.apache.beam.sdk.schemas.transforms.DropFields;
+import org.apache.beam.sdk.schemas.utils.AvroUtils;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.ToJson;
import org.apache.beam.sdk.transforms.WithTimestamps;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.Row;
-import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
+import org.checkerframework.checker.nullness.qual.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,15 +52,23 @@ import org.slf4j.LoggerFactory;
class RowToPubsubMessage extends PTransform<PCollection<Row>, PCollection<PubsubMessage>> {
private static final Logger LOG = LoggerFactory.getLogger(RowToPubsubMessage.class);
private final boolean useTimestampAttribute;
- private final PayloadSerializer serializer;
+ private final PayloadFormat payloadFormat;
+ private final @Nullable Schema payloadSchema;
- private RowToPubsubMessage(boolean useTimestampAttribute, PayloadSerializer serializer) {
+ private RowToPubsubMessage(
+ boolean useTimestampAttribute, PayloadFormat payloadFormat, @Nullable Schema schema) {
this.useTimestampAttribute = useTimestampAttribute;
- this.serializer = serializer;
+ this.payloadFormat = payloadFormat;
+ this.payloadSchema = schema == null ? null : stripFromTimestampField(schema);
}
- public static RowToPubsubMessage of(boolean useTimestampAttribute, PayloadSerializer serializer) {
- return new RowToPubsubMessage(useTimestampAttribute, serializer);
+ public static RowToPubsubMessage of(boolean useTimestampAttribute, PayloadFormat payloadFormat) {
+ return new RowToPubsubMessage(useTimestampAttribute, payloadFormat, null);
+ }
+
+ public static RowToPubsubMessage of(
+ boolean useTimestampAttribute, PayloadFormat payloadFormat, Schema schema) {
+ return new RowToPubsubMessage(useTimestampAttribute, payloadFormat, schema);
}
@Override
@@ -82,10 +96,26 @@ class RowToPubsubMessage extends PTransform<PCollection<Row>, PCollection<Pubsub
rows = withTimestamp;
}
- return rows.apply(
- "MapRowToBytes",
- MapElements.into(new TypeDescriptor<byte[]>() {}).via(serializer::serialize))
- .apply("MapToPubsubMessage", MapElements.via(new ToPubsubMessage()));
+ switch (payloadFormat) {
+ case JSON:
+ return rows.apply("MapRowToJsonString", ToJson.of())
+ .apply("MapToJsonBytes", MapElements.via(new StringToBytes()))
+ .apply("MapToPubsubMessage", MapElements.via(new ToPubsubMessage()));
+ case AVRO:
+ return rows.apply(
+ "MapRowToAvroBytes",
+ MapElements.via(AvroUtils.getRowToAvroBytesFunction(payloadSchema)))
+ .apply("MapToPubsubMessage", MapElements.via(new ToPubsubMessage()));
+ default:
+ throw new IllegalArgumentException("Unsupported payload format: " + payloadFormat);
+ }
+ }
+
+ private static class StringToBytes extends SimpleFunction<String, byte[]> {
+ @Override
+ public byte[] apply(String s) {
+ return s.getBytes(UTF_8);
+ }
}
private static class ToPubsubMessage extends SimpleFunction<byte[], PubsubMessage> {
@@ -94,4 +124,12 @@ class RowToPubsubMessage extends PTransform<PCollection<Row>, PCollection<Pubsub
return new PubsubMessage(bytes, ImmutableMap.of());
}
}
+
+ private Schema stripFromTimestampField(Schema schema) {
+ List<Schema.Field> selectedFields =
+ schema.getFields().stream()
+ .filter(field -> !TIMESTAMP_FIELD.equals(field.getName()))
+ .collect(toList());
+ return Schema.of(selectedFields.toArray(new Schema.Field[0]));
+ }
}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRowTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRowTest.java
index 3d07e02..e98d4e8 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRowTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRowTest.java
@@ -21,6 +21,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.stream.Collectors.toSet;
import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.DLQ_TAG;
import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.MAIN_TAG;
+import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.getParsePayloadFn;
import static org.apache.beam.sdk.io.gcp.pubsub.PubsubSchemaIOProvider.VARCHAR;
import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.size;
import static org.junit.Assert.assertEquals;
@@ -30,13 +31,14 @@ import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.StreamSupport;
-import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.SerializerProvider;
+import org.apache.beam.sdk.io.gcp.pubsub.PubsubSchemaIOProvider.PayloadFormat;
import org.apache.beam.sdk.schemas.Schema;
import org.apache.beam.sdk.schemas.Schema.FieldType;
-import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers;
+import org.apache.beam.sdk.schemas.utils.AvroUtils;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.SimpleFunction;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.Row;
@@ -54,8 +56,6 @@ import org.junit.Test;
"nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402)
})
public class PubsubMessageToRowTest implements Serializable {
- private static final SerializerProvider JSON_SERIALIZER_PROVIDER =
- schema -> PayloadSerializers.getSerializer("json", schema, ImmutableMap.of());
@Rule public transient TestPipeline pipeline = TestPipeline.create();
@@ -90,7 +90,6 @@ public class PubsubMessageToRowTest implements Serializable {
.messageSchema(messageSchema)
.useDlq(false)
.useFlatSchema(false)
- .serializerProvider(JSON_SERIALIZER_PROVIDER)
.build());
PAssert.that(rows.get(MAIN_TAG))
@@ -140,7 +139,6 @@ public class PubsubMessageToRowTest implements Serializable {
.messageSchema(messageSchema)
.useDlq(true)
.useFlatSchema(false)
- .serializerProvider(JSON_SERIALIZER_PROVIDER)
.build());
PCollection<Row> rows = outputs.get(MAIN_TAG);
@@ -197,7 +195,6 @@ public class PubsubMessageToRowTest implements Serializable {
.messageSchema(messageSchema)
.useDlq(false)
.useFlatSchema(true)
- .serializerProvider(JSON_SERIALIZER_PROVIDER)
.build());
PAssert.that(rows.get(MAIN_TAG))
@@ -245,7 +242,6 @@ public class PubsubMessageToRowTest implements Serializable {
.messageSchema(messageSchema)
.useDlq(true)
.useFlatSchema(true)
- .serializerProvider(JSON_SERIALIZER_PROVIDER)
.build());
PCollection<Row> rows = outputs.get(MAIN_TAG);
@@ -298,12 +294,10 @@ public class PubsubMessageToRowTest implements Serializable {
.messageSchema(messageSchema)
.useDlq(false)
.useFlatSchema(true)
- .serializerProvider(JSON_SERIALIZER_PROVIDER)
.build());
Exception exception = Assert.assertThrows(RuntimeException.class, () -> pipeline.run());
- Assert.assertTrue(
- exception.toString(), exception.getMessage().contains("Non-nullable field 'id'"));
+ Assert.assertTrue(exception.getMessage().contains("Error parsing message"));
}
@Test
@@ -329,12 +323,30 @@ public class PubsubMessageToRowTest implements Serializable {
.messageSchema(messageSchema)
.useDlq(false)
.useFlatSchema(false)
- .serializerProvider(JSON_SERIALIZER_PROVIDER)
.build());
Exception exception = Assert.assertThrows(RuntimeException.class, () -> pipeline.run());
- Assert.assertTrue(
- exception.toString(), exception.getMessage().contains("Non-nullable field 'id'"));
+ Assert.assertTrue(exception.getMessage().contains("Error parsing message"));
+ }
+
+ @Test
+ public void testParsesAvroPayload() {
+ Schema payloadSchema = getParserSchema();
+ Row row = row(payloadSchema, 3, "Dovahkiin", 5.5, 5L);
+ byte[] payload = AvroUtils.getRowToAvroBytesFunction(payloadSchema).apply(row);
+ SimpleFunction<PubsubMessage, Row> messageToRowFn =
+ getParsePayloadFn(PayloadFormat.AVRO, payloadSchema);
+ Row parsedRow = messageToRowFn.apply(new PubsubMessage(payload, null));
+ assertEquals(row, parsedRow);
+ }
+
+ private Schema getParserSchema() {
+ return Schema.builder()
+ .addInt32Field("id")
+ .addNullableField("name", FieldType.STRING)
+ .addNullableField("real", FieldType.DOUBLE)
+ .addNullableField("number", FieldType.INT64)
+ .build();
}
private Row row(Schema schema, Object... objects) {