You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ch...@apache.org on 2024/05/31 19:51:36 UTC
(beam) branch master updated: Updates Expansion Service Container to support upgrading using the schema-transform ID (#31451)
This is an automated email from the ASF dual-hosted git repository.
chamikara pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 7b6f9415c10 Updates Expansion Service Container to support upgrading using the schema-transform ID (#31451)
7b6f9415c10 is described below
commit 7b6f9415c108924732f10b4645cd2ecc2a81482a
Author: Chamikara Jayalath <ch...@gmail.com>
AuthorDate: Fri May 31 12:51:30 2024 -0700
Updates Expansion Service Container to support upgrading using the schema-transform ID (#31451)
* Updates Expansion Service to support upgrading using the schema-transform ID
* Fixes a test failure
* Triggerring Iceberg test suite
* Adding a unit test
---
.../IO_Iceberg_Integration_Tests.json | 3 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +
.../construction/ExternalTranslationOptions.java | 4 +-
.../util/construction/PTransformTranslation.java | 2 +-
.../sdk/util/construction/TransformUpgrader.java | 56 ++++++---
.../util/construction/TransformUpgraderTest.java | 139 ++++++++++++++++++---
.../container/expansion_service_config.yml | 3 +
.../sdk/expansion/service/ExpansionService.java | 67 +++++++---
sdks/java/io/expansion-service/build.gradle | 5 +
sdks/java/io/iceberg/build.gradle | 4 +-
.../managed/ManagedSchemaTransformProvider.java | 63 ++++++----
11 files changed, 270 insertions(+), 78 deletions(-)
diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests.json b/.github/trigger_files/IO_Iceberg_Integration_Tests.json
new file mode 100644
index 00000000000..a03c067d2c4
--- /dev/null
+++ b/.github/trigger_files/IO_Iceberg_Integration_Tests.json
@@ -0,0 +1,3 @@
+{
+ "comment": "Modify this file in a trivial way to cause this test suite to run"
+}
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 32747f4efb1..78a6b371e6f 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -692,7 +692,9 @@ class BeamModulePlugin implements Plugin<Project> {
aws_java_sdk2_profiles : "software.amazon.awssdk:profiles:$aws_java_sdk2_version",
azure_sdk_bom : "com.azure:azure-sdk-bom:1.2.14",
bigdataoss_gcsio : "com.google.cloud.bigdataoss:gcsio:$google_cloud_bigdataoss_version",
+ bigdataoss_gcs_connector : "com.google.cloud.bigdataoss:gcs-connector:hadoop2-$google_cloud_bigdataoss_version",
bigdataoss_util : "com.google.cloud.bigdataoss:util:$google_cloud_bigdataoss_version",
+ bigdataoss_util_hadoop : "com.google.cloud.bigdataoss:util-hadoop:hadoop2-$google_cloud_bigdataoss_version",
byte_buddy : "net.bytebuddy:byte-buddy:1.14.12",
cassandra_driver_core : "com.datastax.cassandra:cassandra-driver-core:$cassandra_driver_version",
cassandra_driver_mapping : "com.datastax.cassandra:cassandra-driver-mapping:$cassandra_driver_version",
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/construction/ExternalTranslationOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/construction/ExternalTranslationOptions.java
index 5ed14faf31e..f444d2e3e30 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/construction/ExternalTranslationOptions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/construction/ExternalTranslationOptions.java
@@ -25,7 +25,9 @@ import org.apache.beam.sdk.transforms.resourcehints.ResourceHintsOptions.EmptyLi
public interface ExternalTranslationOptions extends PipelineOptions {
- @Description("Set of URNs of transforms to be overriden using the transform service.")
+ @Description(
+ "Set of URNs of transforms to be overriden using the transform service. The provided strings "
+ + "can be transform URNs of schema-transform IDs")
@Default.InstanceFactory(EmptyListDefault.class)
List<String> getTransformsToOverride();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/construction/PTransformTranslation.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/construction/PTransformTranslation.java
index e2b6d95057f..3d914272373 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/construction/PTransformTranslation.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/construction/PTransformTranslation.java
@@ -535,7 +535,7 @@ public class PTransformTranslation {
if (underlyingIdentifier == null) {
throw new IllegalStateException(
String.format(
- "Encountered a Managed Transform that has an empty \"transform_identifier\": \n%s",
+ "Encountered a Managed Transform that has an empty \"transform_identifier\": %n%s",
configRow));
}
transformBuilder.putAnnotations(
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/construction/TransformUpgrader.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/construction/TransformUpgrader.java
index 941a5daf689..f88d55a1f5c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/construction/TransformUpgrader.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/construction/TransformUpgrader.java
@@ -17,6 +17,8 @@
*/
package org.apache.beam.sdk.util.construction;
+import static org.apache.beam.model.pipeline.v1.ExternalTransforms.ExpansionMethods.Enum.SCHEMA_TRANSFORM;
+
import com.fasterxml.jackson.core.Version;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -51,6 +53,7 @@ import org.apache.beam.sdk.util.construction.PTransformTranslation.TransformPayl
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.POutput;
import org.apache.beam.vendor.grpc.v1p60p1.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.grpc.v1p60p1.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.beam.vendor.grpc.v1p60p1.io.grpc.ManagedChannelBuilder;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter;
@@ -113,6 +116,22 @@ public class TransformUpgrader implements AutoCloseable {
if (urn != null && urnsToOverride.contains(urn)) {
return true;
}
+
+ // Also check if the URN is a schema-transform ID.
+ if (urn.equals(BeamUrns.getUrn(SCHEMA_TRANSFORM))) {
+ try {
+ ExternalTransforms.SchemaTransformPayload schemaTransformPayload =
+ ExternalTransforms.SchemaTransformPayload.parseFrom(
+ entry.getValue().getSpec().getPayload());
+ String schemaTransformId = schemaTransformPayload.getIdentifier();
+ if (urnsToOverride.contains(schemaTransformId)) {
+ return true;
+ }
+ } catch (InvalidProtocolBufferException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
return false;
})
.map(
@@ -184,20 +203,27 @@ public class TransformUpgrader implements AutoCloseable {
if (transformToUpgrade == null) {
throw new IllegalArgumentException("Could not find a transform with the ID " + transformId);
}
- ByteString configRowBytes =
- transformToUpgrade.getAnnotationsOrThrow(
- BeamUrns.getConstant(ExternalTransforms.Annotations.Enum.CONFIG_ROW_KEY));
- ByteString configRowSchemaBytes =
- transformToUpgrade.getAnnotationsOrThrow(
- BeamUrns.getConstant(ExternalTransforms.Annotations.Enum.CONFIG_ROW_SCHEMA_KEY));
- SchemaApi.Schema configRowSchemaProto =
- SchemaApi.Schema.parseFrom(configRowSchemaBytes.toByteArray());
-
- ExternalTransforms.ExternalConfigurationPayload payload =
- ExternalTransforms.ExternalConfigurationPayload.newBuilder()
- .setSchema(configRowSchemaProto)
- .setPayload(configRowBytes)
- .build();
+
+ byte[] payloadBytes = null;
+
+ if (!transformToUpgrade.getSpec().getUrn().equals(BeamUrns.getUrn(SCHEMA_TRANSFORM))) {
+ ByteString configRowBytes =
+ transformToUpgrade.getAnnotationsOrThrow(
+ BeamUrns.getConstant(ExternalTransforms.Annotations.Enum.CONFIG_ROW_KEY));
+ ByteString configRowSchemaBytes =
+ transformToUpgrade.getAnnotationsOrThrow(
+ BeamUrns.getConstant(ExternalTransforms.Annotations.Enum.CONFIG_ROW_SCHEMA_KEY));
+ SchemaApi.Schema configRowSchemaProto =
+ SchemaApi.Schema.parseFrom(configRowSchemaBytes.toByteArray());
+ payloadBytes =
+ ExternalTransforms.ExternalConfigurationPayload.newBuilder()
+ .setSchema(configRowSchemaProto)
+ .setPayload(configRowBytes)
+ .build()
+ .toByteArray();
+ } else {
+ payloadBytes = transformToUpgrade.getSpec().getPayload().toByteArray();
+ }
RunnerApi.PTransform.Builder ptransformBuilder =
RunnerApi.PTransform.newBuilder()
@@ -205,7 +231,7 @@ public class TransformUpgrader implements AutoCloseable {
.setSpec(
RunnerApi.FunctionSpec.newBuilder()
.setUrn(transformToUpgrade.getSpec().getUrn())
- .setPayload(ByteString.copyFrom(payload.toByteArray()))
+ .setPayload(ByteString.copyFrom(payloadBytes))
.build());
for (Map.Entry<String, String> entry : transformToUpgrade.getInputsMap().entrySet()) {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/construction/TransformUpgraderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/construction/TransformUpgraderTest.java
index 65b3e8e89ca..a54383b46e7 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/construction/TransformUpgraderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/construction/TransformUpgraderTest.java
@@ -39,16 +39,22 @@ import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.transforms.SchemaTransform;
+import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider;
+import org.apache.beam.sdk.schemas.transforms.SchemaTransformTranslation.SchemaTransformPayloadTranslator;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.SimpleFunction;
import org.apache.beam.sdk.transforms.ToString;
import org.apache.beam.sdk.util.ByteStringOutputStream;
+import org.apache.beam.sdk.util.construction.PTransformTranslation.TransformPayloadTranslator;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionRowTuple;
import org.apache.beam.sdk.values.Row;
import org.apache.beam.vendor.grpc.v1p60p1.com.google.protobuf.ByteString;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -158,6 +164,60 @@ public class TransformUpgraderTest {
}
}
+ public static class TestSchemaTransformProvider implements SchemaTransformProvider {
+
+ @Override
+ public String identifier() {
+ return "dummy_schema_transform";
+ }
+
+ @Override
+ public Schema configurationSchema() {
+ return Schema.builder().build();
+ }
+
+ @Override
+ public SchemaTransform from(Row configuration) {
+ return new TestSchemaTransform();
+ }
+ }
+
+ public static class TestSchemaTransform extends SchemaTransform {
+
+ @Override
+ public PCollectionRowTuple expand(PCollectionRowTuple input) {
+ return input;
+ }
+ }
+
+ static class TestSchemaTransformTranslator
+ extends SchemaTransformPayloadTranslator<TestSchemaTransform> {
+ @Override
+ public SchemaTransformProvider provider() {
+ return new TestSchemaTransformProvider();
+ }
+
+ @Override
+ public Row toConfigRow(TestSchemaTransform transform) {
+ return Row.withSchema(Schema.builder().build()).build();
+ }
+ }
+
+ @AutoService(TransformPayloadTranslatorRegistrar.class)
+ public static class TestSchemaTransformPayloadTranslatorRegistrar
+ implements TransformPayloadTranslatorRegistrar {
+ @Override
+ @SuppressWarnings({
+ "rawtypes",
+ })
+ public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+ getTransformPayloadTranslators() {
+ return ImmutableMap.<Class<? extends PTransform>, TransformPayloadTranslator>builder()
+ .put(TestSchemaTransform.class, new TestSchemaTransformTranslator())
+ .build();
+ }
+ }
+
static class TestExpansionServiceClientFactory implements ExpansionServiceClientFactory {
ExpansionApi.ExpansionResponse response;
@@ -183,6 +243,18 @@ public class TransformUpgraderTest {
.getTransformsMap()
.get("TransformUpgraderTest-TestTransform2");
}
+
+ boolean schemaTransformTest = false;
+ if (transformToUpgrade == null) {
+ // This is running a schema-transform test.
+ transformToUpgrade =
+ request
+ .getComponents()
+ .getTransformsMap()
+ .get("TransformUpgraderTest-TestSchemaTransform");
+ schemaTransformTest = true;
+ }
+
if (!transformToUpgrade
.getSpec()
.getUrn()
@@ -190,27 +262,30 @@ public class TransformUpgraderTest {
throw new RuntimeException("Could not find a valid transform to upgrade");
}
- Integer oldParam;
- try {
- ByteArrayInputStream byteArrayInputStream =
- new ByteArrayInputStream(transformToUpgrade.getSpec().getPayload().toByteArray());
- ObjectInputStream objectInputStream = new ObjectInputStream(byteArrayInputStream);
- oldParam = (Integer) objectInputStream.readObject();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
RunnerApi.PTransform.Builder upgradedTransform = transformToUpgrade.toBuilder();
FunctionSpec.Builder specBuilder = upgradedTransform.getSpecBuilder();
- ByteStringOutputStream byteStringOutputStream = new ByteStringOutputStream();
- try {
- ObjectOutputStream objectOutputStream = new ObjectOutputStream(byteStringOutputStream);
- objectOutputStream.writeObject(oldParam * 2);
- objectOutputStream.flush();
- specBuilder.setPayload(byteStringOutputStream.toByteString());
- } catch (IOException e) {
- throw new RuntimeException(e);
+ if (!schemaTransformTest) {
+ Integer oldParam;
+ try {
+ ByteArrayInputStream byteArrayInputStream =
+ new ByteArrayInputStream(transformToUpgrade.getSpec().getPayload().toByteArray());
+ ObjectInputStream objectInputStream = new ObjectInputStream(byteArrayInputStream);
+ oldParam = (Integer) objectInputStream.readObject();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ ByteStringOutputStream byteStringOutputStream = new ByteStringOutputStream();
+ try {
+ ObjectOutputStream objectOutputStream =
+ new ObjectOutputStream(byteStringOutputStream);
+ objectOutputStream.writeObject(oldParam * 2);
+ objectOutputStream.flush();
+ specBuilder.setPayload(byteStringOutputStream.toByteString());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
upgradedTransform.setSpec(specBuilder.build());
@@ -291,6 +366,34 @@ public class TransformUpgraderTest {
assertTrue(upgradedTransform.getAnnotationsMap().containsKey(TransformUpgrader.UPGRADE_KEY));
}
+ @Test
+ public void testTransformUpgradeSchemaTransform() throws Exception {
+ Pipeline pipeline = Pipeline.create();
+
+ // Build the pipeline
+ PCollectionRowTuple.empty(pipeline).apply(new TestSchemaTransform());
+
+ RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(pipeline, false);
+ ExternalTranslationOptions options =
+ PipelineOptionsFactory.create().as(ExternalTranslationOptions.class);
+ List<String> urnsToOverride = ImmutableList.of("dummy_schema_transform");
+ options.setTransformsToOverride(urnsToOverride);
+ options.setTransformServiceAddress("dummyaddress");
+
+ RunnerApi.Pipeline upgradedPipelineProto =
+ TransformUpgrader.of(new TestExpansionServiceClientFactory())
+ .upgradeTransformsViaTransformService(pipelineProto, urnsToOverride, options);
+
+ RunnerApi.PTransform upgradedTransform =
+ upgradedPipelineProto
+ .getComponents()
+ .getTransformsMap()
+ .get("TransformUpgraderTest-TestSchemaTransform");
+
+ // Confirm that the upgraded transform includes the upgrade annotation.
+ assertTrue(upgradedTransform.getAnnotationsMap().containsKey(TransformUpgrader.UPGRADE_KEY));
+ }
+
@Test
public void testTransformUpgradeMultipleOccurrences() throws Exception {
Pipeline pipeline = Pipeline.create();
diff --git a/sdks/java/expansion-service/container/expansion_service_config.yml b/sdks/java/expansion-service/container/expansion_service_config.yml
index 653629aa153..4f48efd5947 100644
--- a/sdks/java/expansion-service/container/expansion_service_config.yml
+++ b/sdks/java/expansion-service/container/expansion_service_config.yml
@@ -17,6 +17,9 @@ allowlist:
- "beam:transform:org.apache.beam:schemaio_jdbc_read:v1"
- "beam:transform:org.apache.beam:schemaio_jdbc_write:v1"
- "beam:schematransform:org.apache.beam:bigquery_storage_write:v1"
+# By default, the Expansion Service container will include all dependencies in
+# the classpath. Following config can be used to override this behavior per
+# transform URN or schema-transform ID.
dependencies:
"beam:transform:org.apache.beam:kafka_read_with_metadata:v1":
- path: "jars/beam-sdks-java-io-expansion-service.jar"
diff --git a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java
index 5d436451178..770da14fa1c 100644
--- a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java
+++ b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java
@@ -81,6 +81,7 @@ import org.apache.beam.sdk.util.construction.RehydratedComponents;
import org.apache.beam.sdk.util.construction.SdkComponents;
import org.apache.beam.sdk.util.construction.SplittableParDo;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionRowTuple;
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.POutput;
import org.apache.beam.sdk.values.Row;
@@ -246,6 +247,11 @@ public class ExpansionService extends ExpansionServiceGrpc.ExpansionServiceImplB
private final TransformPayloadTranslator<PTransform<InputT, OutputT>> payloadTranslator;
+ // Returns true if the underlying transform represented by this is a schema-aware transform.
+ private boolean isSchemaTransform() {
+ return (payloadTranslator instanceof SchemaTransformPayloadTranslator);
+ }
+
private TransformProviderForPayloadTranslator(
TransformPayloadTranslator<PTransform<InputT, OutputT>> payloadTranslator) {
this.payloadTranslator = payloadTranslator;
@@ -254,28 +260,51 @@ public class ExpansionService extends ExpansionServiceGrpc.ExpansionServiceImplB
@Override
public PTransform<InputT, OutputT> getTransform(
RunnerApi.FunctionSpec spec, PipelineOptions options) {
- try {
- ExternalConfigurationPayload payload =
- ExternalConfigurationPayload.parseFrom(spec.getPayload());
- Row configRow =
- RowCoder.of(SchemaTranslation.schemaFromProto(payload.getSchema()))
- .decode(new ByteArrayInputStream(payload.getPayload().toByteArray()));
- PTransform transformFromRow = payloadTranslator.fromConfigRow(configRow, options);
- if (transformFromRow != null) {
- return transformFromRow;
- } else {
+ if (isSchemaTransform()) {
+ return ExpansionServiceSchemaTransformProvider.of().getTransform(spec, options);
+ } else {
+ try {
+ ExternalConfigurationPayload payload =
+ ExternalConfigurationPayload.parseFrom(spec.getPayload());
+ Row configRow =
+ RowCoder.of(SchemaTranslation.schemaFromProto(payload.getSchema()))
+ .decode(new ByteArrayInputStream(payload.getPayload().toByteArray()));
+ PTransform transformFromRow = payloadTranslator.fromConfigRow(configRow, options);
+ if (transformFromRow != null) {
+ return transformFromRow;
+ } else {
+ throw new RuntimeException(
+ String.format(
+ "A transform cannot be initiated using the provided config row %s and the"
+ + " TransformPayloadTranslator %s",
+ configRow, payloadTranslator));
+ }
+ } catch (Exception e) {
throw new RuntimeException(
String.format(
- "A transform cannot be initiated using the provided config row %s and the"
- + " TransformPayloadTranslator %s",
- configRow, payloadTranslator));
+ "Failed to build transform %s from spec %s: %s",
+ spec.getUrn(), spec, e.getMessage()),
+ e);
}
- } catch (Exception e) {
- throw new RuntimeException(
- String.format(
- "Failed to build transform %s from spec %s: %s",
- spec.getUrn(), spec, e.getMessage()),
- e);
+ }
+ }
+
+ @Override
+ public InputT createInput(Pipeline p, Map<String, PCollection<?>> inputs) {
+ if (isSchemaTransform()) {
+ return (InputT) ExpansionServiceSchemaTransformProvider.of().createInput(p, inputs);
+ } else {
+ return TransformProvider.super.createInput(p, inputs);
+ }
+ }
+
+ @Override
+ public Map<String, PCollection<?>> extractOutputs(OutputT output) {
+ if (isSchemaTransform()) {
+ return ExpansionServiceSchemaTransformProvider.of()
+ .extractOutputs((PCollectionRowTuple) output);
+ } else {
+ return TransformProvider.super.extractOutputs(output);
}
}
diff --git a/sdks/java/io/expansion-service/build.gradle b/sdks/java/io/expansion-service/build.gradle
index e9fe4c1fe63..95a843e51fd 100644
--- a/sdks/java/io/expansion-service/build.gradle
+++ b/sdks/java/io/expansion-service/build.gradle
@@ -43,6 +43,11 @@ dependencies {
permitUnusedDeclared project(":sdks:java:io:kafka") // BEAM-11761
implementation project(":sdks:java:io:kafka:upgrade")
permitUnusedDeclared project(":sdks:java:io:kafka:upgrade") // BEAM-11761
+
+ // Needed by Iceberg I/O users that use GCS for the warehouse location.
+ implementation library.java.bigdataoss_gcs_connector
+ permitUnusedDeclared library.java.bigdataoss_gcs_connector
+
runtimeOnly library.java.kafka_clients
runtimeOnly library.java.slf4j_jdk14
}
diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle
index 765fa494875..7965cde86e7 100644
--- a/sdks/java/io/iceberg/build.gradle
+++ b/sdks/java/io/iceberg/build.gradle
@@ -57,8 +57,8 @@ dependencies {
testImplementation library.java.hadoop_client
testImplementation library.java.bigdataoss_gcsio
- testImplementation "com.google.cloud.bigdataoss:gcs-connector:hadoop2-2.2.16"
- testImplementation "com.google.cloud.bigdataoss:util-hadoop:hadoop2-2.2.16"
+ testImplementation library.java.bigdataoss_gcs_connector
+ testImplementation library.java.bigdataoss_util_hadoop
testImplementation "org.apache.iceberg:iceberg-gcp:$iceberg_version"
testImplementation "org.apache.iceberg:iceberg-data:$iceberg_version"
testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")
diff --git a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProvider.java b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProvider.java
index 54e1404c650..0702137cffd 100644
--- a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProvider.java
+++ b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProvider.java
@@ -63,28 +63,19 @@ public class ManagedSchemaTransformProvider
return "beam:transform:managed:v1";
}
- private final Map<String, SchemaTransformProvider> schemaTransformProviders = new HashMap<>();
+ // Use 'getAllProviders()' instead of this cache.
+ private final Map<String, SchemaTransformProvider> schemaTransformProvidersCache =
+ new HashMap<>();
+ private boolean providersCached = false;
- public ManagedSchemaTransformProvider() {}
+ private @Nullable Collection<String> supportedIdentifiers;
+
+ public ManagedSchemaTransformProvider() {
+ this(null);
+ }
ManagedSchemaTransformProvider(@Nullable Collection<String> supportedIdentifiers) {
- try {
- for (SchemaTransformProvider schemaTransformProvider :
- ServiceLoader.load(SchemaTransformProvider.class)) {
- if (schemaTransformProviders.containsKey(schemaTransformProvider.identifier())) {
- throw new IllegalArgumentException(
- "Found multiple SchemaTransformProvider implementations with the same identifier "
- + schemaTransformProvider.identifier());
- }
- if (supportedIdentifiers == null
- || supportedIdentifiers.contains(schemaTransformProvider.identifier())) {
- schemaTransformProviders.put(
- schemaTransformProvider.identifier(), schemaTransformProvider);
- }
- }
- } catch (Exception e) {
- throw new RuntimeException(e.getMessage());
- }
+ this.supportedIdentifiers = supportedIdentifiers;
}
@DefaultSchema(AutoValueSchema.class)
@@ -149,7 +140,7 @@ public class ManagedSchemaTransformProvider
managedConfig.validate();
SchemaTransformProvider schemaTransformProvider =
Preconditions.checkNotNull(
- schemaTransformProviders.get(managedConfig.getTransformIdentifier()),
+ getAllProviders().get(managedConfig.getTransformIdentifier()),
"Could not find a transform with the identifier "
+ "%s. This could be either due to the dependency with the "
+ "transform not being available in the classpath or due to "
@@ -236,8 +227,36 @@ public class ManagedSchemaTransformProvider
return YamlUtils.toBeamRow(configMap, transformSchema, false);
}
- Map<String, SchemaTransformProvider> getAllProviders() {
- return schemaTransformProviders;
+ // We load providers seperately, after construction, to prevent the
+ // 'ManagedSchemaTransformProvider' from being initialized in a recursive loop
+ // when being loaded using 'AutoValue'.
+ synchronized Map<String, SchemaTransformProvider> getAllProviders() {
+ if (this.providersCached) {
+ return schemaTransformProvidersCache;
+ }
+ try {
+ for (SchemaTransformProvider schemaTransformProvider :
+ ServiceLoader.load(SchemaTransformProvider.class)) {
+ if (schemaTransformProvidersCache.containsKey(schemaTransformProvider.identifier())) {
+ throw new IllegalArgumentException(
+ "Found multiple SchemaTransformProvider implementations with the same identifier "
+ + schemaTransformProvider.identifier());
+ }
+ if (supportedIdentifiers == null
+ || supportedIdentifiers.contains(schemaTransformProvider.identifier())) {
+ if (schemaTransformProvider.identifier().equals("beam:transform:managed:v1")) {
+ // Prevent recursively adding the 'ManagedSchemaTransformProvider'.
+ continue;
+ }
+ schemaTransformProvidersCache.put(
+ schemaTransformProvider.identifier(), schemaTransformProvider);
+ }
+ }
+ this.providersCached = true;
+ return schemaTransformProvidersCache;
+ } catch (Exception e) {
+ throw new RuntimeException(e.getMessage());
+ }
}
// TODO: set global snake_case naming convention and remove these special cases