You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2025/05/01 21:04:32 UTC

(beam) branch release-2.65 updated: Revert "[ManagedIO] Fail expansion when encountering extra or unknown configu…" (#34802) (#34805)

This is an automated email from the ASF dual-hosted git repository.

yhu pushed a commit to branch release-2.65
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/release-2.65 by this push:
     new 7ca30688029 Revert "[ManagedIO] Fail expansion when encountering extra or unknown configu…" (#34802) (#34805)
7ca30688029 is described below

commit 7ca30688029ed7e8d8885ff60cce1ab48e46ceae
Author: Ahmed Abualsaud <65...@users.noreply.github.com>
AuthorDate: Thu May 1 17:04:27 2025 -0400

    Revert "[ManagedIO] Fail expansion when encountering extra or unknown configu…" (#34802) (#34805)
    
    This reverts commit 363bcc0d3286ccfb4f3ea39641d769c12c6d9eb8.
---
 .../KafkaReadSchemaTransformProviderTest.java      |   5 +
 .../java/org/apache/beam/sdk/managed/Managed.java  |  13 ---
 .../managed/ManagedSchemaTransformProvider.java    |  69 +++----------
 .../testing/TestSchemaTransformProvider.java       |  13 +--
 .../ManagedSchemaTransformProviderTest.java        | 112 +--------------------
 sdks/python/apache_beam/transforms/managed.py      |  10 +-
 .../transforms/managed_iceberg_it_test.py          |   9 --
 7 files changed, 24 insertions(+), 207 deletions(-)

diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java
index 5ee8e76b2c4..a65583b5663 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java
@@ -304,6 +304,11 @@ public class KafkaReadSchemaTransformProviderTest {
                 + "bootstrap_servers: some bootstrap\n"
                 + "schema: '{\"type\":\"record\",\"name\":\"my_record\",\"fields\":[{\"name\":\"bool\",\"type\":\"boolean\"}]}'",
             "topic: topic_3\n"
+                + "bootstrap_servers: some bootstrap\n"
+                + "schema_registry_url: some-url\n"
+                + "schema_registry_subject: some-subject\n"
+                + "format: RAW",
+            "topic: topic_4\n"
                 + "bootstrap_servers: some bootstrap\n"
                 + "format: PROTO\n"
                 + "schema: '"
diff --git a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/Managed.java b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/Managed.java
index 4496aae48b6..3417c35233a 100644
--- a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/Managed.java
+++ b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/Managed.java
@@ -176,8 +176,6 @@ public class Managed {
     @VisibleForTesting
     abstract List<String> getSupportedIdentifiers();
 
-    abstract @Nullable Boolean getSkipConfigValidation();
-
     abstract Builder toBuilder();
 
     @AutoValue.Builder
@@ -191,8 +189,6 @@ public class Managed {
       @VisibleForTesting
       abstract Builder setSupportedIdentifiers(List<String> supportedIdentifiers);
 
-      abstract Builder setSkipConfigValidation(boolean skip);
-
       abstract ManagedTransform build();
     }
 
@@ -219,14 +215,6 @@ public class Managed {
       return toBuilder().setSupportedIdentifiers(supportedIdentifiers).build();
     }
 
-    /**
-     * Skips configuration validation. If unset, the pipeline will fail at construction time if the
-     * configuration includes unknown fields or missing required fields.
-     */
-    public ManagedTransform skipConfigValidation() {
-      return toBuilder().setSkipConfigValidation(true).build();
-    }
-
     @Override
     public PCollectionRowTuple expand(PInput input) {
       PCollectionRowTuple inputTuple = resolveInput(input);
@@ -236,7 +224,6 @@ public class Managed {
               .setTransformIdentifier(getIdentifier())
               .setConfig(YamlUtils.yamlStringFromMap(getConfig()))
               .setConfigUrl(getConfigUrl())
-              .setSkipConfigValidation(getSkipConfigValidation())
               .build();
 
       SchemaTransform underlyingTransform =
diff --git a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProvider.java b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProvider.java
index 9c7816ad539..f4be3f0c9a6 100644
--- a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProvider.java
+++ b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProvider.java
@@ -26,19 +26,15 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.ServiceLoader;
-import java.util.Set;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.MatchResult;
-import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.schemas.AutoValueSchema;
 import org.apache.beam.sdk.schemas.NoSuchSchemaException;
 import org.apache.beam.sdk.schemas.Schema;
@@ -102,11 +98,6 @@ public class ManagedSchemaTransformProvider
     @SchemaFieldDescription("YAML string config used to build the underlying SchemaTransform.")
     public abstract @Nullable String getConfig();
 
-    @SchemaFieldDescription(
-        "Skips configuration validation. If unset, the pipeline will fail at construction "
-            + "time if the configuration includes unknown fields or missing required fields.")
-    public abstract @Nullable Boolean getSkipConfigValidation();
-
     @AutoValue.Builder
     public abstract static class Builder {
       public abstract Builder setTransformIdentifier(String identifier);
@@ -115,8 +106,6 @@ public class ManagedSchemaTransformProvider
 
       public abstract Builder setConfig(@Nullable String yamlConfig);
 
-      public abstract Builder setSkipConfigValidation(@Nullable Boolean skip);
-
       public abstract ManagedConfig build();
     }
 
@@ -165,21 +154,28 @@ public class ManagedSchemaTransformProvider
 
   static class ManagedSchemaTransform extends SchemaTransform {
     private final ManagedConfig managedConfig;
+    private final Row underlyingRowConfig;
     private final SchemaTransformProvider underlyingTransformProvider;
 
     ManagedSchemaTransform(
         ManagedConfig managedConfig, SchemaTransformProvider underlyingTransformProvider) {
+      // parse config before expansion to check if it matches underlying transform's config schema
+      Schema transformConfigSchema = underlyingTransformProvider.configurationSchema();
+      Row underlyingRowConfig;
+      try {
+        underlyingRowConfig = getRowConfig(managedConfig, transformConfigSchema);
+      } catch (Exception e) {
+        throw new IllegalArgumentException(
+            "Encountered an error when retrieving a configuration", e);
+      }
+
+      this.underlyingRowConfig = underlyingRowConfig;
       this.underlyingTransformProvider = underlyingTransformProvider;
       this.managedConfig = managedConfig;
     }
 
     @Override
     public PCollectionRowTuple expand(PCollectionRowTuple input) {
-      Row underlyingRowConfig =
-          getRowConfig(
-              managedConfig,
-              underlyingTransformProvider.configurationSchema(),
-              input.getPipeline().getOptions());
       LOG.debug(
           "Building transform \"{}\" with configuration: {}",
           underlyingTransformProvider.identifier(),
@@ -188,7 +184,6 @@ public class ManagedSchemaTransformProvider
       return input.apply(underlyingTransformProvider.from(underlyingRowConfig));
     }
 
-    @VisibleForTesting
     public ManagedConfig getManagedConfig() {
       return this.managedConfig;
     }
@@ -211,8 +206,7 @@ public class ManagedSchemaTransformProvider
   // May return an empty row (perhaps the underlying transform doesn't have any required
   // parameters)
   @VisibleForTesting
-  static Row getRowConfig(
-      ManagedConfig config, Schema transformConfigSchema, PipelineOptions options) {
+  static Row getRowConfig(ManagedConfig config, Schema transformSchema) {
     Map<String, Object> configMap = config.resolveUnderlyingConfig();
     // Build a config Row that will be used to build the underlying SchemaTransform.
     // If a mapping for the SchemaTransform exists, we use it to update parameter names to align
@@ -231,42 +225,7 @@ public class ManagedSchemaTransformProvider
       configMap = remappedConfig;
     }
 
-    @Nullable Boolean skipValidation = config.getSkipConfigValidation();
-    if (skipValidation == null || !skipValidation) {
-      validateUserConfig(
-          config.getTransformIdentifier(),
-          new HashSet<>(configMap.keySet()),
-          transformConfigSchema);
-    }
-
-    return YamlUtils.toBeamRow(configMap, transformConfigSchema, false);
-  }
-
-  static void validateUserConfig(
-      String transformId, Set<String> userParams, Schema transformConfigSchema) {
-    List<String> missingRequiredFields = new ArrayList<>();
-    for (Schema.Field field : transformConfigSchema.getFields()) {
-      boolean inUserConfig = userParams.remove(field.getName());
-      if (!field.getType().getNullable() && !inUserConfig) {
-        missingRequiredFields.add(field.getName());
-      }
-    }
-
-    if (!missingRequiredFields.isEmpty() || !userParams.isEmpty()) {
-      String msg = "Invalid config for transform '" + transformId + "':";
-      if (!missingRequiredFields.isEmpty()) {
-        msg += " Missing required fields: " + missingRequiredFields + ".";
-      }
-      if (!userParams.isEmpty()) {
-        msg +=
-            " Contains unknown fields: "
-                + userParams
-                + ". If you'd still like to pass "
-                + "these fields, use '.skipConfigValidation()'";
-      }
-
-      throw new IllegalArgumentException(msg);
-    }
+    return YamlUtils.toBeamRow(configMap, transformSchema, false);
   }
 
   public static Map<String, Map<String, String>> getAliases() {
diff --git a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/testing/TestSchemaTransformProvider.java b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/testing/TestSchemaTransformProvider.java
index 7a28050d475..2771ecd0164 100644
--- a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/testing/TestSchemaTransformProvider.java
+++ b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/testing/TestSchemaTransformProvider.java
@@ -17,8 +17,6 @@
  */
 package org.apache.beam.sdk.managed.testing;
 
-import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull;
-
 import com.google.auto.service.AutoService;
 import com.google.auto.value.AutoValue;
 import org.apache.beam.sdk.schemas.AutoValueSchema;
@@ -33,7 +31,6 @@ import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionRowTuple;
 import org.apache.beam.sdk.values.Row;
 import org.apache.beam.sdk.values.TypeDescriptors;
-import org.checkerframework.checker.nullness.qual.Nullable;
 
 @AutoService(SchemaTransformProvider.class)
 public class TestSchemaTransformProvider
@@ -55,27 +52,19 @@ public class TestSchemaTransformProvider
     @SchemaFieldDescription("Integer to add to each row element.")
     public abstract Integer getExtraInteger();
 
-    @SchemaFieldDescription("If true, will upper case the extra string. Default is false.")
-    public abstract @Nullable Boolean getToggleUppercase();
-
     @AutoValue.Builder
     public abstract static class Builder {
       public abstract Builder setExtraString(String extraString);
 
       public abstract Builder setExtraInteger(Integer extraInteger);
 
-      public abstract Builder setToggleUppercase(Boolean toggleUppercase);
-
       public abstract Config build();
     }
   }
 
   @Override
   public SchemaTransform from(Config config) {
-    String extraString =
-        firstNonNull(config.getToggleUppercase(), false)
-            ? config.getExtraString().toUpperCase()
-            : config.getExtraString();
+    String extraString = config.getExtraString();
     Integer extraInteger = config.getExtraInteger();
     return new SchemaTransform() {
       @Override
diff --git a/sdks/java/managed/src/test/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProviderTest.java b/sdks/java/managed/src/test/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProviderTest.java
index a9565400333..a287ec6260c 100644
--- a/sdks/java/managed/src/test/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProviderTest.java
+++ b/sdks/java/managed/src/test/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProviderTest.java
@@ -24,16 +24,9 @@ import static org.junit.Assert.assertTrue;
 import java.net.URISyntaxException;
 import java.nio.file.Paths;
 import java.util.Arrays;
-import java.util.Map;
-import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.managed.testing.TestSchemaTransformProvider;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.utils.YamlUtils;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.values.PCollectionRowTuple;
 import org.apache.beam.sdk.values.Row;
-import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -43,8 +36,6 @@ import org.junit.runners.JUnit4;
 @RunWith(JUnit4.class)
 public class ManagedSchemaTransformProviderTest {
   @Rule public transient ExpectedException thrown = ExpectedException.none();
-  private static final Schema EMPTY_SCHEMA = Schema.builder().build();
-  private static final Row EMPTY_ROW = Row.nullRow(EMPTY_SCHEMA);
 
   @Test
   public void testFailWhenNoConfigSpecified() {
@@ -58,103 +49,6 @@ public class ManagedSchemaTransformProviderTest {
     config.validate();
   }
 
-  @Test
-  public void testFailWhenUnknownFieldsSpecified() {
-    Map<String, Object> config =
-        ImmutableMap.of(
-            "extra_string",
-            "str",
-            "extra_integer",
-            123,
-            "toggle_uppercase",
-            true,
-            "unknown_field",
-            "unknown");
-    ManagedSchemaTransformProvider.ManagedConfig managedConfig =
-        ManagedSchemaTransformProvider.ManagedConfig.builder()
-            .setTransformIdentifier(TestSchemaTransformProvider.IDENTIFIER)
-            .setConfig(YamlUtils.yamlStringFromMap(config))
-            .build();
-
-    thrown.expect(IllegalArgumentException.class);
-    thrown.expectMessage("Invalid config for transform");
-    thrown.expectMessage(TestSchemaTransformProvider.IDENTIFIER);
-    thrown.expectMessage("Contains unknown fields");
-    thrown.expectMessage("unknown_field");
-    Pipeline p = Pipeline.create();
-    new ManagedSchemaTransformProvider(null)
-        .from(managedConfig)
-        .expand(
-            PCollectionRowTuple.of(
-                "input", p.apply(Create.of(EMPTY_ROW).withRowSchema(EMPTY_SCHEMA))));
-  }
-
-  @Test
-  public void testFailWhenMissingRequiredFields() {
-    Map<String, Object> config = ImmutableMap.of("extra_string", "str", "toggle_uppercase", true);
-    ManagedSchemaTransformProvider.ManagedConfig managedConfig =
-        ManagedSchemaTransformProvider.ManagedConfig.builder()
-            .setTransformIdentifier(TestSchemaTransformProvider.IDENTIFIER)
-            .setConfig(YamlUtils.yamlStringFromMap(config))
-            .build();
-
-    thrown.expect(IllegalArgumentException.class);
-    thrown.expectMessage("Invalid config for transform");
-    thrown.expectMessage(TestSchemaTransformProvider.IDENTIFIER);
-    thrown.expectMessage("Missing required fields");
-    thrown.expectMessage("extra_integer");
-    Pipeline p = Pipeline.create();
-    new ManagedSchemaTransformProvider(null)
-        .from(managedConfig)
-        .expand(
-            PCollectionRowTuple.of(
-                "input", p.apply(Create.of(EMPTY_ROW).withRowSchema(EMPTY_SCHEMA))));
-  }
-
-  @Test
-  public void testPassWhenMissingNullableFields() {
-    Map<String, Object> config = ImmutableMap.of("extra_string", "str", "extra_integer", 123);
-    ManagedSchemaTransformProvider.ManagedConfig managedConfig =
-        ManagedSchemaTransformProvider.ManagedConfig.builder()
-            .setTransformIdentifier(TestSchemaTransformProvider.IDENTIFIER)
-            .setConfig(YamlUtils.yamlStringFromMap(config))
-            .build();
-
-    Pipeline p = Pipeline.create();
-    new ManagedSchemaTransformProvider(null)
-        .from(managedConfig)
-        .expand(
-            PCollectionRowTuple.of(
-                "input", p.apply(Create.of(EMPTY_ROW).withRowSchema(EMPTY_SCHEMA))));
-  }
-
-  @Test
-  public void testSkipConfigValidationWithUnknownFields() {
-    Map<String, Object> config =
-        ImmutableMap.of(
-            "extra_string",
-            "str",
-            "extra_integer",
-            123,
-            "toggle_uppercase",
-            true,
-            "unknown_field",
-            "unknown");
-    ManagedSchemaTransformProvider.ManagedConfig managedConfig =
-        ManagedSchemaTransformProvider.ManagedConfig.builder()
-            .setTransformIdentifier(TestSchemaTransformProvider.IDENTIFIER)
-            .setConfig(YamlUtils.yamlStringFromMap(config))
-            .setSkipConfigValidation(true)
-            .build();
-
-    Pipeline p = Pipeline.create();
-    new ManagedSchemaTransformProvider(null)
-        .from(managedConfig)
-        .expand(
-            PCollectionRowTuple.of(
-                "input", p.apply(Create.of(EMPTY_ROW).withRowSchema(EMPTY_SCHEMA))));
-  }
-
   @Test
   public void testGetConfigRowFromYamlString() {
     String yamlString = "extra_string: abc\n" + "extra_integer: 123";
@@ -171,8 +65,7 @@ public class ManagedSchemaTransformProviderTest {
             .build();
 
     Row returnedRow =
-        ManagedSchemaTransformProvider.getRowConfig(
-            config, TestSchemaTransformProvider.SCHEMA, PipelineOptionsFactory.create());
+        ManagedSchemaTransformProvider.getRowConfig(config, TestSchemaTransformProvider.SCHEMA);
 
     assertEquals(expectedRow, returnedRow);
   }
@@ -195,8 +88,7 @@ public class ManagedSchemaTransformProviderTest {
             .withFieldValue("extra_integer", 123)
             .build();
     Row configRow =
-        ManagedSchemaTransformProvider.getRowConfig(
-            config, TestSchemaTransformProvider.SCHEMA, PipelineOptionsFactory.create());
+        ManagedSchemaTransformProvider.getRowConfig(config, TestSchemaTransformProvider.SCHEMA);
 
     assertEquals(expectedRow, configRow);
   }
diff --git a/sdks/python/apache_beam/transforms/managed.py b/sdks/python/apache_beam/transforms/managed.py
index 609a27b3713..2356cca2ef2 100644
--- a/sdks/python/apache_beam/transforms/managed.py
+++ b/sdks/python/apache_beam/transforms/managed.py
@@ -107,7 +107,6 @@ class Read(PTransform):
       source: str,
       config: Optional[dict[str, Any]] = None,
       config_url: Optional[str] = None,
-      skip_config_validation: bool = False,
       expansion_service=None):
     super().__init__()
     self._source = source
@@ -122,7 +121,6 @@ class Read(PTransform):
     self._underlying_identifier = identifier
     self._yaml_config = yaml.dump(config)
     self._config_url = config_url
-    self._skip_config_validation = skip_config_validation
 
   def expand(self, input):
     return input | SchemaAwareExternalTransform(
@@ -131,8 +129,7 @@ class Read(PTransform):
         rearrange_based_on_discovery=True,
         transform_identifier=self._underlying_identifier,
         config=self._yaml_config,
-        config_url=self._config_url,
-        skip_config_validation=self._skip_config_validation)
+        config_url=self._config_url)
 
   def default_label(self) -> str:
     return "Managed Read(%s)" % self._source.upper()
@@ -151,7 +148,6 @@ class Write(PTransform):
       sink: str,
       config: Optional[dict[str, Any]] = None,
       config_url: Optional[str] = None,
-      skip_config_validation: bool = False,
       expansion_service=None):
     super().__init__()
     self._sink = sink
@@ -166,7 +162,6 @@ class Write(PTransform):
     self._underlying_identifier = identifier
     self._yaml_config = yaml.dump(config)
     self._config_url = config_url
-    self._skip_config_validation = skip_config_validation
 
   def expand(self, input):
     return input | SchemaAwareExternalTransform(
@@ -175,8 +170,7 @@ class Write(PTransform):
         rearrange_based_on_discovery=True,
         transform_identifier=self._underlying_identifier,
         config=self._yaml_config,
-        config_url=self._config_url,
-        skip_config_validation=self._skip_config_validation)
+        config_url=self._config_url)
 
   def default_label(self) -> str:
     return "Managed Write(%s)" % self._sink.upper()
diff --git a/sdks/python/apache_beam/transforms/managed_iceberg_it_test.py b/sdks/python/apache_beam/transforms/managed_iceberg_it_test.py
index d402692199b..20cb52335c7 100644
--- a/sdks/python/apache_beam/transforms/managed_iceberg_it_test.py
+++ b/sdks/python/apache_beam/transforms/managed_iceberg_it_test.py
@@ -75,16 +75,7 @@ class ManagedIcebergIT(unittest.TestCase):
           | beam.managed.Read(beam.managed.ICEBERG, config=iceberg_config)
           | beam.Map(lambda row: row._asdict()))
 
-      output_dicts_skipped_config = (
-          read_pipeline
-          | beam.managed.Read(
-              beam.managed.ICEBERG,
-              config=iceberg_config,
-              skip_config_validation=True)
-          | beam.Map(lambda row: row._asdict()))
-
       assert_that(output_dicts, equal_to(expected_dicts))
-      assert_that(output_dicts_skipped_config, equal_to(expected_dicts))
 
 
 if __name__ == '__main__':