You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2025/05/01 21:04:32 UTC
(beam) branch release-2.65 updated: Revert "[ManagedIO] Fail expansion when encountering extra or unknown configu…" (#34802) (#34805)
This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch release-2.65
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.65 by this push:
new 7ca30688029 Revert "[ManagedIO] Fail expansion when encountering extra or unknown configu…" (#34802) (#34805)
7ca30688029 is described below
commit 7ca30688029ed7e8d8885ff60cce1ab48e46ceae
Author: Ahmed Abualsaud <65...@users.noreply.github.com>
AuthorDate: Thu May 1 17:04:27 2025 -0400
Revert "[ManagedIO] Fail expansion when encountering extra or unknown configu…" (#34802) (#34805)
This reverts commit 363bcc0d3286ccfb4f3ea39641d769c12c6d9eb8.
---
.../KafkaReadSchemaTransformProviderTest.java | 5 +
.../java/org/apache/beam/sdk/managed/Managed.java | 13 ---
.../managed/ManagedSchemaTransformProvider.java | 69 +++----------
.../testing/TestSchemaTransformProvider.java | 13 +--
.../ManagedSchemaTransformProviderTest.java | 112 +--------------------
sdks/python/apache_beam/transforms/managed.py | 10 +-
.../transforms/managed_iceberg_it_test.py | 9 --
7 files changed, 24 insertions(+), 207 deletions(-)
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java
index 5ee8e76b2c4..a65583b5663 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java
@@ -304,6 +304,11 @@ public class KafkaReadSchemaTransformProviderTest {
+ "bootstrap_servers: some bootstrap\n"
+ "schema: '{\"type\":\"record\",\"name\":\"my_record\",\"fields\":[{\"name\":\"bool\",\"type\":\"boolean\"}]}'",
"topic: topic_3\n"
+ + "bootstrap_servers: some bootstrap\n"
+ + "schema_registry_url: some-url\n"
+ + "schema_registry_subject: some-subject\n"
+ + "format: RAW",
+ "topic: topic_4\n"
+ "bootstrap_servers: some bootstrap\n"
+ "format: PROTO\n"
+ "schema: '"
diff --git a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/Managed.java b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/Managed.java
index 4496aae48b6..3417c35233a 100644
--- a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/Managed.java
+++ b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/Managed.java
@@ -176,8 +176,6 @@ public class Managed {
@VisibleForTesting
abstract List<String> getSupportedIdentifiers();
- abstract @Nullable Boolean getSkipConfigValidation();
-
abstract Builder toBuilder();
@AutoValue.Builder
@@ -191,8 +189,6 @@ public class Managed {
@VisibleForTesting
abstract Builder setSupportedIdentifiers(List<String> supportedIdentifiers);
- abstract Builder setSkipConfigValidation(boolean skip);
-
abstract ManagedTransform build();
}
@@ -219,14 +215,6 @@ public class Managed {
return toBuilder().setSupportedIdentifiers(supportedIdentifiers).build();
}
- /**
- * Skips configuration validation. If unset, the pipeline will fail at construction time if the
- * configuration includes unknown fields or missing required fields.
- */
- public ManagedTransform skipConfigValidation() {
- return toBuilder().setSkipConfigValidation(true).build();
- }
-
@Override
public PCollectionRowTuple expand(PInput input) {
PCollectionRowTuple inputTuple = resolveInput(input);
@@ -236,7 +224,6 @@ public class Managed {
.setTransformIdentifier(getIdentifier())
.setConfig(YamlUtils.yamlStringFromMap(getConfig()))
.setConfigUrl(getConfigUrl())
- .setSkipConfigValidation(getSkipConfigValidation())
.build();
SchemaTransform underlyingTransform =
diff --git a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProvider.java b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProvider.java
index 9c7816ad539..f4be3f0c9a6 100644
--- a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProvider.java
+++ b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProvider.java
@@ -26,19 +26,15 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.ServiceLoader;
-import java.util.Set;
import javax.annotation.Nullable;
import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.io.fs.MatchResult;
-import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.schemas.AutoValueSchema;
import org.apache.beam.sdk.schemas.NoSuchSchemaException;
import org.apache.beam.sdk.schemas.Schema;
@@ -102,11 +98,6 @@ public class ManagedSchemaTransformProvider
@SchemaFieldDescription("YAML string config used to build the underlying SchemaTransform.")
public abstract @Nullable String getConfig();
- @SchemaFieldDescription(
- "Skips configuration validation. If unset, the pipeline will fail at construction "
- + "time if the configuration includes unknown fields or missing required fields.")
- public abstract @Nullable Boolean getSkipConfigValidation();
-
@AutoValue.Builder
public abstract static class Builder {
public abstract Builder setTransformIdentifier(String identifier);
@@ -115,8 +106,6 @@ public class ManagedSchemaTransformProvider
public abstract Builder setConfig(@Nullable String yamlConfig);
- public abstract Builder setSkipConfigValidation(@Nullable Boolean skip);
-
public abstract ManagedConfig build();
}
@@ -165,21 +154,28 @@ public class ManagedSchemaTransformProvider
static class ManagedSchemaTransform extends SchemaTransform {
private final ManagedConfig managedConfig;
+ private final Row underlyingRowConfig;
private final SchemaTransformProvider underlyingTransformProvider;
ManagedSchemaTransform(
ManagedConfig managedConfig, SchemaTransformProvider underlyingTransformProvider) {
+ // parse config before expansion to check if it matches underlying transform's config schema
+ Schema transformConfigSchema = underlyingTransformProvider.configurationSchema();
+ Row underlyingRowConfig;
+ try {
+ underlyingRowConfig = getRowConfig(managedConfig, transformConfigSchema);
+ } catch (Exception e) {
+ throw new IllegalArgumentException(
+ "Encountered an error when retrieving a configuration", e);
+ }
+
+ this.underlyingRowConfig = underlyingRowConfig;
this.underlyingTransformProvider = underlyingTransformProvider;
this.managedConfig = managedConfig;
}
@Override
public PCollectionRowTuple expand(PCollectionRowTuple input) {
- Row underlyingRowConfig =
- getRowConfig(
- managedConfig,
- underlyingTransformProvider.configurationSchema(),
- input.getPipeline().getOptions());
LOG.debug(
"Building transform \"{}\" with configuration: {}",
underlyingTransformProvider.identifier(),
@@ -188,7 +184,6 @@ public class ManagedSchemaTransformProvider
return input.apply(underlyingTransformProvider.from(underlyingRowConfig));
}
- @VisibleForTesting
public ManagedConfig getManagedConfig() {
return this.managedConfig;
}
@@ -211,8 +206,7 @@ public class ManagedSchemaTransformProvider
// May return an empty row (perhaps the underlying transform doesn't have any required
// parameters)
@VisibleForTesting
- static Row getRowConfig(
- ManagedConfig config, Schema transformConfigSchema, PipelineOptions options) {
+ static Row getRowConfig(ManagedConfig config, Schema transformSchema) {
Map<String, Object> configMap = config.resolveUnderlyingConfig();
// Build a config Row that will be used to build the underlying SchemaTransform.
// If a mapping for the SchemaTransform exists, we use it to update parameter names to align
@@ -231,42 +225,7 @@ public class ManagedSchemaTransformProvider
configMap = remappedConfig;
}
- @Nullable Boolean skipValidation = config.getSkipConfigValidation();
- if (skipValidation == null || !skipValidation) {
- validateUserConfig(
- config.getTransformIdentifier(),
- new HashSet<>(configMap.keySet()),
- transformConfigSchema);
- }
-
- return YamlUtils.toBeamRow(configMap, transformConfigSchema, false);
- }
-
- static void validateUserConfig(
- String transformId, Set<String> userParams, Schema transformConfigSchema) {
- List<String> missingRequiredFields = new ArrayList<>();
- for (Schema.Field field : transformConfigSchema.getFields()) {
- boolean inUserConfig = userParams.remove(field.getName());
- if (!field.getType().getNullable() && !inUserConfig) {
- missingRequiredFields.add(field.getName());
- }
- }
-
- if (!missingRequiredFields.isEmpty() || !userParams.isEmpty()) {
- String msg = "Invalid config for transform '" + transformId + "':";
- if (!missingRequiredFields.isEmpty()) {
- msg += " Missing required fields: " + missingRequiredFields + ".";
- }
- if (!userParams.isEmpty()) {
- msg +=
- " Contains unknown fields: "
- + userParams
- + ". If you'd still like to pass "
- + "these fields, use '.skipConfigValidation()'";
- }
-
- throw new IllegalArgumentException(msg);
- }
+ return YamlUtils.toBeamRow(configMap, transformSchema, false);
}
public static Map<String, Map<String, String>> getAliases() {
diff --git a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/testing/TestSchemaTransformProvider.java b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/testing/TestSchemaTransformProvider.java
index 7a28050d475..2771ecd0164 100644
--- a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/testing/TestSchemaTransformProvider.java
+++ b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/testing/TestSchemaTransformProvider.java
@@ -17,8 +17,6 @@
*/
package org.apache.beam.sdk.managed.testing;
-import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull;
-
import com.google.auto.service.AutoService;
import com.google.auto.value.AutoValue;
import org.apache.beam.sdk.schemas.AutoValueSchema;
@@ -33,7 +31,6 @@ import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionRowTuple;
import org.apache.beam.sdk.values.Row;
import org.apache.beam.sdk.values.TypeDescriptors;
-import org.checkerframework.checker.nullness.qual.Nullable;
@AutoService(SchemaTransformProvider.class)
public class TestSchemaTransformProvider
@@ -55,27 +52,19 @@ public class TestSchemaTransformProvider
@SchemaFieldDescription("Integer to add to each row element.")
public abstract Integer getExtraInteger();
- @SchemaFieldDescription("If true, will upper case the extra string. Default is false.")
- public abstract @Nullable Boolean getToggleUppercase();
-
@AutoValue.Builder
public abstract static class Builder {
public abstract Builder setExtraString(String extraString);
public abstract Builder setExtraInteger(Integer extraInteger);
- public abstract Builder setToggleUppercase(Boolean toggleUppercase);
-
public abstract Config build();
}
}
@Override
public SchemaTransform from(Config config) {
- String extraString =
- firstNonNull(config.getToggleUppercase(), false)
- ? config.getExtraString().toUpperCase()
- : config.getExtraString();
+ String extraString = config.getExtraString();
Integer extraInteger = config.getExtraInteger();
return new SchemaTransform() {
@Override
diff --git a/sdks/java/managed/src/test/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProviderTest.java b/sdks/java/managed/src/test/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProviderTest.java
index a9565400333..a287ec6260c 100644
--- a/sdks/java/managed/src/test/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProviderTest.java
+++ b/sdks/java/managed/src/test/java/org/apache/beam/sdk/managed/ManagedSchemaTransformProviderTest.java
@@ -24,16 +24,9 @@ import static org.junit.Assert.assertTrue;
import java.net.URISyntaxException;
import java.nio.file.Paths;
import java.util.Arrays;
-import java.util.Map;
-import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.managed.testing.TestSchemaTransformProvider;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.sdk.schemas.utils.YamlUtils;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.values.PCollectionRowTuple;
import org.apache.beam.sdk.values.Row;
-import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -43,8 +36,6 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class ManagedSchemaTransformProviderTest {
@Rule public transient ExpectedException thrown = ExpectedException.none();
- private static final Schema EMPTY_SCHEMA = Schema.builder().build();
- private static final Row EMPTY_ROW = Row.nullRow(EMPTY_SCHEMA);
@Test
public void testFailWhenNoConfigSpecified() {
@@ -58,103 +49,6 @@ public class ManagedSchemaTransformProviderTest {
config.validate();
}
- @Test
- public void testFailWhenUnknownFieldsSpecified() {
- Map<String, Object> config =
- ImmutableMap.of(
- "extra_string",
- "str",
- "extra_integer",
- 123,
- "toggle_uppercase",
- true,
- "unknown_field",
- "unknown");
- ManagedSchemaTransformProvider.ManagedConfig managedConfig =
- ManagedSchemaTransformProvider.ManagedConfig.builder()
- .setTransformIdentifier(TestSchemaTransformProvider.IDENTIFIER)
- .setConfig(YamlUtils.yamlStringFromMap(config))
- .build();
-
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("Invalid config for transform");
- thrown.expectMessage(TestSchemaTransformProvider.IDENTIFIER);
- thrown.expectMessage("Contains unknown fields");
- thrown.expectMessage("unknown_field");
- Pipeline p = Pipeline.create();
- new ManagedSchemaTransformProvider(null)
- .from(managedConfig)
- .expand(
- PCollectionRowTuple.of(
- "input", p.apply(Create.of(EMPTY_ROW).withRowSchema(EMPTY_SCHEMA))));
- }
-
- @Test
- public void testFailWhenMissingRequiredFields() {
- Map<String, Object> config = ImmutableMap.of("extra_string", "str", "toggle_uppercase", true);
- ManagedSchemaTransformProvider.ManagedConfig managedConfig =
- ManagedSchemaTransformProvider.ManagedConfig.builder()
- .setTransformIdentifier(TestSchemaTransformProvider.IDENTIFIER)
- .setConfig(YamlUtils.yamlStringFromMap(config))
- .build();
-
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("Invalid config for transform");
- thrown.expectMessage(TestSchemaTransformProvider.IDENTIFIER);
- thrown.expectMessage("Missing required fields");
- thrown.expectMessage("extra_integer");
- Pipeline p = Pipeline.create();
- new ManagedSchemaTransformProvider(null)
- .from(managedConfig)
- .expand(
- PCollectionRowTuple.of(
- "input", p.apply(Create.of(EMPTY_ROW).withRowSchema(EMPTY_SCHEMA))));
- }
-
- @Test
- public void testPassWhenMissingNullableFields() {
- Map<String, Object> config = ImmutableMap.of("extra_string", "str", "extra_integer", 123);
- ManagedSchemaTransformProvider.ManagedConfig managedConfig =
- ManagedSchemaTransformProvider.ManagedConfig.builder()
- .setTransformIdentifier(TestSchemaTransformProvider.IDENTIFIER)
- .setConfig(YamlUtils.yamlStringFromMap(config))
- .build();
-
- Pipeline p = Pipeline.create();
- new ManagedSchemaTransformProvider(null)
- .from(managedConfig)
- .expand(
- PCollectionRowTuple.of(
- "input", p.apply(Create.of(EMPTY_ROW).withRowSchema(EMPTY_SCHEMA))));
- }
-
- @Test
- public void testSkipConfigValidationWithUnknownFields() {
- Map<String, Object> config =
- ImmutableMap.of(
- "extra_string",
- "str",
- "extra_integer",
- 123,
- "toggle_uppercase",
- true,
- "unknown_field",
- "unknown");
- ManagedSchemaTransformProvider.ManagedConfig managedConfig =
- ManagedSchemaTransformProvider.ManagedConfig.builder()
- .setTransformIdentifier(TestSchemaTransformProvider.IDENTIFIER)
- .setConfig(YamlUtils.yamlStringFromMap(config))
- .setSkipConfigValidation(true)
- .build();
-
- Pipeline p = Pipeline.create();
- new ManagedSchemaTransformProvider(null)
- .from(managedConfig)
- .expand(
- PCollectionRowTuple.of(
- "input", p.apply(Create.of(EMPTY_ROW).withRowSchema(EMPTY_SCHEMA))));
- }
-
@Test
public void testGetConfigRowFromYamlString() {
String yamlString = "extra_string: abc\n" + "extra_integer: 123";
@@ -171,8 +65,7 @@ public class ManagedSchemaTransformProviderTest {
.build();
Row returnedRow =
- ManagedSchemaTransformProvider.getRowConfig(
- config, TestSchemaTransformProvider.SCHEMA, PipelineOptionsFactory.create());
+ ManagedSchemaTransformProvider.getRowConfig(config, TestSchemaTransformProvider.SCHEMA);
assertEquals(expectedRow, returnedRow);
}
@@ -195,8 +88,7 @@ public class ManagedSchemaTransformProviderTest {
.withFieldValue("extra_integer", 123)
.build();
Row configRow =
- ManagedSchemaTransformProvider.getRowConfig(
- config, TestSchemaTransformProvider.SCHEMA, PipelineOptionsFactory.create());
+ ManagedSchemaTransformProvider.getRowConfig(config, TestSchemaTransformProvider.SCHEMA);
assertEquals(expectedRow, configRow);
}
diff --git a/sdks/python/apache_beam/transforms/managed.py b/sdks/python/apache_beam/transforms/managed.py
index 609a27b3713..2356cca2ef2 100644
--- a/sdks/python/apache_beam/transforms/managed.py
+++ b/sdks/python/apache_beam/transforms/managed.py
@@ -107,7 +107,6 @@ class Read(PTransform):
source: str,
config: Optional[dict[str, Any]] = None,
config_url: Optional[str] = None,
- skip_config_validation: bool = False,
expansion_service=None):
super().__init__()
self._source = source
@@ -122,7 +121,6 @@ class Read(PTransform):
self._underlying_identifier = identifier
self._yaml_config = yaml.dump(config)
self._config_url = config_url
- self._skip_config_validation = skip_config_validation
def expand(self, input):
return input | SchemaAwareExternalTransform(
@@ -131,8 +129,7 @@ class Read(PTransform):
rearrange_based_on_discovery=True,
transform_identifier=self._underlying_identifier,
config=self._yaml_config,
- config_url=self._config_url,
- skip_config_validation=self._skip_config_validation)
+ config_url=self._config_url)
def default_label(self) -> str:
return "Managed Read(%s)" % self._source.upper()
@@ -151,7 +148,6 @@ class Write(PTransform):
sink: str,
config: Optional[dict[str, Any]] = None,
config_url: Optional[str] = None,
- skip_config_validation: bool = False,
expansion_service=None):
super().__init__()
self._sink = sink
@@ -166,7 +162,6 @@ class Write(PTransform):
self._underlying_identifier = identifier
self._yaml_config = yaml.dump(config)
self._config_url = config_url
- self._skip_config_validation = skip_config_validation
def expand(self, input):
return input | SchemaAwareExternalTransform(
@@ -175,8 +170,7 @@ class Write(PTransform):
rearrange_based_on_discovery=True,
transform_identifier=self._underlying_identifier,
config=self._yaml_config,
- config_url=self._config_url,
- skip_config_validation=self._skip_config_validation)
+ config_url=self._config_url)
def default_label(self) -> str:
return "Managed Write(%s)" % self._sink.upper()
diff --git a/sdks/python/apache_beam/transforms/managed_iceberg_it_test.py b/sdks/python/apache_beam/transforms/managed_iceberg_it_test.py
index d402692199b..20cb52335c7 100644
--- a/sdks/python/apache_beam/transforms/managed_iceberg_it_test.py
+++ b/sdks/python/apache_beam/transforms/managed_iceberg_it_test.py
@@ -75,16 +75,7 @@ class ManagedIcebergIT(unittest.TestCase):
| beam.managed.Read(beam.managed.ICEBERG, config=iceberg_config)
| beam.Map(lambda row: row._asdict()))
- output_dicts_skipped_config = (
- read_pipeline
- | beam.managed.Read(
- beam.managed.ICEBERG,
- config=iceberg_config,
- skip_config_validation=True)
- | beam.Map(lambda row: row._asdict()))
-
assert_that(output_dicts, equal_to(expected_dicts))
- assert_that(output_dicts_skipped_config, equal_to(expected_dicts))
if __name__ == '__main__':