You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ku...@apache.org on 2026/02/26 08:12:48 UTC
(flink-cdc) 01/01: [FLINK-38729] Add support for Flink 2.2.0
This is an automated email from the ASF dual-hosted git repository.
kunni pushed a commit to branch FLINK-38729-2
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
commit cb27d5acf8c5327dc9d1dc2e0901678e3d2e8b9e
Author: lvyanquan <lv...@alibaba-inc.com>
AuthorDate: Wed Feb 25 19:03:57 2026 +0800
[FLINK-38729] Add support for Flink 2.2.0
---
flink-cdc-cli-2.x/pom.xml | 141 +
.../java/org/apache/flink/cdc/cli/CliFrontend.java | 262 ++
.../flink/cdc/cli/utils/ConfigurationUtils.java | 6 +-
flink-cdc-common-2.x/pom.xml | 141 +
.../cdc/common/utils/SchemaMergingUtilsTest.java | 1228 ++++++++
.../apache/flink/cdc/common/types/VariantType.java | 2 +-
.../flink/cdc/common/utils/ThreadLocalCache.java | 2 +-
flink-cdc-composer-2.x/pom.xml | 247 ++
.../flink/translator/DataSinkTranslator.java | 246 ++
.../composer/flink/FlinkEnvironmentUtilsTest.java | 55 +
.../flink/FlinkPipelineBatchComposerITCase.java | 1553 +++++++++
.../flink/FlinkPipelineComposerITCase.java | 1673 ++++++++++
.../flink/FlinkPipelineComposerLenientITCase.java | 1143 +++++++
.../composer/flink/FlinkPipelineComposerTest.java | 226 ++
.../flink/FlinkPipelineTransformITCase.java | 3321 ++++++++++++++++++++
.../cdc/composer/flink/FlinkPipelineUdfITCase.java | 1025 ++++++
.../flink/deployment/ComposeDeploymentTest.java | 40 +
.../flink/translator/DataSinkTranslatorTest.java | 95 +
.../flink/translator/DataSourceTranslatorTest.java | 63 +
.../flink/translator/OperatorUidGeneratorTest.java | 58 +
.../cdc/composer/specs/TransformSpecsITCase.java | 524 +++
.../source/DistributedSourceFunction.java | 276 ++
.../composer/utils/FactoryDiscoveryUtilsTest.java | 51 +
.../composer/utils/factory/DataSinkFactory1.java | 76 +
.../composer/utils/factory/DataSinkFactory2.java | 60 +
.../composer/utils/factory/DataSourceFactory1.java | 76 +
.../composer/utils/factory/DataSourceFactory2.java | 61 +
.../cdc/composer/utils/factory/TestOptions.java | 36 +
.../org.apache.flink.cdc.common.factories.Factory | 20 +
.../src/test/resources/log4j2-test.properties | 25 +
.../src/test/resources/specs/arithmetic.yaml | 209 ++
.../src/test/resources/specs/basic.yaml | 145 +
.../src/test/resources/specs/casting.yaml | 271 ++
.../src/test/resources/specs/comparison.yaml | 281 ++
.../src/test/resources/specs/condition.yaml | 102 +
.../src/test/resources/specs/decimal.yaml | 115 +
.../src/test/resources/specs/logical.yaml | 48 +
.../src/test/resources/specs/meta.yaml | 49 +
.../src/test/resources/specs/nested.yaml | 365 +++
.../src/test/resources/specs/string.yaml | 149 +
.../src/test/resources/specs/temporal.yaml | 278 ++
.../pom.xml | 121 +
.../cdc/connectors/values/sink/ValuesDataSink.java | 187 ++
.../pom.xml | 24 +-
.../flink-cdc-pipeline-connector-values/pom.xml | 12 +
flink-cdc-connect/pom.xml | 1 +
flink-cdc-dist-2.x/pom.xml | 131 +
flink-cdc-dist-2.x/src/main/assembly/assembly.xml | 83 +
flink-cdc-runtime-2.x/pom.xml | 158 +
.../sink/BatchDataSinkWriterOperator.java | 39 +-
.../operators/sink/DataSinkWriterOperator.java | 45 +-
.../sink/DataSinkWriterOperatorFactory.java | 86 +
.../cdc/runtime/serializer/EnumSerializer.java | 298 ++
.../NestedSerializersSnapshotDelegate.java | 204 ++
.../serializer/NullableSerializerWrapper.java | 192 ++
.../serializer/data/ArrayDataSerializer.java | 286 ++
.../serializer/data/DecimalDataSerializer.java | 198 ++
.../data/LocalZonedTimestampDataSerializer.java | 199 ++
.../runtime/serializer/data/MapDataSerializer.java | 328 ++
.../serializer/data/TimestampDataSerializer.java | 192 ++
.../data/ZonedTimestampDataSerializer.java | 202 ++
.../flink/cdc/runtime/typeutils/EventTypeInfo.java | 85 +
.../typeutils/PartitioningEventTypeInfo.java | 85 +
.../schema/distributed/SchemaOperator.java | 2 +-
.../schema/regular/BatchSchemaOperator.java | 10 +-
.../operators/schema/regular/SchemaOperator.java | 12 +-
.../sink/BatchDataSinkFunctionOperator.java | 11 +-
.../sink/BatchDataSinkWriterOperator.java | 10 +-
.../operators/sink/DataSinkFunctionOperator.java | 17 +-
.../operators/sink/DataSinkWriterOperator.java | 16 +-
.../operators/transform/PreTransformOperator.java | 10 +-
.../BatchRegularPrePartitionOperator.java | 10 +-
.../DistributedPrePartitionOperator.java | 12 +-
.../partitioning/RegularPrePartitionOperator.java | 10 +-
.../operators/sink/DataSinkOperatorAdapter.java | 16 +-
pom.xml | 14 +-
76 files changed, 18006 insertions(+), 44 deletions(-)
diff --git a/flink-cdc-cli-2.x/pom.xml b/flink-cdc-cli-2.x/pom.xml
new file mode 100644
index 000000000..a4a4057b0
--- /dev/null
+++ b/flink-cdc-cli-2.x/pom.xml
@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>flink-cdc-parent</artifactId>
+ <groupId>org.apache.flink</groupId>
+ <version>${revision}</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>flink-cdc-cli-2.x</artifactId>
+
+ <properties>
+ <flink.version>${flink.2.x.version}</flink.version>
+ <shaded.guava.version>${flink.2.x.shaded.guava.version}</shaded.guava.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-cli</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>test-jar</id>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- Shade plugin configuration to maintain API compatibility with Flink 2.x -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${maven.shade.plugin.version}</version>
+ <executions>
+ <execution>
+ <id>shade-for-package</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <filters combine.children="append">
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>module-info.class</exclude>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-cdc-common</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <!-- Relocate Guava dependency to align with Flink 2.x shaded version -->
+ <relocation>
+ <pattern>org.apache.flink.shaded.guava31</pattern>
+ <shadedPattern>org.apache.flink.shaded.guava33</shadedPattern>
+ </relocation>
+ <!-- Relocate SourceFunction to legacy package for Flink 2.x API compatibility -->
+ <relocation>
+ <pattern>org.apache.flink.streaming.api.functions.source.SourceFunction</pattern>
+ <shadedPattern>org.apache.flink.streaming.api.functions.source.legacy.SourceFunction</shadedPattern>
+ </relocation>
+ <!-- Relocate SinkFunction to legacy package for Flink 2.x API compatibility -->
+ <relocation>
+ <pattern>org.apache.flink.streaming.api.functions.sink.SinkFunction</pattern>
+ <shadedPattern>org.apache.flink.streaming.api.functions.sink.legacy.SinkFunction</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!--
+ Skip tests in the test phase.
+ Tests will run in the integration-test phase after shading is complete.
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!--
+ Configure failsafe to run integration tests AFTER the package phase.
+ This ensures tests use the shaded artifacts where Guava and Flink APIs
+ have been relocated.
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>3.0.0-M5</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/flink-cdc-cli-2.x/src/main/java/org/apache/flink/cdc/cli/CliFrontend.java b/flink-cdc-cli-2.x/src/main/java/org/apache/flink/cdc/cli/CliFrontend.java
new file mode 100644
index 000000000..b6355dc6c
--- /dev/null
+++ b/flink-cdc-cli-2.x/src/main/java/org/apache/flink/cdc/cli/CliFrontend.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.cli;
+
+import org.apache.flink.cdc.cli.utils.ConfigurationUtils;
+import org.apache.flink.cdc.cli.utils.FlinkEnvironmentUtils;
+import org.apache.flink.cdc.common.annotation.VisibleForTesting;
+import org.apache.flink.cdc.common.configuration.ConfigOption;
+import org.apache.flink.cdc.common.configuration.ConfigOptions;
+import org.apache.flink.cdc.common.configuration.Configuration;
+import org.apache.flink.cdc.common.utils.StringUtils;
+import org.apache.flink.cdc.composer.PipelineExecution;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+
+import org.apache.flink.shaded.guava33.com.google.common.base.Joiner;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.cdc.cli.CliFrontendOptions.FLINK_CONFIG;
+import static org.apache.flink.cdc.cli.CliFrontendOptions.SAVEPOINT_ALLOW_NON_RESTORED_OPTION;
+import static org.apache.flink.cdc.cli.CliFrontendOptions.SAVEPOINT_CLAIM_MODE;
+import static org.apache.flink.cdc.cli.CliFrontendOptions.SAVEPOINT_PATH_OPTION;
+import static org.apache.flink.cdc.cli.CliFrontendOptions.TARGET;
+import static org.apache.flink.cdc.cli.CliFrontendOptions.USE_MINI_CLUSTER;
+import static org.apache.flink.cdc.composer.flink.deployment.ComposeDeployment.LOCAL;
+import static org.apache.flink.cdc.composer.flink.deployment.ComposeDeployment.REMOTE;
+import static org.apache.flink.configuration.StateRecoveryOptions.RESTORE_MODE;
+
+/** The frontend entrypoint for the command-line interface of Flink CDC. */
+public class CliFrontend {
+ private static final Logger LOG = LoggerFactory.getLogger(CliFrontend.class);
+ private static final String FLINK_HOME_ENV_VAR = "FLINK_HOME";
+ private static final String FLINK_CDC_HOME_ENV_VAR = "FLINK_CDC_HOME";
+
+ public static void main(String[] args) throws Exception {
+ Options cliOptions = CliFrontendOptions.initializeOptions();
+ CommandLineParser parser = new DefaultParser();
+ CommandLine commandLine = parser.parse(cliOptions, args);
+
+ // Help message
+ if (args.length == 0 || commandLine.hasOption(CliFrontendOptions.HELP)) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.setLeftPadding(4);
+ formatter.setWidth(80);
+ formatter.printHelp(" ", cliOptions);
+ return;
+ }
+
+ // Create executor and execute the pipeline
+ PipelineExecution.ExecutionInfo result = createExecutor(commandLine).run();
+
+ // Print execution result
+ printExecutionInfo(result);
+ }
+
+ @VisibleForTesting
+ static CliExecutor createExecutor(CommandLine commandLine) throws Exception {
+ // The pipeline definition file would remain unparsed
+ List<String> unparsedArgs = commandLine.getArgList();
+ if (unparsedArgs.isEmpty()) {
+ throw new IllegalArgumentException(
+ "Missing pipeline definition file path in arguments. ");
+ }
+
+ Path pipelineDefPath = new Path(unparsedArgs.get(0));
+ // Take the first unparsed argument as the pipeline definition file
+ LOG.info("Real Path pipelineDefPath {}", pipelineDefPath);
+ // Global pipeline configuration
+ Configuration globalPipelineConfig = getGlobalConfig(commandLine);
+
+ // Load Flink environment
+ Path flinkHome = getFlinkHome(commandLine);
+ Configuration configuration = FlinkEnvironmentUtils.loadFlinkConfiguration(flinkHome);
+
+ // To override the Flink configuration
+ overrideFlinkConfiguration(configuration, commandLine);
+
+ org.apache.flink.configuration.Configuration flinkConfig =
+ org.apache.flink.configuration.Configuration.fromMap(configuration.toMap());
+
+ // Savepoint
+ SavepointRestoreSettings savepointSettings = createSavepointRestoreSettings(commandLine);
+ SavepointRestoreSettings.toConfiguration(savepointSettings, flinkConfig);
+
+ // Additional JARs
+ List<Path> additionalJars =
+ Arrays.stream(
+ Optional.ofNullable(
+ commandLine.getOptionValues(CliFrontendOptions.JAR))
+ .orElse(new String[0]))
+ .map(Path::new)
+ .collect(Collectors.toList());
+
+ // Build executor
+ return new CliExecutor(
+ commandLine,
+ pipelineDefPath,
+ flinkConfig,
+ globalPipelineConfig,
+ additionalJars,
+ flinkHome);
+ }
+
+ private static void overrideFlinkConfiguration(
+ Configuration flinkConfig, CommandLine commandLine) {
+
+ String target =
+ commandLine.hasOption(USE_MINI_CLUSTER)
+ ? LOCAL.getName()
+ : commandLine.getOptionValue(TARGET, REMOTE.getName());
+ flinkConfig.set(
+ ConfigOptions.key(DeploymentOptions.TARGET.key()).stringType().defaultValue(target),
+ target);
+
+ Properties properties = commandLine.getOptionProperties(FLINK_CONFIG.getOpt());
+ LOG.info("Dynamic flink config items found: {}", properties);
+ for (String key : properties.stringPropertyNames()) {
+ String value = properties.getProperty(key);
+ if (StringUtils.isNullOrWhitespaceOnly(key)
+ || StringUtils.isNullOrWhitespaceOnly(value)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "null or white space argument for key or value: %s=%s",
+ key, value));
+ }
+ ConfigOption<String> configOption =
+ ConfigOptions.key(key.trim()).stringType().defaultValue(value.trim());
+ flinkConfig.set(configOption, value.trim());
+ }
+ }
+
+ private static SavepointRestoreSettings createSavepointRestoreSettings(
+ CommandLine commandLine) {
+ if (commandLine.hasOption(SAVEPOINT_PATH_OPTION.getOpt())) {
+ String savepointPath = commandLine.getOptionValue(SAVEPOINT_PATH_OPTION.getOpt());
+ boolean allowNonRestoredState =
+ commandLine.hasOption(SAVEPOINT_ALLOW_NON_RESTORED_OPTION.getOpt());
+ final Object restoreMode;
+ if (commandLine.hasOption(SAVEPOINT_CLAIM_MODE)) {
+ restoreMode =
+ org.apache.flink.configuration.ConfigurationUtils.convertValue(
+ commandLine.getOptionValue(SAVEPOINT_CLAIM_MODE),
+ ConfigurationUtils.getClaimModeClass());
+ } else {
+ restoreMode = RESTORE_MODE.defaultValue();
+ }
+ // allowNonRestoredState is always false because all operators are predefined.
+
+ return (SavepointRestoreSettings)
+ Arrays.stream(SavepointRestoreSettings.class.getMethods())
+ .filter(
+ method ->
+ method.getName().equals("forPath")
+ && method.getParameterCount() == 3)
+ .findFirst()
+ .map(
+ method -> {
+ try {
+ return method.invoke(
+ null,
+ savepointPath,
+ allowNonRestoredState,
+ restoreMode);
+ } catch (IllegalAccessException
+ | InvocationTargetException e) {
+ throw new RuntimeException(
+ "Failed to invoke SavepointRestoreSettings#forPath nethod.",
+ e);
+ }
+ })
+ .orElseThrow(
+ () ->
+ new RuntimeException(
+ "Failed to resolve SavepointRestoreSettings#forPath method."));
+ } else {
+ return SavepointRestoreSettings.none();
+ }
+ }
+
+ private static Path getFlinkHome(CommandLine commandLine) {
+ // Check command line arguments first
+ String flinkHomeFromArgs = commandLine.getOptionValue(CliFrontendOptions.FLINK_HOME);
+ if (flinkHomeFromArgs != null) {
+ LOG.debug("Flink home is loaded by command-line argument: {}", flinkHomeFromArgs);
+ return new Path(flinkHomeFromArgs);
+ }
+
+ // Fallback to environment variable
+ String flinkHomeFromEnvVar = System.getenv(FLINK_HOME_ENV_VAR);
+ if (flinkHomeFromEnvVar != null) {
+ LOG.debug("Flink home is loaded by environment variable: {}", flinkHomeFromEnvVar);
+ return new Path(flinkHomeFromEnvVar);
+ }
+
+ throw new IllegalArgumentException(
+ "Cannot find Flink home from either command line arguments \"--flink-home\" "
+ + "or the environment variable \"FLINK_HOME\". "
+ + "Please make sure Flink home is properly set. ");
+ }
+
+ private static Configuration getGlobalConfig(CommandLine commandLine) throws Exception {
+ // Try to get global config path from command line
+ String globalConfig = commandLine.getOptionValue(CliFrontendOptions.GLOBAL_CONFIG);
+ if (globalConfig != null) {
+ Path globalConfigPath = new Path(globalConfig);
+ LOG.info("Using global config in command line: {}", globalConfigPath);
+ return ConfigurationUtils.loadConfigFile(globalConfigPath);
+ }
+
+ // Fallback to Flink CDC home
+ String flinkCdcHome = System.getenv(FLINK_CDC_HOME_ENV_VAR);
+ if (flinkCdcHome != null) {
+ Path globalConfigPath =
+ new Path(
+ flinkCdcHome, Joiner.on(File.separator).join("conf", "flink-cdc.yaml"));
+ LOG.info("Using global config in FLINK_CDC_HOME: {}", globalConfigPath);
+ return ConfigurationUtils.loadConfigFile(globalConfigPath);
+ }
+
+ // Fallback to empty configuration
+ LOG.warn(
+ "Cannot find global configuration in command-line or FLINK_CDC_HOME. Will use empty global configuration.");
+ return new Configuration();
+ }
+
+ private static void printExecutionInfo(PipelineExecution.ExecutionInfo info) {
+ System.out.println("Pipeline has been submitted to cluster.");
+ System.out.printf("Job ID: %s\n", info.getId());
+ System.out.printf("Job Description: %s\n", info.getDescription());
+ }
+}
diff --git a/flink-cdc-cli/src/main/java/org/apache/flink/cdc/cli/utils/ConfigurationUtils.java b/flink-cdc-cli/src/main/java/org/apache/flink/cdc/cli/utils/ConfigurationUtils.java
index 7280c0679..75b8c9019 100644
--- a/flink-cdc-cli/src/main/java/org/apache/flink/cdc/cli/utils/ConfigurationUtils.java
+++ b/flink-cdc-cli/src/main/java/org/apache/flink/cdc/cli/utils/ConfigurationUtils.java
@@ -72,7 +72,11 @@ public class ConfigurationUtils {
try {
return Class.forName("org.apache.flink.runtime.jobgraph.RestoreMode");
} catch (ClassNotFoundException e) {
- throw new RuntimeException(e);
+ try {
+ return Class.forName("org.apache.flink.core.execution.RecoveryClaimMode");
+ } catch (ClassNotFoundException classNotFoundException) {
+ throw new RuntimeException(classNotFoundException);
+ }
}
}
}
diff --git a/flink-cdc-common-2.x/pom.xml b/flink-cdc-common-2.x/pom.xml
new file mode 100644
index 000000000..4b36a09b9
--- /dev/null
+++ b/flink-cdc-common-2.x/pom.xml
@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>flink-cdc-parent</artifactId>
+ <groupId>org.apache.flink</groupId>
+ <version>${revision}</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>flink-cdc-common-2.x</artifactId>
+
+ <properties>
+ <flink.version>${flink.2.x.version}</flink.version>
+ <shaded.guava.version>${flink.2.x.shaded.guava.version}</shaded.guava.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>test-jar</id>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- Shade plugin configuration to maintain API compatibility with Flink 2.x -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${maven.shade.plugin.version}</version>
+ <executions>
+ <execution>
+ <id>shade-for-package</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <filters combine.children="append">
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>module-info.class</exclude>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-cdc-common</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <!-- Relocate Guava dependency to align with Flink 2.x shaded version -->
+ <relocation>
+ <pattern>org.apache.flink.shaded.guava31</pattern>
+ <shadedPattern>org.apache.flink.shaded.guava33</shadedPattern>
+ </relocation>
+ <!-- Relocate SourceFunction to legacy package for Flink 2.x API compatibility -->
+ <relocation>
+ <pattern>org.apache.flink.streaming.api.functions.source.SourceFunction</pattern>
+ <shadedPattern>org.apache.flink.streaming.api.functions.source.legacy.SourceFunction</shadedPattern>
+ </relocation>
+ <!-- Relocate SinkFunction to legacy package for Flink 2.x API compatibility -->
+ <relocation>
+ <pattern>org.apache.flink.streaming.api.functions.sink.SinkFunction</pattern>
+ <shadedPattern>org.apache.flink.streaming.api.functions.sink.legacy.SinkFunction</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!--
+ Skip tests in the test phase.
+ Tests will run in the integration-test phase after shading is complete.
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!--
+ Configure failsafe to run integration tests AFTER the package phase.
+ This ensures tests use the shaded artifacts where Guava and Flink APIs
+ have been relocated.
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>3.0.0-M5</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/flink-cdc-common-2.x/src/test/java/org/apache/flink/cdc/common/utils/SchemaMergingUtilsTest.java b/flink-cdc-common-2.x/src/test/java/org/apache/flink/cdc/common/utils/SchemaMergingUtilsTest.java
new file mode 100644
index 000000000..791c5fe7e
--- /dev/null
+++ b/flink-cdc-common-2.x/src/test/java/org/apache/flink/cdc/common/utils/SchemaMergingUtilsTest.java
@@ -0,0 +1,1228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.common.utils;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.cdc.common.data.DateData;
+import org.apache.flink.cdc.common.data.DecimalData;
+import org.apache.flink.cdc.common.data.LocalZonedTimestampData;
+import org.apache.flink.cdc.common.data.TimeData;
+import org.apache.flink.cdc.common.data.TimestampData;
+import org.apache.flink.cdc.common.data.ZonedTimestampData;
+import org.apache.flink.cdc.common.data.binary.BinaryStringData;
+import org.apache.flink.cdc.common.event.AddColumnEvent;
+import org.apache.flink.cdc.common.event.AlterColumnTypeEvent;
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DropColumnEvent;
+import org.apache.flink.cdc.common.event.SchemaChangeEvent;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.schema.Column;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.types.DataType;
+import org.apache.flink.cdc.common.types.DataTypes;
+import org.apache.flink.cdc.common.types.DecimalType;
+import org.apache.flink.cdc.common.types.LocalZonedTimestampType;
+import org.apache.flink.cdc.common.types.TimestampType;
+import org.apache.flink.cdc.common.types.ZonedTimestampType;
+
+import org.apache.flink.shaded.guava33.com.google.common.collect.ImmutableMap;
+
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import javax.annotation.Nullable;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static org.apache.flink.cdc.common.types.DataTypes.DECIMAL;
+import static org.apache.flink.cdc.common.types.DataTypes.VARCHAR;
+import static org.apache.flink.cdc.common.utils.SchemaMergingUtils.coerceObject;
+import static org.apache.flink.cdc.common.utils.SchemaMergingUtils.coerceRow;
+import static org.apache.flink.cdc.common.utils.SchemaMergingUtils.getLeastCommonSchema;
+import static org.apache.flink.cdc.common.utils.SchemaMergingUtils.getLeastCommonType;
+import static org.apache.flink.cdc.common.utils.SchemaMergingUtils.getSchemaDifference;
+import static org.apache.flink.cdc.common.utils.SchemaMergingUtils.isDataTypeCompatible;
+import static org.apache.flink.cdc.common.utils.SchemaMergingUtils.isSchemaCompatible;
+
+/** A test for the {@link SchemaMergingUtils}. */
+class SchemaMergingUtilsTest {
+
+ private static final TableId TABLE_ID = TableId.tableId("foo", "bar", "baz");
+
+ private static final DataType CHAR = DataTypes.CHAR(17);
+ private static final DataType VARCHAR = DataTypes.VARCHAR(17);
+ private static final DataType STRING = DataTypes.STRING();
+
+ private static final DataType BOOLEAN = DataTypes.BOOLEAN();
+ private static final DataType BINARY = DataTypes.BINARY(17);
+ private static final DataType VARBINARY = DataTypes.VARBINARY(17);
+ private static final DataType SMALLINT = DataTypes.SMALLINT();
+ private static final DataType TINYINT = DataTypes.TINYINT();
+ private static final DataType INT = DataTypes.INT();
+ private static final DataType BIGINT = DataTypes.BIGINT();
+ private static final DataType DECIMAL =
+ DECIMAL(DecimalType.MAX_PRECISION, DecimalType.DEFAULT_SCALE);
+ private static final DataType FLOAT = DataTypes.FLOAT();
+ private static final DataType DOUBLE = DataTypes.DOUBLE();
+
+ private static final DataType TIMESTAMP_TZ =
+ DataTypes.TIMESTAMP_TZ(ZonedTimestampType.MAX_PRECISION);
+ private static final DataType TIMESTAMP_LTZ =
+ DataTypes.TIMESTAMP_LTZ(LocalZonedTimestampType.MAX_PRECISION);
+ private static final DataType TIMESTAMP = DataTypes.TIMESTAMP(TimestampType.MAX_PRECISION);
+ private static final DataType DATE = DataTypes.DATE();
+ private static final DataType TIME = DataTypes.TIME();
+
+ private static final DataType ROW = DataTypes.ROW(INT, STRING);
+ private static final DataType ARRAY = DataTypes.ARRAY(STRING);
+ private static final DataType MAP = DataTypes.MAP(INT, STRING);
+ private static final DataType VARIANT = DataTypes.VARIANT();
+
+ private static final List<DataType> ALL_TYPES =
+ Arrays.asList(
+ // Binary types
+ STRING,
+ CHAR,
+ VARCHAR,
+ BINARY,
+ VARBINARY,
+ // Exact numeric types
+ TINYINT,
+ SMALLINT,
+ INT,
+ BIGINT,
+ DECIMAL,
+ // Inexact numeric types
+ FLOAT,
+ DOUBLE,
+ // Date and time types
+ TIMESTAMP,
+ TIMESTAMP_LTZ,
+ TIMESTAMP_TZ,
+ TIME,
+ // Complex types
+ ROW,
+ ARRAY,
+ MAP,
+ VARIANT);
+
+ private static final Map<DataType, Object> DUMMY_OBJECTS =
+ ImmutableMap.of(
+ TINYINT,
+ (byte) 17,
+ SMALLINT,
+ (short) 17,
+ INT,
+ 17,
+ BIGINT,
+ 17L,
+ DECIMAL,
+ decOf(17),
+ FLOAT,
+ 17.0f,
+ DOUBLE,
+ 17.0);
+
+ @Test
+ void testIsSchemaCompatible() {
+ Assertions.assertThat(isSchemaCompatible(null, of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test merging into an empty schema")
+ .isFalse();
+
+ Assertions.assertThat(
+ isSchemaCompatible(
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test identical schema")
+ .isTrue();
+
+ Assertions.assertThat(
+ isSchemaCompatible(
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("name", VARCHAR(17), "id", BIGINT)))
+ .as("swapping sequence is ok")
+ .isTrue();
+
+ Assertions.assertThat(
+ isSchemaCompatible(of("id", BIGINT, "name", VARCHAR(17)), of("id", BIGINT)))
+ .as("test a wider upcoming schema")
+ .isTrue();
+
+ Assertions.assertThat(
+ isSchemaCompatible(of("id", BIGINT), of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test a narrower upcoming schema")
+ .isFalse();
+
+ Assertions.assertThat(
+ isSchemaCompatible(
+ of("id", BIGINT, "name", STRING),
+ of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test a wider typed upcoming schema")
+ .isTrue();
+
+ Assertions.assertThat(
+ isSchemaCompatible(
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("id", BIGINT, "name", STRING)))
+ .as("test a narrower typed upcoming schema")
+ .isFalse();
+
+ Stream.of(TINYINT, SMALLINT, INT)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ isSchemaCompatible(
+ of("id", BIGINT, "number", BIGINT),
+ of("id", BIGINT, "number", type)))
+ .as("test fitting %s into BIGINT", type)
+ .isTrue());
+
+ Stream.of(TINYINT, SMALLINT, INT)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ isSchemaCompatible(
+ of("id", BIGINT, "number", type),
+ of("id", BIGINT, "number", BIGINT)))
+ .as("test fitting BIGINT into %s", type)
+ .isFalse());
+
+ Stream.of(TINYINT, SMALLINT, INT, BIGINT, DECIMAL, STRING)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ isSchemaCompatible(
+ of("id", BIGINT, "number", STRING),
+ of("id", BIGINT, "number", type)))
+ .as("test fitting %s into STRING", type)
+ .isTrue());
+
+ Stream.of(TINYINT, SMALLINT, INT, BIGINT, DECIMAL)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ isSchemaCompatible(
+ of("id", BIGINT, "number", type),
+ of("id", BIGINT, "number", STRING)))
+ .as("test fitting STRING into %s", type)
+ .isFalse());
+
+ Stream.of(FLOAT, DOUBLE, STRING)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ isSchemaCompatible(
+ of("id", BIGINT, "number", STRING),
+ of("id", BIGINT, "number", type)))
+ .as("test fitting %s into STRING", type)
+ .isTrue());
+
+ Stream.of(FLOAT, DOUBLE)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ isSchemaCompatible(
+ of("id", BIGINT, "number", type),
+ of("id", BIGINT, "number", STRING)))
+ .as("test fitting STRING into %s", type)
+ .isFalse());
+
+ Assertions.assertThat(
+ isSchemaCompatible(
+ of("id", BIGINT, "foo", INT), of("id", BIGINT, "bar", INT)))
+ .as("columns with different names")
+ .isFalse();
+ }
+
+ @Test
+ void testGetLeastCommonSchema() {
+ Assertions.assertThat(getLeastCommonSchema(null, of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test merging into an empty schema")
+ .isEqualTo(of("id", BIGINT, "name", VARCHAR(17)));
+
+ Assertions.assertThat(
+ getLeastCommonSchema(
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test identical schema")
+ .isEqualTo(of("id", BIGINT, "name", VARCHAR(17)));
+
+ Assertions.assertThat(
+ getLeastCommonSchema(
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("name", VARCHAR(17), "id", BIGINT)))
+ .as("swapping sequence is ok")
+ .isEqualTo(of("id", BIGINT, "name", VARCHAR(17)));
+
+ Assertions.assertThat(
+ getLeastCommonSchema(
+ of("id", BIGINT, "name", VARCHAR(17)), of("id", BIGINT)))
+ .as("test a wider upcoming schema")
+ .isEqualTo(of("id", BIGINT, "name", VARCHAR(17)));
+
+ Assertions.assertThat(
+ getLeastCommonSchema(
+ of("id", BIGINT), of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test a narrower upcoming schema")
+ .isEqualTo(of("id", BIGINT, "name", VARCHAR(17)));
+
+ Assertions.assertThat(
+ getLeastCommonSchema(
+ of("id", BIGINT, "name", STRING),
+ of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test a wider typed upcoming schema")
+ .isEqualTo(of("id", BIGINT, "name", STRING));
+
+ Assertions.assertThat(
+ getLeastCommonSchema(
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("id", BIGINT, "name", STRING)))
+ .as("test a narrower typed upcoming schema")
+ .isEqualTo(of("id", BIGINT, "name", STRING));
+
+ Stream.of(TINYINT, SMALLINT, INT)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ getLeastCommonSchema(
+ of("id", BIGINT, "number", BIGINT),
+ of("id", BIGINT, "number", type)))
+ .as("test fitting %s into BIGINT", type)
+ .isEqualTo(of("id", BIGINT, "number", BIGINT)));
+
+ Stream.of(TINYINT, SMALLINT, INT)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ getLeastCommonSchema(
+ of("id", BIGINT, "number", type),
+ of("id", BIGINT, "number", BIGINT)))
+ .as("test fitting BIGINT into %s", type)
+ .isEqualTo(of("id", BIGINT, "number", BIGINT)));
+
+ Stream.of(TINYINT, SMALLINT, INT, BIGINT, DECIMAL, STRING)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ getLeastCommonSchema(
+ of("id", BIGINT, "number", STRING),
+ of("id", BIGINT, "number", type)))
+ .as("test fitting %s into STRING", type)
+ .isEqualTo(of("id", BIGINT, "number", STRING)));
+
+ Stream.of(TINYINT, SMALLINT, INT, BIGINT, DECIMAL, STRING)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ getLeastCommonSchema(
+ of("id", BIGINT, "number", type),
+ of("id", BIGINT, "number", STRING)))
+ .as("test fitting STRING into %s", type)
+ .isEqualTo(of("id", BIGINT, "number", STRING)));
+
+ Assertions.assertThat(
+ getLeastCommonSchema(
+ of("id", BIGINT, "foo", INT), of("id", BIGINT, "bar", INT)))
+ .as("columns with different names")
+ .isEqualTo(of("id", BIGINT, "foo", INT, "bar", INT));
+
+ Assertions.assertThat(
+ getLeastCommonSchema(
+ of("id", BIGINT, "foo", INT, "baz", FLOAT),
+ of("id", BIGINT, "bar", INT, "baz", DOUBLE)))
+ .as("mixed schema differences")
+ .isEqualTo(of("id", BIGINT, "foo", INT, "baz", DOUBLE, "bar", INT));
+ }
+
+ @Test
+ void testGetSchemaDifference() {
+ Assertions.assertThat(
+ getSchemaDifference(TABLE_ID, null, of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test merging into an empty schema")
+ .containsExactly(
+ new CreateTableEvent(TABLE_ID, of("id", BIGINT, "name", VARCHAR(17))));
+
+ Assertions.assertThat(
+ getSchemaDifference(
+ TABLE_ID,
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test identical schema")
+ .isEmpty();
+
+ Assertions.assertThat(
+ getSchemaDifference(
+ TABLE_ID,
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("name", VARCHAR(17), "id", BIGINT)))
+ .as("swapping sequence is ok")
+ .isEmpty();
+
+ Assertions.assertThat(
+ getSchemaDifference(
+ TABLE_ID, of("id", BIGINT), of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test a widening upcoming schema")
+ .containsExactly(
+ new AddColumnEvent(
+ TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn("name", VARCHAR(17)),
+ AddColumnEvent.ColumnPosition.AFTER,
+ "id"))));
+
+ Assertions.assertThat(
+ getSchemaDifference(
+ TABLE_ID, of("id", BIGINT), of("name", VARCHAR(17), "id", BIGINT)))
+ .as("test a widening upcoming schema at first")
+ .containsExactly(
+ new AddColumnEvent(
+ TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn("name", VARCHAR(17)),
+ AddColumnEvent.ColumnPosition.FIRST,
+ null))));
+
+ Assertions.assertThat(
+ getSchemaDifference(
+ TABLE_ID,
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("id", BIGINT, "name", STRING)))
+ .as("test a type-widening typed upcoming schema")
+ .containsExactly(
+ new AlterColumnTypeEvent(
+ TABLE_ID,
+ Collections.singletonMap("name", STRING),
+ Collections.singletonMap("name", VARCHAR(17))));
+ Assertions.assertThat(
+ getSchemaDifference(
+ TABLE_ID,
+ of("id", BIGINT, "name", STRING, "number", BIGINT),
+ of("id", BIGINT)))
+ .as("test remove id while add gentle")
+ .containsExactly(new DropColumnEvent(TABLE_ID, Arrays.asList("number", "name")));
+ Assertions.assertThat(
+ getSchemaDifference(
+ TABLE_ID,
+ of("id", BIGINT, "name", STRING, "number", BIGINT),
+ of("id", BIGINT, "name", STRING, "gentle", STRING)))
+ .as("test remove id while add gentle")
+ .containsExactly(
+ new AddColumnEvent(
+ TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn("gentle", STRING),
+ AddColumnEvent.ColumnPosition.AFTER,
+ "name"))),
+ new DropColumnEvent(TABLE_ID, Collections.singletonList("number")));
+ Stream.of(TINYINT, SMALLINT, INT)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ getSchemaDifference(
+ TABLE_ID,
+ of("id", BIGINT, "number", type),
+ of("id", BIGINT, "number", BIGINT)))
+ .as("test escalating %s to BIGINT", type)
+ .containsExactly(
+ new AlterColumnTypeEvent(
+ TABLE_ID,
+ Collections.singletonMap("number", BIGINT),
+ Collections.singletonMap("number", type))));
+
+ Stream.of(TINYINT, SMALLINT, INT, BIGINT, DECIMAL, FLOAT)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ getSchemaDifference(
+ TABLE_ID,
+ of("id", BIGINT, "number", type),
+ of("id", BIGINT, "number", DOUBLE)))
+ .as("test escalating %s to DOUBLE", type)
+ .containsExactly(
+ new AlterColumnTypeEvent(
+ TABLE_ID,
+ Collections.singletonMap("number", DOUBLE),
+ Collections.singletonMap("number", type))));
+
+ Assertions.assertThat(
+ getSchemaDifference(
+ TABLE_ID,
+ of("id", BIGINT, "foo", INT, "baz", FLOAT),
+ of("id", BIGINT, "foo", BIGINT, "bar", INT, "baz", DOUBLE)))
+ .as("mixed schema differences")
+ .containsExactly(
+ new AddColumnEvent(
+ TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn("bar", INT),
+ AddColumnEvent.ColumnPosition.AFTER,
+ "foo"))),
+ new AlterColumnTypeEvent(
+ TABLE_ID,
+ ImmutableMap.of("foo", BIGINT, "baz", DOUBLE),
+ ImmutableMap.of("foo", INT, "baz", FLOAT)));
+ }
+
+ @Test
+ void testMergeAndDiff() {
+ Assertions.assertThat(mergeAndDiff(null, of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test merging into an empty schema")
+ .containsExactly(
+ new CreateTableEvent(TABLE_ID, of("id", BIGINT, "name", VARCHAR(17))));
+
+ Assertions.assertThat(
+ mergeAndDiff(
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test identical schema")
+ .isEmpty();
+
+ Assertions.assertThat(
+ mergeAndDiff(
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("name", VARCHAR(17), "id", BIGINT)))
+ .as("swapping sequence is ok")
+ .isEmpty();
+
+ Assertions.assertThat(mergeAndDiff(of("id", BIGINT, "name", VARCHAR(17)), of("id", BIGINT)))
+ .as("test a wider upcoming schema")
+ .isEmpty();
+
+ Assertions.assertThat(mergeAndDiff(of("id", BIGINT), of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test a narrower upcoming schema")
+ .containsExactly(
+ new AddColumnEvent(
+ TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn("name", VARCHAR(17)),
+ AddColumnEvent.ColumnPosition.AFTER,
+ "id"))));
+
+ Assertions.assertThat(
+ mergeAndDiff(
+ of("id", BIGINT, "name", STRING),
+ of("id", BIGINT, "name", VARCHAR(17))))
+ .as("test a wider typed upcoming schema")
+ .isEmpty();
+
+ Assertions.assertThat(
+ mergeAndDiff(
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("id", BIGINT, "name", STRING)))
+ .as("test a narrower typed upcoming schema")
+ .containsExactly(
+ new AlterColumnTypeEvent(
+ TABLE_ID,
+ Collections.singletonMap("name", STRING),
+ Collections.singletonMap("name", VARCHAR(17))));
+
+ Stream.of(TINYINT, SMALLINT, INT)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ mergeAndDiff(
+ of("id", BIGINT, "number", BIGINT),
+ of("id", BIGINT, "number", type)))
+ .as("test fitting %s into BIGINT", type)
+ .isEmpty());
+
+ Stream.of(TINYINT, SMALLINT, INT)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ mergeAndDiff(
+ of("id", BIGINT, "number", type),
+ of("id", BIGINT, "number", BIGINT)))
+ .as("test fitting BIGINT into %s", type)
+ .containsExactly(
+ new AlterColumnTypeEvent(
+ TABLE_ID,
+ Collections.singletonMap("number", BIGINT),
+ Collections.singletonMap("number", type))));
+
+ Stream.of(TINYINT, SMALLINT, INT, BIGINT, DECIMAL, STRING)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ mergeAndDiff(
+ of("id", BIGINT, "number", STRING),
+ of("id", BIGINT, "number", type)))
+ .as("test fitting %s into STRING", type)
+ .isEmpty());
+
+ Stream.of(TINYINT, SMALLINT, INT, BIGINT, DECIMAL)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ mergeAndDiff(
+ of("id", BIGINT, "number", type),
+ of("id", BIGINT, "number", STRING)))
+ .as("test fitting STRING into %s", type)
+ .containsExactly(
+ new AlterColumnTypeEvent(
+ TABLE_ID,
+ Collections.singletonMap("number", STRING),
+ Collections.singletonMap("number", type))));
+
+ Assertions.assertThat(
+ mergeAndDiff(
+ of("id", BIGINT, "foo", INT, "baz", FLOAT),
+ of("id", BIGINT, "bar", INT, "baz", DOUBLE)))
+ .as("mixed schema differences")
+ .containsExactly(
+ new AddColumnEvent(
+ TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn("bar", INT),
+ AddColumnEvent.ColumnPosition.AFTER,
+ "baz"))),
+ new AlterColumnTypeEvent(
+ TABLE_ID,
+ Collections.singletonMap("baz", DOUBLE),
+ Collections.singletonMap("baz", FLOAT)));
+ }
+
+ @Test
+ void testIsDataTypeCompatible() {
+ List<Tuple2<DataType, DataType>> viableConversions =
+ Arrays.asList(
+ Tuple2.of(CHAR, STRING),
+ Tuple2.of(VARCHAR, STRING),
+ Tuple2.of(BOOLEAN, STRING),
+ Tuple2.of(BINARY, STRING),
+ Tuple2.of(DOUBLE, STRING),
+ Tuple2.of(FLOAT, STRING),
+ Tuple2.of(DECIMAL, STRING),
+ Tuple2.of(BIGINT, STRING),
+ Tuple2.of(INT, STRING),
+ Tuple2.of(SMALLINT, STRING),
+ Tuple2.of(TINYINT, STRING),
+ Tuple2.of(TIMESTAMP_TZ, STRING),
+ Tuple2.of(TIMESTAMP_LTZ, STRING),
+ Tuple2.of(TIMESTAMP, STRING),
+ Tuple2.of(DATE, STRING),
+ Tuple2.of(TIME, STRING),
+ Tuple2.of(ROW, STRING),
+ Tuple2.of(ARRAY, STRING),
+ Tuple2.of(MAP, STRING),
+ Tuple2.of(TINYINT, SMALLINT),
+ Tuple2.of(SMALLINT, INT),
+ Tuple2.of(INT, BIGINT),
+ Tuple2.of(BIGINT, DECIMAL),
+ Tuple2.of(DECIMAL, STRING),
+ Tuple2.of(FLOAT, DOUBLE),
+ Tuple2.of(DATE, TIMESTAMP),
+ Tuple2.of(TIMESTAMP, TIMESTAMP_LTZ),
+ Tuple2.of(TIMESTAMP_LTZ, TIMESTAMP_TZ));
+
+ List<Tuple2<DataType, DataType>> infeasibleConversions =
+ Arrays.asList(
+ Tuple2.of(CHAR, BOOLEAN),
+ Tuple2.of(BOOLEAN, BINARY),
+ Tuple2.of(BINARY, DOUBLE),
+ Tuple2.of(DOUBLE, TIMESTAMP_TZ),
+ Tuple2.of(TIMESTAMP_TZ, TIME),
+ Tuple2.of(TIME, ROW),
+ Tuple2.of(ROW, ARRAY),
+ Tuple2.of(ARRAY, MAP));
+
+ viableConversions.forEach(
+ conv ->
+ Assertions.assertThat(isDataTypeCompatible(conv.f1, conv.f0))
+ .as("test fitting %s into %s", conv.f0, conv.f1)
+ .isTrue());
+
+ viableConversions.forEach(
+ conv ->
+ Assertions.assertThat(isDataTypeCompatible(conv.f0, conv.f1))
+ .as("test fitting %s into %s", conv.f1, conv.f0)
+ .isFalse());
+
+ infeasibleConversions.forEach(
+ conv ->
+ Assertions.assertThat(isDataTypeCompatible(conv.f1, conv.f0))
+ .as("test fitting %s into %s", conv.f0, conv.f1)
+ .isFalse());
+
+ infeasibleConversions.forEach(
+ conv ->
+ Assertions.assertThat(isDataTypeCompatible(conv.f0, conv.f1))
+ .as("test fitting %s into %s", conv.f1, conv.f0)
+ .isFalse());
+ }
+
+ @Test
+ void testCoerceObject() {
+ Stream<Tuple4<DataType, Object, DataType, Object>> conversionExpects =
+ Stream.of(
+ // From TINYINT
+ Tuple4.of(TINYINT, (byte) 0, TINYINT, (byte) 0),
+ Tuple4.of(TINYINT, (byte) 1, SMALLINT, (short) 1),
+ Tuple4.of(TINYINT, (byte) 2, INT, 2),
+ Tuple4.of(TINYINT, (byte) 3, BIGINT, 3L),
+ Tuple4.of(TINYINT, (byte) 4, DECIMAL, decOf(4)),
+ Tuple4.of(TINYINT, (byte) 5, FLOAT, 5.0f),
+ Tuple4.of(TINYINT, (byte) 6, DOUBLE, 6.0),
+ Tuple4.of(TINYINT, (byte) 7, STRING, binStrOf("7")),
+
+ // From SMALLINT
+ Tuple4.of(SMALLINT, (short) 1, SMALLINT, (short) 1),
+ Tuple4.of(SMALLINT, (short) 2, INT, 2),
+ Tuple4.of(SMALLINT, (short) 3, BIGINT, 3L),
+ Tuple4.of(SMALLINT, (short) 4, DECIMAL, decOf(4)),
+ Tuple4.of(SMALLINT, (short) 5, FLOAT, 5.0f),
+ Tuple4.of(SMALLINT, (short) 6, DOUBLE, 6.0),
+ Tuple4.of(SMALLINT, (short) 7, STRING, binStrOf("7")),
+
+ // From INT
+ Tuple4.of(INT, 2, INT, 2),
+ Tuple4.of(INT, 3, BIGINT, 3L),
+ Tuple4.of(INT, 4, DECIMAL, decOf(4)),
+ Tuple4.of(INT, 5, FLOAT, 5.0f),
+ Tuple4.of(INT, 6, DOUBLE, 6.0),
+ Tuple4.of(INT, 7, STRING, binStrOf("7")),
+
+ // From BIGINT
+ Tuple4.of(BIGINT, 3L, BIGINT, 3L),
+ Tuple4.of(BIGINT, 4L, DECIMAL, decOf(4)),
+ Tuple4.of(BIGINT, 5L, FLOAT, 5.0f),
+ Tuple4.of(BIGINT, 6L, DOUBLE, 6.0),
+ Tuple4.of(BIGINT, 7L, STRING, binStrOf("7")),
+
+ // From DECIMAL
+ Tuple4.of(DECIMAL, decOf(4), DECIMAL, decOf(4)),
+ Tuple4.of(DECIMAL, decOf(5), FLOAT, 5.0f),
+ Tuple4.of(DECIMAL, decOf(6), DOUBLE, 6.0),
+ Tuple4.of(DECIMAL, decOf(7), STRING, binStrOf("7")),
+
+ // From FLOAT
+ Tuple4.of(FLOAT, 5.0f, FLOAT, 5.0f),
+ Tuple4.of(FLOAT, 6.0f, DOUBLE, 6.0),
+ Tuple4.of(FLOAT, 7.0f, STRING, binStrOf("7.0")),
+
+ // From DOUBLE
+ Tuple4.of(DOUBLE, 6.0f, DOUBLE, 6.0),
+ Tuple4.of(DOUBLE, 7.0f, STRING, binStrOf("7.0")),
+
+ // From STRING
+ Tuple4.of(STRING, binStrOf("AtoZ"), STRING, binStrOf("AtoZ")),
+ Tuple4.of(STRING, binStrOf("lie"), STRING, binStrOf("lie")),
+
+ // From CHAR
+ Tuple4.of(
+ CHAR, binStrOf("les miserables"), CHAR, binStrOf("les miserables")),
+ Tuple4.of(CHAR, binStrOf("notre dame"), STRING, binStrOf("notre dame")),
+
+ // From Binary
+ Tuple4.of(BINARY, binOf("les miserables"), BINARY, binOf("les miserables")),
+ Tuple4.of(
+ BINARY, binOf("notre dame"), STRING, binStrOf("bm90cmUgZGFtZQ==")),
+
+ // From BOOLEAN
+ Tuple4.of(BOOLEAN, true, BOOLEAN, true),
+ Tuple4.of(BOOLEAN, false, BOOLEAN, false),
+ Tuple4.of(BOOLEAN, true, STRING, binStrOf("true")),
+ Tuple4.of(BOOLEAN, false, STRING, binStrOf("false")),
+
+ // From DATE
+ Tuple4.of(DATE, dateOf(2017, 1, 1), DATE, dateOf(2017, 1, 1)),
+ Tuple4.of(DATE, dateOf(2018, 2, 2), TIMESTAMP, tsOf("2018", "02", "02")),
+ Tuple4.of(
+ DATE,
+ dateOf(2019, 3, 3),
+ TIMESTAMP_LTZ,
+ ltzTsOf("2019", "03", "03")),
+ Tuple4.of(
+ DATE, dateOf(2020, 4, 4), TIMESTAMP_TZ, zTsOf("2020", "04", "04")),
+ Tuple4.of(DATE, dateOf(2021, 5, 5), STRING, binStrOf("2021-05-05")),
+
+ // From TIME
+ Tuple4.of(TIME, timeOf(21, 48, 25), TIME, timeOf(21, 48, 25)),
+ Tuple4.of(TIME, timeOf(21, 48, 25), STRING, binStrOf("21:48:25")),
+
+ // From TIMESTAMP
+ Tuple4.of(
+ TIMESTAMP,
+ tsOf("2022", "06", "06"),
+ TIMESTAMP,
+ tsOf("2022", "06", "06")),
+ Tuple4.of(
+ TIMESTAMP,
+ tsOf("2023", "07", "07"),
+ TIMESTAMP_LTZ,
+ ltzTsOf("2023", "07", "07")),
+ Tuple4.of(
+ TIMESTAMP,
+ tsOf("2024", "08", "08"),
+ TIMESTAMP_TZ,
+ zTsOf("2024", "08", "08")),
+ Tuple4.of(
+ TIMESTAMP,
+ tsOf("2025", "09", "09"),
+ STRING,
+ binStrOf("2025-09-09T00:00")),
+
+ // From TIMESTAMP_LTZ
+ Tuple4.of(
+ TIMESTAMP_LTZ,
+ ltzTsOf("2026", "10", "10"),
+ TIMESTAMP_LTZ,
+ ltzTsOf("2026", "10", "10")),
+ Tuple4.of(
+ TIMESTAMP_LTZ,
+ ltzTsOf("2027", "11", "11"),
+ TIMESTAMP_TZ,
+ zTsOf("2027", "11", "11")),
+ Tuple4.of(
+ TIMESTAMP_LTZ,
+ ltzTsOf("2028", "12", "12"),
+ STRING,
+ binStrOf("2028-12-12T00:00")),
+
+ // From TIMESTAMP_TZ
+ Tuple4.of(
+ TIMESTAMP_TZ,
+ zTsOf("2018", "01", "01"),
+ TIMESTAMP_TZ,
+ zTsOf("2018", "01", "01")),
+ Tuple4.of(
+ TIMESTAMP_TZ,
+ zTsOf("2019", "02", "02"),
+ STRING,
+ binStrOf("2019-02-02T00:00:00Z")));
+
+ conversionExpects.forEach(
+ rule ->
+ Assertions.assertThat(coerceObject("UTC", rule.f1, rule.f0, rule.f2))
+ .as("Try coercing %s (%s) to %s type", rule.f1, rule.f0, rule.f2)
+ .isEqualTo(rule.f3));
+ }
+
+ @Test
+ void testCoerceRow() {
+ Assertions.assertThat(
+ coerceRow(
+ "UTC",
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("id", BIGINT, "name", VARCHAR(17)),
+ Arrays.asList(2L, binStrOf("Bob"))))
+ .as("test identical schema")
+ .containsExactly(2L, binStrOf("Bob"));
+
+ Assertions.assertThat(
+ coerceRow(
+ "UTC",
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("name", VARCHAR(17), "id", BIGINT),
+ Arrays.asList(binStrOf("Cecily"), 3L)))
+ .as("swapping sequence is ok")
+ .containsExactly(3L, binStrOf("Cecily"));
+
+ Assertions.assertThat(
+ coerceRow(
+ "UTC",
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("id", BIGINT),
+ Collections.singletonList(4L)))
+ .as("test a wider upcoming schema")
+ .containsExactly(4L, null);
+
+ Assertions.assertThat(
+ coerceRow(
+ "UTC",
+ of("id", BIGINT, "name", STRING),
+ of("id", BIGINT, "name", VARCHAR(17)),
+ Arrays.asList(4L, "Derrida")))
+ .as("test a wider typed upcoming schema")
+ .containsExactly(4L, binStrOf("Derrida"));
+
+ Stream.of(TINYINT, SMALLINT, INT)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ coerceRow(
+ "UTC",
+ of("id", BIGINT, "number", BIGINT),
+ of("id", BIGINT, "number", type),
+ Arrays.asList(5L, DUMMY_OBJECTS.get(type))))
+ .as("test fitting %s into BIGINT", type)
+ .containsExactly(5L, 17L));
+
+ Stream.of(TINYINT, SMALLINT, INT, BIGINT, DECIMAL, FLOAT)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ coerceRow(
+ "UTC",
+ of("id", BIGINT, "number", DOUBLE),
+ of("id", BIGINT, "number", type),
+ Arrays.asList(6L, DUMMY_OBJECTS.get(type))))
+ .as("test fitting %s into DOUBLE", type)
+ .containsExactly(6L, 17.0));
+
+ // Test coercing with NULL
+ Assertions.assertThat(
+ coerceRow(
+ "UTC",
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("id", BIGINT, "name", VARCHAR(17)),
+ Arrays.asList(2L, null)))
+ .as("test identical schema")
+ .containsExactly(2L, null);
+
+ Assertions.assertThat(
+ coerceRow(
+ "UTC",
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("name", VARCHAR(17), "id", BIGINT),
+ Arrays.asList(null, 3L)))
+ .as("swapping sequence is ok")
+ .containsExactly(3L, null);
+
+ Assertions.assertThat(
+ coerceRow(
+ "UTC",
+ of("id", BIGINT, "name", VARCHAR(17)),
+ of("id", BIGINT),
+ Collections.singletonList(4L)))
+ .as("test a wider upcoming schema")
+ .containsExactly(4L, null);
+
+ Assertions.assertThat(
+ coerceRow(
+ "UTC",
+ of("id", BIGINT, "name", STRING),
+ of("id", BIGINT, "name", VARCHAR(17)),
+ Arrays.asList(4L, null)))
+ .as("test a wider typed upcoming schema")
+ .containsExactly(4L, null);
+
+ Stream.of(TINYINT, SMALLINT, INT)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ coerceRow(
+ "UTC",
+ of("id", BIGINT, "number", BIGINT),
+ of("id", BIGINT, "number", type),
+ Arrays.asList(5L, null)))
+ .as("test fitting %s into BIGINT", type)
+ .containsExactly(5L, null));
+
+ Stream.of(TINYINT, SMALLINT, INT, BIGINT, DECIMAL, FLOAT)
+ .forEach(
+ type ->
+ Assertions.assertThat(
+ coerceRow(
+ "UTC",
+ of("id", BIGINT, "number", DOUBLE),
+ of("id", BIGINT, "number", type),
+ Arrays.asList(6L, null)))
+ .as("test fitting %s into DOUBLE", type)
+ .containsExactly(6L, null));
+ }
+
+ @Test
+ void testGetLeastCommonType() {
+ // To-be-merged types are:
+ // STRING, CHAR, VARCHAR, BINARY, VARBINARY, TINYINT, SMALLINT, INT, BIGINT,
+ // DECIMAL, FLOAT, DOUBLE, TIMESTAMP, TIMESTAMP_LTZ, TIMESTAMP_TZ, TIME, ROW, ARRAY,
+ // MAP
+
+ assertTypeMergingVector(
+ STRING,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING));
+
+ assertTypeMergingVector(
+ CHAR,
+ Arrays.asList(
+ STRING, CHAR, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING));
+
+ assertTypeMergingVector(
+ VARCHAR,
+ Arrays.asList(
+ STRING, STRING, VARCHAR, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING));
+
+ assertTypeMergingVector(
+ BINARY,
+ Arrays.asList(
+ STRING, STRING, STRING, BINARY, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING));
+
+ assertTypeMergingVector(
+ VARBINARY,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, VARBINARY, STRING, STRING, STRING, STRING,
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING));
+
+ // 8-bit TINYINT could fit into FLOAT (24 sig bits) or DOUBLE (53 sig bits)
+ assertTypeMergingVector(
+ TINYINT,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, STRING, TINYINT, SMALLINT, INT, BIGINT,
+ DECIMAL, FLOAT, DOUBLE, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING));
+
+ // 16-bit SMALLINT could fit into FLOAT (24 sig bits) or DOUBLE (53 sig bits)
+ assertTypeMergingVector(
+ SMALLINT,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, STRING, SMALLINT, SMALLINT, INT, BIGINT,
+ DECIMAL, FLOAT, DOUBLE, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING));
+
+ // 32-bit INT could fit into DOUBLE (53 sig bits)
+ assertTypeMergingVector(
+ INT,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, STRING, INT, INT, INT, BIGINT, DECIMAL,
+ DOUBLE, DOUBLE, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING));
+
+ assertTypeMergingVector(
+ BIGINT,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, STRING, BIGINT, BIGINT, BIGINT, BIGINT,
+ DECIMAL, DOUBLE, DOUBLE, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING));
+
+ assertTypeMergingVector(
+ DECIMAL,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, STRING, DECIMAL, DECIMAL, DECIMAL, DECIMAL,
+ DECIMAL, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING));
+
+ assertTypeMergingVector(
+ FLOAT,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, STRING, FLOAT, FLOAT, DOUBLE, DOUBLE,
+ STRING, FLOAT, DOUBLE, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING));
+
+ assertTypeMergingVector(
+ DOUBLE,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, STRING, DOUBLE, DOUBLE, DOUBLE, DOUBLE,
+ STRING, DOUBLE, DOUBLE, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING));
+
+ assertTypeMergingVector(
+ TIMESTAMP,
+ Arrays.asList(
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ TIMESTAMP,
+ TIMESTAMP_LTZ,
+ TIMESTAMP_TZ,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING));
+
+ assertTypeMergingVector(
+ TIMESTAMP_LTZ,
+ Arrays.asList(
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ TIMESTAMP_LTZ,
+ TIMESTAMP_LTZ,
+ TIMESTAMP_TZ,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING));
+
+ assertTypeMergingVector(
+ TIMESTAMP_TZ,
+ Arrays.asList(
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ TIMESTAMP_TZ,
+ TIMESTAMP_TZ,
+ TIMESTAMP_TZ,
+ STRING,
+ STRING,
+ STRING,
+ STRING,
+ STRING));
+
+ assertTypeMergingVector(
+ TIME,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING, STRING, STRING, STRING, STRING, TIME, STRING, STRING,
+ STRING, STRING));
+
+ assertTypeMergingVector(
+ ROW,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, ROW, STRING, STRING,
+ STRING));
+
+ assertTypeMergingVector(
+ ARRAY,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, ARRAY,
+ STRING, STRING));
+
+ assertTypeMergingVector(
+ MAP,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, MAP,
+ STRING));
+
+ assertTypeMergingVector(
+ VARIANT,
+ Arrays.asList(
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
+ STRING, VARIANT));
+ }
+
+ private static void assertTypeMergingVector(DataType incomingType, List<DataType> resultTypes) {
+ Assertions.assertThat(ALL_TYPES)
+ .map(type -> getLeastCommonType(type, incomingType))
+ .containsExactlyElementsOf(resultTypes)
+ // Flip LHS and RHS should emit same outputs
+ .map(type -> getLeastCommonType(incomingType, type))
+ .containsExactlyElementsOf(resultTypes);
+ }
+
+ // Some testing utility methods.
+
+ private static List<SchemaChangeEvent> mergeAndDiff(
+ @Nullable Schema currentSchema, Schema upcomingSchema) {
+ Schema afterSchema = getLeastCommonSchema(currentSchema, upcomingSchema);
+ return getSchemaDifference(TABLE_ID, currentSchema, afterSchema);
+ }
+
+ private static Schema of(Object... args) {
+ List<Object> argList = new ArrayList<>(Arrays.asList(args));
+ Preconditions.checkState(argList.size() % 2 == 0);
+ Schema.Builder builder = Schema.newBuilder();
+ while (!argList.isEmpty()) {
+ String colName = (String) argList.remove(0);
+ DataType colType = (DataType) argList.remove(0);
+ builder.physicalColumn(colName, colType);
+ }
+ return builder.build();
+ }
+
+ private static DateData dateOf(int year, int month, int dayOfMonth) {
+ return DateData.fromLocalDate(LocalDate.of(year, month, dayOfMonth));
+ }
+
+ private static TimeData timeOf(int hour, int minute, int second) {
+ return TimeData.fromLocalTime(LocalTime.of(hour, minute, second));
+ }
+
+ private static TimestampData tsOf(String year, String month, String dayOfMonth) {
+ return TimestampData.fromTimestamp(
+ Timestamp.valueOf(String.format("%s-%s-%s 00:00:00", year, month, dayOfMonth)));
+ }
+
+ private static LocalZonedTimestampData ltzTsOf(String year, String month, String dayOfMonth) {
+ return LocalZonedTimestampData.fromEpochMillis(
+ Instant.parse(String.format("%s-%s-%sT00:00:00Z", year, month, dayOfMonth))
+ .toEpochMilli());
+ }
+
+ private static ZonedTimestampData zTsOf(String year, String month, String dayOfMonth) {
+ return ZonedTimestampData.fromZonedDateTime(
+ ZonedDateTime.ofInstant(
+ Instant.parse(String.format("%s-%s-%sT00:00:00Z", year, month, dayOfMonth)),
+ ZoneId.of("UTC")));
+ }
+
+ private static DecimalData decOf(long value) {
+ return DecimalData.fromBigDecimal(
+ BigDecimal.valueOf(value), DecimalType.MAX_PRECISION, DecimalType.DEFAULT_SCALE);
+ }
+
+ private static BinaryStringData binStrOf(String str) {
+ return BinaryStringData.fromString(str);
+ }
+
+ private static byte[] binOf(String str) {
+ return str.getBytes();
+ }
+}
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/types/VariantType.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/types/VariantType.java
index d07907c44..70ab05a7d 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/types/VariantType.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/types/VariantType.java
@@ -17,7 +17,7 @@
package org.apache.flink.cdc.common.types;
-import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.cdc.common.annotation.PublicEvolving;
import java.util.Collections;
import java.util.List;
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/utils/ThreadLocalCache.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/utils/ThreadLocalCache.java
index f4b8101ed..03aef6916 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/utils/ThreadLocalCache.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/utils/ThreadLocalCache.java
@@ -17,7 +17,7 @@
package org.apache.flink.cdc.common.utils;
-import org.apache.flink.annotation.Internal;
+import org.apache.flink.cdc.common.annotation.Internal;
import java.util.LinkedHashMap;
import java.util.Map;
diff --git a/flink-cdc-composer-2.x/pom.xml b/flink-cdc-composer-2.x/pom.xml
new file mode 100644
index 000000000..fa670633b
--- /dev/null
+++ b/flink-cdc-composer-2.x/pom.xml
@@ -0,0 +1,247 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>flink-cdc-parent</artifactId>
+ <groupId>org.apache.flink</groupId>
+ <version>${revision}</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>flink-cdc-composer-2.x</artifactId>
+
+ <properties>
+ <flink.version>${flink.2.x.version}</flink.version>
+ <shaded.guava.version>${flink.2.x.shaded.guava.version}</shaded.guava.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-composer</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-runtime</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-common-2.x</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-runtime-2.x</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-pipeline-connector-values-2.x</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <version>${flink.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils</artifactId>
+ <version>${flink.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-kubernetes</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-yarn</artifactId>
+ <version>${flink.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-pipeline-udf-examples</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-pipeline-model</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- This is for testing Scala UDF.-->
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>${scala.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>test-jar</id>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!--
+ Skip tests in the test phase.
+ Tests will run in the integration-test phase after shading is complete.
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!--
+ Configure failsafe to run integration tests AFTER the package phase.
+ This ensures tests use the shaded artifacts where Guava and Flink APIs
+ have been relocated.
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>3.0.0-M5</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <includes>
+ <include>**/*ITCase.java</include>
+ <include>**/*ITCase.scala</include>
+ </includes>
+ <reuseForks>false</reuseForks>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- Shade plugin configuration to maintain API compatibility with Flink 2.x -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${maven.shade.plugin.version}</version>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <!-- Shading test jar have bug in some previous version, so close this configuration here,
+ see https://issues.apache.org/jira/browse/MSHADE-284 -->
+ <shadeTestJar>false</shadeTestJar>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <filters combine.children="append">
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>module-info.class</exclude>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-shaded-force-shading</include>
+ </includes>
+ </artifactSet>
+ </configuration>
+ </execution>
+ <execution>
+ <id>shade-for-package</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <filters combine.children="append">
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>module-info.class</exclude>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-cdc-composer</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <!-- Relocate Guava dependency to align with Flink 2.x shaded version -->
+ <relocation>
+ <pattern>org.apache.flink.shaded.guava31</pattern>
+ <shadedPattern>org.apache.flink.shaded.guava33</shadedPattern>
+ </relocation>
+ <!-- Relocate SourceFunction to legacy package for Flink 2.x API compatibility -->
+ <relocation>
+ <pattern>org.apache.flink.streaming.api.functions.source.SourceFunction</pattern>
+ <shadedPattern>org.apache.flink.streaming.api.functions.source.legacy.SourceFunction</shadedPattern>
+ </relocation>
+ <!-- Relocate SinkFunction to legacy package for Flink 2.x API compatibility -->
+ <relocation>
+ <pattern>org.apache.flink.streaming.api.functions.sink.SinkFunction</pattern>
+ <shadedPattern>org.apache.flink.streaming.api.functions.sink.legacy.SinkFunction</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/flink-cdc-composer-2.x/src/main/java/org/apache/flink/cdc/composer/flink/translator/DataSinkTranslator.java b/flink-cdc-composer-2.x/src/main/java/org/apache/flink/cdc/composer/flink/translator/DataSinkTranslator.java
new file mode 100644
index 000000000..17c6c0047
--- /dev/null
+++ b/flink-cdc-composer-2.x/src/main/java/org/apache/flink/cdc/composer/flink/translator/DataSinkTranslator.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.composer.flink.translator;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.sink2.SupportsCommitter;
+import org.apache.flink.cdc.common.annotation.Internal;
+import org.apache.flink.cdc.common.annotation.VisibleForTesting;
+import org.apache.flink.cdc.common.configuration.Configuration;
+import org.apache.flink.cdc.common.event.Event;
+import org.apache.flink.cdc.common.factories.DataSinkFactory;
+import org.apache.flink.cdc.common.factories.FactoryHelper;
+import org.apache.flink.cdc.common.sink.DataSink;
+import org.apache.flink.cdc.common.sink.EventSinkProvider;
+import org.apache.flink.cdc.common.sink.FlinkSinkFunctionProvider;
+import org.apache.flink.cdc.common.sink.FlinkSinkProvider;
+import org.apache.flink.cdc.composer.definition.SinkDef;
+import org.apache.flink.cdc.composer.flink.FlinkEnvironmentUtils;
+import org.apache.flink.cdc.composer.utils.FactoryDiscoveryUtils;
+import org.apache.flink.cdc.runtime.operators.sink.BatchDataSinkFunctionOperator;
+import org.apache.flink.cdc.runtime.operators.sink.DataSinkFunctionOperator;
+import org.apache.flink.cdc.runtime.operators.sink.DataSinkWriterOperatorFactory;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.runtime.jobgraph.OperatorID;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessageTypeInfo;
+import org.apache.flink.streaming.api.connector.sink2.SupportsPostCommitTopology;
+import org.apache.flink.streaming.api.connector.sink2.SupportsPreWriteTopology;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.legacy.SinkFunction;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.StreamSink;
+import org.apache.flink.streaming.api.transformations.LegacySinkTransformation;
+import org.apache.flink.streaming.api.transformations.PhysicalTransformation;
+
+import java.lang.reflect.InvocationTargetException;
+
+/** Translator used to build {@link DataSink} for given {@link DataStream}. */
+@Internal
+public class DataSinkTranslator {
+
+ private static final String SINK_WRITER_PREFIX = "Sink Writer: ";
+ private static final String SINK_COMMITTER_PREFIX = "Sink Committer: ";
+
+ public DataSink createDataSink(
+ SinkDef sinkDef, Configuration pipelineConfig, StreamExecutionEnvironment env) {
+ // Search the data sink factory
+ DataSinkFactory sinkFactory =
+ FactoryDiscoveryUtils.getFactoryByIdentifier(
+ sinkDef.getType(), DataSinkFactory.class);
+
+ // Include sink connector JAR
+ FactoryDiscoveryUtils.getJarPathByIdentifier(sinkFactory)
+ .ifPresent(jar -> FlinkEnvironmentUtils.addJar(env, jar));
+
+ // Create data sink
+ return sinkFactory.createDataSink(
+ new FactoryHelper.DefaultContext(
+ sinkDef.getConfig(),
+ pipelineConfig,
+ Thread.currentThread().getContextClassLoader(),
+ env.getConfiguration()));
+ }
+
+ public void translate(
+ SinkDef sinkDef,
+ DataStream<Event> input,
+ DataSink dataSink,
+ OperatorID schemaOperatorID,
+ OperatorUidGenerator operatorUidGenerator) {
+ translate(sinkDef, input, dataSink, false, schemaOperatorID, operatorUidGenerator);
+ }
+
+ public void translate(
+ SinkDef sinkDef,
+ DataStream<Event> input,
+ DataSink dataSink,
+ boolean isBatchMode,
+ OperatorID schemaOperatorID,
+ OperatorUidGenerator operatorUidGenerator) {
+ // Get sink provider
+ EventSinkProvider eventSinkProvider = dataSink.getEventSinkProvider();
+ String sinkName = generateSinkName(sinkDef);
+ if (eventSinkProvider instanceof FlinkSinkProvider) {
+ // Sink V2
+ FlinkSinkProvider sinkProvider = (FlinkSinkProvider) eventSinkProvider;
+ Sink<Event> sink = sinkProvider.getSink();
+ sinkTo(input, sink, sinkName, isBatchMode, schemaOperatorID, operatorUidGenerator);
+ } else if (eventSinkProvider instanceof FlinkSinkFunctionProvider) {
+ // SinkFunction
+ FlinkSinkFunctionProvider sinkFunctionProvider =
+ (FlinkSinkFunctionProvider) eventSinkProvider;
+ SinkFunction<Event> sinkFunction = sinkFunctionProvider.getSinkFunction();
+ sinkTo(
+ input,
+ sinkFunction,
+ sinkName,
+ isBatchMode,
+ schemaOperatorID,
+ operatorUidGenerator);
+ }
+ }
+
+ @VisibleForTesting
+ void sinkTo(
+ DataStream<Event> input,
+ Sink<Event> sink,
+ String sinkName,
+ boolean isBatchMode,
+ OperatorID schemaOperatorID,
+ OperatorUidGenerator operatorUidGenerator) {
+ DataStream<Event> stream = input;
+ // Pre-write topology
+ if (sink instanceof SupportsPreWriteTopology) {
+ stream = ((SupportsPreWriteTopology<Event>) sink).addPreWriteTopology(stream);
+ }
+
+ if (sink instanceof SupportsCommitter) {
+ addCommittingTopology(
+ sink, stream, sinkName, isBatchMode, schemaOperatorID, operatorUidGenerator);
+ } else {
+ stream.transform(
+ SINK_WRITER_PREFIX + sinkName,
+ CommittableMessageTypeInfo.noOutput(),
+ new DataSinkWriterOperatorFactory<>(
+ sink, isBatchMode, schemaOperatorID))
+ .uid(operatorUidGenerator.generateUid("sink-writer"));
+ }
+ }
+
+ private void sinkTo(
+ DataStream<Event> input,
+ SinkFunction<Event> sinkFunction,
+ String sinkName,
+ boolean isBatchMode,
+ OperatorID schemaOperatorID,
+ OperatorUidGenerator operatorUidGenerator) {
+ StreamSink<Event> sinkOperator;
+ if (isBatchMode) {
+ sinkOperator = new BatchDataSinkFunctionOperator(sinkFunction);
+ } else {
+ sinkOperator = new DataSinkFunctionOperator(sinkFunction, schemaOperatorID);
+ }
+ final StreamExecutionEnvironment executionEnvironment = input.getExecutionEnvironment();
+ PhysicalTransformation<Event> transformation =
+ new LegacySinkTransformation<>(
+ input.getTransformation(),
+ SINK_WRITER_PREFIX + sinkName,
+ sinkOperator,
+ executionEnvironment.getParallelism(),
+ false);
+ transformation.setUid(operatorUidGenerator.generateUid("sink-writer"));
+ executionEnvironment.addOperator(transformation);
+ }
+
+ private <CommT> void addCommittingTopology(
+ Sink<Event> sink,
+ DataStream<Event> inputStream,
+ String sinkName,
+ boolean isBatchMode,
+ OperatorID schemaOperatorID,
+ OperatorUidGenerator operatorUidGenerator) {
+ TypeInformation<CommittableMessage<CommT>> typeInformation =
+ CommittableMessageTypeInfo.of(() -> getCommittableSerializer(sink));
+ DataStream<CommittableMessage<CommT>> written =
+ inputStream
+ .transform(
+ SINK_WRITER_PREFIX + sinkName,
+ typeInformation,
+ new DataSinkWriterOperatorFactory<>(
+ sink, isBatchMode, schemaOperatorID))
+ .uid(operatorUidGenerator.generateUid("sink-writer"));
+
+ DataStream<CommittableMessage<CommT>> preCommitted = written;
+
+ // TODO: Hard coding checkpoint
+ boolean isCheckpointingEnabled = true;
+ DataStream<CommittableMessage<CommT>> committed =
+ preCommitted
+ .transform(
+ SINK_COMMITTER_PREFIX + sinkName,
+ typeInformation,
+ getCommitterOperatorFactory(
+ sink, isBatchMode, isCheckpointingEnabled))
+ .uid(operatorUidGenerator.generateUid("sink-committer"));
+
+ if (sink instanceof SupportsPostCommitTopology) {
+ ((SupportsPostCommitTopology<CommT>) sink).addPostCommitTopology(committed);
+ }
+ }
+
+ private String generateSinkName(SinkDef sinkDef) {
+ return sinkDef.getName()
+ .orElse(String.format("Flink CDC Event Sink: %s", sinkDef.getType()));
+ }
+
+ private static <CommT> SimpleVersionedSerializer<CommT> getCommittableSerializer(Object sink) {
+ // FIX ME: TwoPhaseCommittingSink has been deprecated, and its signature has changed
+ // during Flink 1.18 to 1.19. Remove this when Flink 1.18 is no longer supported.
+ try {
+ return (SimpleVersionedSerializer<CommT>)
+ sink.getClass().getDeclaredMethod("getCommittableSerializer").invoke(sink);
+ } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
+ throw new RuntimeException("Failed to get CommittableSerializer", e);
+ }
+ }
+
+ private static <CommT>
+ OneInputStreamOperatorFactory<CommittableMessage<CommT>, CommittableMessage<CommT>>
+ getCommitterOperatorFactory(
+ Sink<Event> sink, boolean isBatchMode, boolean isCheckpointingEnabled) {
+ // FIX ME: OneInputStreamOperatorFactory is an @Internal class, and its signature has
+ // changed during Flink 1.18 to 1.19. Remove this when Flink 1.18 is no longer supported.
+ try {
+ return (OneInputStreamOperatorFactory<
+ CommittableMessage<CommT>, CommittableMessage<CommT>>)
+ Class.forName(
+ "org.apache.flink.streaming.runtime.operators.sink.CommitterOperatorFactory")
+ .getDeclaredConstructors()[0]
+ .newInstance(sink, isBatchMode, isCheckpointingEnabled);
+
+ } catch (ClassNotFoundException
+ | InstantiationException
+ | IllegalAccessException
+ | InvocationTargetException e) {
+ throw new RuntimeException("Failed to create CommitterOperatorFactory", e);
+ }
+ }
+}
diff --git a/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkEnvironmentUtilsTest.java b/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkEnvironmentUtilsTest.java
new file mode 100644
index 000000000..3e89a863b
--- /dev/null
+++ b/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkEnvironmentUtilsTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.composer.flink;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+
+import org.apache.flink.shaded.guava33.com.google.common.collect.Lists;
+
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.net.URL;
+import java.util.Collections;
+import java.util.List;
+
+/** Test for {@link FlinkEnvironmentUtils}. */
+class FlinkEnvironmentUtilsTest {
+
+ @Test
+ void testAddJars() throws Exception {
+ Configuration configuration = new Configuration();
+ configuration.set(PipelineOptions.JARS, Collections.emptyList());
+ StreamExecutionEnvironment env =
+ StreamExecutionEnvironment.createLocalEnvironment(configuration);
+
+ FlinkEnvironmentUtils.addJar(
+ env, Lists.newArrayList(new URL("file://a.jar"), new URL("file://a.jar")));
+ List<String> expectedJars = Lists.newArrayList("file://a.jar");
+ Assertions.assertThat(env.getConfiguration().get(PipelineOptions.JARS))
+ .isEqualTo(expectedJars);
+
+ FlinkEnvironmentUtils.addJar(
+ env, Lists.newArrayList(new URL("file://b.jar"), new URL("file://a.jar")));
+ expectedJars.add("file://b.jar");
+ Assertions.assertThat(env.getConfiguration().get(PipelineOptions.JARS))
+ .isEqualTo(expectedJars);
+ }
+}
diff --git a/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineBatchComposerITCase.java b/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineBatchComposerITCase.java
new file mode 100644
index 000000000..0074039c9
--- /dev/null
+++ b/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineBatchComposerITCase.java
@@ -0,0 +1,1553 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.composer.flink;
+
+import org.apache.flink.cdc.common.configuration.Configuration;
+import org.apache.flink.cdc.common.data.DecimalData;
+import org.apache.flink.cdc.common.data.LocalZonedTimestampData;
+import org.apache.flink.cdc.common.data.TimestampData;
+import org.apache.flink.cdc.common.data.ZonedTimestampData;
+import org.apache.flink.cdc.common.data.binary.BinaryRecordData;
+import org.apache.flink.cdc.common.data.binary.BinaryStringData;
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DataChangeEvent;
+import org.apache.flink.cdc.common.event.Event;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.pipeline.PipelineOptions;
+import org.apache.flink.cdc.common.pipeline.RuntimeExecutionMode;
+import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.types.DataType;
+import org.apache.flink.cdc.common.types.DataTypes;
+import org.apache.flink.cdc.composer.PipelineExecution;
+import org.apache.flink.cdc.composer.definition.PipelineDef;
+import org.apache.flink.cdc.composer.definition.RouteDef;
+import org.apache.flink.cdc.composer.definition.SinkDef;
+import org.apache.flink.cdc.composer.definition.SourceDef;
+import org.apache.flink.cdc.composer.definition.TransformDef;
+import org.apache.flink.cdc.connectors.values.ValuesDatabase;
+import org.apache.flink.cdc.connectors.values.factory.ValuesDataFactory;
+import org.apache.flink.cdc.connectors.values.sink.ValuesDataSink;
+import org.apache.flink.cdc.connectors.values.sink.ValuesDataSinkOptions;
+import org.apache.flink.cdc.connectors.values.source.ValuesDataSourceHelper;
+import org.apache.flink.cdc.connectors.values.source.ValuesDataSourceOptions;
+import org.apache.flink.cdc.runtime.typeutils.BinaryRecordDataGenerator;
+import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
+import org.apache.flink.test.junit5.MiniClusterExtension;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.flink.cdc.connectors.values.source.ValuesDataSourceHelper.TABLE_1;
+import static org.apache.flink.cdc.connectors.values.source.ValuesDataSourceHelper.TABLE_2;
+import static org.apache.flink.configuration.CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Integration test for {@link FlinkPipelineComposer} in batch mode. */
+public class FlinkPipelineBatchComposerITCase {
+
+ private static final int MAX_PARALLELISM = 4;
+
+ // Always use parent-first classloader for CDC classes.
+ // The reason is that ValuesDatabase uses static field for holding data, we need to make sure
+ // the class is loaded by AppClassloader so that we can verify data in the test case.
+ private static final org.apache.flink.configuration.Configuration MINI_CLUSTER_CONFIG =
+ new org.apache.flink.configuration.Configuration();
+
+ static {
+ MINI_CLUSTER_CONFIG.set(
+ ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL,
+ Collections.singletonList("org.apache.flink.cdc"));
+ }
+
+ /**
+ * Use {@link MiniClusterExtension} to reduce the overhead of restarting the MiniCluster for
+ * every test case.
+ */
+ @RegisterExtension
+ static final MiniClusterExtension MINI_CLUSTER_RESOURCE =
+ new MiniClusterExtension(
+ new MiniClusterResourceConfiguration.Builder()
+ .setNumberTaskManagers(1)
+ .setNumberSlotsPerTaskManager(MAX_PARALLELISM)
+ .setConfiguration(MINI_CLUSTER_CONFIG)
+ .build());
+
+ private final PrintStream standardOut = System.out;
+ private final ByteArrayOutputStream outCaptor = new ByteArrayOutputStream();
+
+ @BeforeEach
+ void init() {
+ // Take over STDOUT as we need to check the output of values sink
+ System.setOut(new PrintStream(outCaptor));
+ // Initialize in-memory database
+ ValuesDatabase.clear();
+ }
+
+ @AfterEach
+ void cleanup() {
+ System.setOut(standardOut);
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testSingleSplitSingleTableInStreamingMode(ValuesDataSink.SinkApi sinkApi)
+ throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_SINGLE_BATCH_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> results = ValuesDatabase.getResults(TABLE_1);
+ assertThat(results)
+ .contains(
+ "default_namespace.default_schema.table1:col1=1;col2=1",
+ "default_namespace.default_schema.table1:col1=2;col2=2",
+ "default_namespace.default_schema.table1:col1=3;col2=3");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testSingleSplitSingleTableInBatchMode(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_SINGLE_BATCH_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> results = ValuesDatabase.getResults(TABLE_1);
+ assertThat(results)
+ .contains(
+ "default_namespace.default_schema.table1:col1=1;col2=1",
+ "default_namespace.default_schema.table1:col1=2;col2=2",
+ "default_namespace.default_schema.table1:col1=3;col2=3");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testSingleSplitMultipleTablesInBatchMode(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_MULTI_BATCH_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> table1Results = ValuesDatabase.getResults(TABLE_1);
+ assertThat(table1Results)
+ .containsExactly(
+ "default_namespace.default_schema.table1:col1=1;col2=1",
+ "default_namespace.default_schema.table1:col1=2;col2=2",
+ "default_namespace.default_schema.table1:col1=3;col2=3");
+ List<String> table2Results = ValuesDatabase.getResults(TABLE_2);
+ assertThat(table2Results)
+ .contains(
+ "default_namespace.default_schema.table2:col1=1;col2=1",
+ "default_namespace.default_schema.table2:col1=2;col2=2",
+ "default_namespace.default_schema.table2:col1=3;col2=3");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.table2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[3, 3], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testMultiSplitsSingleTableInBatchMode(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.MULTI_SPLITS_SINGLE_BATCH_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, MAX_PARALLELISM);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> table1Results = ValuesDatabase.getResults(TABLE_1);
+ assertThat(table1Results)
+ .contains(
+ "default_namespace.default_schema.table1:col1=1;col2=1",
+ "default_namespace.default_schema.table1:col1=3;col2=3",
+ "default_namespace.default_schema.table1:col1=5;col2=5");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testTransformInBatchMode(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.TRANSFORM_BATCH_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ TransformDef transformDef =
+ new TransformDef(
+ "default_namespace.default_schema.table1",
+ "*,concat(col1,'0') as col12",
+ "col1 <> '3'",
+ "col1",
+ "col12",
+ "key1=value1",
+ "",
+ null);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ new ArrayList<>(Arrays.asList(transformDef)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING NOT NULL,`col2` STRING,`col12` STRING}, primaryKeys=col1, partitionKeys=col12, options=({key1=value1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1, 10], op=INSERT, meta=({op_ts=1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2, 20], op=INSERT, meta=({op_ts=2})}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testTwoTransformInBatchMode(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+ sourceConfig.set(ValuesDataSourceOptions.BATCH_MODE_ENABLED, true);
+
+ TableId myTable1 = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ TableId myTable2 = TableId.tableId("default_namespace", "default_schema", "mytable2");
+ Schema table1Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .primaryKey("id")
+ .build();
+ Schema table2Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT())
+ .physicalColumn("name", DataTypes.VARCHAR(255))
+ .physicalColumn("description", DataTypes.STRING())
+ .primaryKey("id")
+ .build();
+ // Create test dataset:
+ // Create table 1 [id, name, age]
+ // Create table 2 [id, name, description]
+ // Table 1: +I[1, Alice, 18]
+ // Table 1: +I[2, Bob, 20]
+ // Table 2: +I[3, Charlie, student]
+ // Table 2: +I[4, Donald, student]
+ List<Event> events = new ArrayList<>();
+ BinaryRecordDataGenerator table1dataGenerator =
+ new BinaryRecordDataGenerator(
+ table1Schema.getColumnDataTypes().toArray(new DataType[0]));
+ BinaryRecordDataGenerator table2dataGenerator =
+ new BinaryRecordDataGenerator(
+ table2Schema.getColumnDataTypes().toArray(new DataType[0]));
+ events.add(new CreateTableEvent(myTable1, table1Schema));
+ events.add(new CreateTableEvent(myTable2, table2Schema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {1, BinaryStringData.fromString("Alice"), 18})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 3L,
+ BinaryStringData.fromString("Charlie"),
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Donald"),
+ BinaryStringData.fromString("student")
+ })));
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ List<TransformDef> transformDef =
+ Arrays.asList(
+ new TransformDef(
+ "default_namespace.default_schema.mytable1",
+ "*,'last_name' as last_name",
+ null,
+ null,
+ null,
+ null,
+ "",
+ null),
+ new TransformDef(
+ "default_namespace.default_schema.mytable2",
+ "*,'new_address' as new_address",
+ null,
+ null,
+ null,
+ null,
+ "",
+ null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ transformDef,
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+ Schema myTable1Schema = ValuesDatabase.getTableSchema(myTable1);
+ assertThat(myTable1Schema)
+ .isEqualTo(
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT().notNull())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .physicalColumn("last_name", DataTypes.STRING())
+ .primaryKey("id")
+ .build());
+ Schema myTable2Schema = ValuesDatabase.getTableSchema(myTable2);
+ assertThat(myTable2Schema)
+ .isEqualTo(
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT().notNull())
+ .physicalColumn("name", DataTypes.VARCHAR(255))
+ .physicalColumn("description", DataTypes.STRING())
+ .physicalColumn("new_address", DataTypes.STRING())
+ .primaryKey("id")
+ .build());
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`last_name` STRING}, primaryKeys=id, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`description` STRING,`new_address` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, last_name], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, last_name], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Charlie, student, new_address], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Donald, student, new_address], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testOpTypeMetadataColumnInBatchMode(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.TRANSFORM_BATCH_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ TransformDef transformDef =
+ new TransformDef(
+ "default_namespace.default_schema.table1",
+ "*,concat(col1,'0') as col12,__data_event_type__ as rk,op_ts as opts",
+ "col1 <> '3'",
+ "col1",
+ "col12",
+ "key1=value1",
+ "",
+ null);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ new ArrayList<>(Collections.singletonList(transformDef)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING NOT NULL,`col2` STRING,`col12` STRING,`rk` STRING NOT NULL,`opts` BIGINT NOT NULL}, primaryKeys=col1, partitionKeys=col12, options=({key1=value1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1, 10, +I, 1], op=INSERT, meta=({op_ts=1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2, 20, +I, 2], op=INSERT, meta=({op_ts=2})}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testTransformTwiceInBatchMode(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.TRANSFORM_BATCH_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ TransformDef transformDef1 =
+ new TransformDef(
+ "default_namespace.default_schema.table1",
+ "*,concat(col1,'1') as col12",
+ "col1 = '1' OR col1 = '999'",
+ "col1",
+ "col12",
+ "key1=value1",
+ "",
+ null);
+ TransformDef transformDef2 =
+ new TransformDef(
+ "default_namespace.default_schema.table1",
+ "*,concat(col1,'2') as col12",
+ "col1 = '2'",
+ null,
+ null,
+ null,
+ "",
+ null);
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ new ArrayList<>(Arrays.asList(transformDef1, transformDef2)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING NOT NULL,`col2` STRING,`col12` STRING}, primaryKeys=col1, partitionKeys=col12, options=({key1=value1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1, 11], op=INSERT, meta=({op_ts=1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2, 22], op=INSERT, meta=({op_ts=2})}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testOneToOneRoutingInBatchMode(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_MULTI_BATCH_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup route
+ TableId routedTable1 = TableId.tableId("default_namespace", "default_schema", "routed1");
+ TableId routedTable2 = TableId.tableId("default_namespace", "default_schema", "routed2");
+ List<RouteDef> routeDef =
+ Arrays.asList(
+ new RouteDef(TABLE_1.toString(), routedTable1.toString(), null, null),
+ new RouteDef(TABLE_2.toString(), routedTable2.toString(), null, null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ routeDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> routed1Results = ValuesDatabase.getResults(routedTable1);
+ assertThat(routed1Results)
+ .contains(
+ "default_namespace.default_schema.routed1:col1=1;col2=1",
+ "default_namespace.default_schema.routed1:col1=2;col2=2",
+ "default_namespace.default_schema.routed1:col1=3;col2=3");
+ List<String> routed2Results = ValuesDatabase.getResults(routedTable2);
+ assertThat(routed2Results)
+ .contains(
+ "default_namespace.default_schema.routed2:col1=1;col2=1",
+ "default_namespace.default_schema.routed2:col1=2;col2=2",
+ "default_namespace.default_schema.routed2:col1=3;col2=3");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.routed1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.routed2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed2, before=[], after=[3, 3], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testIdenticalOneToOneRoutingInBatchMode(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_MULTI_BATCH_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup route
+ TableId routedTable1 = TABLE_1;
+ TableId routedTable2 = TABLE_2;
+ List<RouteDef> routeDef =
+ Arrays.asList(
+ new RouteDef(TABLE_1.toString(), routedTable1.toString(), null, null),
+ new RouteDef(TABLE_2.toString(), routedTable2.toString(), null, null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ routeDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> routed1Results = ValuesDatabase.getResults(routedTable1);
+ assertThat(routed1Results)
+ .contains(
+ "default_namespace.default_schema.table1:col1=1;col2=1",
+ "default_namespace.default_schema.table1:col1=2;col2=2",
+ "default_namespace.default_schema.table1:col1=3;col2=3");
+ List<String> routed2Results = ValuesDatabase.getResults(routedTable2);
+ assertThat(routed2Results)
+ .contains(
+ "default_namespace.default_schema.table2:col1=1;col2=1",
+ "default_namespace.default_schema.table2:col1=2;col2=2",
+ "default_namespace.default_schema.table2:col1=3;col2=3");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.table2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[3, 3], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testMergingWithRouteInBatchMode(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+ sourceConfig.set(ValuesDataSourceOptions.BATCH_MODE_ENABLED, true);
+
+ TableId myTable1 = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ TableId myTable2 = TableId.tableId("default_namespace", "default_schema", "mytable2");
+ Schema table1Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .primaryKey("id")
+ .build();
+ Schema table2Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT())
+ .physicalColumn("name", DataTypes.VARCHAR(255))
+ .physicalColumn("age", DataTypes.TINYINT())
+ .physicalColumn("description", DataTypes.STRING())
+ .primaryKey("id")
+ .build();
+
+ // Create test dataset:
+ // Create table 1 [id, name, age]
+ // Create table 2 [id, name, age, description]
+ // Table 1: +I[1, Alice, 18]
+ // Table 1: +I[2, Bob, 20]
+ // Table 2: +I[3, Charlie, 15, student]
+ // Table 2: +I[4, Donald, 25, student]
+ List<Event> events = new ArrayList<>();
+ BinaryRecordDataGenerator table1dataGenerator =
+ new BinaryRecordDataGenerator(
+ table1Schema.getColumnDataTypes().toArray(new DataType[0]));
+ BinaryRecordDataGenerator table2dataGenerator =
+ new BinaryRecordDataGenerator(
+ table2Schema.getColumnDataTypes().toArray(new DataType[0]));
+ events.add(new CreateTableEvent(myTable1, table1Schema));
+ events.add(new CreateTableEvent(myTable2, table2Schema));
+
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {1, BinaryStringData.fromString("Alice"), 18})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20})));
+
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 3L,
+ BinaryStringData.fromString("Charlie"),
+ (byte) 15,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Donald"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup route
+ TableId mergedTable = TableId.tableId("default_namespace", "default_schema", "merged");
+ List<RouteDef> routeDef =
+ Collections.singletonList(
+ new RouteDef(
+ "default_namespace.default_schema.mytable[0-9]",
+ mergedTable.toString(),
+ null,
+ null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ routeDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.merged, schema=columns={`id` BIGINT,`name` STRING,`age` INT,`description` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[1, Alice, 18, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[2, Bob, 20, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[3, Charlie, 15, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[4, Donald, 25, student], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testTransformMergingWithRoute(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+ sourceConfig.set(ValuesDataSourceOptions.BATCH_MODE_ENABLED, true);
+
+ TableId myTable1 = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ TableId myTable2 = TableId.tableId("default_namespace", "default_schema", "mytable2");
+ TableId myTable3 = TableId.tableId("default_namespace", "default_schema", "mytable_other");
+ Schema table1Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .primaryKey("id")
+ .build();
+ Schema table2Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT())
+ .physicalColumn("name", DataTypes.VARCHAR(255))
+ .physicalColumn("age", DataTypes.TINYINT())
+ .physicalColumn("description", DataTypes.STRING())
+ .primaryKey("id")
+ .build();
+ Schema table3Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT().notNull())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("address", DataTypes.STRING())
+ .primaryKey("id")
+ .build();
+ // Create test dataset:
+ // Create table 1 [id, name, age]
+ // Create table 2 [id, name, age, description]
+ // Create table 3 [id, name, address]
+ // Table 1: +I[1, Alice, 18]
+ // Table 1: +I[2, Bob, 20]
+ // Table 2: +I[3, Charlie, 15, student]
+ // Table 2: +I[4, Donald, 25, student]
+ // Table 3: +I[5, E, Beijing]
+ // Table 3: +I[6, F, Hangzhou]
+ List<Event> events = new ArrayList<>();
+ BinaryRecordDataGenerator table1dataGenerator =
+ new BinaryRecordDataGenerator(
+ table1Schema.getColumnDataTypes().toArray(new DataType[0]));
+ BinaryRecordDataGenerator table2dataGenerator =
+ new BinaryRecordDataGenerator(
+ table2Schema.getColumnDataTypes().toArray(new DataType[0]));
+ BinaryRecordDataGenerator table3dataGenerator =
+ new BinaryRecordDataGenerator(
+ table3Schema.getColumnDataTypes().toArray(new DataType[0]));
+ events.add(new CreateTableEvent(myTable1, table1Schema));
+ events.add(new CreateTableEvent(myTable2, table2Schema));
+ events.add(new CreateTableEvent(myTable3, table3Schema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {1, BinaryStringData.fromString("Alice"), 18})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 3L,
+ BinaryStringData.fromString("Charlie"),
+ (byte) 15,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Donald"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable3,
+ table3dataGenerator.generate(
+ new Object[] {
+ 5,
+ BinaryStringData.fromString("E"),
+ BinaryStringData.fromString("Beijing")
+ })));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable3,
+ table3dataGenerator.generate(
+ new Object[] {
+ 6,
+ BinaryStringData.fromString("F"),
+ BinaryStringData.fromString("Hangzhou")
+ })));
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ List<TransformDef> transformDef =
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.mytable[0-9]",
+ "*,'last_name' as last_name",
+ null,
+ null,
+ null,
+ null,
+ "",
+ null));
+
+ // Setup route
+ TableId mergedTable = TableId.tableId("default_namespace", "default_schema", "merged");
+ List<RouteDef> routeDef =
+ Collections.singletonList(
+ new RouteDef(
+ "default_namespace.default_schema.mytable[0-9]",
+ mergedTable.toString(),
+ null,
+ null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ routeDef,
+ transformDef,
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+ Schema mergedTableSchema = ValuesDatabase.getTableSchema(mergedTable);
+ assertThat(mergedTableSchema)
+ .isEqualTo(
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT().notNull())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .physicalColumn("last_name", DataTypes.STRING())
+ .physicalColumn("description", DataTypes.STRING())
+ .primaryKey("id")
+ .build());
+ Schema myTable3Schema = ValuesDatabase.getTableSchema(myTable3);
+ assertThat(myTable3Schema)
+ .isEqualTo(
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT().notNull())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("address", DataTypes.STRING())
+ .primaryKey("id")
+ .build());
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.merged, schema=columns={`id` BIGINT NOT NULL,`name` STRING,`age` INT,`last_name` STRING,`description` STRING}, primaryKeys=id, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable_other, schema=columns={`id` INT NOT NULL,`name` STRING,`address` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[1, Alice, 18, last_name, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[2, Bob, 20, last_name, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[3, Charlie, 15, last_name, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[4, Donald, 25, last_name, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable_other, before=[], after=[5, E, Beijing], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable_other, before=[], after=[6, F, Hangzhou], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testRouteWithReplaceSymbol(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_MULTI_BATCH_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.singletonList(
+ new RouteDef(
+ "default_namespace.default_schema.table[0-9]",
+ "replaced_namespace.replaced_schema.__$__",
+ "__$__",
+ null)),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=replaced_namespace.replaced_schema.table2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=replaced_namespace.replaced_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table2, before=[], after=[3, 3], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testMergingTemporalTypesWithPromotedPrecisions(ValuesDataSink.SinkApi sinkApi)
+ throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+ sourceConfig.set(ValuesDataSourceOptions.BATCH_MODE_ENABLED, true);
+
+ List<Event> events = generateTemporalColumnEvents("default_table_");
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ pipelineConfig.set(PipelineOptions.PIPELINE_LOCAL_TIME_ZONE, "America/New_York");
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Arrays.asList(
+ new RouteDef(
+ "default_namespace.default_schema.default_table_ts_\\.*",
+ "default_namespace.default_schema.default_table_timestamp_merged",
+ null,
+ "Merge timestamp columns with different precision"),
+ new RouteDef(
+ "default_namespace.default_schema.default_table_tz_\\.*",
+ "default_namespace.default_schema.default_table_zoned_timestamp_merged",
+ null,
+ "Merge timestamp_tz columns with different precision"),
+ new RouteDef(
+ "default_namespace.default_schema.default_table_ltz_\\.*",
+ "default_namespace.default_schema.default_table_local_zoned_timestamp_merged",
+ null,
+ "Merge timestamp_ltz columns with different precision"),
+ new RouteDef(
+ "default_namespace.default_schema.default_table_\\.*",
+ "default_namespace.default_schema.default_everything_merged",
+ null,
+ "Merge all timestamp family columns with different precision")),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ String[] expected =
+ Stream.of(
+ // Merging timestamp with different precision
+ "CreateTableEvent{tableId={}_table_timestamp_merged, schema=columns={`id` INT,`name` STRING,`age` INT,`birthday` TIMESTAMP(9) WITH TIME ZONE}, primaryKeys=id, options=()}",
+ // Merging zoned timestamp with different precision
+ "CreateTableEvent{tableId={}_table_zoned_timestamp_merged, schema=columns={`id` INT,`name` STRING,`age` INT,`birthday` TIMESTAMP(9) WITH TIME ZONE}, primaryKeys=id, options=()}",
+ // Merging local-zoned timestamp with different precision
+ "CreateTableEvent{tableId={}_table_local_zoned_timestamp_merged, schema=columns={`id` INT,`name` STRING,`age` INT,`birthday` TIMESTAMP(9) WITH TIME ZONE}, primaryKeys=id, options=()}",
+ // Merging all
+ "CreateTableEvent{tableId={}_everything_merged, schema=columns={`id` INT,`name` STRING,`age` INT,`birthday` TIMESTAMP(9) WITH TIME ZONE}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_table_timestamp_merged, before=[], after=[1, Alice, 17, 2020-01-01T09:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[1, Alice, 17, 2020-01-01T09:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_table_timestamp_merged, before=[], after=[2, Alice, 17, 2020-01-01T09:28:57.123456789-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[2, Alice, 17, 2020-01-01T09:28:57.123456789-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_table_zoned_timestamp_merged, before=[], after=[3, Alice, 17, 2020-01-01T14:28:57Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[3, Alice, 17, 2020-01-01T14:28:57Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_table_zoned_timestamp_merged, before=[], after=[4, Alice, 17, 2020-01-01T14:28:57.123456789Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[4, Alice, 17, 2020-01-01T14:28:57.123456789Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_table_local_zoned_timestamp_merged, before=[], after=[5, Alice, 17, 2020-01-01T04:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[5, Alice, 17, 2020-01-01T04:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_table_local_zoned_timestamp_merged, before=[], after=[6, Alice, 17, 2020-01-01T04:28:57.123456789-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[6, Alice, 17, 2020-01-01T04:28:57.123456789-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_table_timestamp_merged, before=[], after=[101, Zen, 19, 2020-01-01T09:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[101, Zen, 19, 2020-01-01T09:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_table_timestamp_merged, before=[], after=[102, Zen, 19, 2020-01-01T09:28:57.123456789-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[102, Zen, 19, 2020-01-01T09:28:57.123456789-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_table_zoned_timestamp_merged, before=[], after=[103, Zen, 19, 2020-01-01T14:28:57Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[103, Zen, 19, 2020-01-01T14:28:57Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_table_zoned_timestamp_merged, before=[], after=[104, Zen, 19, 2020-01-01T14:28:57.123456789Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[104, Zen, 19, 2020-01-01T14:28:57.123456789Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_table_local_zoned_timestamp_merged, before=[], after=[105, Zen, 19, 2020-01-01T04:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[105, Zen, 19, 2020-01-01T04:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_table_local_zoned_timestamp_merged, before=[], after=[106, Zen, 19, 2020-01-01T04:28:57.123456789-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[106, Zen, 19, 2020-01-01T04:28:57.123456789-05:00], op=INSERT, meta=()}")
+ .map(s -> s.replace("{}", "default_namespace.default_schema.default"))
+ .toArray(String[]::new);
+
+ assertThat(outputEvents).containsExactlyInAnyOrder(expected);
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testMergingDecimalWithVariousPrecisions(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+ sourceConfig.set(ValuesDataSourceOptions.BATCH_MODE_ENABLED, true);
+
+ List<Event> events = generateDecimalColumnEvents("default_table_");
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_EXECUTION_RUNTIME_MODE, RuntimeExecutionMode.BATCH);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.singletonList(
+ new RouteDef(
+ "default_namespace.default_schema.default_table_\\.*",
+ "default_namespace.default_schema.default_everything_merged",
+ null,
+ "Merge all decimal columns with different precision")),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ String[] expected =
+ Stream.of(
+ "CreateTableEvent{tableId=default_namespace.default_schema.default_everything_merged, schema=columns={`id` INT,`name` STRING,`age` INT,`fav_num` DECIMAL(38, 19)}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[1, Alice, 17, 1.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[2, Alice, 17, 22.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[3, Alice, 17, 3333.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[4, Alice, 17, 44444444.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[5, Alice, 17, 555555555555555.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[6, Alice, 17, 66666.6666600000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[7, Alice, 17, 77777777.1700000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[8, Alice, 17, 888888888.8888888888888888888], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[101, Zen, 19, 1.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[102, Zen, 19, 22.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[103, Zen, 19, 3333.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[104, Zen, 19, 44444444.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[105, Zen, 19, 555555555555555.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[106, Zen, 19, 66666.6666600000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[107, Zen, 19, 77777777.1700000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.default_everything_merged, before=[], after=[108, Zen, 19, 888888888.8888888888888888888], op=INSERT, meta=()}")
+ .map(
+ s ->
+ s.replace(
+ "{}",
+ "default_namespace.default_schema.default_everything_merged"))
+ .toArray(String[]::new);
+
+ assertThat(outputEvents).containsExactlyInAnyOrder(expected);
+ }
+
+ private List<Event> generateTemporalColumnEvents(String tableNamePrefix) {
+ List<Event> events = new ArrayList<>();
+ List<Event> dataChangeEvents = new ArrayList<>();
+
+ // Initialize schemas
+ List<String> names = Arrays.asList("ts_0", "ts_9", "tz_0", "tz_9", "ltz_0", "ltz_9");
+
+ List<DataType> types =
+ Arrays.asList(
+ DataTypes.TIMESTAMP(0),
+ DataTypes.TIMESTAMP(9),
+ DataTypes.TIMESTAMP_TZ(0),
+ DataTypes.TIMESTAMP_TZ(9),
+ DataTypes.TIMESTAMP_LTZ(0),
+ DataTypes.TIMESTAMP_LTZ(9));
+
+ Instant lowPrecisionTimestamp = Instant.parse("2020-01-01T14:28:57Z");
+ Instant highPrecisionTimestamp = Instant.parse("2020-01-01T14:28:57.123456789Z");
+
+ List<Object> values =
+ Arrays.asList(
+ TimestampData.fromLocalDateTime(
+ LocalDateTime.ofInstant(lowPrecisionTimestamp, ZoneId.of("UTC"))),
+ TimestampData.fromLocalDateTime(
+ LocalDateTime.ofInstant(highPrecisionTimestamp, ZoneId.of("UTC"))),
+ ZonedTimestampData.fromZonedDateTime(
+ ZonedDateTime.ofInstant(lowPrecisionTimestamp, ZoneId.of("UTC"))),
+ ZonedTimestampData.fromZonedDateTime(
+ ZonedDateTime.ofInstant(highPrecisionTimestamp, ZoneId.of("UTC"))),
+ LocalZonedTimestampData.fromInstant(lowPrecisionTimestamp),
+ LocalZonedTimestampData.fromInstant(highPrecisionTimestamp));
+
+ List<Schema> schemas =
+ types.stream()
+ .map(
+ temporalColumnType ->
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .physicalColumn("birthday", temporalColumnType)
+ .primaryKey("id")
+ .build())
+ .collect(Collectors.toList());
+
+ for (int i = 0; i < names.size(); i++) {
+ TableId generatedTableId =
+ TableId.tableId(
+ "default_namespace", "default_schema", tableNamePrefix + names.get(i));
+ Schema generatedSchema = schemas.get(i);
+ events.add(new CreateTableEvent(generatedTableId, generatedSchema));
+ dataChangeEvents.add(
+ DataChangeEvent.insertEvent(
+ generatedTableId,
+ generate(generatedSchema, 1 + i, "Alice", 17, values.get(i))));
+ }
+
+ for (int i = 0; i < names.size(); i++) {
+ TableId generatedTableId =
+ TableId.tableId(
+ "default_namespace", "default_schema", tableNamePrefix + names.get(i));
+ Schema generatedSchema = schemas.get(i);
+ dataChangeEvents.add(
+ DataChangeEvent.insertEvent(
+ generatedTableId,
+ generate(generatedSchema, 101 + i, "Zen", 19, values.get(i))));
+ }
+
+ events.addAll(dataChangeEvents);
+ return events;
+ }
+
+ private List<Event> generateDecimalColumnEvents(String tableNamePrefix) {
+ List<Event> events = new ArrayList<>();
+ List<Event> dataChangeEvents = new ArrayList<>();
+
+ // Initialize schemas
+ List<String> names =
+ Arrays.asList(
+ "tiny",
+ "small",
+ "vanilla",
+ "big",
+ "dec_15_0",
+ "decimal_10_10",
+ "decimal_16_2",
+ "decimal_29_19");
+
+ List<DataType> types =
+ Arrays.asList(
+ DataTypes.TINYINT(),
+ DataTypes.SMALLINT(),
+ DataTypes.INT(),
+ DataTypes.BIGINT(),
+ DataTypes.DECIMAL(15, 0),
+ DataTypes.DECIMAL(10, 5),
+ DataTypes.DECIMAL(16, 2),
+ DataTypes.DECIMAL(29, 19));
+
+ List<Object> values =
+ Arrays.asList(
+ (byte) 1,
+ (short) 22,
+ 3333,
+ (long) 44444444,
+ DecimalData.fromBigDecimal(new BigDecimal("555555555555555"), 15, 0),
+ DecimalData.fromBigDecimal(new BigDecimal("66666.66666"), 10, 5),
+ DecimalData.fromBigDecimal(new BigDecimal("77777777.17"), 16, 2),
+ DecimalData.fromBigDecimal(
+ new BigDecimal("888888888.8888888888888888888"), 29, 19));
+
+ List<Schema> schemas =
+ types.stream()
+ .map(
+ temporalColumnType ->
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .physicalColumn("fav_num", temporalColumnType)
+ .primaryKey("id")
+ .build())
+ .collect(Collectors.toList());
+
+ for (int i = 0; i < names.size(); i++) {
+ TableId generatedTableId =
+ TableId.tableId(
+ "default_namespace", "default_schema", tableNamePrefix + names.get(i));
+ Schema generatedSchema = schemas.get(i);
+ events.add(new CreateTableEvent(generatedTableId, generatedSchema));
+ dataChangeEvents.add(
+ DataChangeEvent.insertEvent(
+ generatedTableId,
+ generate(generatedSchema, 1 + i, "Alice", 17, values.get(i))));
+ }
+
+ for (int i = 0; i < names.size(); i++) {
+ TableId generatedTableId =
+ TableId.tableId(
+ "default_namespace", "default_schema", tableNamePrefix + names.get(i));
+ Schema generatedSchema = schemas.get(i);
+ dataChangeEvents.add(
+ DataChangeEvent.insertEvent(
+ generatedTableId,
+ generate(generatedSchema, 101 + i, "Zen", 19, values.get(i))));
+ }
+
+ events.addAll(dataChangeEvents);
+ return events;
+ }
+
+ BinaryRecordData generate(Schema schema, Object... fields) {
+ return (new BinaryRecordDataGenerator(schema.getColumnDataTypes().toArray(new DataType[0])))
+ .generate(
+ Arrays.stream(fields)
+ .map(
+ e ->
+ (e instanceof String)
+ ? BinaryStringData.fromString((String) e)
+ : e)
+ .toArray());
+ }
+}
diff --git a/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerITCase.java b/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerITCase.java
new file mode 100644
index 000000000..13cf54e7f
--- /dev/null
+++ b/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerITCase.java
@@ -0,0 +1,1673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.composer.flink;
+
+import org.apache.flink.cdc.common.configuration.Configuration;
+import org.apache.flink.cdc.common.data.DecimalData;
+import org.apache.flink.cdc.common.data.LocalZonedTimestampData;
+import org.apache.flink.cdc.common.data.TimestampData;
+import org.apache.flink.cdc.common.data.ZonedTimestampData;
+import org.apache.flink.cdc.common.data.binary.BinaryRecordData;
+import org.apache.flink.cdc.common.data.binary.BinaryStringData;
+import org.apache.flink.cdc.common.event.AddColumnEvent;
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DataChangeEvent;
+import org.apache.flink.cdc.common.event.Event;
+import org.apache.flink.cdc.common.event.RenameColumnEvent;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.pipeline.PipelineOptions;
+import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
+import org.apache.flink.cdc.common.schema.Column;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.types.DataType;
+import org.apache.flink.cdc.common.types.DataTypes;
+import org.apache.flink.cdc.composer.PipelineExecution;
+import org.apache.flink.cdc.composer.definition.PipelineDef;
+import org.apache.flink.cdc.composer.definition.RouteDef;
+import org.apache.flink.cdc.composer.definition.SinkDef;
+import org.apache.flink.cdc.composer.definition.SourceDef;
+import org.apache.flink.cdc.composer.definition.TransformDef;
+import org.apache.flink.cdc.connectors.values.ValuesDatabase;
+import org.apache.flink.cdc.connectors.values.factory.ValuesDataFactory;
+import org.apache.flink.cdc.connectors.values.sink.ValuesDataSink;
+import org.apache.flink.cdc.connectors.values.sink.ValuesDataSinkOptions;
+import org.apache.flink.cdc.connectors.values.source.ValuesDataSourceHelper;
+import org.apache.flink.cdc.connectors.values.source.ValuesDataSourceOptions;
+import org.apache.flink.cdc.runtime.typeutils.BinaryRecordDataGenerator;
+import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
+import org.apache.flink.test.junit5.MiniClusterExtension;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.flink.cdc.connectors.values.source.ValuesDataSourceHelper.TABLE_1;
+import static org.apache.flink.cdc.connectors.values.source.ValuesDataSourceHelper.TABLE_2;
+import static org.apache.flink.configuration.CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Integration test for {@link FlinkPipelineComposer}. */
+class FlinkPipelineComposerITCase {
+
+ private static final int MAX_PARALLELISM = 4;
+
+ // Always use parent-first classloader for CDC classes.
+ // The reason is that ValuesDatabase uses static field for holding data, we need to make sure
+ // the class is loaded by AppClassloader so that we can verify data in the test case.
+ private static final org.apache.flink.configuration.Configuration MINI_CLUSTER_CONFIG =
+ new org.apache.flink.configuration.Configuration();
+
+ static {
+ MINI_CLUSTER_CONFIG.set(
+ ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL,
+ Collections.singletonList("org.apache.flink.cdc"));
+ }
+
+ /**
+ * Use {@link MiniClusterExtension} to reduce the overhead of restarting the MiniCluster for
+ * every test case.
+ */
+ @RegisterExtension
+ static final MiniClusterExtension MINI_CLUSTER_RESOURCE =
+ new MiniClusterExtension(
+ new MiniClusterResourceConfiguration.Builder()
+ .setNumberTaskManagers(1)
+ .setNumberSlotsPerTaskManager(MAX_PARALLELISM)
+ .setConfiguration(MINI_CLUSTER_CONFIG)
+ .build());
+
+ private final PrintStream standardOut = System.out;
+ private final ByteArrayOutputStream outCaptor = new ByteArrayOutputStream();
+
+ @BeforeEach
+ void init() {
+ // Take over STDOUT as we need to check the output of values sink
+ System.setOut(new PrintStream(outCaptor));
+ // Initialize in-memory database
+ ValuesDatabase.clear();
+ }
+
+ @AfterEach
+ void cleanup() {
+ System.setOut(standardOut);
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testSingleSplitSingleTable(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_SINGLE_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> results = ValuesDatabase.getResults(TABLE_1);
+ assertThat(results)
+ .contains(
+ "default_namespace.default_schema.table1:col1=2;newCol3=x",
+ "default_namespace.default_schema.table1:col1=3;newCol3=");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "RenameColumnEvent{tableId=default_namespace.default_schema.table1, nameMapping={col2=newCol2, col3=newCol3}}",
+ "DropColumnEvent{tableId=default_namespace.default_schema.table1, droppedColumnNames=[newCol2]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, ], after=[2, x], op=UPDATE, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testSingleSplitMultipleTables(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_MULTI_TABLES);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> table1Results = ValuesDatabase.getResults(TABLE_1);
+ assertThat(table1Results)
+ .containsExactly(
+ "default_namespace.default_schema.table1:col1=2;newCol3=x",
+ "default_namespace.default_schema.table1:col1=3;newCol3=");
+ List<String> table2Results = ValuesDatabase.getResults(TABLE_2);
+ assertThat(table2Results)
+ .contains(
+ "default_namespace.default_schema.table2:col1=1;col2=1",
+ "default_namespace.default_schema.table2:col1=2;col2=2",
+ "default_namespace.default_schema.table2:col1=3;col2=3");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.table2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "RenameColumnEvent{tableId=default_namespace.default_schema.table1, nameMapping={col2=newCol2, col3=newCol3}}",
+ "DropColumnEvent{tableId=default_namespace.default_schema.table1, droppedColumnNames=[newCol2]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, 2], after=[2, x], op=UPDATE, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testMultiSplitsSingleTable(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.MULTI_SPLITS_SINGLE_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, MAX_PARALLELISM);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> table1Results = ValuesDatabase.getResults(TABLE_1);
+ assertThat(table1Results)
+ .contains(
+ "default_namespace.default_schema.table1:col1=1;col2=1;col3=x",
+ "default_namespace.default_schema.table1:col1=3;col2=3;col3=x",
+ "default_namespace.default_schema.table1:col1=5;col2=5;col3=");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testTransform(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.TRANSFORM_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ TransformDef transformDef =
+ new TransformDef(
+ "default_namespace.default_schema.table1",
+ "*,concat(col1,'0') as col12",
+ "col1 <> '3'",
+ "col1",
+ "col12",
+ "key1=value1",
+ "",
+ null);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ new ArrayList<>(Collections.singletonList(transformDef)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING NOT NULL,`col2` STRING,`col12` STRING}, primaryKeys=col1, partitionKeys=col12, options=({key1=value1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1, 10], op=INSERT, meta=({op_ts=1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2, 20], op=INSERT, meta=({op_ts=2})}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=AFTER, existedColumnName=col2}]}",
+ "RenameColumnEvent{tableId=default_namespace.default_schema.table1, nameMapping={col2=newCol2, col3=newCol3}}",
+ "DropColumnEvent{tableId=default_namespace.default_schema.table1, droppedColumnNames=[newCol2]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, 1, 10], after=[], op=DELETE, meta=({op_ts=4})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, , 20], after=[2, x, 20], op=UPDATE, meta=({op_ts=5})}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testOpTypeMetadataColumn(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.TRANSFORM_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ TransformDef transformDef =
+ new TransformDef(
+ "default_namespace.default_schema.table1",
+ "*,concat(col1,'0') as col12,__data_event_type__ as rk,op_ts as opts",
+ "col1 <> '3'",
+ "col1",
+ "col12",
+ "key1=value1",
+ "",
+ null);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ new ArrayList<>(Collections.singletonList(transformDef)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING NOT NULL,`col2` STRING,`col12` STRING,`rk` STRING NOT NULL,`opts` BIGINT NOT NULL}, primaryKeys=col1, partitionKeys=col12, options=({key1=value1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1, 10, +I, 1], op=INSERT, meta=({op_ts=1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2, 20, +I, 2], op=INSERT, meta=({op_ts=2})}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=AFTER, existedColumnName=col2}]}",
+ "RenameColumnEvent{tableId=default_namespace.default_schema.table1, nameMapping={col2=newCol2, col3=newCol3}}",
+ "DropColumnEvent{tableId=default_namespace.default_schema.table1, droppedColumnNames=[newCol2]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, 1, 10, -D, 4], after=[], op=DELETE, meta=({op_ts=4})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, , 20, -U, 5], after=[2, x, 20, +U, 5], op=UPDATE, meta=({op_ts=5})}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testTransformTwice(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.TRANSFORM_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ TransformDef transformDef1 =
+ new TransformDef(
+ "default_namespace.default_schema.table1",
+ "*,concat(col1,'1') as col12",
+ "col1 = '1' OR col1 = '999'",
+ "col1",
+ "col12",
+ "key1=value1",
+ "",
+ null);
+ TransformDef transformDef2 =
+ new TransformDef(
+ "default_namespace.default_schema.table1",
+ "*,concat(col1,'2') as col12",
+ "col1 = '2'",
+ null,
+ null,
+ null,
+ "",
+ null);
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ new ArrayList<>(Arrays.asList(transformDef1, transformDef2)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING NOT NULL,`col2` STRING,`col12` STRING}, primaryKeys=col1, partitionKeys=col12, options=({key1=value1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1, 11], op=INSERT, meta=({op_ts=1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2, 22], op=INSERT, meta=({op_ts=2})}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=AFTER, existedColumnName=col2}]}",
+ "RenameColumnEvent{tableId=default_namespace.default_schema.table1, nameMapping={col2=newCol2, col3=newCol3}}",
+ "DropColumnEvent{tableId=default_namespace.default_schema.table1, droppedColumnNames=[newCol2]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, 1, 11], after=[], op=DELETE, meta=({op_ts=4})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, , 22], after=[2, x, 22], op=UPDATE, meta=({op_ts=5})}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testOneToOneRouting(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_MULTI_TABLES);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup route
+ TableId routedTable1 = TableId.tableId("default_namespace", "default_schema", "routed1");
+ TableId routedTable2 = TableId.tableId("default_namespace", "default_schema", "routed2");
+ List<RouteDef> routeDef =
+ Arrays.asList(
+ new RouteDef(TABLE_1.toString(), routedTable1.toString(), null, null),
+ new RouteDef(TABLE_2.toString(), routedTable2.toString(), null, null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ routeDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> routed1Results = ValuesDatabase.getResults(routedTable1);
+ assertThat(routed1Results)
+ .contains(
+ "default_namespace.default_schema.routed1:col1=2;newCol3=x",
+ "default_namespace.default_schema.routed1:col1=3;newCol3=");
+ List<String> routed2Results = ValuesDatabase.getResults(routedTable2);
+ assertThat(routed2Results)
+ .contains(
+ "default_namespace.default_schema.routed2:col1=1;col2=1",
+ "default_namespace.default_schema.routed2:col1=2;col2=2",
+ "default_namespace.default_schema.routed2:col1=3;col2=3");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.routed1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.routed2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.routed1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed2, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "RenameColumnEvent{tableId=default_namespace.default_schema.routed1, nameMapping={col2=newCol2, col3=newCol3}}",
+ "DropColumnEvent{tableId=default_namespace.default_schema.routed1, droppedColumnNames=[newCol2]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed1, before=[1, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed1, before=[2, 2], after=[2, x], op=UPDATE, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testIdenticalOneToOneRouting(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_MULTI_TABLES);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup route
+ TableId routedTable1 = TABLE_1;
+ TableId routedTable2 = TABLE_2;
+ List<RouteDef> routeDef =
+ Arrays.asList(
+ new RouteDef(TABLE_1.toString(), routedTable1.toString(), null, null),
+ new RouteDef(TABLE_2.toString(), routedTable2.toString(), null, null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ routeDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> routed1Results = ValuesDatabase.getResults(routedTable1);
+ assertThat(routed1Results)
+ .contains(
+ "default_namespace.default_schema.table1:col1=2;newCol3=x",
+ "default_namespace.default_schema.table1:col1=3;newCol3=");
+ List<String> routed2Results = ValuesDatabase.getResults(routedTable2);
+ assertThat(routed2Results)
+ .contains(
+ "default_namespace.default_schema.table2:col1=1;col2=1",
+ "default_namespace.default_schema.table2:col1=2;col2=2",
+ "default_namespace.default_schema.table2:col1=3;col2=3");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.table2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "RenameColumnEvent{tableId=default_namespace.default_schema.table1, nameMapping={col2=newCol2, col3=newCol3}}",
+ "DropColumnEvent{tableId=default_namespace.default_schema.table1, droppedColumnNames=[newCol2]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, 2], after=[2, x], op=UPDATE, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testMergingWithRoute(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId myTable1 = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ TableId myTable2 = TableId.tableId("default_namespace", "default_schema", "mytable2");
+ Schema table1Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .primaryKey("id")
+ .build();
+ Schema table2Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT())
+ .physicalColumn("name", DataTypes.VARCHAR(255))
+ .physicalColumn("age", DataTypes.TINYINT())
+ .physicalColumn("description", DataTypes.STRING())
+ .primaryKey("id")
+ .build();
+
+ // Create test dataset:
+ // Create table 1 [id, name, age]
+ // Table 1: +I[1, Alice, 18]
+ // Table 1: +I[2, Bob, 20]
+ // Table 1: -U[2, Bob, 20] +U[2, Bob, 30]
+ // Create table 2 [id, name, age, description]
+ // Table 2: +I[3, Charlie, 15, student]
+ // Table 2: +I[4, Donald, 25, student]
+ // Table 2: -D[4, Donald, 25, student]
+ // Rename column for table 1: name -> last_name
+ // Add column for table 2: gender
+ // Table 1: +I[5, Eliza, 24]
+ // Table 2: +I[6, Frank, 30, student, male]
+ List<Event> events = new ArrayList<>();
+ BinaryRecordDataGenerator table1dataGenerator =
+ new BinaryRecordDataGenerator(
+ table1Schema.getColumnDataTypes().toArray(new DataType[0]));
+ BinaryRecordDataGenerator table2dataGenerator =
+ new BinaryRecordDataGenerator(
+ table2Schema.getColumnDataTypes().toArray(new DataType[0]));
+ events.add(new CreateTableEvent(myTable1, table1Schema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {1, BinaryStringData.fromString("Alice"), 18})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20})));
+ events.add(
+ DataChangeEvent.updateEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20}),
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 30})));
+ events.add(new CreateTableEvent(myTable2, table2Schema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 3L,
+ BinaryStringData.fromString("Charlie"),
+ (byte) 15,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Donald"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.deleteEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Donald"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(new RenameColumnEvent(myTable1, ImmutableMap.of("name", "last_name")));
+ events.add(
+ new AddColumnEvent(
+ myTable2,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn("gender", DataTypes.STRING())))));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {5, BinaryStringData.fromString("Eliza"), 24})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ new BinaryRecordDataGenerator(
+ new DataType[] {
+ DataTypes.BIGINT(),
+ DataTypes.VARCHAR(255),
+ DataTypes.TINYINT(),
+ DataTypes.STRING(),
+ DataTypes.STRING()
+ })
+ .generate(
+ new Object[] {
+ 6L,
+ BinaryStringData.fromString("Frank"),
+ (byte) 30,
+ BinaryStringData.fromString("student"),
+ BinaryStringData.fromString("male")
+ })));
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup route
+ TableId mergedTable = TableId.tableId("default_namespace", "default_schema", "merged");
+ List<RouteDef> routeDef =
+ Collections.singletonList(
+ new RouteDef(
+ "default_namespace.default_schema.mytable[0-9]",
+ mergedTable.toString(),
+ null,
+ null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ routeDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+ Schema mergedTableSchema = ValuesDatabase.getTableSchema(mergedTable);
+ assertThat(mergedTableSchema)
+ .isEqualTo(
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .physicalColumn("description", DataTypes.STRING())
+ .physicalColumn("last_name", DataTypes.STRING())
+ .physicalColumn("gender", DataTypes.STRING())
+ .primaryKey("id")
+ .build());
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.merged, schema=columns={`id` INT,`name` STRING,`age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[1, Alice, 18], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[2, Bob, 20], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[2, Bob, 20], after=[2, Bob, 30], op=UPDATE, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`description` STRING, position=AFTER, existedColumnName=age}]}",
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.merged, typeMapping={id=BIGINT}, oldTypeMapping={id=INT}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[3, Charlie, 15, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[4, Donald, 25, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[4, Donald, 25, student], after=[], op=DELETE, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`last_name` STRING, position=AFTER, existedColumnName=description}]}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`gender` STRING, position=AFTER, existedColumnName=last_name}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[5, null, 24, null, Eliza, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[6, Frank, 30, student, null, male], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testTransformMergingWithRoute(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId myTable1 = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ TableId myTable2 = TableId.tableId("default_namespace", "default_schema", "mytable2");
+ Schema table1Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .primaryKey("id")
+ .build();
+ Schema table2Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT())
+ .physicalColumn("name", DataTypes.VARCHAR(255))
+ .physicalColumn("age", DataTypes.TINYINT())
+ .physicalColumn("description", DataTypes.STRING())
+ .primaryKey("id")
+ .build();
+
+ // Create test dataset:
+ // Create table 1 [id, name, age]
+ // Table 1: +I[1, Alice, 18]
+ // Table 1: +I[2, Bob, 20]
+ // Table 1: -U[2, Bob, 20] +U[2, Bob, 30]
+ // Create table 2 [id, name, age, description]
+ // Table 2: +I[3, Charlie, 15, student]
+ // Table 2: +I[4, Donald, 25, student]
+ // Table 2: -D[4, Donald, 25, student]
+ // Rename column for table 1: name -> last_name
+ // Add column for table 2: gender
+ // Table 1: +I[5, Eliza, 24]
+ // Table 2: +I[6, Frank, 30, student, male]
+ List<Event> events = new ArrayList<>();
+ BinaryRecordDataGenerator table1dataGenerator =
+ new BinaryRecordDataGenerator(
+ table1Schema.getColumnDataTypes().toArray(new DataType[0]));
+ BinaryRecordDataGenerator table2dataGenerator =
+ new BinaryRecordDataGenerator(
+ table2Schema.getColumnDataTypes().toArray(new DataType[0]));
+ events.add(new CreateTableEvent(myTable1, table1Schema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {1, BinaryStringData.fromString("Alice"), 18})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20})));
+ events.add(
+ DataChangeEvent.updateEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20}),
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 30})));
+ events.add(new CreateTableEvent(myTable2, table2Schema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 3L,
+ BinaryStringData.fromString("Charlie"),
+ (byte) 15,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Donald"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.deleteEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Donald"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+ // events.add(new RenameColumnEvent(myTable1, ImmutableMap.of("name", "last_name")));
+ events.add(
+ new AddColumnEvent(
+ myTable2,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn("gender", DataTypes.STRING())))));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {5, BinaryStringData.fromString("Eliza"), 24})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ new BinaryRecordDataGenerator(
+ new DataType[] {
+ DataTypes.BIGINT(),
+ DataTypes.VARCHAR(255),
+ DataTypes.TINYINT(),
+ DataTypes.STRING(),
+ DataTypes.STRING()
+ })
+ .generate(
+ new Object[] {
+ 6L,
+ BinaryStringData.fromString("Frank"),
+ (byte) 30,
+ BinaryStringData.fromString("student"),
+ BinaryStringData.fromString("male")
+ })));
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ List<TransformDef> transformDef =
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.mytable[0-9]",
+ "*,'last_name' as last_name",
+ null,
+ null,
+ null,
+ null,
+ "",
+ null));
+
+ // Setup route
+ TableId mergedTable = TableId.tableId("default_namespace", "default_schema", "merged");
+ List<RouteDef> routeDef =
+ Collections.singletonList(
+ new RouteDef(
+ "default_namespace.default_schema.mytable[0-9]",
+ mergedTable.toString(),
+ null,
+ null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ routeDef,
+ transformDef,
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+ Schema mergedTableSchema = ValuesDatabase.getTableSchema(mergedTable);
+ assertThat(mergedTableSchema)
+ .isEqualTo(
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT().notNull())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .physicalColumn("last_name", DataTypes.STRING())
+ .physicalColumn("description", DataTypes.STRING())
+ .physicalColumn("gender", DataTypes.STRING())
+ .primaryKey("id")
+ .build());
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.merged, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`last_name` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[1, Alice, 18, last_name], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[2, Bob, 20, last_name], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[2, Bob, 20, last_name], after=[2, Bob, 30, last_name], op=UPDATE, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`description` STRING, position=AFTER, existedColumnName=last_name}]}",
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.merged, typeMapping={id=BIGINT NOT NULL}, oldTypeMapping={id=INT NOT NULL}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[3, Charlie, 15, last_name, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[4, Donald, 25, last_name, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[4, Donald, 25, last_name, student], after=[], op=DELETE, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`gender` STRING, position=AFTER, existedColumnName=description}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[5, Eliza, 24, last_name, null, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[6, Frank, 30, last_name, student, male], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testTransformMergingWithRouteChangeOrder(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId myTable1 = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ TableId myTable2 = TableId.tableId("default_namespace", "default_schema", "mytable2");
+ Schema table1Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .primaryKey("id")
+ .build();
+ Schema table2Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT())
+ .physicalColumn("name", DataTypes.VARCHAR(255))
+ .physicalColumn("age", DataTypes.TINYINT())
+ .physicalColumn("description", DataTypes.STRING())
+ .primaryKey("id")
+ .build();
+
+ List<Event> events = new ArrayList<>();
+ BinaryRecordDataGenerator table1dataGenerator =
+ new BinaryRecordDataGenerator(
+ table1Schema.getColumnDataTypes().toArray(new DataType[0]));
+ BinaryRecordDataGenerator table2dataGenerator =
+ new BinaryRecordDataGenerator(
+ table2Schema.getColumnDataTypes().toArray(new DataType[0]));
+ events.add(new CreateTableEvent(myTable1, table1Schema));
+ events.add(new CreateTableEvent(myTable2, table2Schema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {1, BinaryStringData.fromString("Alice"), 18})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20})));
+ events.add(
+ DataChangeEvent.updateEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20}),
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 30})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 3L,
+ BinaryStringData.fromString("Charlie"),
+ (byte) 15,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Donald"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.deleteEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Donald"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+ // events.add(new RenameColumnEvent(myTable1, ImmutableMap.of("name", "last_name")));
+ events.add(
+ new AddColumnEvent(
+ myTable2,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn("gender", DataTypes.STRING())))));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {5, BinaryStringData.fromString("Eliza"), 24})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ new BinaryRecordDataGenerator(
+ new DataType[] {
+ DataTypes.BIGINT(),
+ DataTypes.VARCHAR(255),
+ DataTypes.TINYINT(),
+ DataTypes.STRING(),
+ DataTypes.STRING()
+ })
+ .generate(
+ new Object[] {
+ 6L,
+ BinaryStringData.fromString("Frank"),
+ (byte) 30,
+ BinaryStringData.fromString("student"),
+ BinaryStringData.fromString("male")
+ })));
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ List<TransformDef> transformDef =
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.mytable[0-9]",
+ "*,'last_name' as last_name",
+ null,
+ null,
+ null,
+ null,
+ "",
+ null));
+
+ // Setup route
+ TableId mergedTable = TableId.tableId("default_namespace", "default_schema", "merged");
+ List<RouteDef> routeDef =
+ Collections.singletonList(
+ new RouteDef(
+ "default_namespace.default_schema.mytable[0-9]",
+ mergedTable.toString(),
+ null,
+ null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ routeDef,
+ transformDef,
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+ Schema mergedTableSchema = ValuesDatabase.getTableSchema(mergedTable);
+ assertThat(mergedTableSchema)
+ .isEqualTo(
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT().notNull())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .physicalColumn("last_name", DataTypes.STRING())
+ .physicalColumn("description", DataTypes.STRING())
+ .physicalColumn("gender", DataTypes.STRING())
+ .primaryKey("id")
+ .build());
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.merged, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`last_name` STRING}, primaryKeys=id, options=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`description` STRING, position=AFTER, existedColumnName=last_name}]}",
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.merged, typeMapping={id=BIGINT NOT NULL}, oldTypeMapping={id=INT NOT NULL}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[1, Alice, 18, last_name, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[2, Bob, 20, last_name, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[2, Bob, 20, last_name, null], after=[2, Bob, 30, last_name, null], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[3, Charlie, 15, last_name, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[4, Donald, 25, last_name, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[4, Donald, 25, last_name, student], after=[], op=DELETE, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`gender` STRING, position=AFTER, existedColumnName=description}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[5, Eliza, 24, last_name, null, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[6, Frank, 30, last_name, student, male], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testRouteWithReplaceSymbol(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_MULTI_TABLES);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.singletonList(
+ new RouteDef(
+ "default_namespace.default_schema.table[0-9]",
+ "replaced_namespace.replaced_schema.__$__",
+ "__$__",
+ null)),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=replaced_namespace.replaced_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=replaced_namespace.replaced_schema.table2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=replaced_namespace.replaced_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table2, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "RenameColumnEvent{tableId=replaced_namespace.replaced_schema.table1, nameMapping={col2=newCol2, col3=newCol3}}",
+ "DropColumnEvent{tableId=replaced_namespace.replaced_schema.table1, droppedColumnNames=[newCol2]}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table1, before=[1, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table1, before=[2, 2], after=[2, x], op=UPDATE, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testMergingTemporalTypesWithPromotedPrecisions(ValuesDataSink.SinkApi sinkApi)
+ throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ List<Event> events = generateTemporalColumnEvents("default_table_");
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ pipelineConfig.set(PipelineOptions.PIPELINE_LOCAL_TIME_ZONE, "America/New_York");
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Arrays.asList(
+ new RouteDef(
+ "default_namespace.default_schema.default_table_ts_\\.*",
+ "default_namespace.default_schema.default_table_timestamp_merged",
+ null,
+ "Merge timestamp columns with different precision"),
+ new RouteDef(
+ "default_namespace.default_schema.default_table_tz_\\.*",
+ "default_namespace.default_schema.default_table_zoned_timestamp_merged",
+ null,
+ "Merge timestamp_tz columns with different precision"),
+ new RouteDef(
+ "default_namespace.default_schema.default_table_ltz_\\.*",
+ "default_namespace.default_schema.default_table_local_zoned_timestamp_merged",
+ null,
+ "Merge timestamp_ltz columns with different precision"),
+ new RouteDef(
+ "default_namespace.default_schema.default_table_\\.*",
+ "default_namespace.default_schema.default_everything_merged",
+ null,
+ "Merge all timestamp family columns with different precision")),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ String[] expected =
+ Stream.of(
+ // Merging timestamp with different precision
+ "CreateTableEvent{tableId={}_table_timestamp_merged, schema=columns={`id` INT,`name` STRING,`age` INT,`birthday` TIMESTAMP(0)}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId={}_table_timestamp_merged, before=[], after=[1, Alice, 17, 2020-01-01T14:28:57], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId={}_table_timestamp_merged, typeMapping={birthday=TIMESTAMP(9)}, oldTypeMapping={birthday=TIMESTAMP(0)}}",
+ "DataChangeEvent{tableId={}_table_timestamp_merged, before=[], after=[2, Alice, 17, 2020-01-01T14:28:57.123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_table_timestamp_merged, before=[], after=[101, Zen, 19, 2020-01-01T14:28:57], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_table_timestamp_merged, before=[], after=[102, Zen, 19, 2020-01-01T14:28:57.123456789], op=INSERT, meta=()}",
+
+ // Merging zoned timestamp with different precision
+ "CreateTableEvent{tableId={}_table_zoned_timestamp_merged, schema=columns={`id` INT,`name` STRING,`age` INT,`birthday` TIMESTAMP(0) WITH TIME ZONE}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId={}_table_zoned_timestamp_merged, before=[], after=[3, Alice, 17, 2020-01-01T14:28:57Z], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId={}_table_zoned_timestamp_merged, typeMapping={birthday=TIMESTAMP(9) WITH TIME ZONE}, oldTypeMapping={birthday=TIMESTAMP(0) WITH TIME ZONE}}",
+ "DataChangeEvent{tableId={}_table_zoned_timestamp_merged, before=[], after=[4, Alice, 17, 2020-01-01T14:28:57.123456789Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_table_zoned_timestamp_merged, before=[], after=[103, Zen, 19, 2020-01-01T14:28:57Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_table_zoned_timestamp_merged, before=[], after=[104, Zen, 19, 2020-01-01T14:28:57.123456789Z], op=INSERT, meta=()}",
+
+ // Merging local-zoned timestamp with different precision
+ "CreateTableEvent{tableId={}_table_local_zoned_timestamp_merged, schema=columns={`id` INT,`name` STRING,`age` INT,`birthday` TIMESTAMP_LTZ(0)}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId={}_table_local_zoned_timestamp_merged, before=[], after=[5, Alice, 17, 2020-01-01T14:28:57], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId={}_table_local_zoned_timestamp_merged, typeMapping={birthday=TIMESTAMP_LTZ(9)}, oldTypeMapping={birthday=TIMESTAMP_LTZ(0)}}",
+ "DataChangeEvent{tableId={}_table_local_zoned_timestamp_merged, before=[], after=[6, Alice, 17, 2020-01-01T14:28:57.123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_table_local_zoned_timestamp_merged, before=[], after=[105, Zen, 19, 2020-01-01T14:28:57], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_table_local_zoned_timestamp_merged, before=[], after=[106, Zen, 19, 2020-01-01T14:28:57.123456789], op=INSERT, meta=()}",
+
+ // Merging all
+ "CreateTableEvent{tableId={}_everything_merged, schema=columns={`id` INT,`name` STRING,`age` INT,`birthday` TIMESTAMP(0)}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[1, Alice, 17, 2020-01-01T14:28:57], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId={}_everything_merged, typeMapping={birthday=TIMESTAMP(9)}, oldTypeMapping={birthday=TIMESTAMP(0)}}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[2, Alice, 17, 2020-01-01T14:28:57.123456789], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId={}_everything_merged, typeMapping={birthday=TIMESTAMP(9) WITH TIME ZONE}, oldTypeMapping={birthday=TIMESTAMP(9)}}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[3, Alice, 17, 2020-01-01T14:28:57Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[4, Alice, 17, 2020-01-01T14:28:57.123456789Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[5, Alice, 17, 2020-01-01T04:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[6, Alice, 17, 2020-01-01T04:28:57.123456789-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[101, Zen, 19, 2020-01-01T09:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[102, Zen, 19, 2020-01-01T09:28:57.123456789-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[103, Zen, 19, 2020-01-01T14:28:57Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[104, Zen, 19, 2020-01-01T14:28:57.123456789Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[105, Zen, 19, 2020-01-01T04:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[106, Zen, 19, 2020-01-01T04:28:57.123456789-05:00], op=INSERT, meta=()}")
+ .map(s -> s.replace("{}", "default_namespace.default_schema.default"))
+ .toArray(String[]::new);
+
+ assertThat(outputEvents).containsExactlyInAnyOrder(expected);
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testMergingDecimalWithVariousPrecisions(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ List<Event> events = generateDecimalColumnEvents("default_table_");
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.singletonList(
+ new RouteDef(
+ "default_namespace.default_schema.default_table_\\.*",
+ "default_namespace.default_schema.default_everything_merged",
+ null,
+ "Merge all decimal columns with different precision")),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ String[] expected =
+ Stream.of(
+ "CreateTableEvent{tableId={}, schema=columns={`id` INT,`name` STRING,`age` INT,`fav_num` TINYINT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId={}, before=[], after=[1, Alice, 17, 1], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId={}, typeMapping={fav_num=SMALLINT}, oldTypeMapping={fav_num=TINYINT}}",
+ "DataChangeEvent{tableId={}, before=[], after=[2, Alice, 17, 22], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId={}, typeMapping={fav_num=INT}, oldTypeMapping={fav_num=SMALLINT}}",
+ "DataChangeEvent{tableId={}, before=[], after=[3, Alice, 17, 3333], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId={}, typeMapping={fav_num=BIGINT}, oldTypeMapping={fav_num=INT}}",
+ "DataChangeEvent{tableId={}, before=[], after=[4, Alice, 17, 44444444], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId={}, typeMapping={fav_num=DECIMAL(19, 0)}, oldTypeMapping={fav_num=BIGINT}}",
+ "DataChangeEvent{tableId={}, before=[], after=[5, Alice, 17, 555555555555555], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId={}, typeMapping={fav_num=DECIMAL(24, 5)}, oldTypeMapping={fav_num=DECIMAL(19, 0)}}",
+ "DataChangeEvent{tableId={}, before=[], after=[6, Alice, 17, 66666.66666], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}, before=[], after=[7, Alice, 17, 77777777.17000], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId={}, typeMapping={fav_num=DECIMAL(38, 19)}, oldTypeMapping={fav_num=DECIMAL(24, 5)}}",
+ "DataChangeEvent{tableId={}, before=[], after=[8, Alice, 17, 888888888.8888888888888888888], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}, before=[], after=[101, Zen, 19, 1.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}, before=[], after=[102, Zen, 19, 22.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}, before=[], after=[103, Zen, 19, 3333.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}, before=[], after=[104, Zen, 19, 44444444.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}, before=[], after=[105, Zen, 19, 555555555555555.0000000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}, before=[], after=[106, Zen, 19, 66666.6666600000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}, before=[], after=[107, Zen, 19, 77777777.1700000000000000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}, before=[], after=[108, Zen, 19, 888888888.8888888888888888888], op=INSERT, meta=()}")
+ .map(
+ s ->
+ s.replace(
+ "{}",
+ "default_namespace.default_schema.default_everything_merged"))
+ .toArray(String[]::new);
+
+ assertThat(outputEvents).containsExactlyInAnyOrder(expected);
+ }
+
+ private List<Event> generateTemporalColumnEvents(String tableNamePrefix) {
+ List<Event> events = new ArrayList<>();
+
+ // Initialize schemas
+ List<String> names = Arrays.asList("ts_0", "ts_9", "tz_0", "tz_9", "ltz_0", "ltz_9");
+
+ List<DataType> types =
+ Arrays.asList(
+ DataTypes.TIMESTAMP(0),
+ DataTypes.TIMESTAMP(9),
+ DataTypes.TIMESTAMP_TZ(0),
+ DataTypes.TIMESTAMP_TZ(9),
+ DataTypes.TIMESTAMP_LTZ(0),
+ DataTypes.TIMESTAMP_LTZ(9));
+
+ Instant lowPrecisionTimestamp = Instant.parse("2020-01-01T14:28:57Z");
+ Instant highPrecisionTimestamp = Instant.parse("2020-01-01T14:28:57.123456789Z");
+
+ List<Object> values =
+ Arrays.asList(
+ TimestampData.fromLocalDateTime(
+ LocalDateTime.ofInstant(lowPrecisionTimestamp, ZoneId.of("UTC"))),
+ TimestampData.fromLocalDateTime(
+ LocalDateTime.ofInstant(highPrecisionTimestamp, ZoneId.of("UTC"))),
+ ZonedTimestampData.fromZonedDateTime(
+ ZonedDateTime.ofInstant(lowPrecisionTimestamp, ZoneId.of("UTC"))),
+ ZonedTimestampData.fromZonedDateTime(
+ ZonedDateTime.ofInstant(highPrecisionTimestamp, ZoneId.of("UTC"))),
+ LocalZonedTimestampData.fromInstant(lowPrecisionTimestamp),
+ LocalZonedTimestampData.fromInstant(highPrecisionTimestamp));
+
+ List<Schema> schemas =
+ types.stream()
+ .map(
+ temporalColumnType ->
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .physicalColumn("birthday", temporalColumnType)
+ .primaryKey("id")
+ .build())
+ .collect(Collectors.toList());
+
+ for (int i = 0; i < names.size(); i++) {
+ TableId generatedTableId =
+ TableId.tableId(
+ "default_namespace", "default_schema", tableNamePrefix + names.get(i));
+ Schema generatedSchema = schemas.get(i);
+ events.add(new CreateTableEvent(generatedTableId, generatedSchema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ generatedTableId,
+ generate(generatedSchema, 1 + i, "Alice", 17, values.get(i))));
+ }
+
+ for (int i = 0; i < names.size(); i++) {
+ TableId generatedTableId =
+ TableId.tableId(
+ "default_namespace", "default_schema", tableNamePrefix + names.get(i));
+ Schema generatedSchema = schemas.get(i);
+ events.add(
+ DataChangeEvent.insertEvent(
+ generatedTableId,
+ generate(generatedSchema, 101 + i, "Zen", 19, values.get(i))));
+ }
+
+ return events;
+ }
+
+ private List<Event> generateDecimalColumnEvents(String tableNamePrefix) {
+ List<Event> events = new ArrayList<>();
+
+ // Initialize schemas
+ List<String> names =
+ Arrays.asList(
+ "tiny",
+ "small",
+ "vanilla",
+ "big",
+ "dec_15_0",
+ "decimal_10_10",
+ "decimal_16_2",
+ "decimal_29_19");
+
+ List<DataType> types =
+ Arrays.asList(
+ DataTypes.TINYINT(),
+ DataTypes.SMALLINT(),
+ DataTypes.INT(),
+ DataTypes.BIGINT(),
+ DataTypes.DECIMAL(15, 0),
+ DataTypes.DECIMAL(10, 5),
+ DataTypes.DECIMAL(16, 2),
+ DataTypes.DECIMAL(29, 19));
+
+ List<Object> values =
+ Arrays.asList(
+ (byte) 1,
+ (short) 22,
+ 3333,
+ (long) 44444444,
+ DecimalData.fromBigDecimal(new BigDecimal("555555555555555"), 15, 0),
+ DecimalData.fromBigDecimal(new BigDecimal("66666.66666"), 10, 5),
+ DecimalData.fromBigDecimal(new BigDecimal("77777777.17"), 16, 2),
+ DecimalData.fromBigDecimal(
+ new BigDecimal("888888888.8888888888888888888"), 29, 19));
+
+ List<Schema> schemas =
+ types.stream()
+ .map(
+ temporalColumnType ->
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .physicalColumn("fav_num", temporalColumnType)
+ .primaryKey("id")
+ .build())
+ .collect(Collectors.toList());
+
+ for (int i = 0; i < names.size(); i++) {
+ TableId generatedTableId =
+ TableId.tableId(
+ "default_namespace", "default_schema", tableNamePrefix + names.get(i));
+ Schema generatedSchema = schemas.get(i);
+ events.add(new CreateTableEvent(generatedTableId, generatedSchema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ generatedTableId,
+ generate(generatedSchema, 1 + i, "Alice", 17, values.get(i))));
+ }
+
+ for (int i = 0; i < names.size(); i++) {
+ TableId generatedTableId =
+ TableId.tableId(
+ "default_namespace", "default_schema", tableNamePrefix + names.get(i));
+ Schema generatedSchema = schemas.get(i);
+ events.add(
+ DataChangeEvent.insertEvent(
+ generatedTableId,
+ generate(generatedSchema, 101 + i, "Zen", 19, values.get(i))));
+ }
+
+ return events;
+ }
+
+ BinaryRecordData generate(Schema schema, Object... fields) {
+ return (new BinaryRecordDataGenerator(schema.getColumnDataTypes().toArray(new DataType[0])))
+ .generate(
+ Arrays.stream(fields)
+ .map(
+ e ->
+ (e instanceof String)
+ ? BinaryStringData.fromString((String) e)
+ : e)
+ .toArray());
+ }
+}
diff --git a/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerLenientITCase.java b/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerLenientITCase.java
new file mode 100644
index 000000000..f6bb4f429
--- /dev/null
+++ b/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerLenientITCase.java
@@ -0,0 +1,1143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.composer.flink;
+
+import org.apache.flink.cdc.common.configuration.Configuration;
+import org.apache.flink.cdc.common.data.binary.BinaryStringData;
+import org.apache.flink.cdc.common.event.AddColumnEvent;
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DataChangeEvent;
+import org.apache.flink.cdc.common.event.Event;
+import org.apache.flink.cdc.common.event.RenameColumnEvent;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.pipeline.PipelineOptions;
+import org.apache.flink.cdc.common.schema.Column;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.types.DataType;
+import org.apache.flink.cdc.common.types.DataTypes;
+import org.apache.flink.cdc.composer.PipelineExecution;
+import org.apache.flink.cdc.composer.definition.PipelineDef;
+import org.apache.flink.cdc.composer.definition.RouteDef;
+import org.apache.flink.cdc.composer.definition.SinkDef;
+import org.apache.flink.cdc.composer.definition.SourceDef;
+import org.apache.flink.cdc.composer.definition.TransformDef;
+import org.apache.flink.cdc.connectors.values.ValuesDatabase;
+import org.apache.flink.cdc.connectors.values.factory.ValuesDataFactory;
+import org.apache.flink.cdc.connectors.values.sink.ValuesDataSink;
+import org.apache.flink.cdc.connectors.values.sink.ValuesDataSinkOptions;
+import org.apache.flink.cdc.connectors.values.source.ValuesDataSourceHelper;
+import org.apache.flink.cdc.connectors.values.source.ValuesDataSourceOptions;
+import org.apache.flink.cdc.runtime.typeutils.BinaryRecordDataGenerator;
+import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
+import org.apache.flink.test.junit5.MiniClusterExtension;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.flink.cdc.connectors.values.source.ValuesDataSourceHelper.TABLE_1;
+import static org.apache.flink.cdc.connectors.values.source.ValuesDataSourceHelper.TABLE_2;
+import static org.apache.flink.configuration.CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Integration test for {@link FlinkPipelineComposer}. */
+class FlinkPipelineComposerLenientITCase {
+
+ private static final int MAX_PARALLELISM = 4;
+
+ private static final String LINE_SEPARATOR = System.lineSeparator();
+
+ // Always use parent-first classloader for CDC classes.
+ // The reason is that ValuesDatabase uses static field for holding data, we need to make sure
+ // the class is loaded by AppClassloader so that we can verify data in the test case.
+ private static final org.apache.flink.configuration.Configuration MINI_CLUSTER_CONFIG =
+ new org.apache.flink.configuration.Configuration();
+
+ static {
+ MINI_CLUSTER_CONFIG.set(
+ ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL,
+ Collections.singletonList("org.apache.flink.cdc"));
+ }
+
+ /**
+ * Use {@link MiniClusterExtension} to reduce the overhead of restarting the MiniCluster for
+ * every test case.
+ */
+ @RegisterExtension
+ static final MiniClusterExtension MINI_CLUSTER_RESOURCE =
+ new MiniClusterExtension(
+ new MiniClusterResourceConfiguration.Builder()
+ .setNumberTaskManagers(1)
+ .setNumberSlotsPerTaskManager(MAX_PARALLELISM)
+ .setConfiguration(MINI_CLUSTER_CONFIG)
+ .build());
+
+ private final PrintStream standardOut = System.out;
+ private final ByteArrayOutputStream outCaptor = new ByteArrayOutputStream();
+
+ @BeforeEach
+ void init() {
+ // Take over STDOUT as we need to check the output of values sink
+ System.setOut(new PrintStream(outCaptor));
+ // Initialize in-memory database
+ ValuesDatabase.clear();
+ }
+
+ @AfterEach
+ void cleanup() {
+ System.setOut(standardOut);
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testSingleSplitSingleTable(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_SINGLE_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> results = ValuesDatabase.getResults(TABLE_1);
+ assertThat(results)
+ .contains(
+ "default_namespace.default_schema.table1:col1=2;col2=;col3=;newCol2=;newCol3=x",
+ "default_namespace.default_schema.table1:col1=3;col2=3;col3=;newCol2=;newCol3=");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split(LINE_SEPARATOR);
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`newCol2` STRING, position=LAST, existedColumnName=null}, ColumnWithPosition{column=`newCol3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, null, null, null, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, null, null, null, ], after=[2, null, null, null, x], op=UPDATE, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testSingleSplitSingleTableWithDefaultValue(ValuesDataSink.SinkApi sinkApi)
+ throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_SINGLE_TABLE_WITH_DEFAULT_VALUE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> results = ValuesDatabase.getResults(TABLE_1);
+ assertThat(results)
+ .containsExactlyInAnyOrder(
+ "default_namespace.default_schema.table1:col1=2;col2=;col3=;newCol2=;newCol3=x;colWithDefault=;newColWithDefault=",
+ "default_namespace.default_schema.table1:col1=3;col2=3;col3=;newCol2=;newCol3=;colWithDefault=;newColWithDefault=");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split(LINE_SEPARATOR);
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`newCol2` STRING, position=LAST, existedColumnName=null}, ColumnWithPosition{column=`newCol3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, null, null, null, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, null, null, null, ], after=[2, null, null, null, x], op=UPDATE, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`colWithDefault` STRING 'flink', position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`newColWithDefault` STRING 'flink', position=LAST, existedColumnName=null}]}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testSingleSplitMultipleTables(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_MULTI_TABLES);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> table1Results = ValuesDatabase.getResults(TABLE_1);
+ assertThat(table1Results)
+ .containsExactly(
+ "default_namespace.default_schema.table1:col1=2;col2=;col3=;newCol2=;newCol3=x",
+ "default_namespace.default_schema.table1:col1=3;col2=3;col3=;newCol2=;newCol3=");
+ List<String> table2Results = ValuesDatabase.getResults(TABLE_2);
+ assertThat(table2Results)
+ .contains(
+ "default_namespace.default_schema.table2:col1=1;col2=1",
+ "default_namespace.default_schema.table2:col1=2;col2=2",
+ "default_namespace.default_schema.table2:col1=3;col2=3");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split(LINE_SEPARATOR);
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.table2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`newCol2` STRING, position=LAST, existedColumnName=null}, ColumnWithPosition{column=`newCol3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, null, null, null, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, null, null, null, 2], after=[2, null, null, null, x], op=UPDATE, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testMultiSplitsSingleTable(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.MULTI_SPLITS_SINGLE_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, MAX_PARALLELISM);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> table1Results = ValuesDatabase.getResults(TABLE_1);
+ assertThat(table1Results)
+ .contains(
+ "default_namespace.default_schema.table1:col1=1;col2=1;col3=x",
+ "default_namespace.default_schema.table1:col1=3;col2=3;col3=x",
+ "default_namespace.default_schema.table1:col1=5;col2=5;col3=");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testTransform(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.TRANSFORM_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ TransformDef transformDef =
+ new TransformDef(
+ "default_namespace.default_schema.table1",
+ "*,concat(col1,'0') as col12",
+ "col1 <> '3'",
+ "col1",
+ "col12",
+ "key1=value1",
+ "",
+ null);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ new ArrayList<>(Collections.singletonList(transformDef)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split(LINE_SEPARATOR);
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING NOT NULL,`col2` STRING,`col12` STRING}, primaryKeys=col1, partitionKeys=col12, options=({key1=value1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1, 10], op=INSERT, meta=({op_ts=1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2, 20], op=INSERT, meta=({op_ts=2})}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`newCol2` STRING, position=LAST, existedColumnName=null}, ColumnWithPosition{column=`newCol3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, null, 10, null, null, 1], after=[], op=DELETE, meta=({op_ts=4})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, null, 20, null, null, ], after=[2, null, 20, null, null, x], op=UPDATE, meta=({op_ts=5})}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testOpTypeMetadataColumn(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.TRANSFORM_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ TransformDef transformDef =
+ new TransformDef(
+ "default_namespace.default_schema.table1",
+ "*,concat(col1,'0') as col12,__data_event_type__ as rk",
+ "col1 <> '3'",
+ "col1",
+ "col12",
+ "key1=value1",
+ "",
+ null);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ new ArrayList<>(Collections.singletonList(transformDef)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split(LINE_SEPARATOR);
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING NOT NULL,`col2` STRING,`col12` STRING,`rk` STRING NOT NULL}, primaryKeys=col1, partitionKeys=col12, options=({key1=value1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1, 10, +I], op=INSERT, meta=({op_ts=1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2, 20, +I], op=INSERT, meta=({op_ts=2})}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`newCol2` STRING, position=LAST, existedColumnName=null}, ColumnWithPosition{column=`newCol3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, null, 10, -D, null, null, 1], after=[], op=DELETE, meta=({op_ts=4})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, null, 20, -U, null, null, ], after=[2, null, 20, +U, null, null, x], op=UPDATE, meta=({op_ts=5})}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testTransformTwice(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.TRANSFORM_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ TransformDef transformDef1 =
+ new TransformDef(
+ "default_namespace.default_schema.table1",
+ "*,concat(col1,'1') as col12",
+ "col1 = '1' OR col1 = '999'",
+ "col1",
+ "col12",
+ "key1=value1",
+ "",
+ null);
+ TransformDef transformDef2 =
+ new TransformDef(
+ "default_namespace.default_schema.table1",
+ "*,concat(col1,'2') as col12",
+ "col1 = '2'",
+ null,
+ null,
+ null,
+ "",
+ null);
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ new ArrayList<>(Arrays.asList(transformDef1, transformDef2)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split(LINE_SEPARATOR);
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING NOT NULL,`col2` STRING,`col12` STRING}, primaryKeys=col1, partitionKeys=col12, options=({key1=value1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1, 11], op=INSERT, meta=({op_ts=1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2, 22], op=INSERT, meta=({op_ts=2})}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`newCol2` STRING, position=LAST, existedColumnName=null}, ColumnWithPosition{column=`newCol3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, null, 11, null, null, 1], after=[], op=DELETE, meta=({op_ts=4})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, null, 22, null, null, ], after=[2, null, 22, null, null, x], op=UPDATE, meta=({op_ts=5})}");
+ }
+
+ @Test
+ void testOneToOneRouting() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_MULTI_TABLES);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup route
+ TableId routedTable1 = TableId.tableId("default_namespace", "default_schema", "routed1");
+ TableId routedTable2 = TableId.tableId("default_namespace", "default_schema", "routed2");
+ List<RouteDef> routeDef =
+ Arrays.asList(
+ new RouteDef(TABLE_1.toString(), routedTable1.toString(), null, null),
+ new RouteDef(TABLE_2.toString(), routedTable2.toString(), null, null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ routeDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> routed1Results = ValuesDatabase.getResults(routedTable1);
+ assertThat(routed1Results)
+ .contains(
+ "default_namespace.default_schema.routed1:col1=2;col2=;col3=;newCol2=;newCol3=x",
+ "default_namespace.default_schema.routed1:col1=3;col2=3;col3=;newCol2=;newCol3=");
+ List<String> routed2Results = ValuesDatabase.getResults(routedTable2);
+ assertThat(routed2Results)
+ .contains(
+ "default_namespace.default_schema.routed2:col1=1;col2=1",
+ "default_namespace.default_schema.routed2:col1=2;col2=2",
+ "default_namespace.default_schema.routed2:col1=3;col2=3");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split(LINE_SEPARATOR);
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.routed1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.routed2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.routed1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed2, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.routed1, addedColumns=[ColumnWithPosition{column=`newCol2` STRING, position=LAST, existedColumnName=null}, ColumnWithPosition{column=`newCol3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed1, before=[1, null, null, null, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.routed1, before=[2, null, null, null, 2], after=[2, null, null, null, x], op=UPDATE, meta=()}");
+ }
+
+ @Test
+ void testIdenticalOneToOneRouting() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_MULTI_TABLES);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup route
+ TableId routedTable1 = TABLE_1;
+ TableId routedTable2 = TABLE_2;
+ List<RouteDef> routeDef =
+ Arrays.asList(
+ new RouteDef(TABLE_1.toString(), routedTable1.toString(), null, null),
+ new RouteDef(TABLE_2.toString(), routedTable2.toString(), null, null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ routeDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check result in ValuesDatabase
+ List<String> routed1Results = ValuesDatabase.getResults(routedTable1);
+ assertThat(routed1Results)
+ .contains(
+ "default_namespace.default_schema.table1:col1=2;col2=;col3=;newCol2=;newCol3=x",
+ "default_namespace.default_schema.table1:col1=3;col2=3;col3=;newCol2=;newCol3=");
+ List<String> routed2Results = ValuesDatabase.getResults(routedTable2);
+ assertThat(routed2Results)
+ .contains(
+ "default_namespace.default_schema.table2:col1=1;col2=1",
+ "default_namespace.default_schema.table2:col1=2;col2=2",
+ "default_namespace.default_schema.table2:col1=3;col2=3");
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split(LINE_SEPARATOR);
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.table2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`newCol2` STRING, position=LAST, existedColumnName=null}, ColumnWithPosition{column=`newCol3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, null, null, null, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, null, null, null, 2], after=[2, null, null, null, x], op=UPDATE, meta=()}");
+ }
+
+ @Test
+ void testMergingWithRoute() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId myTable1 = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ TableId myTable2 = TableId.tableId("default_namespace", "default_schema", "mytable2");
+ Schema table1Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .primaryKey("id")
+ .build();
+ Schema table2Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT())
+ .physicalColumn("name", DataTypes.VARCHAR(255))
+ .physicalColumn("age", DataTypes.TINYINT())
+ .physicalColumn("description", DataTypes.STRING())
+ .primaryKey("id")
+ .build();
+
+ // Create test dataset:
+ // Create table 1 [id, name, age]
+ // Table 1: +I[1, Alice, 18]
+ // Table 1: +I[2, Bob, 20]
+ // Table 1: -U[2, Bob, 20] +U[2, Bob, 30]
+ // Create table 2 [id, name, age]
+ // Table 2: +I[3, Charlie, 15, student]
+ // Table 2: +I[4, Donald, 25, student]
+ // Table 2: -D[4, Donald, 25, student]
+ // Rename column for table 1: name -> last_name
+ // Add column for table 2: gender
+ // Table 1: +I[5, Eliza, 24]
+ // Table 2: +I[6, Frank, 30, student, male]
+ List<Event> events = new ArrayList<>();
+ BinaryRecordDataGenerator table1dataGenerator =
+ new BinaryRecordDataGenerator(
+ table1Schema.getColumnDataTypes().toArray(new DataType[0]));
+ BinaryRecordDataGenerator table2dataGenerator =
+ new BinaryRecordDataGenerator(
+ table2Schema.getColumnDataTypes().toArray(new DataType[0]));
+ events.add(new CreateTableEvent(myTable1, table1Schema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {1, BinaryStringData.fromString("Alice"), 18})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20})));
+ events.add(
+ DataChangeEvent.updateEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20}),
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 30})));
+ events.add(new CreateTableEvent(myTable2, table2Schema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 3L,
+ BinaryStringData.fromString("Charlie"),
+ (byte) 15,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Donald"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.deleteEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Donald"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(new RenameColumnEvent(myTable1, ImmutableMap.of("name", "last_name")));
+ events.add(
+ new AddColumnEvent(
+ myTable2,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn("gender", DataTypes.STRING())))));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {5, BinaryStringData.fromString("Eliza"), 24})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ new BinaryRecordDataGenerator(
+ new DataType[] {
+ DataTypes.BIGINT(),
+ DataTypes.VARCHAR(255),
+ DataTypes.TINYINT(),
+ DataTypes.STRING(),
+ DataTypes.STRING()
+ })
+ .generate(
+ new Object[] {
+ 6L,
+ BinaryStringData.fromString("Frank"),
+ (byte) 30,
+ BinaryStringData.fromString("student"),
+ BinaryStringData.fromString("male")
+ })));
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup route
+ TableId mergedTable = TableId.tableId("default_namespace", "default_schema", "merged");
+ List<RouteDef> routeDef =
+ Collections.singletonList(
+ new RouteDef(
+ "default_namespace.default_schema.mytable[0-9]",
+ mergedTable.toString(),
+ null,
+ null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ routeDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+ Schema mergedTableSchema = ValuesDatabase.getTableSchema(mergedTable);
+ assertThat(mergedTableSchema)
+ .isEqualTo(
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .physicalColumn("description", DataTypes.STRING())
+ .physicalColumn("last_name", DataTypes.STRING())
+ .physicalColumn("gender", DataTypes.STRING())
+ .primaryKey("id")
+ .build());
+ String[] outputEvents = outCaptor.toString().trim().split(LINE_SEPARATOR);
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.merged, schema=columns={`id` INT,`name` STRING,`age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[1, Alice, 18], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[2, Bob, 20], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[2, Bob, 20], after=[2, Bob, 30], op=UPDATE, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`description` STRING, position=LAST, existedColumnName=null}]}",
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.merged, typeMapping={id=BIGINT}, oldTypeMapping={id=INT}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[3, Charlie, 15, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[4, Donald, 25, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[4, Donald, 25, student], after=[], op=DELETE, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`last_name` STRING, position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`gender` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[5, null, 24, null, Eliza, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[6, Frank, 30, student, null, male], op=INSERT, meta=()}");
+ }
+
+ @Test
+ void testTransformMergingWithRoute() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId myTable1 = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ TableId myTable2 = TableId.tableId("default_namespace", "default_schema", "mytable2");
+ Schema table1Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .primaryKey("id")
+ .build();
+ Schema table2Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT())
+ .physicalColumn("name", DataTypes.VARCHAR(255))
+ .physicalColumn("age", DataTypes.TINYINT())
+ .physicalColumn("description", DataTypes.STRING())
+ .primaryKey("id")
+ .build();
+
+ // Create test dataset:
+ // Create table 1 [id, name, age]
+ // Table 1: +I[1, Alice, 18]
+ // Table 1: +I[2, Bob, 20]
+ // Table 1: -U[2, Bob, 20] +U[2, Bob, 30]
+ // Create table 2 [id, name, age, description]
+ // Table 2: +I[3, Charlie, 15, student]
+ // Table 2: +I[4, Donald, 25, student]
+ // Table 2: -D[4, Donald, 25, student]
+ // Rename column for table 1: name -> last_name
+ // Add column for table 2: gender
+ // Table 1: +I[5, Eliza, 24]
+ // Table 2: +I[6, Frank, 30, student, male]
+ List<Event> events = new ArrayList<>();
+ BinaryRecordDataGenerator table1dataGenerator =
+ new BinaryRecordDataGenerator(
+ table1Schema.getColumnDataTypes().toArray(new DataType[0]));
+ BinaryRecordDataGenerator table2dataGenerator =
+ new BinaryRecordDataGenerator(
+ table2Schema.getColumnDataTypes().toArray(new DataType[0]));
+ events.add(new CreateTableEvent(myTable1, table1Schema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {1, BinaryStringData.fromString("Alice"), 18})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20})));
+ events.add(
+ DataChangeEvent.updateEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20}),
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 30})));
+ events.add(new CreateTableEvent(myTable2, table2Schema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 3L,
+ BinaryStringData.fromString("Charlie"),
+ (byte) 15,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Donald"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.deleteEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Donald"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+ // events.add(new RenameColumnEvent(myTable1, ImmutableMap.of("name", "last_name")));
+ events.add(
+ new AddColumnEvent(
+ myTable2,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn("gender", DataTypes.STRING())))));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {5, BinaryStringData.fromString("Eliza"), 24})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ new BinaryRecordDataGenerator(
+ new DataType[] {
+ DataTypes.BIGINT(),
+ DataTypes.VARCHAR(255),
+ DataTypes.TINYINT(),
+ DataTypes.STRING(),
+ DataTypes.STRING()
+ })
+ .generate(
+ new Object[] {
+ 6L,
+ BinaryStringData.fromString("Frank"),
+ (byte) 30,
+ BinaryStringData.fromString("student"),
+ BinaryStringData.fromString("male")
+ })));
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ List<TransformDef> transformDef =
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.mytable[0-9]",
+ "*,'last_name' as last_name",
+ null,
+ null,
+ null,
+ null,
+ "",
+ null));
+
+ // Setup route
+ TableId mergedTable = TableId.tableId("default_namespace", "default_schema", "merged");
+ List<RouteDef> routeDef =
+ Collections.singletonList(
+ new RouteDef(
+ "default_namespace.default_schema.mytable[0-9]",
+ mergedTable.toString(),
+ null,
+ null));
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ routeDef,
+ transformDef,
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+ Schema mergedTableSchema = ValuesDatabase.getTableSchema(mergedTable);
+ assertThat(mergedTableSchema)
+ .isEqualTo(
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT().notNull())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .physicalColumn("last_name", DataTypes.STRING())
+ .physicalColumn("description", DataTypes.STRING())
+ .physicalColumn("gender", DataTypes.STRING())
+ .primaryKey("id")
+ .build());
+ String[] outputEvents = outCaptor.toString().trim().split(LINE_SEPARATOR);
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.merged, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`last_name` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[1, Alice, 18, last_name], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[2, Bob, 20, last_name], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[2, Bob, 20, last_name], after=[2, Bob, 30, last_name], op=UPDATE, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`description` STRING, position=LAST, existedColumnName=null}]}",
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.merged, typeMapping={id=BIGINT NOT NULL}, oldTypeMapping={id=INT NOT NULL}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[3, Charlie, 15, last_name, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[4, Donald, 25, last_name, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[4, Donald, 25, last_name, student], after=[], op=DELETE, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`gender` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[5, Eliza, 24, last_name, null, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[6, Frank, 30, last_name, student, male], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testRouteWithReplaceSymbol(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_MULTI_TABLES);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.singletonList(
+ new RouteDef(
+ "default_namespace.default_schema.table[0-9]",
+ "replaced_namespace.replaced_schema.__$__",
+ "__$__",
+ null)),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split(LINE_SEPARATOR);
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=replaced_namespace.replaced_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=replaced_namespace.replaced_schema.table2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=replaced_namespace.replaced_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table2, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=replaced_namespace.replaced_schema.table1, addedColumns=[ColumnWithPosition{column=`newCol2` STRING, position=LAST, existedColumnName=null}, ColumnWithPosition{column=`newCol3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table1, before=[1, null, null, null, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=replaced_namespace.replaced_schema.table1, before=[2, null, null, null, 2], after=[2, null, null, null, x], op=UPDATE, meta=()}");
+ }
+}
diff --git a/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerTest.java b/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerTest.java
new file mode 100644
index 000000000..b292b456a
--- /dev/null
+++ b/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerTest.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.composer.flink;
+
+import org.apache.flink.cdc.common.configuration.ConfigOption;
+import org.apache.flink.cdc.common.configuration.Configuration;
+import org.apache.flink.cdc.common.factories.DataSinkFactory;
+import org.apache.flink.cdc.common.factories.DataSourceFactory;
+import org.apache.flink.cdc.common.factories.FactoryHelper;
+import org.apache.flink.cdc.common.pipeline.PipelineOptions;
+import org.apache.flink.cdc.common.sink.DataSink;
+import org.apache.flink.cdc.common.sink.EventSinkProvider;
+import org.apache.flink.cdc.common.sink.MetadataApplier;
+import org.apache.flink.cdc.common.source.DataSource;
+import org.apache.flink.cdc.composer.definition.PipelineDef;
+import org.apache.flink.cdc.composer.definition.SinkDef;
+import org.apache.flink.cdc.composer.definition.SourceDef;
+import org.apache.flink.cdc.composer.utils.FactoryDiscoveryUtils;
+import org.apache.flink.cdc.composer.utils.factory.DataSinkFactory1;
+import org.apache.flink.cdc.connectors.values.factory.ValuesDataFactory;
+import org.apache.flink.cdc.connectors.values.source.ValuesDataSource;
+import org.apache.flink.cdc.connectors.values.source.ValuesDataSourceHelper;
+import org.apache.flink.configuration.DeploymentOptions;
+
+import org.apache.flink.shaded.guava33.com.google.common.collect.ImmutableMap;
+
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** A test for the {@link FlinkPipelineComposer}. */
+class FlinkPipelineComposerTest {
+
+ @Test
+ void testCreateDataSinkFromSinkDef() {
+ SinkDef sinkDef =
+ new SinkDef(
+ "data-sink-factory-1",
+ "sink-database",
+ Configuration.fromMap(
+ ImmutableMap.<String, String>builder()
+ .put("host", "0.0.0.0")
+ .build()));
+
+ DataSinkFactory sinkFactory =
+ FactoryDiscoveryUtils.getFactoryByIdentifier(
+ sinkDef.getType(), DataSinkFactory.class);
+
+ DataSink dataSink =
+ sinkFactory.createDataSink(
+ new FactoryHelper.DefaultContext(
+ sinkDef.getConfig(),
+ new Configuration(),
+ Thread.currentThread().getContextClassLoader()));
+
+ Assertions.assertThat(dataSink).isExactlyInstanceOf(DataSinkFactory1.TestDataSink.class);
+ Assertions.assertThat(((DataSinkFactory1.TestDataSink) dataSink).getHost())
+ .isEqualTo("0.0.0.0");
+ }
+
+ @Test
+ void testGettingFlinkConfiguration() {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ new SourceDef(TestDataSourceFactory.IDENTIFIER, null, new Configuration()),
+ new SinkDef(TestDataSinkFactory.IDENTIFIER, null, new Configuration()),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ new Configuration());
+
+ assertThatCode(() -> composer.compose(pipelineDef)).doesNotThrowAnyException();
+ }
+
+ /** A dummy {@link DataSinkFactory} that validates the execution target. */
+ public static class TestDataSinkFactory implements DataSinkFactory {
+
+ public static final String IDENTIFIER = "test-sink-factory";
+
+ @Override
+ public DataSink createDataSink(Context context) {
+ // This option has no default value.
+ String target = context.getFlinkConf().get(DeploymentOptions.TARGET);
+ if (!"local".equals(target)) {
+ throw new IllegalArgumentException(
+ "The flink configuration is invalid. Please check the pipeline configuration.");
+ }
+ return new DataSink() {
+ @Override
+ public EventSinkProvider getEventSinkProvider() {
+ return null;
+ }
+
+ @Override
+ public MetadataApplier getMetadataApplier() {
+ return schemaChangeEvent -> {};
+ }
+ };
+ }
+
+ @Override
+ public String identifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ return new HashSet<>();
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ return new HashSet<>();
+ }
+ }
+
+ /** A dummy {@link DataSourceFactory} that validates the execution target. */
+ public static class TestDataSourceFactory implements DataSourceFactory {
+
+ public static final String IDENTIFIER = "test-source-factory";
+
+ @Override
+ public DataSource createDataSource(Context context) {
+ // This option has no default value.
+ String target = context.getFlinkConf().get(DeploymentOptions.TARGET);
+ if (!"local".equals(target)) {
+ throw new IllegalArgumentException(
+ "The flink configuration is invalid. Please check the pipeline configuration.");
+ }
+ return new ValuesDataSource(
+ ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_SINGLE_TABLE, Integer.MAX_VALUE);
+ }
+
+ @Override
+ public String identifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ return new HashSet<>();
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ return new HashSet<>();
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource
+ void testInvalidPipelineConfiguration(Configuration pipelineConfig) {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+ PipelineDef pipelineDef = buildPipelineDefinitionFromConfiguration(pipelineConfig);
+
+ assertThatThrownBy(() -> composer.compose(pipelineDef))
+ .isExactlyInstanceOf(IllegalArgumentException.class);
+ }
+
+ static Stream<Arguments> testInvalidPipelineConfiguration() {
+ Configuration configuration1 = new Configuration();
+ configuration1.set(PipelineOptions.PIPELINE_OPERATOR_UID_PREFIX, "junit");
+ configuration1.set(PipelineOptions.PIPELINE_SCHEMA_OPERATOR_UID, "schema-operator");
+
+ return Stream.of(Arguments.of(configuration1));
+ }
+
+ @ParameterizedTest
+ @MethodSource
+ void testValidPipelineConfiguration(Configuration pipelineConfig) {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+ PipelineDef pipelineDef = buildPipelineDefinitionFromConfiguration(pipelineConfig);
+ composer.compose(pipelineDef);
+ }
+
+ static Stream<Arguments> testValidPipelineConfiguration() {
+ Configuration configuration1 = new Configuration();
+
+ Configuration configuration2 = new Configuration();
+ configuration2.set(PipelineOptions.PIPELINE_SCHEMA_OPERATOR_UID, "schema-operator");
+
+ Configuration configuration3 = new Configuration();
+ configuration3.set(PipelineOptions.PIPELINE_OPERATOR_UID_PREFIX, "junit");
+
+ return Stream.of(
+ Arguments.of(configuration1),
+ Arguments.of(configuration2),
+ Arguments.of(configuration3));
+ }
+
+ PipelineDef buildPipelineDefinitionFromConfiguration(Configuration pipelineConfig) {
+ return new PipelineDef(
+ new SourceDef(ValuesDataFactory.IDENTIFIER, null, new Configuration()),
+ new SinkDef(ValuesDataFactory.IDENTIFIER, null, new Configuration()),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+ }
+}
diff --git a/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineTransformITCase.java b/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineTransformITCase.java
new file mode 100644
index 000000000..830221b81
--- /dev/null
+++ b/flink-cdc-composer-2.x/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineTransformITCase.java
@@ -0,0 +1,3321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.composer.flink;
+
+import org.apache.flink.cdc.common.configuration.Configuration;
+import org.apache.flink.cdc.common.data.DateData;
+import org.apache.flink.cdc.common.data.DecimalData;
+import org.apache.flink.cdc.common.data.LocalZonedTimestampData;
+import org.apache.flink.cdc.common.data.TimeData;
+import org.apache.flink.cdc.common.data.TimestampData;
+import org.apache.flink.cdc.common.data.binary.BinaryRecordData;
+import org.apache.flink.cdc.common.data.binary.BinaryStringData;
+import org.apache.flink.cdc.common.event.AddColumnEvent;
+import org.apache.flink.cdc.common.event.AlterColumnTypeEvent;
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DataChangeEvent;
+import org.apache.flink.cdc.common.event.DropColumnEvent;
+import org.apache.flink.cdc.common.event.Event;
+import org.apache.flink.cdc.common.event.RenameColumnEvent;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.pipeline.PipelineOptions;
+import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
+import org.apache.flink.cdc.common.schema.Column;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.types.DataType;
+import org.apache.flink.cdc.common.types.DataTypes;
+import org.apache.flink.cdc.composer.PipelineExecution;
+import org.apache.flink.cdc.composer.definition.PipelineDef;
+import org.apache.flink.cdc.composer.definition.SinkDef;
+import org.apache.flink.cdc.composer.definition.SourceDef;
+import org.apache.flink.cdc.composer.definition.TransformDef;
+import org.apache.flink.cdc.connectors.values.ValuesDatabase;
+import org.apache.flink.cdc.connectors.values.factory.ValuesDataFactory;
+import org.apache.flink.cdc.connectors.values.sink.ValuesDataSink;
+import org.apache.flink.cdc.connectors.values.sink.ValuesDataSinkOptions;
+import org.apache.flink.cdc.connectors.values.source.ValuesDataSourceHelper;
+import org.apache.flink.cdc.connectors.values.source.ValuesDataSourceOptions;
+import org.apache.flink.cdc.runtime.typeutils.BinaryRecordDataGenerator;
+import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
+import org.apache.flink.test.junit5.MiniClusterExtension;
+
+import org.apache.flink.shaded.guava33.com.google.common.collect.ImmutableMap;
+
+import org.assertj.core.api.Assertions;
+import org.assertj.core.api.ThrowableAssert;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.flink.configuration.CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Integration test for {@link FlinkPipelineComposer}. */
+class FlinkPipelineTransformITCase {
+
+ private static final int MAX_PARALLELISM = 4;
+
+ // Always use parent-first classloader for CDC classes.
+ // The reason is that ValuesDatabase uses static field for holding data, we need to make sure
+ // the class is loaded by AppClassloader so that we can verify data in the test case.
+ private static final org.apache.flink.configuration.Configuration MINI_CLUSTER_CONFIG =
+ new org.apache.flink.configuration.Configuration();
+
+ static {
+ MINI_CLUSTER_CONFIG.set(
+ ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL,
+ Collections.singletonList("org.apache.flink.cdc"));
+ }
+
+ /**
+ * Use {@link MiniClusterExtension} to reduce the overhead of restarting the MiniCluster for
+ * every test case.
+ */
+ @RegisterExtension
+ static final MiniClusterExtension MINI_CLUSTER_RESOURCE =
+ new MiniClusterExtension(
+ new MiniClusterResourceConfiguration.Builder()
+ .setNumberTaskManagers(1)
+ .setNumberSlotsPerTaskManager(MAX_PARALLELISM)
+ .setConfiguration(MINI_CLUSTER_CONFIG)
+ .build());
+
+ private final PrintStream standardOut = System.out;
+ private final ByteArrayOutputStream outCaptor = new ByteArrayOutputStream();
+
+ @BeforeEach
+ void init() {
+ // Take over STDOUT as we need to check the output of values sink
+ System.setOut(new PrintStream(outCaptor));
+ // Initialize in-memory database
+ ValuesDatabase.clear();
+ }
+
+ @AfterEach
+ void cleanup() {
+ System.setOut(standardOut);
+ }
+
+ /** This tests if we can append calculated columns based on existing columns. */
+ @ParameterizedTest
+ @EnumSource
+ void testCalculatedColumns(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, id || name AS uid, age * 2 AS double_age",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`uid` STRING,`double_age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, 1Alice, 36], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, 2Bob, 40], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, 2Bob, 40], after=[2, Bob, 30, 2Bob, 60], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`uid` STRING,`double_age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student, 3Carol, 30], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student, 4Derrida, 50], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student, 4Derrida, 50], after=[], op=DELETE, meta=()}"));
+ }
+
+ /** This tests if we can reference a column more than once in projection expressions. */
+ @ParameterizedTest
+ @EnumSource
+ void testMultipleReferencedColumnsInProjection(ValuesDataSink.SinkApi sinkApi)
+ throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, CAST(age * age * age AS INT) AS cubic_age",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`cubic_age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, 5832], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, 8000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, 8000], after=[2, Bob, 30, 27000], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`cubic_age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student, 3375], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student, 15625], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student, 15625], after=[], op=DELETE, meta=()}"));
+ }
+
+ /** This tests if we can reference a column more than once in filtering expressions. */
+ @ParameterizedTest
+ @EnumSource
+ void testMultipleReferencedColumnsInFilter(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ null,
+ "id > 2 AND id < 4",
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT}, primaryKeys=id, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student], op=INSERT, meta=()}"));
+ }
+
+ /** This tests if we can filter out source records by rule. */
+ @ParameterizedTest
+ @EnumSource
+ void testFilteringRules(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ null,
+ "CHAR_LENGTH(name) > 3",
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18], op=INSERT, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student], after=[], op=DELETE, meta=()}"));
+ }
+
+ /**
+ * This tests if transform rule could be used to classify source records based on filtering
+ * rules.
+ */
+ @ParameterizedTest
+ @EnumSource
+ void testMultipleDispatchTransform(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Arrays.asList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, 'YOUNG' AS category",
+ "age < 20",
+ null,
+ null,
+ null,
+ null,
+ null),
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, 'OLD' AS category",
+ "age >= 20",
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`category` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, YOUNG], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, OLD], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, OLD], after=[2, Bob, 30, OLD], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`category` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student, YOUNG], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student, OLD], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student, OLD], after=[], op=DELETE, meta=()}"));
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testMultipleTransformWithDiffRefColumn(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Arrays.asList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "id,age,'Juvenile' AS roleName",
+ "age < 18",
+ null,
+ null,
+ null,
+ null,
+ null),
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "id,age,name AS roleName",
+ "age >= 18",
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`age` INT,`roleName` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, 18, Alice], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, 20, Bob], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, 20, Bob], after=[2, 30, Bob], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`age` TINYINT,`roleName` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, 15, Juvenile], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, 25, Derrida], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, 25, Derrida], after=[], op=DELETE, meta=()}"));
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testMultiTransformWithAsterisk(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Arrays.asList(
+ new TransformDef(
+ "default_namespace.default_schema.mytable2",
+ "*,'Juvenile' AS roleName",
+ "age < 18",
+ null,
+ null,
+ null,
+ null,
+ null),
+ new TransformDef(
+ "default_namespace.default_schema.mytable2",
+ "id,name,age,description,name AS roleName",
+ "age >= 18",
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT,`name` STRING,`age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20], after=[2, Bob, 30], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`roleName` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student, Juvenile], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student, Derrida], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student, Derrida], after=[], op=DELETE, meta=()}"));
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testMultiTransformMissingProjection(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Arrays.asList(
+ new TransformDef(
+ "default_namespace.default_schema.mytable2",
+ null,
+ "age < 18",
+ null,
+ null,
+ null,
+ null,
+ null),
+ new TransformDef(
+ "default_namespace.default_schema.mytable2",
+ "id,UPPER(name) AS name,age,description",
+ "age >= 18",
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT,`name` STRING,`age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20], after=[2, Bob, 30], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` STRING,`age` TINYINT,`description` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, DERRIDA, 25, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, DERRIDA, 25, student], after=[], op=DELETE, meta=()}"));
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ @Disabled("to be fixed in FLINK-37132")
+ void testMultiTransformSchemaColumnsCompatibilityWithNullProjection(
+ ValuesDataSink.SinkApi sinkApi) {
+ TransformDef nullProjection =
+ new TransformDef(
+ "default_namespace.default_schema.mytable2",
+ null,
+ "age < 18",
+ null,
+ null,
+ null,
+ null,
+ null);
+
+ assertThatThrownBy(
+ () ->
+ runGenericTransformTest(
+ sinkApi,
+ Arrays.asList(
+ nullProjection,
+ new TransformDef(
+ "default_namespace.default_schema.mytable2",
+ // reference part column
+ "id,UPPER(name) AS name",
+ "age >= 18",
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList()))
+ .rootCause()
+ .isExactlyInstanceOf(IllegalStateException.class)
+ .hasMessage(
+ "Unable to merge schema columns={`id` BIGINT,`name` VARCHAR(255),`age` TINYINT,`description` STRING}, primaryKeys=id, options=() "
+ + "and columns={`id` BIGINT,`name` STRING}, primaryKeys=id, options=() with different column counts.");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ @Disabled("to be fixed in FLINK-37132")
+ void testMultiTransformSchemaColumnsCompatibilityWithEmptyProjection(
+ ValuesDataSink.SinkApi sinkApi) {
+ TransformDef emptyProjection =
+ new TransformDef(
+ "default_namespace.default_schema.mytable2",
+ "",
+ "age < 18",
+ null,
+ null,
+ null,
+ null,
+ null);
+
+ assertThatThrownBy(
+ () ->
+ runGenericTransformTest(
+ sinkApi,
+ Arrays.asList(
+ emptyProjection,
+ new TransformDef(
+ "default_namespace.default_schema.mytable2",
+ // reference part column
+ "id,UPPER(name) AS name",
+ "age >= 18",
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList()))
+ .rootCause()
+ .isExactlyInstanceOf(IllegalStateException.class)
+ .hasMessage(
+ "Unable to merge schema columns={`id` BIGINT,`name` VARCHAR(255),`age` TINYINT,`description` STRING}, primaryKeys=id, options=() "
+ + "and columns={`id` BIGINT,`name` STRING}, primaryKeys=id, options=() with different column counts.");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testMultiTransformWithNullEmptyAsteriskProjections(ValuesDataSink.SinkApi sinkApi)
+ throws Exception {
+ TransformDef nullProjection =
+ new TransformDef(
+ "default_namespace.default_schema.mytable2",
+ null,
+ "age < 18",
+ null,
+ null,
+ null,
+ null,
+ null);
+
+ TransformDef emptyProjection =
+ new TransformDef(
+ "default_namespace.default_schema.mytable2",
+ "",
+ "age < 18",
+ null,
+ null,
+ null,
+ null,
+ null);
+
+ TransformDef asteriskProjection =
+ new TransformDef(
+ "default_namespace.default_schema.mytable2",
+ "*",
+ "age < 18",
+ null,
+ null,
+ null,
+ null,
+ null);
+
+ runGenericTransformTest(
+ sinkApi,
+ Arrays.asList(
+ // Setting projection as null, '', or * should be equivalent
+ nullProjection,
+ emptyProjection,
+ asteriskProjection,
+ new TransformDef(
+ "default_namespace.default_schema.mytable2",
+ // reference all column
+ "id,UPPER(name) AS name,age,description",
+ "age >= 18",
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT,`name` STRING,`age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20], after=[2, Bob, 30], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` STRING,`age` TINYINT,`description` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, DERRIDA, 25, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, DERRIDA, 25, student], after=[], op=DELETE, meta=()}"));
+ }
+
+ /** This tests if transform generates metadata info correctly. */
+ @ParameterizedTest
+ @EnumSource
+ void testMetadataInfo(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*",
+ null,
+ "id,name",
+ "id",
+ "replication_num=1,bucket=17",
+ "Just a Transform Block",
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING NOT NULL,`age` INT}, primaryKeys=id;name, partitionKeys=id, options=({bucket=17, replication_num=1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20], after=[2, Bob, 30], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255) NOT NULL,`age` TINYINT,`description` STRING}, primaryKeys=id;name, partitionKeys=id, options=({bucket=17, replication_num=1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student], after=[], op=DELETE, meta=()}"));
+ }
+
+ /**
+ * This tests if transform generates metadata info correctly without specifying projection /
+ * filtering rules.
+ */
+ @ParameterizedTest
+ @EnumSource
+ void testMetadataInfoWithoutChangingSchema(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ null,
+ null,
+ "id,name",
+ "id",
+ "replication_num=1,bucket=17",
+ "A Transform Block without projection or filter",
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING NOT NULL,`age` INT}, primaryKeys=id;name, partitionKeys=id, options=({bucket=17, replication_num=1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20], after=[2, Bob, 30], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255) NOT NULL,`age` TINYINT,`description` STRING}, primaryKeys=id;name, partitionKeys=id, options=({bucket=17, replication_num=1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student], after=[], op=DELETE, meta=()}"));
+ }
+
+ /** This tests if projection rule could reference metadata info correctly. */
+ @ParameterizedTest
+ @EnumSource
+ void testMetadataColumn(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "id, name, age, __namespace_name__, __schema_name__, __table_name__",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`__namespace_name__` STRING NOT NULL,`__schema_name__` STRING NOT NULL,`__table_name__` STRING NOT NULL}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, default_namespace, default_schema, mytable1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, default_namespace, default_schema, mytable1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, default_namespace, default_schema, mytable1], after=[2, Bob, 30, default_namespace, default_schema, mytable1], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`__namespace_name__` STRING NOT NULL,`__schema_name__` STRING NOT NULL,`__table_name__` STRING NOT NULL}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, default_namespace, default_schema, mytable2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, default_namespace, default_schema, mytable2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, default_namespace, default_schema, mytable2], after=[], op=DELETE, meta=()}"));
+ }
+
+ /** This tests if projection rule could reference metadata info correctly with wildcard (*). */
+ @ParameterizedTest
+ @EnumSource
+ void testMetadataColumnWithWildcard(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, __namespace_name__, __schema_name__, __table_name__",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`__namespace_name__` STRING NOT NULL,`__schema_name__` STRING NOT NULL,`__table_name__` STRING NOT NULL}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, default_namespace, default_schema, mytable1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, default_namespace, default_schema, mytable1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, default_namespace, default_schema, mytable1], after=[2, Bob, 30, default_namespace, default_schema, mytable1], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`__namespace_name__` STRING NOT NULL,`__schema_name__` STRING NOT NULL,`__table_name__` STRING NOT NULL}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student, default_namespace, default_schema, mytable2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student, default_namespace, default_schema, mytable2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student, default_namespace, default_schema, mytable2], after=[], op=DELETE, meta=()}"));
+ }
+
+ /**
+ * This tests if transform operator could distinguish metadata column identifiers and string
+ * literals.
+ */
+ @ParameterizedTest
+ @EnumSource
+ void testUsingMetadataColumnLiteralWithWildcard(ValuesDataSink.SinkApi sinkApi)
+ throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, '__namespace_name____schema_name____table_name__' AS string_literal",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`string_literal` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, __namespace_name____schema_name____table_name__], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, __namespace_name____schema_name____table_name__], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, __namespace_name____schema_name____table_name__], after=[2, Bob, 30, __namespace_name____schema_name____table_name__], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`string_literal` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student, __namespace_name____schema_name____table_name__], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student, __namespace_name____schema_name____table_name__], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student, __namespace_name____schema_name____table_name__], after=[], op=DELETE, meta=()}"));
+ }
+
+ /** This tests if projection rule could reference metadata info correctly. */
+ @ParameterizedTest
+ @EnumSource
+ void testConvertDeleteAsInsert(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "id, name, age, __namespace_name__, __schema_name__, __table_name__, __data_event_type__",
+ null,
+ null,
+ null,
+ null,
+ null,
+ "SOFT_DELETE")),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`__namespace_name__` STRING NOT NULL,`__schema_name__` STRING NOT NULL,`__table_name__` STRING NOT NULL,`__data_event_type__` STRING NOT NULL}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, default_namespace, default_schema, mytable1, +I], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, default_namespace, default_schema, mytable1, +I], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, default_namespace, default_schema, mytable1, -U], after=[2, Bob, 30, default_namespace, default_schema, mytable1, +U], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`__namespace_name__` STRING NOT NULL,`__schema_name__` STRING NOT NULL,`__table_name__` STRING NOT NULL,`__data_event_type__` STRING NOT NULL}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, default_namespace, default_schema, mytable2, +I], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, default_namespace, default_schema, mytable2, +I], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, default_namespace, default_schema, mytable2, -D], op=INSERT, meta=()}"));
+ }
+
+ /** This tests if built-in comparison functions work as expected. */
+ @ParameterizedTest
+ @EnumSource
+ void testBuiltinComparisonFunctions(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, "
+ + "id = 2 AS col1, id <> 3 AS col2, id > 2 as col3, "
+ + "id >= 2 as col4, id < 3 as col5, id <= 4 as col6, "
+ + "name IS NULL as col7, name IS NOT NULL as col8, "
+ + "id BETWEEN 1 AND 3 as col9, id NOT BETWEEN 2 AND 4 as col10, "
+ + "name LIKE 'li' as col11, name LIKE 'ro' as col12, "
+ + "CAST(id AS INT) IN (1, 3, 5) as col13, name IN ('Bob', 'Derrida') AS col14",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`col1` BOOLEAN,`col2` BOOLEAN,`col3` BOOLEAN,`col4` BOOLEAN,`col5` BOOLEAN,`col6` BOOLEAN,`col7` BOOLEAN,`col8` BOOLEAN,`col9` BOOLEAN,`col10` BOOLEAN,`col11` BOOLEAN,`col12` BOOLEAN,`col13` BOOLEAN,`col14` BOOLEAN}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, false, true, false, false, true, true, false, true, true, true, true, false, true, false], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, true, true, false, true, true, true, false, true, true, false, false, false, false, true], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, true, true, false, true, true, true, false, true, true, false, false, false, false, true], after=[2, Bob, 30, true, true, false, true, true, true, false, true, true, false, false, false, false, true], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`col1` BOOLEAN,`col2` BOOLEAN,`col3` BOOLEAN,`col4` BOOLEAN,`col5` BOOLEAN,`col6` BOOLEAN,`col7` BOOLEAN,`col8` BOOLEAN,`col9` BOOLEAN,`col10` BOOLEAN,`col11` BOOLEAN,`col12` BOOLEAN,`col13` BOOLEAN,`col14` BOOLEAN}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student, false, true, true, true, false, true, false, true, true, false, false, true, true, false], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student, false, true, true, true, false, true, false, true, false, false, false, false, false, true], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student, false, true, true, true, false, true, false, true, false, false, false, false, false, true], after=[], op=DELETE, meta=()}"));
+ }
+
+ /** This tests if built-in logical functions work as expected. */
+ @ParameterizedTest
+ @EnumSource
+ void testBuiltinLogicalFunctions(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, "
+ + "id = 2 OR true as col1, id <> 3 OR false as col2, "
+ + "name = 'Alice' AND true as col4, name <> 'Bob' AND false as col5, "
+ + "NOT id = 1 as col6, id = 3 IS FALSE as col7, "
+ + "name = 'Derrida' IS TRUE as col8, "
+ + "name <> 'Carol' IS NOT FALSE as col9, "
+ + "name <> 'Eve' IS NOT TRUE as col10",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`col1` BOOLEAN,`col2` BOOLEAN,`col4` BOOLEAN,`col5` BOOLEAN,`col6` BOOLEAN,`col7` BOOLEAN,`col8` BOOLEAN,`col9` BOOLEAN,`col10` BOOLEAN}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, true, true, true, false, false, true, false, true, false], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, true, true, false, false, true, true, false, true, false], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, true, true, false, false, true, true, false, true, false], after=[2, Bob, 30, true, true, false, false, true, true, false, true, false], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`col1` BOOLEAN,`col2` BOOLEAN,`col4` BOOLEAN,`col5` BOOLEAN,`col6` BOOLEAN,`col7` BOOLEAN,`col8` BOOLEAN,`col9` BOOLEAN,`col10` BOOLEAN}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student, true, true, false, false, true, true, false, false, false], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student, true, true, false, false, true, true, true, true, false], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student, true, true, false, false, true, true, true, true, false], after=[], op=DELETE, meta=()}"));
+ }
+
+ /** This tests if built-in arithmetic functions work as expected. */
+ @ParameterizedTest
+ @EnumSource
+ void testBuiltinArithmeticFunctions(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, "
+ + "id + 17 AS col1, id - 17 AS col2, id * 17 AS col3, "
+ + "CAST(id AS DOUBLE) / 1.7 AS col4, "
+ + "CAST(id AS INT) % 3 AS col5, ABS(id - 17) AS col6, "
+ + "CEIL(CAST(id AS DOUBLE) / 1.7) AS col7, "
+ + "FLOOR(CAST(id AS DOUBLE) / 1.7) AS col8, "
+ + "ROUND(CAST(id AS DOUBLE) / 1.7, 0) AS col9, "
+ + "CHAR_LENGTH(UUID()) AS col10",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`col1` INT,`col2` INT,`col3` INT,`col4` DOUBLE,`col5` INT,`col6` INT,`col7` DOUBLE,`col8` DOUBLE,`col9` DOUBLE,`col10` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, 18, -16, 17, 0.5882352941176471, 1, 16, 1.0, 0.0, 1.0, 36], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, 19, -15, 34, 1.1764705882352942, 2, 15, 2.0, 1.0, 1.0, 36], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, 19, -15, 34, 1.1764705882352942, 2, 15, 2.0, 1.0, 1.0, 36], after=[2, Bob, 30, 19, -15, 34, 1.1764705882352942, 2, 15, 2.0, 1.0, 1.0, 36], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`col1` BIGINT,`col2` BIGINT,`col3` BIGINT,`col4` DOUBLE,`col5` INT,`col6` BIGINT,`col7` DOUBLE,`col8` DOUBLE,`col9` DOUBLE,`col10` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student, 20, -14, 51, 1.7647058823529411, 0, 14, 2.0, 1.0, 2.0, 36], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student, 21, -13, 68, 2.3529411764705883, 1, 13, 3.0, 2.0, 2.0, 36], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student, 21, -13, 68, 2.3529411764705883, 1, 13, 3.0, 2.0, 2.0, 36], after=[], op=DELETE, meta=()}"));
+ }
+
+ /** This tests if built-in string functions work as expected. */
+ @ParameterizedTest
+ @EnumSource
+ void testBuiltinStringFunctions(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, "
+ + "'Dear ' || name AS col1, "
+ + "CHAR_LENGTH(name) AS col2, "
+ + "UPPER(name) AS col3, "
+ + "LOWER(name) AS col4, "
+ + "TRIM(name) AS col5, "
+ + "REGEXP_REPLACE(name, 'Al|Bo', '**') AS col6, "
+ + "SUBSTR(name, 1, 1) AS col7, "
+ + "SUBSTR(name, 2, 1) AS col8, "
+ + "SUBSTR(name, 3) AS col9, "
+ + "CONCAT(name, ' - ', CAST(id AS VARCHAR)) AS col10",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`col1` STRING,`col2` INT,`col3` STRING,`col4` STRING,`col5` STRING,`col6` STRING,`col7` STRING,`col8` STRING,`col9` STRING,`col10` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, Dear Alice, 5, ALICE, alice, Alice, **ice, A, l, ice, Alice - 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, Dear Bob, 3, BOB, bob, Bob, **b, B, o, b, Bob - 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, Dear Bob, 3, BOB, bob, Bob, **b, B, o, b, Bob - 2], after=[2, Bob, 30, Dear Bob, 3, BOB, bob, Bob, **b, B, o, b, Bob - 2], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`col1` STRING,`col2` INT,`col3` STRING,`col4` STRING,`col5` STRING,`col6` STRING,`col7` STRING,`col8` STRING,`col9` STRING,`col10` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student, Dear Carol, 5, CAROL, carol, Carol, Carol, C, a, rol, Carol - 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student, Dear Derrida, 7, DERRIDA, derrida, Derrida, Derrida, D, e, rrida, Derrida - 4], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student, Dear Derrida, 7, DERRIDA, derrida, Derrida, Derrida, D, e, rrida, Derrida - 4], after=[], op=DELETE, meta=()}"));
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ @Disabled("SUBSTRING ... FROM ... FOR ... isn't available until we close FLINK-35985.")
+ void testSubstringFunctions(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, "
+ + "SUBSTR(name, 0, 1) AS col1, "
+ + "SUBSTR(name, 2, 1) AS col2, "
+ + "SUBSTR(name, 3) AS col3, "
+ + "SUBSTRING(name FROM 0 FOR 1) AS col4, "
+ + "SUBSTRING(name FROM 2 FOR 1) AS col5, "
+ + "SUBSTRING(name FROM 3) AS col6",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList("To", "be", "added"));
+ }
+
+ /** This tests if built-in conditional functions work as expected. */
+ @ParameterizedTest
+ @EnumSource
+ @Disabled("This case will not run until we close FLINK-35986.")
+ void testConditionalFunctions(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, "
+ + "CASE UPPER(name)"
+ + " WHEN 'ALICE' THEN 'A - Alice'"
+ + " WHEN 'BOB' THEN 'B - Bob'"
+ + " WHEN 'CAROL' THEN 'C - Carol'"
+ + " ELSE 'D - Derrida' END AS col1, "
+ + "CASE"
+ + " WHEN id = 1 THEN '1 - One'"
+ + " WHEN id = 2 THEN '2 - Two'"
+ + " WHEN id = 3 THEN '3 - Three'"
+ + " ELSE '4 - Four' END AS col2, "
+ + "COALESCE(name, 'FALLBACK') AS col3, "
+ + "COALESCE(NULL, NULL, id, 42, NULL) AS col4, "
+ + "COALESCE(NULL, NULL, NULL, NULL, NULL) AS col5, "
+ + "IF(TRUE, 'true', 'false') AS col6, "
+ + "IF(id < 3, 'ID < 3', 'ID >= 3') AS col7, "
+ + "IF(name = 'Alice', IF(id = 1, 'YES', 'NO'), 'NO') AS col8",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Arrays.asList("Foo", "Bar", "Baz"));
+ }
+
+ /** This tests if transform temporal functions works as expected. */
+ @Test
+ void testTransformWithTemporalFunction() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId myTable1 = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ TableId myTable2 = TableId.tableId("default_namespace", "default_schema", "mytable2");
+ Schema table1Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .primaryKey("id")
+ .build();
+ Schema table2Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT())
+ .physicalColumn("name", DataTypes.VARCHAR(255))
+ .physicalColumn("age", DataTypes.TINYINT())
+ .physicalColumn("description", DataTypes.STRING())
+ .primaryKey("id")
+ .build();
+
+ List<Event> events = getTestEvents(table1Schema, table2Schema, myTable1, myTable2);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(PipelineOptions.PIPELINE_LOCAL_TIME_ZONE, "America/Los_Angeles");
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, LOCALTIME as lcl_t, CURRENT_TIME as cur_t, CAST(CURRENT_TIMESTAMP AS TIMESTAMP) as cur_ts, CAST(NOW() AS TIMESTAMP) as now_ts, LOCALTIMESTAMP as lcl_ts, CURRENT_DATE as cur_dt",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ Arrays.stream(outputEvents).forEach(this::extractDataLines);
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ public void testTransformWithColumnNameMap(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.COMPLEX_COLUMN_NAME_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup transform
+ TransformDef transformDef =
+ new TransformDef(
+ "default_namespace.default_schema.table1",
+ "*, `timestamp-type`",
+ "`foo-bar` > 0",
+ null,
+ null,
+ null,
+ null,
+ null);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(transformDef),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`class` STRING NOT NULL,`foo-bar` INT,`bar-foo` INT,`timestamp-type` STRING NOT NULL}, primaryKeys=class, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[class1, 1, 10, type1], op=INSERT, meta=({timestamp-type=type1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[class2, 2, 100, type2], op=INSERT, meta=({timestamp-type=type2})}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`import-package` STRING, position=AFTER, existedColumnName=bar-foo}]}",
+ "RenameColumnEvent{tableId=default_namespace.default_schema.table1, nameMapping={bar-foo=bar-baz}}",
+ "DropColumnEvent{tableId=default_namespace.default_schema.table1, droppedColumnNames=[bar-baz]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[class1, 1, , type1], after=[], op=DELETE, meta=({timestamp-type=type1})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[class2, 2, , type2], after=[new-class2, 20, new-package2, type2], op=UPDATE, meta=({timestamp-type=type2})}");
+ }
+
+ /** This tests if transform temporal functions works as expected. */
+ @ParameterizedTest
+ @ValueSource(strings = {"America/Los_Angeles", "UTC", "Asia/Shanghai"})
+ void testTransformWithTimestamps(String timezone) throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId myTable = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ Schema tableSchema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT().notNull())
+ .physicalColumn("ts", DataTypes.TIMESTAMP(6))
+ .physicalColumn("ts_ltz", DataTypes.TIMESTAMP_LTZ(6))
+ .primaryKey("id")
+ .build();
+
+ BinaryRecordDataGenerator generator =
+ new BinaryRecordDataGenerator(
+ tableSchema.getColumnDataTypes().toArray(new DataType[0]));
+
+ List<Event> events =
+ Arrays.asList(
+ new CreateTableEvent(myTable, tableSchema),
+ DataChangeEvent.insertEvent(
+ myTable,
+ generator.generate(
+ new Object[] {
+ 1,
+ TimestampData.fromTimestamp(
+ Timestamp.valueOf("2023-11-27 20:12:31")),
+ LocalZonedTimestampData.fromInstant(
+ toInstant("2020-07-17 18:00:22", timezone)),
+ })),
+ DataChangeEvent.insertEvent(
+ myTable,
+ generator.generate(
+ new Object[] {
+ 2,
+ TimestampData.fromTimestamp(
+ Timestamp.valueOf("2018-02-01 04:14:01")),
+ LocalZonedTimestampData.fromInstant(
+ toInstant("2019-12-31 21:00:22", timezone)),
+ })),
+ DataChangeEvent.insertEvent(
+ myTable, generator.generate(new Object[] {3, null, null})));
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(PipelineOptions.PIPELINE_LOCAL_TIME_ZONE, timezone);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "id, "
+ + "DATE_FORMAT(ts, 'yyyy~MM~dd') AS df1, "
+ + "DATE_FORMAT(ts_ltz, 'yyyy~MM~dd') AS df2, "
+ + "DATE_FORMAT(ts, 'yyyy->MM->dd / HH->mm->ss') AS df3, "
+ + "DATE_FORMAT(ts_ltz, 'yyyy->MM->dd / HH->mm->ss') AS df4, "
+ + "DATE_FORMAT(TIMESTAMPADD(SECOND, 17, ts), 'yyyy->MM->dd / HH->mm->ss') AS df5, "
+ + "DATE_FORMAT(TIMESTAMPADD(SECOND, 17, ts_ltz), 'yyyy->MM->dd / HH->mm->ss') AS df6",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ Assertions.assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`df1` STRING,`df2` STRING,`df3` STRING,`df4` STRING,`df5` STRING,`df6` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, 2023~11~27, 2020~07~17, 2023->11->27 / 20->12->31, 2020->07->17 / 18->00->22, 2023->11->27 / 20->12->48, 2020->07->17 / 18->00->39], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, 2018~02~01, 2019~12~31, 2018->02->01 / 04->14->01, 2019->12->31 / 21->00->22, 2018->02->01 / 04->14->18, 2019->12->31 / 21->00->39], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3, null, null, null, null, null, null], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testTransformMergingIncompatibleRules(ValuesDataSink.SinkApi apiVersion) {
+ Assertions.assertThatThrownBy(
+ () ->
+ runGenericTransformTest(
+ apiVersion,
+ Arrays.asList(
+ new TransformDef(
+ "\\.*.\\.*.mytable1",
+ "*, 'rule_1_matched' AS rule_1_matched",
+ "id > 0",
+ null,
+ "id",
+ null,
+ null,
+ null),
+ new TransformDef(
+ "\\.*.\\.*.\\.*",
+ "*, 'rule_fallback' AS rule_fallback",
+ null,
+ null,
+ "id",
+ null,
+ null,
+ null)),
+ Collections.emptyList()))
+ .rootCause()
+ .isExactlyInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Trying to merge transformed schemas [columns={`id` INT,`name` STRING,`age` INT,`rule_1_matched` STRING}, primaryKeys=id, partitionKeys=id, options=(), columns={`id` INT,`name` STRING,`age` INT,`rule_fallback` STRING}, primaryKeys=id, partitionKeys=id, options=()], but got more than one column name views: [[id, name, age, rule_1_matched], [id, name, age, rule_fallback]]");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testTransformWithFallbackRules(ValuesDataSink.SinkApi apiVersion) throws Exception {
+ runGenericTransformTest(
+ apiVersion,
+ Arrays.asList(
+ new TransformDef(
+ "\\.*.\\.*.mytable1",
+ "*, 'rule_1_matched' AS rule_1_matched",
+ null,
+ null,
+ "id",
+ null,
+ null,
+ null),
+ new TransformDef(
+ "\\.*.\\.*.\\.*",
+ "*, 'rule_fallback' AS rule_fallback",
+ null,
+ null,
+ "id",
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`rule_1_matched` STRING}, primaryKeys=id, partitionKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, rule_1_matched], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, rule_1_matched], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, rule_1_matched], after=[2, Bob, 30, rule_1_matched], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`rule_fallback` STRING}, primaryKeys=id, partitionKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student, rule_fallback], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student, rule_fallback], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student, rule_fallback], after=[], op=DELETE, meta=()}"));
+ }
+
+ void runGenericTransformTest(
+ ValuesDataSink.SinkApi sinkApi,
+ List<TransformDef> transformDefs,
+ List<String> expectedResults)
+ throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId myTable1 = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ TableId myTable2 = TableId.tableId("default_namespace", "default_schema", "mytable2");
+ Schema table1Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.STRING())
+ .physicalColumn("age", DataTypes.INT())
+ .primaryKey("id")
+ .build();
+ Schema table2Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.BIGINT())
+ .physicalColumn("name", DataTypes.VARCHAR(255))
+ .physicalColumn("age", DataTypes.TINYINT())
+ .physicalColumn("description", DataTypes.STRING())
+ .primaryKey("id")
+ .build();
+
+ List<Event> events = getTestEvents(table1Schema, table2Schema, myTable1, myTable2);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ transformDefs,
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ assertThat(outputEvents).containsExactly(expectedResults.toArray(new String[0]));
+ }
+
+ private static List<Event> getTestEvents(
+ Schema table1Schema, Schema table2Schema, TableId myTable1, TableId myTable2) {
+ List<Event> events = new ArrayList<>();
+ BinaryRecordDataGenerator table1dataGenerator =
+ new BinaryRecordDataGenerator(
+ table1Schema.getColumnDataTypes().toArray(new DataType[0]));
+ BinaryRecordDataGenerator table2dataGenerator =
+ new BinaryRecordDataGenerator(
+ table2Schema.getColumnDataTypes().toArray(new DataType[0]));
+ events.add(new CreateTableEvent(myTable1, table1Schema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {1, BinaryStringData.fromString("Alice"), 18})));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20})));
+ events.add(
+ DataChangeEvent.updateEvent(
+ myTable1,
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 20}),
+ table1dataGenerator.generate(
+ new Object[] {2, BinaryStringData.fromString("Bob"), 30})));
+ events.add(new CreateTableEvent(myTable2, table2Schema));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 3L,
+ BinaryStringData.fromString("Carol"),
+ (byte) 15,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.insertEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Derrida"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+ events.add(
+ DataChangeEvent.deleteEvent(
+ myTable2,
+ table2dataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Derrida"),
+ (byte) 25,
+ BinaryStringData.fromString("student")
+ })));
+ return events;
+ }
+
+ @Test
+ void testVanillaTransformWithSchemaEvolution() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId tableId = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ List<Event> events = generateSchemaEvolutionEvents(tableId);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ assertThat(outputEvents)
+ .containsExactly(
+ // Initial stage
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT,`name` STRING,`age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 21], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Barcarolle, 22], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3, Cecily, 23], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3, Cecily, 23], after=[3, Colin, 24], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Barcarolle, 22], after=[], op=DELETE, meta=()}",
+
+ // Add column stage
+ "AddColumnEvent{tableId=default_namespace.default_schema.mytable1, addedColumns=[ColumnWithPosition{column=`rank` STRING, position=FIRST, existedColumnName=null}, ColumnWithPosition{column=`gender` TINYINT, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1st, 4, Derrida, 24, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2nd, 5, Eve, 25, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2nd, 5, Eve, 25, 1], after=[2nd, 5, Eva, 20, 2], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3rd, 6, Fiona, 26, 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3rd, 6, Fiona, 26, 3], after=[], op=DELETE, meta=()}",
+
+ // Alter column type stage
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.mytable1, typeMapping={gender=INT, name=VARCHAR(17), age=DOUBLE}, oldTypeMapping={gender=TINYINT, name=STRING, age=INT}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[4th, 7, Gem, 19.0, -1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[5th, 8, Helen, 18.0, -2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[5th, 8, Helen, 18.0, -2], after=[5th, 8, Harry, 18.0, -3], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[6th, 9, IINA, 17.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[6th, 9, IINA, 17.0, 0], after=[], op=DELETE, meta=()}",
+
+ // Rename column stage
+ "RenameColumnEvent{tableId=default_namespace.default_schema.mytable1, nameMapping={gender=biological_sex, age=toshi}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[7th, 10, Julia, 24.0, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[8th, 11, Kalle, 23.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[8th, 11, Kalle, 23.0, 0], after=[8th, 11, Kella, 18.0, 0], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[9th, 12, Lynx, 17.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[9th, 12, Lynx, 17.0, 0], after=[], op=DELETE, meta=()}",
+
+ // Drop column stage
+ "DropColumnEvent{tableId=default_namespace.default_schema.mytable1, droppedColumnNames=[biological_sex, toshi]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[10th, 13, Munroe], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[11th, 14, Neko], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[11th, 14, Neko], after=[11th, 14, Nein], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[12th, 15, Oops], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[12th, 15, Oops], after=[], op=DELETE, meta=()}");
+ }
+
+ @Test
+ void testWildcardTransformWithSchemaEvolution() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId tableId = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ List<Event> events = generateSchemaEvolutionEvents(tableId);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ tableId.toString(),
+ "*",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ assertThat(outputEvents)
+ .containsExactly(
+ // Initial stage
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 21], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Barcarolle, 22], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3, Cecily, 23], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3, Cecily, 23], after=[3, Colin, 24], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Barcarolle, 22], after=[], op=DELETE, meta=()}",
+
+ // Add column stage
+ "AddColumnEvent{tableId=default_namespace.default_schema.mytable1, addedColumns=[ColumnWithPosition{column=`rank` STRING, position=BEFORE, existedColumnName=id}, ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1st, 4, Derrida, 24, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2nd, 5, Eve, 25, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2nd, 5, Eve, 25, 1], after=[2nd, 5, Eva, 20, 2], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3rd, 6, Fiona, 26, 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3rd, 6, Fiona, 26, 3], after=[], op=DELETE, meta=()}",
+
+ // Alter column type stage
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.mytable1, typeMapping={gender=INT, name=VARCHAR(17), age=DOUBLE}, oldTypeMapping={gender=TINYINT, name=STRING, age=INT}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[4th, 7, Gem, 19.0, -1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[5th, 8, Helen, 18.0, -2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[5th, 8, Helen, 18.0, -2], after=[5th, 8, Harry, 18.0, -3], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[6th, 9, IINA, 17.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[6th, 9, IINA, 17.0, 0], after=[], op=DELETE, meta=()}",
+
+ // Rename column stage
+ "RenameColumnEvent{tableId=default_namespace.default_schema.mytable1, nameMapping={gender=biological_sex, age=toshi}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[7th, 10, Julia, 24.0, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[8th, 11, Kalle, 23.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[8th, 11, Kalle, 23.0, 0], after=[8th, 11, Kella, 18.0, 0], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[9th, 12, Lynx, 17.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[9th, 12, Lynx, 17.0, 0], after=[], op=DELETE, meta=()}",
+
+ // Drop column stage
+ "DropColumnEvent{tableId=default_namespace.default_schema.mytable1, droppedColumnNames=[biological_sex, toshi]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[10th, 13, Munroe], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[11th, 14, Neko], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[11th, 14, Neko], after=[11th, 14, Nein], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[12th, 15, Oops], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[12th, 15, Oops], after=[], op=DELETE, meta=()}");
+ }
+
+ @Test
+ void testExplicitTransformWithSchemaEvolution() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId tableId = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ List<Event> events = generateSchemaEvolutionEvents(tableId);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ tableId.toString(),
+ "id, name, CAST(id AS VARCHAR) || ' -> ' || name AS extend_id",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`extend_id` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 1 -> Alice], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Barcarolle, 2 -> Barcarolle], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3, Cecily, 3 -> Cecily], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3, Cecily, 3 -> Cecily], after=[3, Colin, 3 -> Colin], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Barcarolle, 2 -> Barcarolle], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[4, Derrida, 4 -> Derrida], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[5, Eve, 5 -> Eve], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[5, Eve, 5 -> Eve], after=[5, Eva, 5 -> Eva], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[6, Fiona, 6 -> Fiona], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[6, Fiona, 6 -> Fiona], after=[], op=DELETE, meta=()}",
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.mytable1, typeMapping={name=VARCHAR(17)}, oldTypeMapping={name=STRING}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[7, Gem, 7 -> Gem], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[8, Helen, 8 -> Helen], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[8, Helen, 8 -> Helen], after=[8, Harry, 8 -> Harry], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[9, IINA, 9 -> IINA], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[9, IINA, 9 -> IINA], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[10, Julia, 10 -> Julia], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[11, Kalle, 11 -> Kalle], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[11, Kalle, 11 -> Kalle], after=[11, Kella, 11 -> Kella], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[12, Lynx, 12 -> Lynx], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[12, Lynx, 12 -> Lynx], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[13, Munroe, 13 -> Munroe], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[14, Neko, 14 -> Neko], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[14, Neko, 14 -> Neko], after=[14, Nein, 14 -> Nein], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[15, Oops, 15 -> Oops], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[15, Oops, 15 -> Oops], after=[], op=DELETE, meta=()}");
+ }
+
+ @Test
+ void testPreAsteriskWithSchemaEvolution() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId tableId = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ List<Event> events = generateSchemaEvolutionEvents(tableId);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ tableId.toString(),
+ "*, CAST(id AS VARCHAR) || ' -> ' || name AS extend_id",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ assertThat(outputEvents)
+ .containsExactly(
+ // Initial stage
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`extend_id` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 21, 1 -> Alice], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Barcarolle, 22, 2 -> Barcarolle], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3, Cecily, 23, 3 -> Cecily], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3, Cecily, 23, 3 -> Cecily], after=[3, Colin, 24, 3 -> Colin], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Barcarolle, 22, 2 -> Barcarolle], after=[], op=DELETE, meta=()}",
+
+ // Add column stage
+ "AddColumnEvent{tableId=default_namespace.default_schema.mytable1, addedColumns=[ColumnWithPosition{column=`rank` STRING, position=BEFORE, existedColumnName=id}, ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1st, 4, Derrida, 24, 0, 4 -> Derrida], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2nd, 5, Eve, 25, 1, 5 -> Eve], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2nd, 5, Eve, 25, 1, 5 -> Eve], after=[2nd, 5, Eva, 20, 2, 5 -> Eva], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3rd, 6, Fiona, 26, 3, 6 -> Fiona], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3rd, 6, Fiona, 26, 3, 6 -> Fiona], after=[], op=DELETE, meta=()}",
+
+ // Alter column type stage
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.mytable1, typeMapping={gender=INT, name=VARCHAR(17), age=DOUBLE}, oldTypeMapping={gender=TINYINT, name=STRING, age=INT}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[4th, 7, Gem, 19.0, -1, 7 -> Gem], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[5th, 8, Helen, 18.0, -2, 8 -> Helen], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[5th, 8, Helen, 18.0, -2, 8 -> Helen], after=[5th, 8, Harry, 18.0, -3, 8 -> Harry], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[6th, 9, IINA, 17.0, 0, 9 -> IINA], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[6th, 9, IINA, 17.0, 0, 9 -> IINA], after=[], op=DELETE, meta=()}",
+
+ // Rename column stage
+ "RenameColumnEvent{tableId=default_namespace.default_schema.mytable1, nameMapping={gender=biological_sex, age=toshi}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[7th, 10, Julia, 24.0, 1, 10 -> Julia], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[8th, 11, Kalle, 23.0, 0, 11 -> Kalle], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[8th, 11, Kalle, 23.0, 0, 11 -> Kalle], after=[8th, 11, Kella, 18.0, 0, 11 -> Kella], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[9th, 12, Lynx, 17.0, 0, 12 -> Lynx], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[9th, 12, Lynx, 17.0, 0, 12 -> Lynx], after=[], op=DELETE, meta=()}",
+
+ // Drop column stage
+ "DropColumnEvent{tableId=default_namespace.default_schema.mytable1, droppedColumnNames=[biological_sex, toshi]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[10th, 13, Munroe, 13 -> Munroe], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[11th, 14, Neko, 14 -> Neko], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[11th, 14, Neko, 14 -> Neko], after=[11th, 14, Nein, 14 -> Nein], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[12th, 15, Oops, 15 -> Oops], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[12th, 15, Oops, 15 -> Oops], after=[], op=DELETE, meta=()}");
+ }
+
+ @Test
+ void testPostAsteriskWithSchemaEvolution() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId tableId = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ List<Event> events = generateSchemaEvolutionEvents(tableId);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ tableId.toString(),
+ "CAST(id AS VARCHAR) || ' -> ' || name AS extend_id, *",
+ "id > 0",
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ assertThat(outputEvents)
+ .containsExactly(
+ // Initial stage
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`extend_id` STRING,`id` INT NOT NULL,`name` STRING,`age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1 -> Alice, 1, Alice, 21], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2 -> Barcarolle, 2, Barcarolle, 22], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3 -> Cecily, 3, Cecily, 23], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3 -> Cecily, 3, Cecily, 23], after=[3 -> Colin, 3, Colin, 24], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2 -> Barcarolle, 2, Barcarolle, 22], after=[], op=DELETE, meta=()}",
+
+ // Add column stage
+ "AddColumnEvent{tableId=default_namespace.default_schema.mytable1, addedColumns=[ColumnWithPosition{column=`rank` STRING, position=BEFORE, existedColumnName=id}, ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[4 -> Derrida, 1st, 4, Derrida, 24, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[5 -> Eve, 2nd, 5, Eve, 25, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[5 -> Eve, 2nd, 5, Eve, 25, 1], after=[5 -> Eva, 2nd, 5, Eva, 20, 2], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[6 -> Fiona, 3rd, 6, Fiona, 26, 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[6 -> Fiona, 3rd, 6, Fiona, 26, 3], after=[], op=DELETE, meta=()}",
+
+ // Alter column type stage
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.mytable1, typeMapping={gender=INT, name=VARCHAR(17), age=DOUBLE}, oldTypeMapping={gender=TINYINT, name=STRING, age=INT}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[7 -> Gem, 4th, 7, Gem, 19.0, -1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[8 -> Helen, 5th, 8, Helen, 18.0, -2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[8 -> Helen, 5th, 8, Helen, 18.0, -2], after=[8 -> Harry, 5th, 8, Harry, 18.0, -3], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[9 -> IINA, 6th, 9, IINA, 17.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[9 -> IINA, 6th, 9, IINA, 17.0, 0], after=[], op=DELETE, meta=()}",
+
+ // Rename column stage
+ "RenameColumnEvent{tableId=default_namespace.default_schema.mytable1, nameMapping={gender=biological_sex, age=toshi}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[10 -> Julia, 7th, 10, Julia, 24.0, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[11 -> Kalle, 8th, 11, Kalle, 23.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[11 -> Kalle, 8th, 11, Kalle, 23.0, 0], after=[11 -> Kella, 8th, 11, Kella, 18.0, 0], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[12 -> Lynx, 9th, 12, Lynx, 17.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[12 -> Lynx, 9th, 12, Lynx, 17.0, 0], after=[], op=DELETE, meta=()}",
+
+ // Drop column stage
+ "DropColumnEvent{tableId=default_namespace.default_schema.mytable1, droppedColumnNames=[biological_sex, toshi]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[13 -> Munroe, 10th, 13, Munroe], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[14 -> Neko, 11th, 14, Neko], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[14 -> Neko, 11th, 14, Neko], after=[14 -> Nein, 11th, 14, Nein], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[15 -> Oops, 12th, 15, Oops], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[15 -> Oops, 12th, 15, Oops], after=[], op=DELETE, meta=()}");
+ }
+
+ @Test
+ void testTransformWithFilterButNoProjection() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId tableId = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ List<Event> events = generateSchemaEvolutionEvents(tableId);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ null,
+ "id > 1",
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ assertThat(outputEvents)
+ .containsExactly(
+ // Initial stage
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Barcarolle, 22], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3, Cecily, 23], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3, Cecily, 23], after=[3, Colin, 24], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Barcarolle, 22], after=[], op=DELETE, meta=()}",
+
+ // Add column stage
+ "AddColumnEvent{tableId=default_namespace.default_schema.mytable1, addedColumns=[ColumnWithPosition{column=`rank` STRING, position=BEFORE, existedColumnName=id}, ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1st, 4, Derrida, 24, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2nd, 5, Eve, 25, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2nd, 5, Eve, 25, 1], after=[2nd, 5, Eva, 20, 2], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3rd, 6, Fiona, 26, 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3rd, 6, Fiona, 26, 3], after=[], op=DELETE, meta=()}",
+
+ // Alter column type stage
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.mytable1, typeMapping={gender=INT, name=VARCHAR(17), age=DOUBLE}, oldTypeMapping={gender=TINYINT, name=STRING, age=INT}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[4th, 7, Gem, 19.0, -1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[5th, 8, Helen, 18.0, -2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[5th, 8, Helen, 18.0, -2], after=[5th, 8, Harry, 18.0, -3], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[6th, 9, IINA, 17.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[6th, 9, IINA, 17.0, 0], after=[], op=DELETE, meta=()}",
+
+ // Rename column stage
+ "RenameColumnEvent{tableId=default_namespace.default_schema.mytable1, nameMapping={gender=biological_sex, age=toshi}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[7th, 10, Julia, 24.0, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[8th, 11, Kalle, 23.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[8th, 11, Kalle, 23.0, 0], after=[8th, 11, Kella, 18.0, 0], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[9th, 12, Lynx, 17.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[9th, 12, Lynx, 17.0, 0], after=[], op=DELETE, meta=()}",
+
+ // Drop column stage
+ "DropColumnEvent{tableId=default_namespace.default_schema.mytable1, droppedColumnNames=[biological_sex, toshi]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[10th, 13, Munroe], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[11th, 14, Neko], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[11th, 14, Neko], after=[11th, 14, Nein], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[12th, 15, Oops], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[12th, 15, Oops], after=[], op=DELETE, meta=()}");
+ }
+
+ @Test
+ void testTransformUnmatchedSchemaEvolution() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId tableId = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ List<Event> events = generateSchemaEvolutionEvents(tableId);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ "foo.bar.baz", // This doesn't match given tableId
+ "*",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ assertThat(outputEvents)
+ .containsExactly(
+ // Initial stage
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT,`name` STRING,`age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 21], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Barcarolle, 22], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3, Cecily, 23], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3, Cecily, 23], after=[3, Colin, 24], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Barcarolle, 22], after=[], op=DELETE, meta=()}",
+
+ // Add column stage
+ "AddColumnEvent{tableId=default_namespace.default_schema.mytable1, addedColumns=[ColumnWithPosition{column=`rank` STRING, position=BEFORE, existedColumnName=id}, ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1st, 4, Derrida, 24, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2nd, 5, Eve, 25, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2nd, 5, Eve, 25, 1], after=[2nd, 5, Eva, 20, 2], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3rd, 6, Fiona, 26, 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3rd, 6, Fiona, 26, 3], after=[], op=DELETE, meta=()}",
+
+ // Alter column type stage
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.mytable1, typeMapping={gender=INT, name=VARCHAR(17), age=DOUBLE}, oldTypeMapping={gender=TINYINT, name=STRING, age=INT}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[4th, 7, Gem, 19.0, -1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[5th, 8, Helen, 18.0, -2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[5th, 8, Helen, 18.0, -2], after=[5th, 8, Harry, 18.0, -3], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[6th, 9, IINA, 17.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[6th, 9, IINA, 17.0, 0], after=[], op=DELETE, meta=()}",
+
+ // Rename column stage
+ "RenameColumnEvent{tableId=default_namespace.default_schema.mytable1, nameMapping={gender=biological_sex, age=toshi}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[7th, 10, Julia, 24.0, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[8th, 11, Kalle, 23.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[8th, 11, Kalle, 23.0, 0], after=[8th, 11, Kella, 18.0, 0], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[9th, 12, Lynx, 17.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[9th, 12, Lynx, 17.0, 0], after=[], op=DELETE, meta=()}",
+
+ // Drop column stage
+ "DropColumnEvent{tableId=default_namespace.default_schema.mytable1, droppedColumnNames=[biological_sex, toshi]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[10th, 13, Munroe], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[11th, 14, Neko], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[11th, 14, Neko], after=[11th, 14, Nein], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[12th, 15, Oops], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[12th, 15, Oops], after=[], op=DELETE, meta=()}");
+ }
+
+ @Test
+ void testExplicitPrimaryKeyWithNullable() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId tableId = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ List<Event> events = generateSchemaEvolutionEvents(tableId);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.mytable1",
+ null,
+ null,
+ "name",
+ "id,name",
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ assertThat(outputEvents)
+ .containsExactly(
+ // Initial stage
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT,`name` STRING NOT NULL,`age` INT}, primaryKeys=name, partitionKeys=id;name, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 21], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Barcarolle, 22], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3, Cecily, 23], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3, Cecily, 23], after=[3, Colin, 24], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Barcarolle, 22], after=[], op=DELETE, meta=()}",
+
+ // Add column stage
+ "AddColumnEvent{tableId=default_namespace.default_schema.mytable1, addedColumns=[ColumnWithPosition{column=`rank` STRING, position=BEFORE, existedColumnName=id}, ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1st, 4, Derrida, 24, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2nd, 5, Eve, 25, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2nd, 5, Eve, 25, 1], after=[2nd, 5, Eva, 20, 2], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3rd, 6, Fiona, 26, 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3rd, 6, Fiona, 26, 3], after=[], op=DELETE, meta=()}",
+
+ // Alter column type stage
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.mytable1, typeMapping={gender=INT, name=VARCHAR(17), age=DOUBLE}, oldTypeMapping={gender=TINYINT, name=STRING NOT NULL, age=INT}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[4th, 7, Gem, 19.0, -1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[5th, 8, Helen, 18.0, -2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[5th, 8, Helen, 18.0, -2], after=[5th, 8, Harry, 18.0, -3], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[6th, 9, IINA, 17.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[6th, 9, IINA, 17.0, 0], after=[], op=DELETE, meta=()}",
+
+ // Rename column stage
+ "RenameColumnEvent{tableId=default_namespace.default_schema.mytable1, nameMapping={gender=biological_sex, age=toshi}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[7th, 10, Julia, 24.0, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[8th, 11, Kalle, 23.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[8th, 11, Kalle, 23.0, 0], after=[8th, 11, Kella, 18.0, 0], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[9th, 12, Lynx, 17.0, 0], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[9th, 12, Lynx, 17.0, 0], after=[], op=DELETE, meta=()}",
+
+ // Drop column stage
+ "DropColumnEvent{tableId=default_namespace.default_schema.mytable1, droppedColumnNames=[biological_sex, toshi]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[10th, 13, Munroe], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[11th, 14, Neko], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[11th, 14, Neko], after=[11th, 14, Nein], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[12th, 15, Oops], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[12th, 15, Oops], after=[], op=DELETE, meta=()}");
+ }
+
+ @Test
+ void testTransformWithCommentsAndDefaultExpr() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId myTable1 = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ TableId myTable2 = TableId.tableId("default_namespace", "default_schema", "mytable2");
+ Schema table1Schema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT(), "id column", "AUTO_INCREMENT()")
+ .physicalColumn("name", DataTypes.STRING(), "name column", "Jane Doe")
+ .physicalColumn("age", DataTypes.INT(), "age column", "17")
+ .partitionKey("id, age")
+ .primaryKey("id")
+ .build();
+ Schema table2Schema =
+ Schema.newBuilder()
+ .physicalColumn(
+ "id", DataTypes.BIGINT(), "column for id", "AUTO_DECREMENT()")
+ .physicalColumn(
+ "name", DataTypes.VARCHAR(255), "column for name", "John Smith")
+ .physicalColumn("age", DataTypes.TINYINT(), "column for age", "91")
+ .physicalColumn(
+ "description",
+ DataTypes.STRING(),
+ "column for descriptions",
+ "not important")
+ .primaryKey("id")
+ .partitionKey("id, name")
+ .build();
+
+ List<Event> events = getTestEvents(table1Schema, table2Schema, myTable1, myTable2);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ "*, name AS new_name, age + 1 AS new_age, 'extras' AS extras",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ Assertions.assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL 'id column' 'AUTO_INCREMENT()',`name` STRING 'name column' 'Jane Doe',`age` INT 'age column' '17',`new_name` STRING 'name column' 'Jane Doe',`new_age` INT,`extras` STRING}, primaryKeys=id, partitionKeys=id, age, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, Alice, 19, extras], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, Bob, 21, extras], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, Bob, 21, extras], after=[2, Bob, 30, Bob, 31, extras], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL 'column for id' 'AUTO_DECREMENT()',`name` VARCHAR(255) 'column for name' 'John Smith',`age` TINYINT 'column for age' '91',`description` STRING 'column for descriptions' 'not important',`new_name` VARCHAR(255) 'column for name' 'John Smith',`new_age` INT,`extras` STRING}, primaryKeys=id, partitionKeys=id, name, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student, Carol, 16, extras], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student, Derrida, 26, extras], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student, Derrida, 26, extras], after=[], op=DELETE, meta=()}");
+ }
+
+ String[] runNumericCastingWith(String expression) throws Exception {
+ try {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId tableId = TableId.tableId("ns", "scm", "tbl");
+ List<Event> events = generateNumericCastingEvents(tableId);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ "ns.scm.tbl",
+ expression,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ return outCaptor.toString().trim().split("\n");
+ } finally {
+ outCaptor.reset();
+ }
+ }
+
+ // Generate a projection expression like CAST(tiny AS <T>) AS tiny, ...
+ private static String generateCastTo(String type) {
+ return "id, "
+ + Stream.of(
+ "tiny",
+ "small",
+ "int",
+ "bigint",
+ "float",
+ "double",
+ "decimal",
+ "valid_char",
+ "invalid_char")
+ .map(col -> String.format("CAST(%s_c AS %s) AS %s_c", col, type, col))
+ .collect(Collectors.joining(", "));
+ }
+
+ @Test
+ void testNumericCastingsWithTruncation() throws Exception {
+ assertThat(runNumericCastingWith("*"))
+ .containsExactly(
+ "CreateTableEvent{tableId=ns.scm.tbl, schema=columns={`id` BIGINT NOT NULL,`tiny_c` TINYINT,`small_c` SMALLINT,`int_c` INT,`bigint_c` BIGINT,`float_c` FLOAT,`double_c` DOUBLE,`decimal_c` DECIMAL(10, 2),`valid_char_c` VARCHAR(17),`invalid_char_c` VARCHAR(17)}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[-1, -2, -3, -4, -5, -6.7, -8.9, -10.11, -12.13, foo], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[0, 0, 0, 0, 0, 0.0, 0.0, 0.00, 0, bar], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[1, 2, 3, 4, 5, 6.7, 8.9, 10.11, 12.13, baz], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[2, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+
+ assertThat(runNumericCastingWith(generateCastTo("BOOLEAN")))
+ .containsExactly(
+ "CreateTableEvent{tableId=ns.scm.tbl, schema=columns={`id` BIGINT NOT NULL,`tiny_c` BOOLEAN,`small_c` BOOLEAN,`int_c` BOOLEAN,`bigint_c` BOOLEAN,`float_c` BOOLEAN,`double_c` BOOLEAN,`decimal_c` BOOLEAN,`valid_char_c` BOOLEAN,`invalid_char_c` BOOLEAN}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[-1, true, true, true, true, true, true, true, false, false], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[0, false, false, false, false, false, false, false, false, false], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[1, true, true, true, true, true, true, true, false, false], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[2, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+
+ assertThat(runNumericCastingWith(generateCastTo("TINYINT")))
+ .containsExactly(
+ "CreateTableEvent{tableId=ns.scm.tbl, schema=columns={`id` BIGINT NOT NULL,`tiny_c` TINYINT,`small_c` TINYINT,`int_c` TINYINT,`bigint_c` TINYINT,`float_c` TINYINT,`double_c` TINYINT,`decimal_c` TINYINT,`valid_char_c` TINYINT,`invalid_char_c` TINYINT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[-1, -2, -3, -4, -5, -6, -8, -10, -12, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[0, 0, 0, 0, 0, 0, 0, 0, 0, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[1, 2, 3, 4, 5, 6, 8, 10, 12, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[2, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+
+ assertThat(runNumericCastingWith(generateCastTo("SMALLINT")))
+ .containsExactly(
+ "CreateTableEvent{tableId=ns.scm.tbl, schema=columns={`id` BIGINT NOT NULL,`tiny_c` SMALLINT,`small_c` SMALLINT,`int_c` SMALLINT,`bigint_c` SMALLINT,`float_c` SMALLINT,`double_c` SMALLINT,`decimal_c` SMALLINT,`valid_char_c` SMALLINT,`invalid_char_c` SMALLINT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[-1, -2, -3, -4, -5, -6, -8, -10, -12, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[0, 0, 0, 0, 0, 0, 0, 0, 0, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[1, 2, 3, 4, 5, 6, 8, 10, 12, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[2, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+
+ assertThat(runNumericCastingWith(generateCastTo("INT")))
+ .containsExactly(
+ "CreateTableEvent{tableId=ns.scm.tbl, schema=columns={`id` BIGINT NOT NULL,`tiny_c` INT,`small_c` INT,`int_c` INT,`bigint_c` INT,`float_c` INT,`double_c` INT,`decimal_c` INT,`valid_char_c` INT,`invalid_char_c` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[-1, -2, -3, -4, -5, -6, -8, -10, -12, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[0, 0, 0, 0, 0, 0, 0, 0, 0, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[1, 2, 3, 4, 5, 6, 8, 10, 12, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[2, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+
+ assertThat(runNumericCastingWith(generateCastTo("BIGINT")))
+ .containsExactly(
+ "CreateTableEvent{tableId=ns.scm.tbl, schema=columns={`id` BIGINT NOT NULL,`tiny_c` BIGINT,`small_c` BIGINT,`int_c` BIGINT,`bigint_c` BIGINT,`float_c` BIGINT,`double_c` BIGINT,`decimal_c` BIGINT,`valid_char_c` BIGINT,`invalid_char_c` BIGINT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[-1, -2, -3, -4, -5, -6, -8, -10, -12, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[0, 0, 0, 0, 0, 0, 0, 0, 0, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[1, 2, 3, 4, 5, 6, 8, 10, 12, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[2, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+
+ assertThat(runNumericCastingWith(generateCastTo("FLOAT")))
+ .containsExactly(
+ "CreateTableEvent{tableId=ns.scm.tbl, schema=columns={`id` BIGINT NOT NULL,`tiny_c` FLOAT,`small_c` FLOAT,`int_c` FLOAT,`bigint_c` FLOAT,`float_c` FLOAT,`double_c` FLOAT,`decimal_c` FLOAT,`valid_char_c` FLOAT,`invalid_char_c` FLOAT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[-1, -2.0, -3.0, -4.0, -5.0, -6.7, -8.9, -10.11, -12.13, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[1, 2.0, 3.0, 4.0, 5.0, 6.7, 8.9, 10.11, 12.13, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[2, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+
+ assertThat(runNumericCastingWith(generateCastTo("DOUBLE")))
+ .containsExactly(
+ "CreateTableEvent{tableId=ns.scm.tbl, schema=columns={`id` BIGINT NOT NULL,`tiny_c` DOUBLE,`small_c` DOUBLE,`int_c` DOUBLE,`bigint_c` DOUBLE,`float_c` DOUBLE,`double_c` DOUBLE,`decimal_c` DOUBLE,`valid_char_c` DOUBLE,`invalid_char_c` DOUBLE}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[-1, -2.0, -3.0, -4.0, -5.0, -6.699999809265137, -8.9, -10.11, -12.13, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[1, 2.0, 3.0, 4.0, 5.0, 6.699999809265137, 8.9, 10.11, 12.13, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[2, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+
+ assertThat(runNumericCastingWith(generateCastTo("DECIMAL(1, 0)")))
+ .containsExactly(
+ "CreateTableEvent{tableId=ns.scm.tbl, schema=columns={`id` BIGINT NOT NULL,`tiny_c` DECIMAL(1, 0),`small_c` DECIMAL(1, 0),`int_c` DECIMAL(1, 0),`bigint_c` DECIMAL(1, 0),`float_c` DECIMAL(1, 0),`double_c` DECIMAL(1, 0),`decimal_c` DECIMAL(1, 0),`valid_char_c` DECIMAL(1, 0),`invalid_char_c` DECIMAL(1, 0)}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[-1, -2, -3, -4, -5, -7, -9, null, null, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[0, 0, 0, 0, 0, 0, 0, 0, 0, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[1, 2, 3, 4, 5, 7, 9, null, null, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[2, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+
+ assertThat(runNumericCastingWith(generateCastTo("DECIMAL(2, 0)")))
+ .containsExactly(
+ "CreateTableEvent{tableId=ns.scm.tbl, schema=columns={`id` BIGINT NOT NULL,`tiny_c` DECIMAL(2, 0),`small_c` DECIMAL(2, 0),`int_c` DECIMAL(2, 0),`bigint_c` DECIMAL(2, 0),`float_c` DECIMAL(2, 0),`double_c` DECIMAL(2, 0),`decimal_c` DECIMAL(2, 0),`valid_char_c` DECIMAL(2, 0),`invalid_char_c` DECIMAL(2, 0)}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[-1, -2, -3, -4, -5, -7, -9, -10, -12, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[0, 0, 0, 0, 0, 0, 0, 0, 0, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[1, 2, 3, 4, 5, 7, 9, 10, 12, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[2, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+
+ assertThat(runNumericCastingWith(generateCastTo("DECIMAL(3, 1)")))
+ .containsExactly(
+ "CreateTableEvent{tableId=ns.scm.tbl, schema=columns={`id` BIGINT NOT NULL,`tiny_c` DECIMAL(3, 1),`small_c` DECIMAL(3, 1),`int_c` DECIMAL(3, 1),`bigint_c` DECIMAL(3, 1),`float_c` DECIMAL(3, 1),`double_c` DECIMAL(3, 1),`decimal_c` DECIMAL(3, 1),`valid_char_c` DECIMAL(3, 1),`invalid_char_c` DECIMAL(3, 1)}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[-1, -2.0, -3.0, -4.0, -5.0, -6.7, -8.9, -10.1, -12.1, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[1, 2.0, 3.0, 4.0, 5.0, 6.7, 8.9, 10.1, 12.1, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[2, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+
+ assertThat(runNumericCastingWith(generateCastTo("DECIMAL(19, 10)")))
+ .containsExactly(
+ "CreateTableEvent{tableId=ns.scm.tbl, schema=columns={`id` BIGINT NOT NULL,`tiny_c` DECIMAL(19, 10),`small_c` DECIMAL(19, 10),`int_c` DECIMAL(19, 10),`bigint_c` DECIMAL(19, 10),`float_c` DECIMAL(19, 10),`double_c` DECIMAL(19, 10),`decimal_c` DECIMAL(19, 10),`valid_char_c` DECIMAL(19, 10),`invalid_char_c` DECIMAL(19, 10)}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[-1, -2.0000000000, -3.0000000000, -4.0000000000, -5.0000000000, -6.7000000000, -8.9000000000, -10.1100000000, -12.1300000000, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[0, 0.0000000000, 0.0000000000, 0.0000000000, 0.0000000000, 0.0000000000, 0.0000000000, 0.0000000000, 0.0000000000, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[1, 2.0000000000, 3.0000000000, 4.0000000000, 5.0000000000, 6.7000000000, 8.9000000000, 10.1100000000, 12.1300000000, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=ns.scm.tbl, before=[], after=[2, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+ }
+
+ @Test
+ void testTransformWithLargeLiterals() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId tableId = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ List<Event> events = generateSchemaEvolutionEvents(tableId);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ "\\.*.\\.*.\\.*",
+ "*, 2147483647 AS int_max, "
+ + "2147483648 AS greater_than_int_max, "
+ + "-2147483648 AS int_min, "
+ + "-2147483649 AS less_than_int_min, "
+ + "CAST(1234567890123456789 AS DECIMAL(19, 0)) AS really_big_decimal",
+ "CAST(id AS BIGINT) + 2147483648 > 2147483649", // equivalent to id > 1
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ assertThat(outputEvents)
+ .containsExactly(
+ // Initial stage
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`int_max` INT,`greater_than_int_max` BIGINT,`int_min` INT,`less_than_int_min` BIGINT,`really_big_decimal` DECIMAL(19, 0)}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Barcarolle, 22, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3, Cecily, 23, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3, Cecily, 23, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], after=[3, Colin, 24, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Barcarolle, 22, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], after=[], op=DELETE, meta=()}",
+
+ // Add Column
+ "AddColumnEvent{tableId=default_namespace.default_schema.mytable1, addedColumns=[ColumnWithPosition{column=`rank` STRING, position=BEFORE, existedColumnName=id}, ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1st, 4, Derrida, 24, 0, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2nd, 5, Eve, 25, 1, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2nd, 5, Eve, 25, 1, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], after=[2nd, 5, Eva, 20, 2, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[3rd, 6, Fiona, 26, 3, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[3rd, 6, Fiona, 26, 3, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], after=[], op=DELETE, meta=()}",
+
+ // Alter column type
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.mytable1, typeMapping={gender=INT, name=VARCHAR(17), age=DOUBLE}, oldTypeMapping={gender=TINYINT, name=STRING, age=INT}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[4th, 7, Gem, 19.0, -1, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[5th, 8, Helen, 18.0, -2, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[5th, 8, Helen, 18.0, -2, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], after=[5th, 8, Harry, 18.0, -3, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[6th, 9, IINA, 17.0, 0, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[6th, 9, IINA, 17.0, 0, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], after=[], op=DELETE, meta=()}",
+
+ // Rename column
+ "RenameColumnEvent{tableId=default_namespace.default_schema.mytable1, nameMapping={gender=biological_sex, age=toshi}}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[7th, 10, Julia, 24.0, 1, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[8th, 11, Kalle, 23.0, 0, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[8th, 11, Kalle, 23.0, 0, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], after=[8th, 11, Kella, 18.0, 0, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[9th, 12, Lynx, 17.0, 0, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[9th, 12, Lynx, 17.0, 0, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], after=[], op=DELETE, meta=()}",
+
+ // Drop column
+ "DropColumnEvent{tableId=default_namespace.default_schema.mytable1, droppedColumnNames=[biological_sex, toshi]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[10th, 13, Munroe, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[11th, 14, Neko, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[11th, 14, Neko, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], after=[11th, 14, Nein, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[12th, 15, Oops, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[12th, 15, Oops, 2147483647, 2147483648, -2147483648, -2147483649, 1234567890123456789], after=[], op=DELETE, meta=()}");
+ }
+
+ @Test
+ void testFloorCeilAndRoundFunction() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId tableId = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ List<Event> events = generateFloorCeilAndRoundEvents(tableId);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ "\\.*.\\.*.\\.*",
+ "*, "
+ + "CEIL(tinyint_col) AS ceil_tinyint,"
+ + "CEIL(smallint_col) AS ceil_smallint,"
+ + "CEIL(int_col) AS ceil_int,"
+ + "CEIL(bigint_col) AS ceil_bigint,"
+ + "CEIL(float_col) AS ceil_float,"
+ + "CEIL(double_col) AS ceil_double,"
+ + "CEIL(decimal_col) AS ceil_decimal,"
+ + "CEILING(tinyint_col) AS ceiling_tinyint,"
+ + "CEILING(smallint_col) AS ceiling_smallint,"
+ + "CEILING(int_col) AS ceiling_int,"
+ + "CEILING(bigint_col) AS ceiling_bigint,"
+ + "CEILING(float_col) AS ceiling_float,"
+ + "CEILING(double_col) AS ceiling_double,"
+ + "CEILING(decimal_col) AS ceiling_decimal,"
+ + "FLOOR(tinyint_col) AS floor_tinyint,"
+ + "FLOOR(smallint_col) AS floor_smallint,"
+ + "FLOOR(int_col) AS floor_int,"
+ + "FLOOR(bigint_col) AS floor_bigint,"
+ + "FLOOR(float_col) AS floor_float,"
+ + "FLOOR(double_col) AS floor_double,"
+ + "FLOOR(decimal_col) AS floor_decimal,"
+ + "ROUND(tinyint_col, 2) AS round_tinyint,"
+ + "ROUND(smallint_col, 2) AS round_smallint,"
+ + "ROUND(int_col, 2) AS round_int,"
+ + "ROUND(bigint_col, 2) AS round_bigint,"
+ + "ROUND(float_col, 2) AS round_float,"
+ + "ROUND(double_col, 2) AS round_double,"
+ + "ROUND(decimal_col, 2) AS round_decimal,"
+ + "ROUND(tinyint_col, 0) AS round_0_tinyint,"
+ + "ROUND(smallint_col, 0) AS round_0_smallint,"
+ + "ROUND(int_col, 0) AS round_0_int,"
+ + "ROUND(bigint_col, 0) AS round_0_bigint,"
+ + "ROUND(float_col, 0) AS round_0_float,"
+ + "ROUND(double_col, 0) AS round_0_double,"
+ + "ROUND(decimal_col, 0) AS round_0_decimal",
+ null,
+ "id",
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`tinyint_col` TINYINT,`smallint_col` SMALLINT,`int_col` INT,`bigint_col` BIGINT,`float_col` FLOAT,`double_col` DOUBLE,`decimal_col` DECIMAL(10, 3),`ceil_tinyint` TINYINT,`ceil_smallint` SMALLINT,`ceil_int` INT,`ceil_bigint` BIGINT,`ceil_float` FLOAT,`ceil_double` DOUBLE,`ceil_decimal` DECIMAL(10, 0),`ceiling_tinyint` TINYINT,`ceiling_smallint` SMALLINT,`ceiling_i [...]
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, 1, 1, 1, 1, 1.1, 1.1, 1.100, 1, 1, 1, 1, 2.0, 2.0, 2, 1, 1, 1, 1, 2.0, 2.0, 2, 1, 1, 1, 1, 1.0, 1.0, 1, 1, 1, 1, 1, 1.1, 1.1, 1.10, 1, 1, 1, 1, 1.0, 1.0, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[4, 4, 4, 4, 4, 4.44, 4.44, 4.440, 4, 4, 4, 4, 5.0, 5.0, 5, 4, 4, 4, 4, 5.0, 5.0, 5, 4, 4, 4, 4, 4.0, 4.0, 4, 4, 4, 4, 4, 4.44, 4.44, 4.44, 4, 4, 4, 4, 4.0, 4.0, 4], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[5, 5, 5, 5, 5, 5.555, 5.555, 5.555, 5, 5, 5, 5, 6.0, 6.0, 6, 5, 5, 5, 5, 6.0, 6.0, 6, 5, 5, 5, 5, 5.0, 5.0, 5, 5, 5, 5, 5, 5.56, 5.56, 5.56, 5, 5, 5, 5, 6.0, 6.0, 6], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[9, 9, 9, 9, 9, 1.0E7, 9999999.999, 9999999.999, 9, 9, 9, 9, 1.0E7, 1.0E7, 10000000, 9, 9, 9, 9, 1.0E7, 1.0E7, 10000000, 9, 9, 9, 9, 1.0E7, 9999999.0, 9999999, 9, 9, 9, 9, 1.0E7, 1.0E7, 10000000.00, 9, 9, 9, 9, 1.0E7, 1.0E7, 10000000], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[0, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+ }
+
+ @Test
+ void testAbsFunction() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId tableId = TableId.tableId("default_namespace", "default_schema", "mytable1");
+ List<Event> events = generateAbsEvents(tableId);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ "\\.*.\\.*.\\.*",
+ "*, "
+ + "ABS(tinyint_col) AS abs_tinyint,"
+ + "ABS(smallint_col) AS abs_smallint,"
+ + "ABS(int_col) AS abs_int,"
+ + "ABS(bigint_col) AS abs_bigint,"
+ + "ABS(float_col) AS abs_float,"
+ + "ABS(double_col) AS abs_double,"
+ + "ABS(decimal_col) AS abs_decimal",
+ null,
+ "id",
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`tinyint_col` TINYINT,`smallint_col` SMALLINT,`int_col` INT,`bigint_col` BIGINT,`float_col` FLOAT,`double_col` DOUBLE,`decimal_col` DECIMAL(10, 2),`abs_tinyint` TINYINT,`abs_smallint` SMALLINT,`abs_int` INT,`abs_bigint` BIGINT,`abs_float` FLOAT,`abs_double` DOUBLE,`abs_decimal` DECIMAL(10, 2)}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, 1, 1, 1, 1, 1.1, 1.1, 1.10, 1, 1, 1, 1, 1.1, 1.1, 1.10], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[-4, -4, -4, -4, -4, -4.44, -4.44, -4.44, 4, 4, 4, 4, 4.44, 4.44, 4.44], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[-9, -9, -9, -9, -9, -1.0E8, -9.999999999E7, -99999999.99, 9, 9, 9, 9, 1.0E8, 9.999999999E7, 99999999.99], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[0, null, null, null, null, null, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+ }
+
+ private ThrowableAssert.ThrowingCallable expectTransformError(
+ String projectionRule, String filterRule, String primaryKeys) {
+ return () ->
+ runGenericTransformTest(
+ ValuesDataSink.SinkApi.SINK_V2,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ projectionRule,
+ filterRule,
+ primaryKeys,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList());
+ }
+
+ @Test
+ void testShadeOriginalColumnsWithDifferentType() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.TRANSFORM_TABLE);
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+ Configuration sinkConfig = new Configuration();
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(
+ new TransformDef(
+ "\\.*.\\.*.\\.*",
+ "*, 0.5 + CAST(col1 AS DOUBLE) AS col1",
+ "col1 > 1.5",
+ null,
+ null,
+ null,
+ null,
+ null)),
+ Collections.emptyList(),
+ pipelineConfig);
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+ assertThat(outputEvents)
+ .containsExactly(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` DOUBLE NOT NULL,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2.5, 2], op=INSERT, meta=({op_ts=2})}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3.5, 3], op=INSERT, meta=({op_ts=3})}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=AFTER, existedColumnName=col2}]}",
+ "RenameColumnEvent{tableId=default_namespace.default_schema.table1, nameMapping={col2=newCol2, col3=newCol3}}",
+ "DropColumnEvent{tableId=default_namespace.default_schema.table1, droppedColumnNames=[newCol2]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2.5, ], after=[2.5, x], op=UPDATE, meta=({op_ts=5})}");
+ }
+
+ private static final String[] UNICODE_STRINGS = {
+ "ascii test!?",
+ "大五",
+ "测试数据",
+ "ひびぴ",
+ "죠주쥬",
+ "ÀÆÉ",
+ "ÓÔŐÖ",
+ "αβγδε",
+ "בבקשה",
+ "твой",
+ "ภาษาไทย",
+ "piedzimst brīvi"
+ };
+
+ @ParameterizedTest
+ @EnumSource
+ void testTransformProjectionWithUnicodeCharacters(ValuesDataSink.SinkApi sinkApi)
+ throws Exception {
+ for (String unicodeString : UNICODE_STRINGS) {
+ List<String> expected =
+ Stream.of(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`prefix` STRING,`id` INT NOT NULL,`name` STRING,`age` INT,`suffix` STRING}, primaryKeys=id, partitionKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[{UNICODE_STRING} -> 1, 1, Alice, 18, 1 <- {UNICODE_STRING}], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[{UNICODE_STRING} -> 2, 2, Bob, 20, 2 <- {UNICODE_STRING}], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[{UNICODE_STRING} -> 2, 2, Bob, 20, 2 <- {UNICODE_STRING}], after=[{UNICODE_STRING} -> 2, 2, Bob, 30, 2 <- {UNICODE_STRING}], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`prefix` STRING,`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`suffix` STRING}, primaryKeys=id, partitionKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[{UNICODE_STRING} -> 3, 3, Carol, 15, student, 3 <- {UNICODE_STRING}], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[{UNICODE_STRING} -> 4, 4, Derrida, 25, student, 4 <- {UNICODE_STRING}], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[{UNICODE_STRING} -> 4, 4, Derrida, 25, student, 4 <- {UNICODE_STRING}], after=[], op=DELETE, meta=()}")
+ .map(tpl -> tpl.replace("{UNICODE_STRING}", unicodeString))
+ .collect(Collectors.toList());
+
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "\\.*.\\.*.\\.*",
+ String.format(
+ "'%s' || ' -> ' || id AS prefix, *, id || ' <- ' || '%s' AS suffix",
+ unicodeString, unicodeString),
+ null,
+ null,
+ "id",
+ null,
+ null,
+ null)),
+ expected);
+ outCaptor.reset();
+ }
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testTransformFilterWithUnicodeCharacters(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ for (String unicodeString : UNICODE_STRINGS) {
+ List<String> expected =
+ Stream.of(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`extras` STRING}, primaryKeys=id, partitionKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18, {UNICODE_STRING}], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20, {UNICODE_STRING}], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20, {UNICODE_STRING}], after=[2, Bob, 30, {UNICODE_STRING}], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`extras` STRING}, primaryKeys=id, partitionKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student, {UNICODE_STRING}], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[4, Derrida, 25, student, {UNICODE_STRING}], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[4, Derrida, 25, student, {UNICODE_STRING}], after=[], op=DELETE, meta=()}")
+ .map(tpl -> tpl.replace("{UNICODE_STRING}", unicodeString))
+ .collect(Collectors.toList());
+
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "\\.*.\\.*.\\.*",
+ String.format("*, '%s' AS extras", unicodeString),
+ String.format("extras = '%s'", unicodeString),
+ null,
+ "id",
+ null,
+ null,
+ null)),
+ expected);
+ outCaptor.reset();
+
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "\\.*.\\.*.\\.*",
+ String.format("*, '%s' AS extras", unicodeString),
+ String.format("extras <> '%s'", unicodeString),
+ null,
+ "id",
+ null,
+ null,
+ null)),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT,`extras` STRING}, primaryKeys=id, partitionKeys=id, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING,`extras` STRING}, primaryKeys=id, partitionKeys=id, options=()}"));
+ outCaptor.reset();
+ }
+ }
+
+ @Test
+ void testDateAndTimeCastingFunctions() throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ ValuesDataSourceOptions.EVENT_SET_ID,
+ ValuesDataSourceHelper.EventSetId.CUSTOM_SOURCE_EVENTS);
+
+ TableId tableId = TableId.tableId("default_namespace", "default_schema", "my_table");
+ Schema tableSchema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("date_0", DataTypes.DATE())
+ .physicalColumn("time_0", DataTypes.TIME(0))
+ .physicalColumn("time_3", DataTypes.TIME(3))
+ .physicalColumn("time_6", DataTypes.TIME(6))
+ .physicalColumn("time_9", DataTypes.TIME(9))
+ .primaryKey("id")
+ .build();
+ BinaryRecordDataGenerator generator =
+ new BinaryRecordDataGenerator(
+ tableSchema.getColumnDataTypes().toArray(new DataType[0]));
+
+ List<Event> events =
+ Arrays.asList(
+ new CreateTableEvent(tableId, tableSchema),
+ DataChangeEvent.insertEvent(
+ tableId,
+ generator.generate(
+ new Object[] {
+ 1,
+ DateData.fromLocalDate(LocalDate.of(1999, 12, 31)),
+ TimeData.fromLocalTime(LocalTime.of(21, 48, 25)),
+ TimeData.fromLocalTime(
+ LocalTime.of(21, 48, 25, 123000000)),
+ TimeData.fromLocalTime(
+ LocalTime.of(21, 48, 25, 123456000)),
+ TimeData.fromLocalTime(
+ LocalTime.of(21, 48, 25, 123456789))
+ })),
+ DataChangeEvent.insertEvent(
+ tableId,
+ generator.generate(
+ new Object[] {2, null, null, null, null, null})));
+
+ TransformDef transformDef =
+ new TransformDef(
+ tableId.toString(),
+ "*,"
+ + "CAST(date_0 AS VARCHAR) AS date_0_str,"
+ + "CAST(time_0 AS VARCHAR) AS time_0_str,"
+ + "CAST(time_3 AS VARCHAR) AS time_3_str,"
+ + "CAST(time_6 AS VARCHAR) AS time_6_str,"
+ + "CAST(time_9 AS VARCHAR) AS time_9_str",
+ null,
+ "id",
+ null,
+ null,
+ null,
+ null);
+
+ ValuesDataSourceHelper.setSourceEvents(Collections.singletonList(events));
+
+ SourceDef sourceDef =
+ new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source", sourceConfig);
+
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, ValuesDataSink.SinkApi.SINK_V2);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.singletonList(transformDef),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ PipelineExecution execution = composer.compose(pipelineDef);
+ execution.execute();
+
+ // Check the order and content of all received events
+ String[] outputEvents = outCaptor.toString().trim().split("\n");
+
+ assertThat(outputEvents)
+ .containsExactlyInAnyOrder(
+ "CreateTableEvent{tableId=default_namespace.default_schema.my_table, schema=columns={`id` INT NOT NULL,`date_0` DATE,`time_0` TIME(0),`time_3` TIME(3),`time_6` TIME(6),`time_9` TIME(9),`date_0_str` STRING,`time_0_str` STRING,`time_3_str` STRING,`time_6_str` STRING,`time_9_str` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.my_table, before=[], after=[1, 1999-12-31, 21:48:25, 21:48:25.123, 21:48:25.123, 21:48:25.123, 1999-12-31, 21:48:25, 21:48:25.123, 21:48:25.123, 21:48:25.123], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.my_table, before=[], after=[2, null, null, null, null, null, null, null, null, null, null], op=INSERT, meta=()}");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testPostTransformConvertersWoProjection(ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runGenericTransformTest(
+ sinkApi,
+ Collections.singletonList(
+ new TransformDef(
+ "default_namespace.default_schema.\\.*",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ "SOFT_DELETE")),
+ Arrays.asList(
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable1, schema=columns={`id` INT NOT NULL,`name` STRING,`age` INT}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[1, Alice, 18], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[], after=[2, Bob, 20], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable1, before=[2, Bob, 20], after=[2, Bob, 30], op=UPDATE, meta=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.mytable2, schema=columns={`id` BIGINT NOT NULL,`name` VARCHAR(255),`age` TINYINT,`description` STRING}, primaryKeys=id, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.mytable2, before=[], after=[3, Carol, 15, student], op=INSERT, meta=()}",
... 8915 lines suppressed ...