You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tw...@apache.org on 2021/08/31 08:16:31 UTC

[flink] branch master updated (8abf8c6 -> 4726878)

This is an automated email from the ASF dual-hosted git repository.

twalthr pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.


    from 8abf8c6  [FLINK-23715][parquet] Support for reading fields that do not exist in Parquet files
     new bc69c1c  [FLINK-24004][table-planner] Introduce TableFactoryHarness
     new c1bf5da  [FLINK-23663][table-planner] Push primary key filters through ChangelogNormalize
     new 4726878  [hotfix][table-planner] Migrate PushProjectIntoTableSourceScanRuleTest to test harness

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../apache/flink/table/api/TableDescriptor.java    |  19 +-
 .../table/api/internal/TableEnvironmentImpl.java   |  26 +-
 .../flink/table/catalog/DefaultCatalogTable.java   |   4 +-
 .../PushFilterPastChangelogNormalizeRule.java      | 250 ++++++++++++
 .../planner/plan/rules/FlinkStreamRuleSets.scala   |   5 +-
 .../planner/factories/TableFactoryHarness.java     | 423 +++++++++++++++++++++
 .../PushProjectIntoTableSourceScanRuleTest.java    | 179 +++------
 .../PushFilterPastChangelogNormalizeRuleTest.java  | 101 +++++
 .../org.apache.flink.table.factories.Factory       |   2 +-
 .../PushFilterPastChangelogNormalizeRuleTest.xml   |  82 ++++
 10 files changed, 944 insertions(+), 147 deletions(-)
 create mode 100644 flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/PushFilterPastChangelogNormalizeRule.java
 create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TableFactoryHarness.java
 create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/physical/stream/PushFilterPastChangelogNormalizeRuleTest.java
 create mode 100644 flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/physical/stream/PushFilterPastChangelogNormalizeRuleTest.xml

[flink] 01/03: [FLINK-24004][table-planner] Introduce TableFactoryHarness

Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit bc69c1c77ec94a57b3603c5eefcbddb8ce83ea64
Author: Ingo Bürk <in...@tngtech.com>
AuthorDate: Mon Aug 30 08:39:53 2021 +0200

    [FLINK-24004][table-planner] Introduce TableFactoryHarness
---
 .../apache/flink/table/api/TableDescriptor.java    |  19 +-
 .../table/api/internal/TableEnvironmentImpl.java   |  26 +-
 .../flink/table/catalog/DefaultCatalogTable.java   |   4 +-
 .../planner/factories/TableFactoryHarness.java     | 423 +++++++++++++++++++++
 .../org.apache.flink.table.factories.Factory       |   1 +
 5 files changed, 445 insertions(+), 28 deletions(-)

diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/TableDescriptor.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/TableDescriptor.java
index da21a8c..e60ad10 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/TableDescriptor.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/TableDescriptor.java
@@ -51,14 +51,14 @@ import java.util.stream.Collectors;
  * TableEnvironment#createTemporaryTable(String, TableDescriptor)}.
  */
 @PublicEvolving
-public final class TableDescriptor {
+public class TableDescriptor {
 
     private final @Nullable Schema schema;
     private final Map<String, String> options;
     private final List<String> partitionKeys;
     private final @Nullable String comment;
 
-    private TableDescriptor(
+    protected TableDescriptor(
             @Nullable Schema schema,
             Map<String, String> options,
             List<String> partitionKeys,
@@ -101,6 +101,21 @@ public final class TableDescriptor {
 
     // ---------------------------------------------------------------------------------------------
 
+    /** Converts this descriptor into a {@link CatalogTable}. */
+    public CatalogTable toCatalogTable() {
+        final Schema schema =
+                getSchema()
+                        .orElseThrow(
+                                () ->
+                                        new ValidationException(
+                                                "Missing schema in TableDescriptor. "
+                                                        + "A schema is typically required. "
+                                                        + "It can only be omitted at certain "
+                                                        + "documented locations."));
+
+        return CatalogTable.of(schema, getComment().orElse(null), getPartitionKeys(), getOptions());
+    }
+
     /** Converts this immutable instance into a mutable {@link Builder}. */
     public Builder toBuilder() {
         return new Builder(this);
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java
index 78e1b35..892645c 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java
@@ -29,7 +29,6 @@ import org.apache.flink.table.api.DataTypes;
 import org.apache.flink.table.api.EnvironmentSettings;
 import org.apache.flink.table.api.ExplainDetail;
 import org.apache.flink.table.api.ResultKind;
-import org.apache.flink.table.api.Schema;
 import org.apache.flink.table.api.SqlParserException;
 import org.apache.flink.table.api.StatementSet;
 import org.apache.flink.table.api.Table;
@@ -470,8 +469,7 @@ public class TableEnvironmentImpl implements TableEnvironmentInternal {
     private void createTemporaryTableInternal(
             UnresolvedIdentifier path, TableDescriptor descriptor) {
         final ObjectIdentifier tableIdentifier = catalogManager.qualifyIdentifier(path);
-        final CatalogTable catalogTable = convertTableDescriptor(descriptor);
-        catalogManager.createTemporaryTable(catalogTable, tableIdentifier, false);
+        catalogManager.createTemporaryTable(descriptor.toCatalogTable(), tableIdentifier, false);
     }
 
     @Override
@@ -481,27 +479,7 @@ public class TableEnvironmentImpl implements TableEnvironmentInternal {
 
         final ObjectIdentifier tableIdentifier =
                 catalogManager.qualifyIdentifier(getParser().parseIdentifier(path));
-        final CatalogTable catalogTable = convertTableDescriptor(descriptor);
-        catalogManager.createTable(catalogTable, tableIdentifier, false);
-    }
-
-    private CatalogTable convertTableDescriptor(TableDescriptor descriptor) {
-        final Schema schema =
-                descriptor
-                        .getSchema()
-                        .orElseThrow(
-                                () ->
-                                        new ValidationException(
-                                                "Missing schema in TableDescriptor. "
-                                                        + "A schema is typically required. "
-                                                        + "It can only be omitted at certain "
-                                                        + "documented locations."));
-
-        return CatalogTable.of(
-                schema,
-                descriptor.getComment().orElse(null),
-                descriptor.getPartitionKeys(),
-                descriptor.getOptions());
+        catalogManager.createTable(descriptor.toCatalogTable(), tableIdentifier, false);
     }
 
     @Override
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/catalog/DefaultCatalogTable.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/catalog/DefaultCatalogTable.java
index 35b713e..c149a67 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/catalog/DefaultCatalogTable.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/catalog/DefaultCatalogTable.java
@@ -33,14 +33,14 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
 
 /** Default implementation of a {@link CatalogTable}. */
 @Internal
-class DefaultCatalogTable implements CatalogTable {
+public class DefaultCatalogTable implements CatalogTable {
 
     private final Schema schema;
     private final @Nullable String comment;
     private final List<String> partitionKeys;
     private final Map<String, String> options;
 
-    DefaultCatalogTable(
+    protected DefaultCatalogTable(
             Schema schema,
             @Nullable String comment,
             List<String> partitionKeys,
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TableFactoryHarness.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TableFactoryHarness.java
new file mode 100644
index 0000000..9d15acf
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TableFactoryHarness.java
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.factories;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.streaming.api.functions.sink.SinkFunction;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.table.api.Schema;
+import org.apache.flink.table.api.TableDescriptor;
+import org.apache.flink.table.api.TableEnvironment;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.catalog.CatalogBaseTable;
+import org.apache.flink.table.catalog.CatalogTable;
+import org.apache.flink.table.catalog.DefaultCatalogTable;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.sink.SinkFunctionProvider;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.connector.source.LookupTableSource;
+import org.apache.flink.table.connector.source.ScanTableSource;
+import org.apache.flink.table.connector.source.SourceFunctionProvider;
+import org.apache.flink.table.connector.source.TableFunctionProvider;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.factories.DynamicTableFactory;
+import org.apache.flink.table.factories.DynamicTableSinkFactory;
+import org.apache.flink.table.factories.DynamicTableSourceFactory;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.functions.TableFunction;
+import org.apache.flink.testutils.junit.SharedObjects;
+
+import javax.annotation.Nullable;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Provides a flexible testing harness for table factories.
+ *
+ * <p>This testing harness allows writing custom sources and sinks which can be directly
+ * instantiated from the test. This avoids having to implement a factory, and enables using the
+ * {@link SharedObjects} rule to get direct access to the underlying source/sink from the test.
+ *
+ * <p>The underlying source/sink must extend from {@link SourceBase} or {@link SinkBase}.
+ *
+ * <p>Example:
+ *
+ * <pre>{@code
+ * public class CustomSourceTest {
+ *    {@literal @}Rule public SharedObjects sharedObjects = SharedObjects.create();
+ *
+ *    {@literal @}Test
+ *    public void test() {
+ *        SharedReference<List<Long>> appliedLimits = sharedObjects.add(new ArrayList<>());
+ *        TableDescriptor sourceDescriptor =
+ *                TableFactoryHarness.newBuilder()
+ *                        .schema(Schema.derived())
+ *                        .source(new CustomSource(appliedLimits))
+ *                        .build();
+ *        tEnv.createTable("T", sourceDescriptor);
+ *
+ *        tEnv.explainSql("SELECT * FROM T LIMIT 42");
+ *
+ *        assertEquals(1, appliedLimits.get().size());
+ *        assertEquals((Long) 42L, appliedLimits.get().get(0));
+ *    }
+ *
+ *    private static class CustomSource extends TableFactoryHarness.ScanSourceBase
+ *            implements SupportsLimitPushDown {
+ *        private final SharedReference<List<Long>> appliedLimits;
+ *
+ *        CustomSource(SharedReference<List<Long>> appliedLimits) {
+ *            this.appliedLimits = appliedLimits;
+ *        }
+ *
+ *        public void applyLimit(long limit) {
+ *            appliedLimits.get().add(limit);
+ *        }
+ *    }
+ * }
+ * }</pre>
+ */
+public class TableFactoryHarness {
+
+    /** Factory identifier for {@link Factory}. */
+    public static final String IDENTIFIER = "harness";
+
+    // ---------------------------------------------------------------------------------------------
+
+    /**
+     * Creates a builder for a new {@link TableDescriptor} specialized for this harness.
+     *
+     * <p>Use this method to create a {@link TableDescriptor} and passing in the source / sink
+     * implementation you want to use. The descriptor can for example be used with {@link
+     * TableEnvironment#createTable(String, TableDescriptor)} to register a table.
+     */
+    public static HarnessTableDescriptor.Builder newBuilder() {
+        return new HarnessTableDescriptor.Builder();
+    }
+
+    // ---------------------------------------------------------------------------------------------
+
+    /**
+     * Factory which creates a source / sink defined in the specialized {@link HarnessCatalogTable}.
+     */
+    public static class Factory implements DynamicTableSourceFactory, DynamicTableSinkFactory {
+
+        @Override
+        public String factoryIdentifier() {
+            return IDENTIFIER;
+        }
+
+        @Override
+        public Set<ConfigOption<?>> requiredOptions() {
+            return Collections.emptySet();
+        }
+
+        @Override
+        public Set<ConfigOption<?>> optionalOptions() {
+            return Collections.emptySet();
+        }
+
+        @Override
+        public DynamicTableSource createDynamicTableSource(Context context) {
+            final FactoryUtil.TableFactoryHelper factoryHelper =
+                    FactoryUtil.createTableFactoryHelper(this, context);
+            factoryHelper.validate();
+
+            final HarnessCatalogTable catalogTable =
+                    (HarnessCatalogTable) context.getCatalogTable().getOrigin();
+            if (catalogTable.source == null) {
+                throw new ValidationException(
+                        String.format(
+                                "Table '%s' has no source configured.",
+                                context.getObjectIdentifier()));
+            }
+
+            catalogTable.source.factoryContext = context;
+            return catalogTable.source;
+        }
+
+        @Override
+        public DynamicTableSink createDynamicTableSink(Context context) {
+            final FactoryUtil.TableFactoryHelper factoryHelper =
+                    FactoryUtil.createTableFactoryHelper(this, context);
+            factoryHelper.validate();
+
+            final HarnessCatalogTable catalogTable =
+                    (HarnessCatalogTable) context.getCatalogTable().getOrigin();
+            if (catalogTable.sink == null) {
+                throw new ValidationException(
+                        String.format(
+                                "Table '%s' has no sink configured.",
+                                context.getObjectIdentifier()));
+            }
+
+            catalogTable.sink.factoryContext = context;
+            return catalogTable.sink;
+        }
+    }
+
+    /**
+     * Specialized version of {@link TableDescriptor} which allows passing a custom source / sink to
+     * the {@link CatalogTable} created from it.
+     */
+    private static class HarnessTableDescriptor extends TableDescriptor {
+
+        private final @Nullable SourceBase source;
+        private final @Nullable SinkBase sink;
+
+        private HarnessTableDescriptor(
+                Schema schema, @Nullable SourceBase source, @Nullable SinkBase sink) {
+            super(
+                    schema,
+                    Collections.singletonMap(FactoryUtil.CONNECTOR.key(), IDENTIFIER),
+                    Collections.emptyList(),
+                    null);
+
+            this.source = source;
+            this.sink = sink;
+        }
+
+        @Override
+        public CatalogTable toCatalogTable() {
+            return new HarnessCatalogTable(super.toCatalogTable(), source, sink);
+        }
+
+        /** Builder for {@link HarnessTableDescriptor}. */
+        public static class Builder {
+            private @Nullable Schema schema;
+            private @Nullable SourceBase source;
+            private @Nullable SinkBase sink;
+
+            /** Define the schema of the {@link TableDescriptor}. */
+            public Builder schema(Schema schema) {
+                this.schema = schema;
+                return this;
+            }
+
+            /** Use a bounded {@link ScanTableSource} which produces no data. */
+            public Builder boundedScanSource() {
+                return source(new ScanSourceBase(true) {});
+            }
+
+            /** Use an unbounded {@link ScanTableSource} which produces no data. */
+            public Builder unboundedScanSource() {
+                return source(new ScanSourceBase(false) {});
+            }
+
+            /**
+             * Use an unbounded {@link ScanTableSource} with the given {@param changelogMode} which
+             * produces no data.
+             */
+            public Builder unboundedScanSource(ChangelogMode changelogMode) {
+                return source(
+                        new ScanSourceBase(false) {
+                            @Override
+                            public ChangelogMode getChangelogMode() {
+                                return changelogMode;
+                            }
+                        });
+            }
+
+            /** Use a {@link LookupTableSource} which produces no data. */
+            public Builder lookupSource() {
+                return source(new LookupSourceBase() {});
+            }
+
+            /** Use a custom {@link DynamicTableSource}. */
+            public Builder source(SourceBase source) {
+                this.source = source;
+                return this;
+            }
+
+            /** Use a {@link DynamicTableSink} which discards all data. */
+            public Builder sink() {
+                return sink(new SinkBase() {});
+            }
+
+            /** Use a custom {@link DynamicTableSink}. */
+            public Builder sink(SinkBase sink) {
+                this.sink = sink;
+                return this;
+            }
+
+            /** Builds a {@link TableDescriptor}. */
+            public TableDescriptor build() {
+                return new HarnessTableDescriptor(schema, source, sink);
+            }
+        }
+    }
+
+    /** Specialized {@link CatalogTable} which contains a custom source / sink. */
+    private static class HarnessCatalogTable extends DefaultCatalogTable {
+
+        private final @Nullable SourceBase source;
+        private final @Nullable SinkBase sink;
+
+        public HarnessCatalogTable(
+                CatalogTable parentTable, @Nullable SourceBase source, @Nullable SinkBase sink) {
+            super(
+                    parentTable.getUnresolvedSchema(),
+                    parentTable.getComment(),
+                    parentTable.getPartitionKeys(),
+                    parentTable.getOptions());
+
+            this.source = source;
+            this.sink = sink;
+        }
+
+        @Override
+        public CatalogBaseTable copy() {
+            return copy(getOptions());
+        }
+
+        @Override
+        public CatalogTable copy(Map<String, String> options) {
+            final CatalogTable parentTable =
+                    CatalogTable.of(
+                            getUnresolvedSchema(), getComment(), getPartitionKeys(), options);
+            return new HarnessCatalogTable(parentTable, source, sink);
+        }
+    }
+
+    // ---------------------------------------------------------------------------------------------
+
+    /**
+     * Base class for custom sources which implement {@link ScanTableSource}.
+     *
+     * <p>Most interface methods are default-implemented for convenience, but can be overridden when
+     * necessary. By default, a {@link ScanRuntimeProvider} is used which doesn't produce anything.
+     *
+     * <p>Sources derived from this base class will also be provided the {@link
+     * DynamicTableFactory.Context} of the factory which gives access to e.g. the {@link
+     * CatalogTable}.
+     */
+    public abstract static class ScanSourceBase extends SourceBase implements ScanTableSource {
+
+        private final boolean bounded;
+
+        public ScanSourceBase() {
+            this(true);
+        }
+
+        public ScanSourceBase(boolean bounded) {
+            this.bounded = bounded;
+        }
+
+        @Override
+        public ChangelogMode getChangelogMode() {
+            return ChangelogMode.insertOnly();
+        }
+
+        @Override
+        public ScanRuntimeProvider getScanRuntimeProvider(ScanContext runtimeProviderContext) {
+            return SourceFunctionProvider.of(
+                    new SourceFunction<RowData>() {
+                        @Override
+                        public void run(SourceContext<RowData> ctx) {}
+
+                        @Override
+                        public void cancel() {}
+                    },
+                    bounded);
+        }
+    }
+
+    /**
+     * Base class for custom sources which implement {@link LookupTableSource}.
+     *
+     * <p>Most interface methods are default-implemented for convenience, but can be overridden when
+     * necessary. By default, a {@link LookupRuntimeProvider} is used which doesn't produce
+     * anything.
+     *
+     * <p>Sources derived from this base class will also be provided the {@link
+     * DynamicTableFactory.Context} of the factory which gives access to e.g. the {@link
+     * CatalogTable}.
+     */
+    public abstract static class LookupSourceBase extends SourceBase implements LookupTableSource {
+        @Override
+        public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) {
+            return TableFunctionProvider.of(new TableFunction<RowData>() {});
+        }
+    }
+
+    /**
+     * Base class for custom sinks.
+     *
+     * <p>Most interface methods are default-implemented for convenience, but can be overridden when
+     * necessary. By default, a {@link SinkRuntimeProvider} is used which does nothing.
+     *
+     * <p>Sinks derived from this base class will also be provided the {@link
+     * DynamicTableFactory.Context} of the factory which gives access to e.g. the {@link
+     * CatalogTable}.
+     */
+    public abstract static class SinkBase implements DynamicTableSink {
+
+        private DynamicTableFactory.Context factoryContext;
+
+        @Override
+        public ChangelogMode getChangelogMode(ChangelogMode requestedMode) {
+            return ChangelogMode.all();
+        }
+
+        @Override
+        public SinkRuntimeProvider getSinkRuntimeProvider(Context context) {
+            return SinkFunctionProvider.of(
+                    new SinkFunction<RowData>() {
+                        @Override
+                        public void invoke(RowData value, Context context1) {}
+                    });
+        }
+
+        @Override
+        public DynamicTableSink copy() {
+            return this;
+        }
+
+        @Override
+        public String asSummaryString() {
+            return "Unspecified Testing Sink";
+        }
+
+        public DynamicTableFactory.Context getFactoryContext() {
+            return factoryContext;
+        }
+    }
+
+    /** Base class for {@link ScanSourceBase} and {{@link LookupSourceBase}}. */
+    private abstract static class SourceBase implements DynamicTableSource {
+        private DynamicTableFactory.Context factoryContext;
+
+        @Override
+        public DynamicTableSource copy() {
+            return this;
+        }
+
+        @Override
+        public String asSummaryString() {
+            return "Unspecified Testing Source";
+        }
+
+        public DynamicTableFactory.Context getFactoryContext() {
+            return factoryContext;
+        }
+    }
+}
diff --git a/flink-table/flink-table-planner/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-table/flink-table-planner/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory
index 05bddfe..616b708 100644
--- a/flink-table/flink-table-planner/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory
+++ b/flink-table/flink-table-planner/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory
@@ -16,3 +16,4 @@
 org.apache.flink.table.planner.factories.TestValuesTableFactory
 org.apache.flink.table.planner.factories.TestFileFactory
 org.apache.flink.table.planner.plan.rules.logical.PushProjectIntoTableSourceScanRuleTest$MetadataNoProjectionPushDownTableFactory
+org.apache.flink.table.planner.factories.TableFactoryHarness$Factory

[flink] 03/03: [hotfix][table-planner] Migrate PushProjectIntoTableSourceScanRuleTest to test harness

Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 47268785de0b14ed9ed36accae29441e11a16c63
Author: Ingo Bürk <in...@tngtech.com>
AuthorDate: Mon Aug 30 09:19:49 2021 +0200

    [hotfix][table-planner] Migrate PushProjectIntoTableSourceScanRuleTest to test harness
---
 .../PushProjectIntoTableSourceScanRuleTest.java    | 179 +++++++--------------
 .../org.apache.flink.table.factories.Factory       |   1 -
 2 files changed, 62 insertions(+), 118 deletions(-)

diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PushProjectIntoTableSourceScanRuleTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PushProjectIntoTableSourceScanRuleTest.java
index 05e1bb2..dc44dd3 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PushProjectIntoTableSourceScanRuleTest.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PushProjectIntoTableSourceScanRuleTest.java
@@ -18,40 +18,32 @@
 
 package org.apache.flink.table.planner.plan.rules.logical;
 
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.ConfigOptions;
-import org.apache.flink.configuration.ReadableConfig;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
 import org.apache.flink.table.api.Schema;
 import org.apache.flink.table.api.TableDescriptor;
-import org.apache.flink.table.connector.ChangelogMode;
-import org.apache.flink.table.connector.source.DynamicTableSource;
-import org.apache.flink.table.connector.source.ScanTableSource;
-import org.apache.flink.table.connector.source.SourceFunctionProvider;
+import org.apache.flink.table.connector.source.abilities.SupportsProjectionPushDown;
 import org.apache.flink.table.connector.source.abilities.SupportsReadingMetadata;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.factories.DynamicTableSourceFactory;
-import org.apache.flink.table.factories.FactoryUtil;
 import org.apache.flink.table.planner.calcite.CalciteConfig;
+import org.apache.flink.table.planner.factories.TableFactoryHarness;
 import org.apache.flink.table.planner.plan.optimize.program.BatchOptimizeContext;
 import org.apache.flink.table.planner.plan.optimize.program.FlinkBatchProgram;
 import org.apache.flink.table.planner.plan.optimize.program.FlinkHepRuleSetProgramBuilder;
 import org.apache.flink.table.planner.plan.optimize.program.HEP_RULES_EXECUTION_TYPE;
 import org.apache.flink.table.planner.utils.TableConfigUtils;
 import org.apache.flink.table.types.DataType;
+import org.apache.flink.testutils.junit.SharedObjects;
+import org.apache.flink.testutils.junit.SharedReference;
 
 import org.apache.calcite.plan.hep.HepMatchOrder;
 import org.apache.calcite.tools.RuleSets;
+import org.junit.Rule;
 import org.junit.Test;
 
-import java.util.Collections;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import static org.apache.flink.table.api.DataTypes.STRING;
-import static org.apache.flink.table.planner.plan.rules.logical.PushProjectIntoTableSourceScanRuleTest.MetadataNoProjectionPushDownTableFactory.SUPPORTS_METADATA_PROJECTION;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.hasSize;
@@ -60,6 +52,8 @@ import static org.hamcrest.Matchers.hasSize;
 public class PushProjectIntoTableSourceScanRuleTest
         extends PushProjectIntoLegacyTableSourceScanRuleTest {
 
+    @Rule public final SharedObjects sharedObjects = SharedObjects.create();
+
     @Override
     public void setup() {
         util().buildBatchProgram(FlinkBatchProgram.DEFAULT_REWRITE());
@@ -273,121 +267,81 @@ public class PushProjectIntoTableSourceScanRuleTest
 
     @Test
     public void testMetadataProjectionWithoutProjectionPushDownWhenSupported() {
-        createMetadataTableWithoutProjectionPushDown("T1", true);
+        final SharedReference<List<String>> appliedKeys = sharedObjects.add(new ArrayList<>());
+        final TableDescriptor sourceDescriptor =
+                TableFactoryHarness.newBuilder()
+                        .schema(NoPushDownSource.SCHEMA)
+                        .source(new NoPushDownSource(true, appliedKeys))
+                        .build();
+        util().tableEnv().createTable("T1", sourceDescriptor);
 
         util().verifyRelPlan("SELECT m1, metadata FROM T1");
-        assertThat(
-                MetadataNoProjectionPushDownTableFactory.appliedMetadataKeys.get(),
-                contains("m1", "m2"));
+        assertThat(appliedKeys.get(), contains("m1", "m2"));
     }
 
     @Test
     public void testMetadataProjectionWithoutProjectionPushDownWhenNotSupported() {
-        createMetadataTableWithoutProjectionPushDown("T2", false);
+        final SharedReference<List<String>> appliedKeys = sharedObjects.add(new ArrayList<>());
+        final TableDescriptor sourceDescriptor =
+                TableFactoryHarness.newBuilder()
+                        .schema(NoPushDownSource.SCHEMA)
+                        .source(new NoPushDownSource(false, appliedKeys))
+                        .build();
+        util().tableEnv().createTable("T2", sourceDescriptor);
 
         util().verifyRelPlan("SELECT m1, metadata FROM T2");
-        assertThat(
-                MetadataNoProjectionPushDownTableFactory.appliedMetadataKeys.get(),
-                contains("m1", "m2", "m3"));
+        assertThat(appliedKeys.get(), contains("m1", "m2", "m3"));
     }
 
     @Test
     public void testMetadataProjectionWithoutProjectionPushDownWhenSupportedAndNoneSelected() {
-        createMetadataTableWithoutProjectionPushDown("T3", true);
+        final SharedReference<List<String>> appliedKeys = sharedObjects.add(new ArrayList<>());
+        final TableDescriptor sourceDescriptor =
+                TableFactoryHarness.newBuilder()
+                        .schema(NoPushDownSource.SCHEMA)
+                        .source(new NoPushDownSource(true, appliedKeys))
+                        .build();
+        util().tableEnv().createTable("T3", sourceDescriptor);
 
         util().verifyRelPlan("SELECT 1 FROM T3");
-        assertThat(MetadataNoProjectionPushDownTableFactory.appliedMetadataKeys.get(), hasSize(0));
+        assertThat(appliedKeys.get(), hasSize(0));
     }
 
     @Test
     public void testMetadataProjectionWithoutProjectionPushDownWhenNotSupportedAndNoneSelected() {
-        createMetadataTableWithoutProjectionPushDown("T4", false);
+        final SharedReference<List<String>> appliedKeys = sharedObjects.add(new ArrayList<>());
+        final TableDescriptor sourceDescriptor =
+                TableFactoryHarness.newBuilder()
+                        .schema(NoPushDownSource.SCHEMA)
+                        .source(new NoPushDownSource(false, appliedKeys))
+                        .build();
+        util().tableEnv().createTable("T4", sourceDescriptor);
 
         util().verifyRelPlan("SELECT 1 FROM T4");
-        assertThat(
-                MetadataNoProjectionPushDownTableFactory.appliedMetadataKeys.get(),
-                contains("m1", "m2", "m3"));
+        assertThat(appliedKeys.get(), contains("m1", "m2", "m3"));
     }
 
     // ---------------------------------------------------------------------------------------------
 
-    private void createMetadataTableWithoutProjectionPushDown(
-            String name, boolean supportsMetadataProjection) {
-        util().tableEnv()
-                .createTable(
-                        name,
-                        TableDescriptor.forConnector(
-                                        MetadataNoProjectionPushDownTableFactory.IDENTIFIER)
-                                .schema(
-                                        Schema.newBuilder()
-                                                .columnByMetadata("m1", STRING())
-                                                .columnByMetadata("metadata", STRING(), "m2")
-                                                .columnByMetadata("m3", STRING())
-                                                .build())
-                                .option(SUPPORTS_METADATA_PROJECTION, supportsMetadataProjection)
-                                .build());
-    }
-
-    // ---------------------------------------------------------------------------------------------
-
-    /** Factory for {@link Source}. */
-    public static class MetadataNoProjectionPushDownTableFactory
-            implements DynamicTableSourceFactory {
-        public static final String IDENTIFIER = "metadataNoProjectionPushDown";
-
-        public static final ConfigOption<Boolean> SUPPORTS_METADATA_PROJECTION =
-                ConfigOptions.key("supports-metadata-projection").booleanType().defaultValue(true);
-
-        public static ThreadLocal<List<String>> appliedMetadataKeys = new ThreadLocal<>();
-
-        @Override
-        public String factoryIdentifier() {
-            return IDENTIFIER;
-        }
-
-        @Override
-        public Set<ConfigOption<?>> requiredOptions() {
-            return Collections.emptySet();
-        }
-
-        @Override
-        public Set<ConfigOption<?>> optionalOptions() {
-            return Collections.singleton(SUPPORTS_METADATA_PROJECTION);
-        }
-
-        @Override
-        public DynamicTableSource createDynamicTableSource(Context context) {
-            FactoryUtil.TableFactoryHelper helper =
-                    FactoryUtil.createTableFactoryHelper(this, context);
-            return new Source(helper.getOptions());
-        }
-    }
-
-    private static class Source implements ScanTableSource, SupportsReadingMetadata {
-
-        private final ReadableConfig options;
-
-        public Source(ReadableConfig options) {
-            this.options = options;
-            MetadataNoProjectionPushDownTableFactory.appliedMetadataKeys.remove();
-        }
-
-        @Override
-        public ChangelogMode getChangelogMode() {
-            return ChangelogMode.insertOnly();
-        }
-
-        @Override
-        public ScanRuntimeProvider getScanRuntimeProvider(ScanContext runtimeProviderContext) {
-            return SourceFunctionProvider.of(
-                    new SourceFunction<RowData>() {
-                        @Override
-                        public void run(SourceContext<RowData> ctx) {}
-
-                        @Override
-                        public void cancel() {}
-                    },
-                    true);
+    /** Source which supports metadata but not {@link SupportsProjectionPushDown}. */
+    private static class NoPushDownSource extends TableFactoryHarness.ScanSourceBase
+            implements SupportsReadingMetadata {
+
+        public static final Schema SCHEMA =
+                Schema.newBuilder()
+                        .columnByMetadata("m1", STRING())
+                        .columnByMetadata("metadata", STRING(), "m2")
+                        .columnByMetadata("m3", STRING())
+                        .build();
+
+        private final boolean supportsMetadataProjection;
+        private final SharedReference<List<String>> appliedMetadataKeys;
+
+        public NoPushDownSource(
+                boolean supportsMetadataProjection,
+                SharedReference<List<String>> appliedMetadataKeys) {
+            this.supportsMetadataProjection = supportsMetadataProjection;
+            this.appliedMetadataKeys = appliedMetadataKeys;
         }
 
         @Override
@@ -401,22 +355,13 @@ public class PushProjectIntoTableSourceScanRuleTest
 
         @Override
         public void applyReadableMetadata(List<String> metadataKeys, DataType producedDataType) {
-            MetadataNoProjectionPushDownTableFactory.appliedMetadataKeys.set(metadataKeys);
+            appliedMetadataKeys.get().clear();
+            appliedMetadataKeys.get().addAll(metadataKeys);
         }
 
         @Override
         public boolean supportsMetadataProjection() {
-            return options.get(SUPPORTS_METADATA_PROJECTION);
-        }
-
-        @Override
-        public DynamicTableSource copy() {
-            return new Source(options);
-        }
-
-        @Override
-        public String asSummaryString() {
-            return getClass().getName();
+            return supportsMetadataProjection;
         }
     }
 }
diff --git a/flink-table/flink-table-planner/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-table/flink-table-planner/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory
index 616b708..7f96306 100644
--- a/flink-table/flink-table-planner/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory
+++ b/flink-table/flink-table-planner/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory
@@ -15,5 +15,4 @@
 
 org.apache.flink.table.planner.factories.TestValuesTableFactory
 org.apache.flink.table.planner.factories.TestFileFactory
-org.apache.flink.table.planner.plan.rules.logical.PushProjectIntoTableSourceScanRuleTest$MetadataNoProjectionPushDownTableFactory
 org.apache.flink.table.planner.factories.TableFactoryHarness$Factory

[flink] 02/03: [FLINK-23663][table-planner] Push primary key filters through ChangelogNormalize

Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit c1bf5da8696b0369a3475d9cda355d149c652cff
Author: Ingo Bürk <in...@tngtech.com>
AuthorDate: Fri Aug 27 12:21:22 2021 +0200

    [FLINK-23663][table-planner] Push primary key filters through ChangelogNormalize
    
    This closes #17011.
---
 .../PushFilterPastChangelogNormalizeRule.java      | 250 +++++++++++++++++++++
 .../planner/plan/rules/FlinkStreamRuleSets.scala   |   5 +-
 .../PushFilterPastChangelogNormalizeRuleTest.java  | 101 +++++++++
 .../PushFilterPastChangelogNormalizeRuleTest.xml   |  82 +++++++
 4 files changed, 437 insertions(+), 1 deletion(-)

diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/PushFilterPastChangelogNormalizeRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/PushFilterPastChangelogNormalizeRule.java
new file mode 100644
index 0000000..e325103
--- /dev/null
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/PushFilterPastChangelogNormalizeRule.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.plan.rules.physical.stream;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalCalc;
+import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize;
+import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExchange;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.RelRule;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rex.RexLocalRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexProgram;
+import org.apache.calcite.rex.RexProgramBuilder;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.calcite.util.Pair;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.apache.flink.table.planner.plan.utils.RexNodeExtractor.extractRefInputFields;
+
+/**
+ * Pushes primary key filters through a {@link StreamPhysicalChangelogNormalize ChangelogNormalize}
+ * operator to reduce its state size.
+ *
+ * <p>This rule looks for Calc → ChangelogNormalize where the {@link StreamPhysicalCalc Calc}
+ * contains a filter condition. The condition is transformed into CNF and then each conjunction is
+ * tested for whether it affects only primary key columns. If such conditions exist, they are moved
+ * into a new, separate Calc and pushed through the ChangelogNormalize operator. ChangelogNormalize
+ * keeps state for every unique key it encounters, thus pushing filters on the primary key in front
+ * of it helps reduce the size of its state.
+ *
+ * <p>Note that pushing primary key filters is safe to do, but pushing any other filters can lead to
+ * incorrect results.
+ */
+@Internal
+public class PushFilterPastChangelogNormalizeRule
+        extends RelRule<PushFilterPastChangelogNormalizeRule.Config> {
+
+    public static final RelOptRule INSTANCE = Config.EMPTY.as(Config.class).onMatch().toRule();
+
+    public PushFilterPastChangelogNormalizeRule(Config config) {
+        super(config);
+    }
+
+    @Override
+    public void onMatch(RelOptRuleCall call) {
+        final StreamPhysicalCalc calc = call.rel(0);
+        final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(1);
+
+        final RexProgram program = calc.getProgram();
+        final RexNode condition =
+                RexUtil.toCnf(
+                        call.builder().getRexBuilder(),
+                        program.expandLocalRef(program.getCondition()));
+
+        final Set<Integer> primaryKeyIndices =
+                IntStream.of(changelogNormalize.uniqueKeys()).boxed().collect(Collectors.toSet());
+
+        // Determine which filters can be pushed (= involve only primary key columns)
+        final List<RexNode> primaryKeyPredicates = new ArrayList<>();
+        final List<RexNode> otherPredicates = new ArrayList<>();
+        partitionPrimaryKeyPredicates(
+                RelOptUtil.conjunctions(condition),
+                primaryKeyIndices,
+                primaryKeyPredicates,
+                otherPredicates);
+
+        // Construct a new ChangelogNormalize which has primary key filters pushed into it
+        final StreamPhysicalChangelogNormalize newChangelogNormalize =
+                pushFiltersThroughChangelogNormalize(call, primaryKeyPredicates);
+
+        // Retain only filters which haven't been pushed
+        transformWithRemainingPredicates(call, newChangelogNormalize, otherPredicates);
+    }
+
+    /**
+     * Separates the given {@param predicates} into filters which affect only the primary key and
+     * anything else.
+     */
+    private void partitionPrimaryKeyPredicates(
+            List<RexNode> predicates,
+            Set<Integer> primaryKeyIndices,
+            List<RexNode> primaryKeyPredicates,
+            List<RexNode> remainingPredicates) {
+        for (RexNode predicate : predicates) {
+            int[] inputRefs = extractRefInputFields(Collections.singletonList(predicate));
+            if (Arrays.stream(inputRefs).allMatch(primaryKeyIndices::contains)) {
+                primaryKeyPredicates.add(predicate);
+            } else {
+                remainingPredicates.add(predicate);
+            }
+        }
+    }
+
+    /** Pushes {@param primaryKeyPredicates} into the {@link StreamPhysicalChangelogNormalize}. */
+    private StreamPhysicalChangelogNormalize pushFiltersThroughChangelogNormalize(
+            RelOptRuleCall call, List<RexNode> primaryKeyPredicates) {
+        final StreamPhysicalChangelogNormalize changelogNormalize = call.rel(1);
+        final StreamPhysicalExchange exchange = call.rel(2);
+
+        if (primaryKeyPredicates.isEmpty()) {
+            // There are no filters which can be pushed, so just return the existing node.
+            return changelogNormalize;
+        }
+
+        final StreamPhysicalCalc pushedFiltersCalc =
+                projectIdentityWithConditions(
+                        call.builder(), exchange.getInput(), primaryKeyPredicates);
+
+        final StreamPhysicalExchange newExchange =
+                (StreamPhysicalExchange)
+                        exchange.copy(
+                                exchange.getTraitSet(),
+                                Collections.singletonList(pushedFiltersCalc));
+
+        return (StreamPhysicalChangelogNormalize)
+                changelogNormalize.copy(
+                        changelogNormalize.getTraitSet(), Collections.singletonList(newExchange));
+    }
+
+    /**
+     * Returns a {@link StreamPhysicalCalc} with the given {@param conditions} and an identity
+     * projection.
+     */
+    private StreamPhysicalCalc projectIdentityWithConditions(
+            RelBuilder relBuilder, RelNode newInput, List<RexNode> conditions) {
+
+        final RexProgramBuilder programBuilder =
+                new RexProgramBuilder(newInput.getRowType(), relBuilder.getRexBuilder());
+        programBuilder.addIdentity();
+
+        final RexNode condition = relBuilder.and(conditions);
+        if (!condition.isAlwaysTrue()) {
+            programBuilder.addCondition(condition);
+        }
+
+        final RexProgram newProgram = programBuilder.getProgram();
+        return new StreamPhysicalCalc(
+                newInput.getCluster(),
+                newInput.getTraitSet(),
+                newInput,
+                newProgram,
+                newProgram.getOutputRowType());
+    }
+
+    /**
+     * Returns a {@link StreamPhysicalCalc} which is a copy of {@param calc}, but with the
+     * projections applied from {@param projectFromCalc}.
+     */
+    private StreamPhysicalCalc projectWith(
+            RelBuilder relBuilder, StreamPhysicalCalc projectFromCalc, StreamPhysicalCalc calc) {
+        final RexProgramBuilder programBuilder =
+                new RexProgramBuilder(calc.getRowType(), relBuilder.getRexBuilder());
+        if (calc.getProgram().getCondition() != null) {
+            programBuilder.addCondition(
+                    calc.getProgram().expandLocalRef(calc.getProgram().getCondition()));
+        }
+
+        for (Pair<RexLocalRef, String> projectRef :
+                projectFromCalc.getProgram().getNamedProjects()) {
+            final RexNode project = projectFromCalc.getProgram().expandLocalRef(projectRef.left);
+            programBuilder.addProject(project, projectRef.right);
+        }
+
+        final RexProgram newProgram = programBuilder.getProgram();
+        return (StreamPhysicalCalc) calc.copy(calc.getTraitSet(), calc.getInput(), newProgram);
+    }
+
+    /**
+     * Transforms the {@link RelOptRuleCall} to use {@param changelogNormalize} as the new input to
+     * a {@link StreamPhysicalCalc} which uses {@param predicates} for the condition.
+     */
+    private void transformWithRemainingPredicates(
+            RelOptRuleCall call,
+            StreamPhysicalChangelogNormalize changelogNormalize,
+            List<RexNode> predicates) {
+        final StreamPhysicalCalc calc = call.rel(0);
+        final RelBuilder relBuilder = call.builder();
+
+        final StreamPhysicalCalc newCalc =
+                projectIdentityWithConditions(relBuilder, changelogNormalize, predicates);
+        final StreamPhysicalCalc newProjectedCalc = projectWith(relBuilder, calc, newCalc);
+
+        if (newProjectedCalc.getProgram().isTrivial()) {
+            call.transformTo(changelogNormalize);
+        } else {
+            call.transformTo(newProjectedCalc);
+        }
+    }
+
+    // ---------------------------------------------------------------------------------------------
+
+    /** Configuration for {@link PushFilterPastChangelogNormalizeRule}. */
+    public interface Config extends RelRule.Config {
+
+        @Override
+        default RelOptRule toRule() {
+            return new PushFilterPastChangelogNormalizeRule(this);
+        }
+
+        default Config onMatch() {
+            final RelRule.OperandTransform exchangeTransform =
+                    operandBuilder ->
+                            operandBuilder.operand(StreamPhysicalExchange.class).anyInputs();
+
+            final RelRule.OperandTransform changelogNormalizeTransform =
+                    operandBuilder ->
+                            operandBuilder
+                                    .operand(StreamPhysicalChangelogNormalize.class)
+                                    .oneInput(exchangeTransform);
+
+            final RelRule.OperandTransform calcTransform =
+                    operandBuilder ->
+                            operandBuilder
+                                    .operand(StreamPhysicalCalc.class)
+                                    .predicate(calc -> calc.getProgram().getCondition() != null)
+                                    .oneInput(changelogNormalizeTransform);
+
+            return withOperandSupplier(calcTransform).as(Config.class);
+        }
+    }
+}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala
index 6a06e6d..da81412 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala
@@ -495,6 +495,9 @@ object FlinkStreamRuleSets {
     SimplifyWindowTableFunctionRules.WITH_LEFT_RIGHT_CALC_WINDOW_JOIN,
     SimplifyWindowTableFunctionRules.WITH_LEFT_CALC_WINDOW_JOIN,
     SimplifyWindowTableFunctionRules.WITH_RIGHT_CALC_WINDOW_JOIN,
-    SimplifyWindowTableFunctionRules.WITH_WINDOW_JOIN)
+    SimplifyWindowTableFunctionRules.WITH_WINDOW_JOIN,
+    // optimize ChangelogNormalize
+    PushFilterPastChangelogNormalizeRule.INSTANCE
+  )
 
 }
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/physical/stream/PushFilterPastChangelogNormalizeRuleTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/physical/stream/PushFilterPastChangelogNormalizeRuleTest.java
new file mode 100644
index 0000000..ca30965
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/physical/stream/PushFilterPastChangelogNormalizeRuleTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.plan.rules.physical.stream;
+
+import org.apache.flink.table.api.Schema;
+import org.apache.flink.table.api.TableConfig;
+import org.apache.flink.table.api.TableDescriptor;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.planner.factories.TableFactoryHarness;
+import org.apache.flink.table.planner.utils.StreamTableTestUtil;
+import org.apache.flink.table.planner.utils.TableTestBase;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.flink.table.api.DataTypes.BIGINT;
+import static org.apache.flink.table.api.DataTypes.INT;
+import static org.apache.flink.table.api.DataTypes.STRING;
+
+/** Tests for {@link PushFilterPastChangelogNormalizeRule}. */
+public class PushFilterPastChangelogNormalizeRuleTest extends TableTestBase {
+
+    private StreamTableTestUtil util;
+
+    @Before
+    public void before() {
+        util = streamTestUtil(TableConfig.getDefault());
+    }
+
+    @Test
+    public void testWithSinglePrimaryKeyFilter() {
+        final TableDescriptor sourceDescriptor =
+                TableFactoryHarness.newBuilder()
+                        .schema(
+                                Schema.newBuilder()
+                                        .column("f0", STRING())
+                                        .column("f1", INT().notNull())
+                                        .primaryKey("f1")
+                                        .build())
+                        .unboundedScanSource(ChangelogMode.upsert())
+                        .build();
+
+        util.tableEnv().createTable("T", sourceDescriptor);
+        util.verifyRelPlan("SELECT * FROM T WHERE f1 < 1");
+    }
+
+    @Test
+    public void testWithMultipleFilters() {
+        final TableDescriptor sourceDescriptor =
+                TableFactoryHarness.newBuilder()
+                        .schema(
+                                Schema.newBuilder()
+                                        .column("f0", STRING())
+                                        .column("f1", INT().notNull())
+                                        .column("f2", STRING())
+                                        .primaryKey("f1")
+                                        .build())
+                        .unboundedScanSource(ChangelogMode.upsert())
+                        .build();
+
+        util.tableEnv().createTable("T", sourceDescriptor);
+
+        // Only the first filter (f1 < 10) can be pushed
+        util.verifyRelPlan(
+                "SELECT f1, SUM(f1) AS `sum` FROM T WHERE f1 < 10 AND (f1 > 3 OR f2 IS NULL) GROUP BY f1");
+    }
+
+    @Test
+    public void testWithMultiplePrimaryKeyColumns() {
+        final TableDescriptor sourceDescriptor =
+                TableFactoryHarness.newBuilder()
+                        .schema(
+                                Schema.newBuilder()
+                                        .column("f0", STRING())
+                                        .column("f1", INT().notNull())
+                                        .column("f2", BIGINT().notNull())
+                                        .primaryKey("f1", "f2")
+                                        .build())
+                        .unboundedScanSource(ChangelogMode.upsert())
+                        .build();
+
+        util.tableEnv().createTable("T", sourceDescriptor);
+        util.verifyRelPlan("SELECT f0, f1 FROM T WHERE (f1 < 1 OR f2 > 10) AND f0 IS NOT NULL");
+    }
+}
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/physical/stream/PushFilterPastChangelogNormalizeRuleTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/physical/stream/PushFilterPastChangelogNormalizeRuleTest.xml
new file mode 100644
index 0000000..8c9d7a9
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/physical/stream/PushFilterPastChangelogNormalizeRuleTest.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0" ?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<Root>
+  <TestCase name="testWithMultipleFilters">
+    <Resource name="sql">
+      <![CDATA[SELECT f1, SUM(f1) AS `sum` FROM T WHERE f1 < 10 AND (f1 > 3 OR f2 IS NULL) GROUP BY f1]]>
+    </Resource>
+    <Resource name="ast">
+      <![CDATA[
+LogicalAggregate(group=[{0}], sum=[SUM($0)])
++- LogicalProject(f1=[$1])
+   +- LogicalFilter(condition=[AND(<($1, 10), OR(>($1, 3), IS NULL($2)))])
+      +- LogicalTableScan(table=[[default_catalog, default_database, T]])
+]]>
+    </Resource>
+    <Resource name="optimized rel plan">
+      <![CDATA[
+Calc(select=[f1, f1 AS sum], where=[OR(>(f1, 3), IS NULL(f2))])
++- ChangelogNormalize(key=[f1])
+   +- Exchange(distribution=[hash[f1]])
+      +- Calc(select=[f0, f1, f2], where=[<(f1, 10)])
+         +- TableSourceScan(table=[[default_catalog, default_database, T]], fields=[f0, f1, f2])
+]]>
+    </Resource>
+  </TestCase>
+  <TestCase name="testWithMultiplePrimaryKeyColumns">
+    <Resource name="sql">
+      <![CDATA[SELECT f0, f1 FROM T WHERE (f1 < 1 OR f2 > 10) AND f0 IS NOT NULL]]>
+    </Resource>
+    <Resource name="ast">
+      <![CDATA[
+LogicalProject(f0=[$0], f1=[$1])
++- LogicalFilter(condition=[AND(OR(<($1, 1), >($2, 10)), IS NOT NULL($0))])
+   +- LogicalTableScan(table=[[default_catalog, default_database, T]])
+]]>
+    </Resource>
+    <Resource name="optimized rel plan">
+      <![CDATA[
+Calc(select=[f0, f1], where=[IS NOT NULL(f0)])
++- ChangelogNormalize(key=[f1, f2])
+   +- Exchange(distribution=[hash[f1, f2]])
+      +- Calc(select=[f0, f1, f2], where=[OR(<(f1, 1), >(f2, 10))])
+         +- TableSourceScan(table=[[default_catalog, default_database, T]], fields=[f0, f1, f2])
+]]>
+    </Resource>
+  </TestCase>
+  <TestCase name="testWithSinglePrimaryKeyFilter">
+    <Resource name="sql">
+      <![CDATA[SELECT * FROM T WHERE f1 < 1]]>
+    </Resource>
+    <Resource name="ast">
+      <![CDATA[
+LogicalProject(f0=[$0], f1=[$1])
++- LogicalFilter(condition=[<($1, 1)])
+   +- LogicalTableScan(table=[[default_catalog, default_database, T]])
+]]>
+    </Resource>
+    <Resource name="optimized rel plan">
+      <![CDATA[
+ChangelogNormalize(key=[f1])
++- Exchange(distribution=[hash[f1]])
+   +- Calc(select=[f0, f1], where=[<(f1, 1)])
+      +- TableSourceScan(table=[[default_catalog, default_database, T]], fields=[f0, f1])
+]]>
+    </Resource>
+  </TestCase>
+</Root>