You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by di...@apache.org on 2021/01/18 05:08:08 UTC

[flink] branch master updated: [FLINK-20933][python] Fix the Python operators of Python DataStream API doesn't use managed memory when mixed use with Python Table API

This is an automated email from the ASF dual-hosted git repository.

dianfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new e1d98e2  [FLINK-20933][python] Fix the Python operators of Python DataStream API doesn't use managed memory when mixed use with Python Table API
e1d98e2 is described below

commit e1d98e2d806493ff921b2984ad2a1eb200835c04
Author: huangxingbo <hx...@gmail.com>
AuthorDate: Tue Jan 12 15:25:30 2021 +0800

    [FLINK-20933][python] Fix the Python operators of Python DataStream API doesn't use managed memory when mixed use with Python Table API
    
    This closes #14621.
---
 flink-python/pyflink/table/table_environment.py    |  5 ++
 .../table/tests/test_table_environment_api.py      |  1 +
 .../apache/flink/python/util/PythonConfigUtil.java | 85 +++++++++++++++++++---
 .../exec/batch/BatchExecPythonGroupAggregate.java  |  2 +-
 .../nodes/exec/common/CommonExecPythonCalc.java    | 11 +--
 .../exec/common/CommonExecPythonCorrelate.java     |  9 +--
 .../stream/StreamExecPythonGroupAggregate.java     |  5 +-
 .../plan/nodes/exec/utils/CommonPythonUtil.java    | 35 +++------
 .../BatchExecPythonGroupWindowAggregate.scala      |  5 +-
 .../exec/batch/BatchExecPythonOverAggregate.scala  |  5 +-
 .../StreamExecPythonGroupTableAggregate.scala      |  5 +-
 .../StreamExecPythonGroupWindowAggregate.scala     |  5 +-
 .../stream/StreamExecPythonOverAggregate.scala     |  5 +-
 .../flink/table/plan/nodes/CommonPythonBase.scala  | 52 +++----------
 .../plan/nodes/dataset/DataSetPythonCalc.scala     |  2 +-
 .../nodes/dataset/DataSetPythonCorrelate.scala     |  2 +-
 .../nodes/datastream/DataStreamPythonCalc.scala    |  5 +-
 .../datastream/DataStreamPythonCorrelate.scala     |  5 +-
 18 files changed, 132 insertions(+), 112 deletions(-)

diff --git a/flink-python/pyflink/table/table_environment.py b/flink-python/pyflink/table/table_environment.py
index fdf6e7f..2a331d2 100644
--- a/flink-python/pyflink/table/table_environment.py
+++ b/flink-python/pyflink/table/table_environment.py
@@ -1736,6 +1736,11 @@ class StreamTableEnvironment(TableEnvironment):
         .. versionadded:: 1.12.0
         """
         j_data_stream = data_stream._j_data_stream
+        JPythonConfigUtil = get_gateway().jvm.org.apache.flink.python.util.PythonConfigUtil
+        JPythonConfigUtil.declareManagedMemory(
+            j_data_stream.getTransformation(),
+            self._get_j_env(),
+            self._j_tenv.getConfig())
         if len(fields) == 0:
             return Table(j_table=self._j_tenv.fromDataStream(j_data_stream), t_env=self)
         elif all(isinstance(f, Expression) for f in fields):
diff --git a/flink-python/pyflink/table/tests/test_table_environment_api.py b/flink-python/pyflink/table/tests/test_table_environment_api.py
index a1d2a00..de5c4e3 100644
--- a/flink-python/pyflink/table/tests/test_table_environment_api.py
+++ b/flink-python/pyflink/table/tests/test_table_environment_api.py
@@ -414,6 +414,7 @@ class StreamTableEnvironmentTests(TableEnvironmentTest, PyFlinkStreamTableTestCa
         expected = ['+I[1, Hi, Hello]', '+I[2, Hello, Hi]']
         self.assert_equals(result, expected)
 
+        ds = ds.map(lambda x: x, Types.ROW([Types.INT(), Types.STRING(), Types.STRING()]))
         table = t_env.from_data_stream(ds, col('a'), col('b'), col('c'))
         t_env.register_table_sink("ExprSink",
                                   source_sink_utils.TestAppendSink(field_names, field_types))
diff --git a/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java b/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java
index c55763b..dde2e98 100644
--- a/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java
+++ b/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java
@@ -18,8 +18,11 @@
 package org.apache.flink.python.util;
 
 import org.apache.flink.api.common.RuntimeExecutionMode;
+import org.apache.flink.api.common.cache.DistributedCache;
 import org.apache.flink.api.connector.source.Boundedness;
 import org.apache.flink.api.dag.Transformation;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.ExecutionOptions;
 import org.apache.flink.configuration.PipelineOptions;
@@ -44,6 +47,8 @@ import org.apache.flink.streaming.api.transformations.OneInputTransformation;
 import org.apache.flink.streaming.api.transformations.TwoInputTransformation;
 import org.apache.flink.streaming.api.transformations.WithBoundedness;
 import org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner;
+import org.apache.flink.table.api.TableConfig;
+import org.apache.flink.table.api.TableException;
 
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
@@ -128,6 +133,27 @@ public class PythonConfigUtil {
         firstStream.setSlotSharingGroup(secondStream.getSlotSharingGroup());
     }
 
+    /** Set Python Operator Use Managed Memory. */
+    public static void declareManagedMemory(
+            Transformation<?> transformation,
+            StreamExecutionEnvironment env,
+            TableConfig tableConfig) {
+        Configuration config = getMergedConfig(env, tableConfig);
+        if (config.getBoolean(PythonOptions.USE_MANAGED_MEMORY)) {
+            declareManagedMemory(transformation);
+        }
+    }
+
+    private static void declareManagedMemory(Transformation<?> transformation) {
+        if (isPythonOperator(transformation)) {
+            transformation.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
+        }
+        List<Transformation<?>> inputTransformations = transformation.getInputs();
+        for (Transformation inputTransformation : inputTransformations) {
+            declareManagedMemory(inputTransformation);
+        }
+    }
+
     /**
      * Generate a {@link StreamGraph} for transformations maintained by current {@link
      * StreamExecutionEnvironment}, and reset the merged env configurations with dependencies to
@@ -153,18 +179,7 @@ public class PythonConfigUtil {
             transformationsField.setAccessible(true);
             for (Transformation transform :
                     (List<Transformation<?>>) transformationsField.get(env)) {
-                if (transform instanceof OneInputTransformation
-                        && isPythonOperator(
-                                ((OneInputTransformation) transform).getOperatorFactory())) {
-                    transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
-                } else if (transform instanceof TwoInputTransformation
-                        && isPythonOperator(
-                                ((TwoInputTransformation) transform).getOperatorFactory())) {
-                    transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
-                } else if (transform instanceof AbstractMultipleInputTransformation
-                        && isPythonOperator(
-                                ((AbstractMultipleInputTransformation) transform)
-                                        .getOperatorFactory())) {
+                if (isPythonOperator(transform)) {
                     transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
                 }
             }
@@ -215,6 +230,19 @@ public class PythonConfigUtil {
         }
     }
 
+    private static boolean isPythonOperator(Transformation<?> transform) {
+        if (transform instanceof OneInputTransformation) {
+            return isPythonOperator(((OneInputTransformation) transform).getOperatorFactory());
+        } else if (transform instanceof TwoInputTransformation) {
+            return isPythonOperator(((TwoInputTransformation) transform).getOperatorFactory());
+        } else if (transform instanceof AbstractMultipleInputTransformation) {
+            return isPythonOperator(
+                    ((AbstractMultipleInputTransformation) transform).getOperatorFactory());
+        } else {
+            return false;
+        }
+    }
+
     private static void setStreamPartitionCustomOperatorNumPartitions(
             Collection<StreamNode> streamNodes, StreamGraph streamGraph) {
         for (StreamNode streamNode : streamNodes) {
@@ -270,4 +298,37 @@ public class PythonConfigUtil {
         }
         return !existsUnboundedSource;
     }
+
+    public static Configuration getMergedConfig(
+            StreamExecutionEnvironment env, TableConfig tableConfig) {
+        try {
+            Configuration config = new Configuration(getEnvironmentConfig(env));
+            config.addAll(tableConfig.getConfiguration());
+            Configuration mergedConfig =
+                    PythonDependencyUtils.configurePythonDependencies(env.getCachedFiles(), config);
+            mergedConfig.setString("table.exec.timezone", tableConfig.getLocalTimeZone().getId());
+            return mergedConfig;
+        } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
+            throw new TableException("Method getMergedConfig failed.", e);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public static Configuration getMergedConfig(ExecutionEnvironment env, TableConfig tableConfig) {
+        try {
+            Field field = ExecutionEnvironment.class.getDeclaredField("cacheFile");
+            field.setAccessible(true);
+            Configuration config = new Configuration(env.getConfiguration());
+            config.addAll(tableConfig.getConfiguration());
+            Configuration mergedConfig =
+                    PythonDependencyUtils.configurePythonDependencies(
+                            (List<Tuple2<String, DistributedCache.DistributedCacheEntry>>)
+                                    field.get(env),
+                            config);
+            mergedConfig.setString("table.exec.timezone", tableConfig.getLocalTimeZone().getId());
+            return mergedConfig;
+        } catch (NoSuchFieldException | IllegalAccessException e) {
+            throw new TableException("Method getMergedConfig failed.", e);
+        }
+    }
 }
diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupAggregate.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupAggregate.java
index f449010..94c2895 100644
--- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupAggregate.java
+++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupAggregate.java
@@ -73,7 +73,7 @@ public class BatchExecPythonGroupAggregate extends CommonExecPythonAggregate
         final RowType inputRowType = (RowType) inputNode.getOutputType();
         final RowType outputRowType = InternalTypeInfo.of(getOutputType()).toRowType();
         Configuration config =
-                CommonPythonUtil.getConfig(planner.getExecEnv(), planner.getTableConfig());
+                CommonPythonUtil.getMergedConfig(planner.getExecEnv(), planner.getTableConfig());
         OneInputTransformation<RowData, RowData> transform =
                 createPythonOneInputTransformation(
                         inputTransform, inputRowType, outputRowType, config);
diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java
index 5917245..33bbb48 100644
--- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java
+++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java
@@ -76,18 +76,15 @@ public abstract class CommonExecPythonCalc extends ExecNodeBase<RowData> {
     protected Transformation<RowData> translateToPlanInternal(PlannerBase planner) {
         final ExecNode<RowData> inputNode = (ExecNode<RowData>) getInputNodes().get(0);
         final Transformation<RowData> inputTransform = inputNode.translateToPlan(planner);
+        final Configuration config =
+                CommonPythonUtil.getMergedConfig(planner.getExecEnv(), planner.getTableConfig());
         OneInputTransformation<RowData, RowData> ret =
-                createPythonOneInputTransformation(
-                        inputTransform,
-                        calcProgram,
-                        getDesc(),
-                        CommonPythonUtil.getConfig(planner.getExecEnv(), planner.getTableConfig()));
+                createPythonOneInputTransformation(inputTransform, calcProgram, getDesc(), config);
         if (inputsContainSingleton()) {
             ret.setParallelism(1);
             ret.setMaxParallelism(1);
         }
-        if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(
-                planner.getTableConfig().getConfiguration())) {
+        if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(config)) {
             ret.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
         }
         return ret;
diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCorrelate.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCorrelate.java
index 7f261a3..a5c35ee 100644
--- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCorrelate.java
+++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCorrelate.java
@@ -73,16 +73,15 @@ public abstract class CommonExecPythonCorrelate extends ExecNodeBase<RowData> {
     protected Transformation<RowData> translateToPlanInternal(PlannerBase planner) {
         final ExecNode<RowData> inputNode = (ExecNode<RowData>) getInputNodes().get(0);
         final Transformation<RowData> inputTransform = inputNode.translateToPlan(planner);
+        final Configuration config =
+                CommonPythonUtil.getMergedConfig(planner.getExecEnv(), planner.getTableConfig());
         OneInputTransformation<RowData, RowData> transform =
-                createPythonOneInputTransformation(
-                        inputTransform,
-                        CommonPythonUtil.getConfig(planner.getExecEnv(), planner.getTableConfig()));
+                createPythonOneInputTransformation(inputTransform, config);
         if (inputsContainSingleton()) {
             transform.setParallelism(1);
             transform.setMaxParallelism(1);
         }
-        if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(
-                planner.getTableConfig().getConfiguration())) {
+        if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(config)) {
             transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
         }
         return transform;
diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupAggregate.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupAggregate.java
index ecbfbac..a213a12 100644
--- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupAggregate.java
+++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupAggregate.java
@@ -111,9 +111,10 @@ public class StreamExecPythonGroupAggregate extends CommonExecPythonAggregate
                         extractPythonAggregateFunctionInfos(aggInfoList, aggCalls);
         PythonAggregateFunctionInfo[] pythonFunctionInfos = aggInfosAndDataViewSpecs.f0;
         DataViewUtils.DataViewSpec[][] dataViewSpecs = aggInfosAndDataViewSpecs.f1;
+        Configuration config = CommonPythonUtil.getMergedConfig(planner.getExecEnv(), tableConfig);
         final OneInputStreamOperator<RowData, RowData> operator =
                 getPythonAggregateFunctionOperator(
-                        CommonPythonUtil.getConfig(planner.getExecEnv(), tableConfig),
+                        config,
                         inputRowType,
                         InternalTypeInfo.of(getOutputType()).toRowType(),
                         pythonFunctionInfos,
@@ -136,7 +137,7 @@ public class StreamExecPythonGroupAggregate extends CommonExecPythonAggregate
             transform.setMaxParallelism(1);
         }
 
-        if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(tableConfig.getConfiguration())) {
+        if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(config)) {
             transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
         }
 
diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java
index d24f739..f035c4f 100644
--- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java
+++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java
@@ -41,15 +41,14 @@ import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 
 /** A utility class used in PyFlink. */
 public class CommonPythonUtil {
     private static final Method convertLiteralToPython;
 
-    private static final String PYTHON_DEPENDENCY_UTILS_CLASS =
-            "org.apache.flink.python.util.PythonDependencyUtils";
+    private static final String PYTHON_CONFIG_UTILS_CLASS =
+            "org.apache.flink.python.util.PythonConfigUtil";
 
     static {
         convertLiteralToPython = loadConvertLiteralToPythonMethod();
@@ -67,26 +66,20 @@ public class CommonPythonUtil {
     }
 
     @SuppressWarnings("unchecked")
-    public static Configuration getConfig(StreamExecutionEnvironment env, TableConfig tableConfig) {
-        Class clazz = loadClass(PYTHON_DEPENDENCY_UTILS_CLASS);
+    public static Configuration getMergedConfig(
+            StreamExecutionEnvironment env, TableConfig tableConfig) {
+        Class clazz = loadClass(PYTHON_CONFIG_UTILS_CLASS);
         try {
-            StreamExecutionEnvironment readEnv = getRealEnvironment(env);
+            StreamExecutionEnvironment realEnv = getRealEnvironment(env);
             Method method =
                     clazz.getDeclaredMethod(
-                            "configurePythonDependencies", List.class, Configuration.class);
-            Configuration config =
-                    (Configuration)
-                            method.invoke(
-                                    null,
-                                    readEnv.getCachedFiles(),
-                                    getMergedConfiguration(readEnv, tableConfig));
-            config.setString("table.exec.timezone", tableConfig.getLocalTimeZone().getId());
-            return config;
+                            "getMergedConfig", StreamExecutionEnvironment.class, TableConfig.class);
+            return (Configuration) method.invoke(null, realEnv, tableConfig);
         } catch (NoSuchFieldException
                 | IllegalAccessException
                 | NoSuchMethodException
                 | InvocationTargetException e) {
-            throw new TableException("Method configurePythonDependencies accessed failed.", e);
+            throw new TableException("Method getMergedConfig accessed failed.", e);
         }
     }
 
@@ -173,14 +166,4 @@ public class CommonPythonUtil {
         }
         return env;
     }
-
-    private static Configuration getMergedConfiguration(
-            StreamExecutionEnvironment env, TableConfig tableConfig)
-            throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
-        Method method = StreamExecutionEnvironment.class.getDeclaredMethod("getConfiguration");
-        method.setAccessible(true);
-        Configuration config = new Configuration((Configuration) method.invoke(env));
-        config.addAll(tableConfig.getConfiguration());
-        return config;
-    }
 }
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupWindowAggregate.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupWindowAggregate.scala
index 7ffdde8..35f450c 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupWindowAggregate.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupWindowAggregate.scala
@@ -71,6 +71,7 @@ class BatchExecPythonGroupWindowAggregate(
     val groupBufferLimitSize = planner.getTableConfig.getConfiguration.getInteger(
       ExecutionConfigOptions.TABLE_EXEC_WINDOW_AGG_BUFFER_SIZE_LIMIT)
 
+    val config = getConfig(planner.getExecEnv, planner.getTableConfig)
     val transform = createPythonOneInputTransformation(
       inputTransform,
       inputNode.getOutputType.asInstanceOf[RowType],
@@ -79,9 +80,9 @@ class BatchExecPythonGroupWindowAggregate(
       groupBufferLimitSize,
       windowSizeAndSlideSize.f0,
       windowSizeAndSlideSize.f1,
-      getConfig(planner.getExecEnv, planner.getTableConfig))
+      config)
 
-    if (isPythonWorkerUsingManagedMemory(planner.getTableConfig.getConfiguration)) {
+    if (isPythonWorkerUsingManagedMemory(config)) {
       transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
     }
     transform
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonOverAggregate.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonOverAggregate.scala
index ff01b4b..cba4ef3 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonOverAggregate.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonOverAggregate.scala
@@ -85,15 +85,16 @@ class BatchExecPythonOverAggregate(
         windowBoundary.append(boundary)
         group.getAggCalls.map((_, index))
     }
+    val config = getConfig(planner.getExecEnv, planner.getTableConfig)
     val transform = createPythonOneInputTransformation(
       input,
       aggCallToWindowIndex,
       windowBoundary.toArray,
       inputNode.getOutputType.asInstanceOf[RowType],
       outputType,
-      getConfig(planner.getExecEnv, planner.getTableConfig))
+      config)
 
-    if (isPythonWorkerUsingManagedMemory(planner.getTableConfig.getConfiguration)) {
+    if (isPythonWorkerUsingManagedMemory(config)) {
       transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
     }
     transform
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupTableAggregate.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupTableAggregate.scala
index 9f6985f..1add83d 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupTableAggregate.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupTableAggregate.scala
@@ -88,8 +88,9 @@ class StreamExecPythonGroupTableAggregate(
       dataViewSpecs = Array(Array())
     }
 
+    val config = getConfig(planner.getExecEnv, tableConfig)
     val operator = getPythonTableAggregateFunctionOperator(
-      getConfig(planner.getExecEnv, tableConfig),
+      config,
       inputRowType,
       outputType,
       pythonFunctionInfos,
@@ -115,7 +116,7 @@ class StreamExecPythonGroupTableAggregate(
       ret.setMaxParallelism(1)
     }
 
-    if (isPythonWorkerUsingManagedMemory(planner.getTableConfig.getConfiguration)) {
+    if (isPythonWorkerUsingManagedMemory(config)) {
       ret.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
     }
 
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupWindowAggregate.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupWindowAggregate.scala
index 8c95e13..9f08aaa 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupWindowAggregate.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupWindowAggregate.scala
@@ -103,6 +103,7 @@ class StreamExecPythonGroupWindowAggregate(
 
     val inputTransform = inputNode.translateToPlan(planner)
     val (windowAssigner, trigger) = generateWindowAssignerAndTrigger()
+    val mergedConfig = getConfig(planner.getExecEnv, planner.getTableConfig)
     val transform = createPythonStreamWindowGroupOneInputTransformation(
       inputTransform,
       inputNode.getOutputType.asInstanceOf[RowType],
@@ -111,7 +112,7 @@ class StreamExecPythonGroupWindowAggregate(
       windowAssigner,
       trigger,
       emitStrategy.getAllowLateness,
-      getConfig(planner.getExecEnv, planner.getTableConfig))
+      mergedConfig)
 
     if (inputsContainSingleton()) {
       transform.setParallelism(1)
@@ -124,7 +125,7 @@ class StreamExecPythonGroupWindowAggregate(
     transform.setStateKeySelector(selector)
     transform.setStateKeyType(selector.getProducedType)
 
-    if (isPythonWorkerUsingManagedMemory(planner.getTableConfig.getConfiguration)) {
+    if (isPythonWorkerUsingManagedMemory(mergedConfig)) {
       transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
     }
     transform
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonOverAggregate.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonOverAggregate.scala
index b6ee4af..539a163 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonOverAggregate.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonOverAggregate.scala
@@ -111,6 +111,7 @@ class StreamExecPythonOverAggregate(
     }
     // bounded OVER window
     val precedingOffset = -1 * boundValue.asInstanceOf[Long]
+    val config = getConfig(planner.getExecEnv, tableConfig)
     val transform = createPythonOneInputTransformation(
       inputTransform,
       inputRowType,
@@ -122,9 +123,9 @@ class StreamExecPythonOverAggregate(
       partitionKeys,
       tableConfig.getMinIdleStateRetentionTime,
       tableConfig.getMaxIdleStateRetentionTime,
-      getConfig(planner.getExecEnv, tableConfig))
+      config)
 
-    if (isPythonWorkerUsingManagedMemory(tableConfig.getConfiguration)) {
+    if (isPythonWorkerUsingManagedMemory(config)) {
       transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
     }
 
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonPythonBase.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonPythonBase.scala
index e01901e..d6d4186 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonPythonBase.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonPythonBase.scala
@@ -86,58 +86,24 @@ trait CommonPythonBase {
     }
   }
 
-  protected def getConfig(
+  protected def getMergedConfig(
       env: ExecutionEnvironment,
       tableConfig: TableConfig): Configuration = {
-    val field = classOf[ExecutionEnvironment].getDeclaredField("cacheFile")
-    field.setAccessible(true)
-    val clazz = loadClass(CommonPythonBase.PYTHON_DEPENDENCY_UTILS_CLASS)
+    val clazz = loadClass(CommonPythonBase.PythonConfigUtil)
     val method = clazz.getDeclaredMethod(
-      "configurePythonDependencies", classOf[java.util.List[_]], classOf[Configuration])
-    val config = method.invoke(
-      null, field.get(env), getMergedConfiguration(env, tableConfig))
-      .asInstanceOf[Configuration]
-    config.setString("table.exec.timezone", tableConfig.getLocalTimeZone.getId)
+      "getMergedConfig", classOf[ExecutionEnvironment], classOf[TableConfig])
+    val config = method.invoke(null, env, tableConfig).asInstanceOf[Configuration]
     config
   }
 
-  protected def getConfig(
+  protected def getMergedConfig(
       env: StreamExecutionEnvironment,
       tableConfig: TableConfig): Configuration = {
-    val clazz = loadClass(CommonPythonBase.PYTHON_DEPENDENCY_UTILS_CLASS)
+    val clazz = loadClass(CommonPythonBase.PythonConfigUtil)
     val realEnv = getRealEnvironment(env)
     val method = clazz.getDeclaredMethod(
-      "configurePythonDependencies", classOf[java.util.List[_]], classOf[Configuration])
-    val config = method.invoke(
-      null, realEnv.getCachedFiles, getMergedConfiguration(realEnv, tableConfig))
-      .asInstanceOf[Configuration]
-    config.setString("table.exec.timezone", tableConfig.getLocalTimeZone.getId)
-    config
-  }
-
-  private def getMergedConfiguration(
-      env: StreamExecutionEnvironment,
-      tableConfig: TableConfig): Configuration = {
-    // As the python dependency configurations may appear in both
-    // `StreamExecutionEnvironment#getConfiguration` (e.g. parsed from flink-conf.yaml and command
-    // line) and `TableConfig#getConfiguration` (e.g. user specified), we need to merge them and
-    // ensure the user specified configuration has priority over others.
-    val method = classOf[StreamExecutionEnvironment].getDeclaredMethod("getConfiguration")
-    method.setAccessible(true)
-    val config = new Configuration(method.invoke(env).asInstanceOf[Configuration])
-    config.addAll(tableConfig.getConfiguration)
-    config
-  }
-
-  private def getMergedConfiguration(
-      env: ExecutionEnvironment,
-      tableConfig: TableConfig): Configuration = {
-    // As the python dependency configurations may appear in both
-    // `ExecutionEnvironment#getConfiguration` (e.g. parsed from flink-conf.yaml and command
-    // line) and `TableConfig#getConfiguration` (e.g. user specified), we need to merge them and
-    // ensure the user specified configuration has priority over others.
-    val config = new Configuration(env.getConfiguration)
-    config.addAll(tableConfig.getConfiguration)
+      "getMergedConfig", classOf[StreamExecutionEnvironment], classOf[TableConfig])
+    val config = method.invoke(null, realEnv, tableConfig).asInstanceOf[Configuration]
     config
   }
 
@@ -159,5 +125,5 @@ trait CommonPythonBase {
 }
 
 object CommonPythonBase {
-  val PYTHON_DEPENDENCY_UTILS_CLASS = "org.apache.flink.python.util.PythonDependencyUtils"
+  val PythonConfigUtil = "org.apache.flink.python.util.PythonConfigUtil"
 }
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCalc.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCalc.scala
index cbc5deb..c4bfca1 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCalc.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCalc.scala
@@ -80,7 +80,7 @@ class DataSetPythonCalc(
     val flatMapFunctionOutputRowType = TypeConversions.fromLegacyInfoToDataType(
       flatMapFunctionResultTypeInfo).getLogicalType.asInstanceOf[RowType]
     val flatMapFunction = getPythonScalarFunctionFlatMap(
-      getConfig(tableEnv.execEnv, tableEnv.getConfig),
+      getMergedConfig(tableEnv.execEnv, tableEnv.getConfig),
       flatMapFunctionInputRowType,
       flatMapFunctionOutputRowType,
       calcProgram)
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCorrelate.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCorrelate.scala
index 17d61d3..cb0efb8 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCorrelate.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCorrelate.scala
@@ -87,7 +87,7 @@ class DataSetPythonCorrelate(
     val sqlFunction = pythonTableFuncRexCall.getOperator.asInstanceOf[TableSqlFunction]
 
     val flatMapFunction = getPythonTableFunctionFlatMap(
-      getConfig(tableEnv.execEnv, tableEnv.getConfig),
+      getMergedConfig(tableEnv.execEnv, tableEnv.getConfig),
       pythonOperatorInputRowType,
       pythonOperatorOutputRowType,
       pythonFunctionInfo,
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCalc.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCalc.scala
index 576e26d..114f530 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCalc.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCalc.scala
@@ -81,8 +81,9 @@ class DataStreamPythonCalc(
       inputSchema.typeInfo).getLogicalType.asInstanceOf[RowType]
     val pythonOperatorOutputRowType = TypeConversions.fromLegacyInfoToDataType(
       pythonOperatorResultTypeInfo).getLogicalType.asInstanceOf[RowType]
+    val config = getMergedConfig(planner.getExecutionEnvironment, planner.getConfig)
     val pythonOperator = getPythonScalarFunctionOperator(
-      getConfig(planner.getExecutionEnvironment, planner.getConfig),
+      config,
       pythonOperatorInputRowType,
       pythonOperatorOutputRowType,
       calcProgram)
@@ -95,7 +96,7 @@ class DataStreamPythonCalc(
       // keep parallelism to ensure order of accumulate and retract messages
       .setParallelism(inputParallelism)
 
-    if (isPythonWorkerUsingManagedMemory(planner.getConfig.getConfiguration)) {
+    if (isPythonWorkerUsingManagedMemory(config)) {
       ret.getTransformation.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
     }
     ret
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCorrelate.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCorrelate.scala
index 45d9b07..6af0c75 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCorrelate.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCorrelate.scala
@@ -93,8 +93,9 @@ class DataStreamPythonCorrelate(
 
     val sqlFunction = pythonTableFuncRexCall.getOperator.asInstanceOf[TableSqlFunction]
 
+    val config = getMergedConfig(planner.getExecutionEnvironment, planner.getConfig)
     val pythonOperator = getPythonTableFunctionOperator(
-      getConfig(planner.getExecutionEnvironment, planner.getConfig),
+      config,
       pythonOperatorInputRowType,
       pythonOperatorOutputRowType,
       pythonFunctionInfo,
@@ -114,7 +115,7 @@ class DataStreamPythonCorrelate(
       // keep parallelism to ensure order of accumulate and retract messages
       .setParallelism(inputDataStream.getParallelism)
 
-    if (isPythonWorkerUsingManagedMemory(planner.getConfig.getConfiguration)) {
+    if (isPythonWorkerUsingManagedMemory(config)) {
       ret.getTransformation.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
     }
     ret