You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by di...@apache.org on 2021/01/18 05:08:08 UTC
[flink] branch master updated: [FLINK-20933][python] Fix the Python
operators of Python DataStream API doesn't use managed memory when mixed
use with Python Table API
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new e1d98e2 [FLINK-20933][python] Fix the Python operators of Python DataStream API doesn't use managed memory when mixed use with Python Table API
e1d98e2 is described below
commit e1d98e2d806493ff921b2984ad2a1eb200835c04
Author: huangxingbo <hx...@gmail.com>
AuthorDate: Tue Jan 12 15:25:30 2021 +0800
[FLINK-20933][python] Fix the Python operators of Python DataStream API doesn't use managed memory when mixed use with Python Table API
This closes #14621.
---
flink-python/pyflink/table/table_environment.py | 5 ++
.../table/tests/test_table_environment_api.py | 1 +
.../apache/flink/python/util/PythonConfigUtil.java | 85 +++++++++++++++++++---
.../exec/batch/BatchExecPythonGroupAggregate.java | 2 +-
.../nodes/exec/common/CommonExecPythonCalc.java | 11 +--
.../exec/common/CommonExecPythonCorrelate.java | 9 +--
.../stream/StreamExecPythonGroupAggregate.java | 5 +-
.../plan/nodes/exec/utils/CommonPythonUtil.java | 35 +++------
.../BatchExecPythonGroupWindowAggregate.scala | 5 +-
.../exec/batch/BatchExecPythonOverAggregate.scala | 5 +-
.../StreamExecPythonGroupTableAggregate.scala | 5 +-
.../StreamExecPythonGroupWindowAggregate.scala | 5 +-
.../stream/StreamExecPythonOverAggregate.scala | 5 +-
.../flink/table/plan/nodes/CommonPythonBase.scala | 52 +++----------
.../plan/nodes/dataset/DataSetPythonCalc.scala | 2 +-
.../nodes/dataset/DataSetPythonCorrelate.scala | 2 +-
.../nodes/datastream/DataStreamPythonCalc.scala | 5 +-
.../datastream/DataStreamPythonCorrelate.scala | 5 +-
18 files changed, 132 insertions(+), 112 deletions(-)
diff --git a/flink-python/pyflink/table/table_environment.py b/flink-python/pyflink/table/table_environment.py
index fdf6e7f..2a331d2 100644
--- a/flink-python/pyflink/table/table_environment.py
+++ b/flink-python/pyflink/table/table_environment.py
@@ -1736,6 +1736,11 @@ class StreamTableEnvironment(TableEnvironment):
.. versionadded:: 1.12.0
"""
j_data_stream = data_stream._j_data_stream
+ JPythonConfigUtil = get_gateway().jvm.org.apache.flink.python.util.PythonConfigUtil
+ JPythonConfigUtil.declareManagedMemory(
+ j_data_stream.getTransformation(),
+ self._get_j_env(),
+ self._j_tenv.getConfig())
if len(fields) == 0:
return Table(j_table=self._j_tenv.fromDataStream(j_data_stream), t_env=self)
elif all(isinstance(f, Expression) for f in fields):
diff --git a/flink-python/pyflink/table/tests/test_table_environment_api.py b/flink-python/pyflink/table/tests/test_table_environment_api.py
index a1d2a00..de5c4e3 100644
--- a/flink-python/pyflink/table/tests/test_table_environment_api.py
+++ b/flink-python/pyflink/table/tests/test_table_environment_api.py
@@ -414,6 +414,7 @@ class StreamTableEnvironmentTests(TableEnvironmentTest, PyFlinkStreamTableTestCa
expected = ['+I[1, Hi, Hello]', '+I[2, Hello, Hi]']
self.assert_equals(result, expected)
+ ds = ds.map(lambda x: x, Types.ROW([Types.INT(), Types.STRING(), Types.STRING()]))
table = t_env.from_data_stream(ds, col('a'), col('b'), col('c'))
t_env.register_table_sink("ExprSink",
source_sink_utils.TestAppendSink(field_names, field_types))
diff --git a/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java b/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java
index c55763b..dde2e98 100644
--- a/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java
+++ b/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java
@@ -18,8 +18,11 @@
package org.apache.flink.python.util;
import org.apache.flink.api.common.RuntimeExecutionMode;
+import org.apache.flink.api.common.cache.DistributedCache;
import org.apache.flink.api.connector.source.Boundedness;
import org.apache.flink.api.dag.Transformation;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.ExecutionOptions;
import org.apache.flink.configuration.PipelineOptions;
@@ -44,6 +47,8 @@ import org.apache.flink.streaming.api.transformations.OneInputTransformation;
import org.apache.flink.streaming.api.transformations.TwoInputTransformation;
import org.apache.flink.streaming.api.transformations.WithBoundedness;
import org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner;
+import org.apache.flink.table.api.TableConfig;
+import org.apache.flink.table.api.TableException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
@@ -128,6 +133,27 @@ public class PythonConfigUtil {
firstStream.setSlotSharingGroup(secondStream.getSlotSharingGroup());
}
+ /** Set Python Operator Use Managed Memory. */
+ public static void declareManagedMemory(
+ Transformation<?> transformation,
+ StreamExecutionEnvironment env,
+ TableConfig tableConfig) {
+ Configuration config = getMergedConfig(env, tableConfig);
+ if (config.getBoolean(PythonOptions.USE_MANAGED_MEMORY)) {
+ declareManagedMemory(transformation);
+ }
+ }
+
+ private static void declareManagedMemory(Transformation<?> transformation) {
+ if (isPythonOperator(transformation)) {
+ transformation.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
+ }
+ List<Transformation<?>> inputTransformations = transformation.getInputs();
+ for (Transformation inputTransformation : inputTransformations) {
+ declareManagedMemory(inputTransformation);
+ }
+ }
+
/**
* Generate a {@link StreamGraph} for transformations maintained by current {@link
* StreamExecutionEnvironment}, and reset the merged env configurations with dependencies to
@@ -153,18 +179,7 @@ public class PythonConfigUtil {
transformationsField.setAccessible(true);
for (Transformation transform :
(List<Transformation<?>>) transformationsField.get(env)) {
- if (transform instanceof OneInputTransformation
- && isPythonOperator(
- ((OneInputTransformation) transform).getOperatorFactory())) {
- transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
- } else if (transform instanceof TwoInputTransformation
- && isPythonOperator(
- ((TwoInputTransformation) transform).getOperatorFactory())) {
- transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
- } else if (transform instanceof AbstractMultipleInputTransformation
- && isPythonOperator(
- ((AbstractMultipleInputTransformation) transform)
- .getOperatorFactory())) {
+ if (isPythonOperator(transform)) {
transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
}
}
@@ -215,6 +230,19 @@ public class PythonConfigUtil {
}
}
+ private static boolean isPythonOperator(Transformation<?> transform) {
+ if (transform instanceof OneInputTransformation) {
+ return isPythonOperator(((OneInputTransformation) transform).getOperatorFactory());
+ } else if (transform instanceof TwoInputTransformation) {
+ return isPythonOperator(((TwoInputTransformation) transform).getOperatorFactory());
+ } else if (transform instanceof AbstractMultipleInputTransformation) {
+ return isPythonOperator(
+ ((AbstractMultipleInputTransformation) transform).getOperatorFactory());
+ } else {
+ return false;
+ }
+ }
+
private static void setStreamPartitionCustomOperatorNumPartitions(
Collection<StreamNode> streamNodes, StreamGraph streamGraph) {
for (StreamNode streamNode : streamNodes) {
@@ -270,4 +298,37 @@ public class PythonConfigUtil {
}
return !existsUnboundedSource;
}
+
+ public static Configuration getMergedConfig(
+ StreamExecutionEnvironment env, TableConfig tableConfig) {
+ try {
+ Configuration config = new Configuration(getEnvironmentConfig(env));
+ config.addAll(tableConfig.getConfiguration());
+ Configuration mergedConfig =
+ PythonDependencyUtils.configurePythonDependencies(env.getCachedFiles(), config);
+ mergedConfig.setString("table.exec.timezone", tableConfig.getLocalTimeZone().getId());
+ return mergedConfig;
+ } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
+ throw new TableException("Method getMergedConfig failed.", e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public static Configuration getMergedConfig(ExecutionEnvironment env, TableConfig tableConfig) {
+ try {
+ Field field = ExecutionEnvironment.class.getDeclaredField("cacheFile");
+ field.setAccessible(true);
+ Configuration config = new Configuration(env.getConfiguration());
+ config.addAll(tableConfig.getConfiguration());
+ Configuration mergedConfig =
+ PythonDependencyUtils.configurePythonDependencies(
+ (List<Tuple2<String, DistributedCache.DistributedCacheEntry>>)
+ field.get(env),
+ config);
+ mergedConfig.setString("table.exec.timezone", tableConfig.getLocalTimeZone().getId());
+ return mergedConfig;
+ } catch (NoSuchFieldException | IllegalAccessException e) {
+ throw new TableException("Method getMergedConfig failed.", e);
+ }
+ }
}
diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupAggregate.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupAggregate.java
index f449010..94c2895 100644
--- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupAggregate.java
+++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupAggregate.java
@@ -73,7 +73,7 @@ public class BatchExecPythonGroupAggregate extends CommonExecPythonAggregate
final RowType inputRowType = (RowType) inputNode.getOutputType();
final RowType outputRowType = InternalTypeInfo.of(getOutputType()).toRowType();
Configuration config =
- CommonPythonUtil.getConfig(planner.getExecEnv(), planner.getTableConfig());
+ CommonPythonUtil.getMergedConfig(planner.getExecEnv(), planner.getTableConfig());
OneInputTransformation<RowData, RowData> transform =
createPythonOneInputTransformation(
inputTransform, inputRowType, outputRowType, config);
diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java
index 5917245..33bbb48 100644
--- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java
+++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java
@@ -76,18 +76,15 @@ public abstract class CommonExecPythonCalc extends ExecNodeBase<RowData> {
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner) {
final ExecNode<RowData> inputNode = (ExecNode<RowData>) getInputNodes().get(0);
final Transformation<RowData> inputTransform = inputNode.translateToPlan(planner);
+ final Configuration config =
+ CommonPythonUtil.getMergedConfig(planner.getExecEnv(), planner.getTableConfig());
OneInputTransformation<RowData, RowData> ret =
- createPythonOneInputTransformation(
- inputTransform,
- calcProgram,
- getDesc(),
- CommonPythonUtil.getConfig(planner.getExecEnv(), planner.getTableConfig()));
+ createPythonOneInputTransformation(inputTransform, calcProgram, getDesc(), config);
if (inputsContainSingleton()) {
ret.setParallelism(1);
ret.setMaxParallelism(1);
}
- if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(
- planner.getTableConfig().getConfiguration())) {
+ if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(config)) {
ret.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
}
return ret;
diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCorrelate.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCorrelate.java
index 7f261a3..a5c35ee 100644
--- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCorrelate.java
+++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCorrelate.java
@@ -73,16 +73,15 @@ public abstract class CommonExecPythonCorrelate extends ExecNodeBase<RowData> {
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner) {
final ExecNode<RowData> inputNode = (ExecNode<RowData>) getInputNodes().get(0);
final Transformation<RowData> inputTransform = inputNode.translateToPlan(planner);
+ final Configuration config =
+ CommonPythonUtil.getMergedConfig(planner.getExecEnv(), planner.getTableConfig());
OneInputTransformation<RowData, RowData> transform =
- createPythonOneInputTransformation(
- inputTransform,
- CommonPythonUtil.getConfig(planner.getExecEnv(), planner.getTableConfig()));
+ createPythonOneInputTransformation(inputTransform, config);
if (inputsContainSingleton()) {
transform.setParallelism(1);
transform.setMaxParallelism(1);
}
- if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(
- planner.getTableConfig().getConfiguration())) {
+ if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(config)) {
transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
}
return transform;
diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupAggregate.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupAggregate.java
index ecbfbac..a213a12 100644
--- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupAggregate.java
+++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupAggregate.java
@@ -111,9 +111,10 @@ public class StreamExecPythonGroupAggregate extends CommonExecPythonAggregate
extractPythonAggregateFunctionInfos(aggInfoList, aggCalls);
PythonAggregateFunctionInfo[] pythonFunctionInfos = aggInfosAndDataViewSpecs.f0;
DataViewUtils.DataViewSpec[][] dataViewSpecs = aggInfosAndDataViewSpecs.f1;
+ Configuration config = CommonPythonUtil.getMergedConfig(planner.getExecEnv(), tableConfig);
final OneInputStreamOperator<RowData, RowData> operator =
getPythonAggregateFunctionOperator(
- CommonPythonUtil.getConfig(planner.getExecEnv(), tableConfig),
+ config,
inputRowType,
InternalTypeInfo.of(getOutputType()).toRowType(),
pythonFunctionInfos,
@@ -136,7 +137,7 @@ public class StreamExecPythonGroupAggregate extends CommonExecPythonAggregate
transform.setMaxParallelism(1);
}
- if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(tableConfig.getConfiguration())) {
+ if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(config)) {
transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
}
diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java
index d24f739..f035c4f 100644
--- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java
+++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java
@@ -41,15 +41,14 @@ import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
-import java.util.List;
import java.util.Map;
/** A utility class used in PyFlink. */
public class CommonPythonUtil {
private static final Method convertLiteralToPython;
- private static final String PYTHON_DEPENDENCY_UTILS_CLASS =
- "org.apache.flink.python.util.PythonDependencyUtils";
+ private static final String PYTHON_CONFIG_UTILS_CLASS =
+ "org.apache.flink.python.util.PythonConfigUtil";
static {
convertLiteralToPython = loadConvertLiteralToPythonMethod();
@@ -67,26 +66,20 @@ public class CommonPythonUtil {
}
@SuppressWarnings("unchecked")
- public static Configuration getConfig(StreamExecutionEnvironment env, TableConfig tableConfig) {
- Class clazz = loadClass(PYTHON_DEPENDENCY_UTILS_CLASS);
+ public static Configuration getMergedConfig(
+ StreamExecutionEnvironment env, TableConfig tableConfig) {
+ Class clazz = loadClass(PYTHON_CONFIG_UTILS_CLASS);
try {
- StreamExecutionEnvironment readEnv = getRealEnvironment(env);
+ StreamExecutionEnvironment realEnv = getRealEnvironment(env);
Method method =
clazz.getDeclaredMethod(
- "configurePythonDependencies", List.class, Configuration.class);
- Configuration config =
- (Configuration)
- method.invoke(
- null,
- readEnv.getCachedFiles(),
- getMergedConfiguration(readEnv, tableConfig));
- config.setString("table.exec.timezone", tableConfig.getLocalTimeZone().getId());
- return config;
+ "getMergedConfig", StreamExecutionEnvironment.class, TableConfig.class);
+ return (Configuration) method.invoke(null, realEnv, tableConfig);
} catch (NoSuchFieldException
| IllegalAccessException
| NoSuchMethodException
| InvocationTargetException e) {
- throw new TableException("Method configurePythonDependencies accessed failed.", e);
+ throw new TableException("Method getMergedConfig accessed failed.", e);
}
}
@@ -173,14 +166,4 @@ public class CommonPythonUtil {
}
return env;
}
-
- private static Configuration getMergedConfiguration(
- StreamExecutionEnvironment env, TableConfig tableConfig)
- throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
- Method method = StreamExecutionEnvironment.class.getDeclaredMethod("getConfiguration");
- method.setAccessible(true);
- Configuration config = new Configuration((Configuration) method.invoke(env));
- config.addAll(tableConfig.getConfiguration());
- return config;
- }
}
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupWindowAggregate.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupWindowAggregate.scala
index 7ffdde8..35f450c 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupWindowAggregate.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonGroupWindowAggregate.scala
@@ -71,6 +71,7 @@ class BatchExecPythonGroupWindowAggregate(
val groupBufferLimitSize = planner.getTableConfig.getConfiguration.getInteger(
ExecutionConfigOptions.TABLE_EXEC_WINDOW_AGG_BUFFER_SIZE_LIMIT)
+ val config = getConfig(planner.getExecEnv, planner.getTableConfig)
val transform = createPythonOneInputTransformation(
inputTransform,
inputNode.getOutputType.asInstanceOf[RowType],
@@ -79,9 +80,9 @@ class BatchExecPythonGroupWindowAggregate(
groupBufferLimitSize,
windowSizeAndSlideSize.f0,
windowSizeAndSlideSize.f1,
- getConfig(planner.getExecEnv, planner.getTableConfig))
+ config)
- if (isPythonWorkerUsingManagedMemory(planner.getTableConfig.getConfiguration)) {
+ if (isPythonWorkerUsingManagedMemory(config)) {
transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
}
transform
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonOverAggregate.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonOverAggregate.scala
index ff01b4b..cba4ef3 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonOverAggregate.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/batch/BatchExecPythonOverAggregate.scala
@@ -85,15 +85,16 @@ class BatchExecPythonOverAggregate(
windowBoundary.append(boundary)
group.getAggCalls.map((_, index))
}
+ val config = getConfig(planner.getExecEnv, planner.getTableConfig)
val transform = createPythonOneInputTransformation(
input,
aggCallToWindowIndex,
windowBoundary.toArray,
inputNode.getOutputType.asInstanceOf[RowType],
outputType,
- getConfig(planner.getExecEnv, planner.getTableConfig))
+ config)
- if (isPythonWorkerUsingManagedMemory(planner.getTableConfig.getConfiguration)) {
+ if (isPythonWorkerUsingManagedMemory(config)) {
transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
}
transform
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupTableAggregate.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupTableAggregate.scala
index 9f6985f..1add83d 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupTableAggregate.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupTableAggregate.scala
@@ -88,8 +88,9 @@ class StreamExecPythonGroupTableAggregate(
dataViewSpecs = Array(Array())
}
+ val config = getConfig(planner.getExecEnv, tableConfig)
val operator = getPythonTableAggregateFunctionOperator(
- getConfig(planner.getExecEnv, tableConfig),
+ config,
inputRowType,
outputType,
pythonFunctionInfos,
@@ -115,7 +116,7 @@ class StreamExecPythonGroupTableAggregate(
ret.setMaxParallelism(1)
}
- if (isPythonWorkerUsingManagedMemory(planner.getTableConfig.getConfiguration)) {
+ if (isPythonWorkerUsingManagedMemory(config)) {
ret.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
}
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupWindowAggregate.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupWindowAggregate.scala
index 8c95e13..9f08aaa 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupWindowAggregate.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonGroupWindowAggregate.scala
@@ -103,6 +103,7 @@ class StreamExecPythonGroupWindowAggregate(
val inputTransform = inputNode.translateToPlan(planner)
val (windowAssigner, trigger) = generateWindowAssignerAndTrigger()
+ val mergedConfig = getConfig(planner.getExecEnv, planner.getTableConfig)
val transform = createPythonStreamWindowGroupOneInputTransformation(
inputTransform,
inputNode.getOutputType.asInstanceOf[RowType],
@@ -111,7 +112,7 @@ class StreamExecPythonGroupWindowAggregate(
windowAssigner,
trigger,
emitStrategy.getAllowLateness,
- getConfig(planner.getExecEnv, planner.getTableConfig))
+ mergedConfig)
if (inputsContainSingleton()) {
transform.setParallelism(1)
@@ -124,7 +125,7 @@ class StreamExecPythonGroupWindowAggregate(
transform.setStateKeySelector(selector)
transform.setStateKeyType(selector.getProducedType)
- if (isPythonWorkerUsingManagedMemory(planner.getTableConfig.getConfiguration)) {
+ if (isPythonWorkerUsingManagedMemory(mergedConfig)) {
transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
}
transform
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonOverAggregate.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonOverAggregate.scala
index b6ee4af..539a163 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonOverAggregate.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecPythonOverAggregate.scala
@@ -111,6 +111,7 @@ class StreamExecPythonOverAggregate(
}
// bounded OVER window
val precedingOffset = -1 * boundValue.asInstanceOf[Long]
+ val config = getConfig(planner.getExecEnv, tableConfig)
val transform = createPythonOneInputTransformation(
inputTransform,
inputRowType,
@@ -122,9 +123,9 @@ class StreamExecPythonOverAggregate(
partitionKeys,
tableConfig.getMinIdleStateRetentionTime,
tableConfig.getMaxIdleStateRetentionTime,
- getConfig(planner.getExecEnv, tableConfig))
+ config)
- if (isPythonWorkerUsingManagedMemory(tableConfig.getConfiguration)) {
+ if (isPythonWorkerUsingManagedMemory(config)) {
transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonPythonBase.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonPythonBase.scala
index e01901e..d6d4186 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonPythonBase.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonPythonBase.scala
@@ -86,58 +86,24 @@ trait CommonPythonBase {
}
}
- protected def getConfig(
+ protected def getMergedConfig(
env: ExecutionEnvironment,
tableConfig: TableConfig): Configuration = {
- val field = classOf[ExecutionEnvironment].getDeclaredField("cacheFile")
- field.setAccessible(true)
- val clazz = loadClass(CommonPythonBase.PYTHON_DEPENDENCY_UTILS_CLASS)
+ val clazz = loadClass(CommonPythonBase.PythonConfigUtil)
val method = clazz.getDeclaredMethod(
- "configurePythonDependencies", classOf[java.util.List[_]], classOf[Configuration])
- val config = method.invoke(
- null, field.get(env), getMergedConfiguration(env, tableConfig))
- .asInstanceOf[Configuration]
- config.setString("table.exec.timezone", tableConfig.getLocalTimeZone.getId)
+ "getMergedConfig", classOf[ExecutionEnvironment], classOf[TableConfig])
+ val config = method.invoke(null, env, tableConfig).asInstanceOf[Configuration]
config
}
- protected def getConfig(
+ protected def getMergedConfig(
env: StreamExecutionEnvironment,
tableConfig: TableConfig): Configuration = {
- val clazz = loadClass(CommonPythonBase.PYTHON_DEPENDENCY_UTILS_CLASS)
+ val clazz = loadClass(CommonPythonBase.PythonConfigUtil)
val realEnv = getRealEnvironment(env)
val method = clazz.getDeclaredMethod(
- "configurePythonDependencies", classOf[java.util.List[_]], classOf[Configuration])
- val config = method.invoke(
- null, realEnv.getCachedFiles, getMergedConfiguration(realEnv, tableConfig))
- .asInstanceOf[Configuration]
- config.setString("table.exec.timezone", tableConfig.getLocalTimeZone.getId)
- config
- }
-
- private def getMergedConfiguration(
- env: StreamExecutionEnvironment,
- tableConfig: TableConfig): Configuration = {
- // As the python dependency configurations may appear in both
- // `StreamExecutionEnvironment#getConfiguration` (e.g. parsed from flink-conf.yaml and command
- // line) and `TableConfig#getConfiguration` (e.g. user specified), we need to merge them and
- // ensure the user specified configuration has priority over others.
- val method = classOf[StreamExecutionEnvironment].getDeclaredMethod("getConfiguration")
- method.setAccessible(true)
- val config = new Configuration(method.invoke(env).asInstanceOf[Configuration])
- config.addAll(tableConfig.getConfiguration)
- config
- }
-
- private def getMergedConfiguration(
- env: ExecutionEnvironment,
- tableConfig: TableConfig): Configuration = {
- // As the python dependency configurations may appear in both
- // `ExecutionEnvironment#getConfiguration` (e.g. parsed from flink-conf.yaml and command
- // line) and `TableConfig#getConfiguration` (e.g. user specified), we need to merge them and
- // ensure the user specified configuration has priority over others.
- val config = new Configuration(env.getConfiguration)
- config.addAll(tableConfig.getConfiguration)
+ "getMergedConfig", classOf[StreamExecutionEnvironment], classOf[TableConfig])
+ val config = method.invoke(null, realEnv, tableConfig).asInstanceOf[Configuration]
config
}
@@ -159,5 +125,5 @@ trait CommonPythonBase {
}
object CommonPythonBase {
- val PYTHON_DEPENDENCY_UTILS_CLASS = "org.apache.flink.python.util.PythonDependencyUtils"
+ val PythonConfigUtil = "org.apache.flink.python.util.PythonConfigUtil"
}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCalc.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCalc.scala
index cbc5deb..c4bfca1 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCalc.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCalc.scala
@@ -80,7 +80,7 @@ class DataSetPythonCalc(
val flatMapFunctionOutputRowType = TypeConversions.fromLegacyInfoToDataType(
flatMapFunctionResultTypeInfo).getLogicalType.asInstanceOf[RowType]
val flatMapFunction = getPythonScalarFunctionFlatMap(
- getConfig(tableEnv.execEnv, tableEnv.getConfig),
+ getMergedConfig(tableEnv.execEnv, tableEnv.getConfig),
flatMapFunctionInputRowType,
flatMapFunctionOutputRowType,
calcProgram)
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCorrelate.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCorrelate.scala
index 17d61d3..cb0efb8 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCorrelate.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetPythonCorrelate.scala
@@ -87,7 +87,7 @@ class DataSetPythonCorrelate(
val sqlFunction = pythonTableFuncRexCall.getOperator.asInstanceOf[TableSqlFunction]
val flatMapFunction = getPythonTableFunctionFlatMap(
- getConfig(tableEnv.execEnv, tableEnv.getConfig),
+ getMergedConfig(tableEnv.execEnv, tableEnv.getConfig),
pythonOperatorInputRowType,
pythonOperatorOutputRowType,
pythonFunctionInfo,
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCalc.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCalc.scala
index 576e26d..114f530 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCalc.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCalc.scala
@@ -81,8 +81,9 @@ class DataStreamPythonCalc(
inputSchema.typeInfo).getLogicalType.asInstanceOf[RowType]
val pythonOperatorOutputRowType = TypeConversions.fromLegacyInfoToDataType(
pythonOperatorResultTypeInfo).getLogicalType.asInstanceOf[RowType]
+ val config = getMergedConfig(planner.getExecutionEnvironment, planner.getConfig)
val pythonOperator = getPythonScalarFunctionOperator(
- getConfig(planner.getExecutionEnvironment, planner.getConfig),
+ config,
pythonOperatorInputRowType,
pythonOperatorOutputRowType,
calcProgram)
@@ -95,7 +96,7 @@ class DataStreamPythonCalc(
// keep parallelism to ensure order of accumulate and retract messages
.setParallelism(inputParallelism)
- if (isPythonWorkerUsingManagedMemory(planner.getConfig.getConfiguration)) {
+ if (isPythonWorkerUsingManagedMemory(config)) {
ret.getTransformation.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
}
ret
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCorrelate.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCorrelate.scala
index 45d9b07..6af0c75 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCorrelate.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCorrelate.scala
@@ -93,8 +93,9 @@ class DataStreamPythonCorrelate(
val sqlFunction = pythonTableFuncRexCall.getOperator.asInstanceOf[TableSqlFunction]
+ val config = getMergedConfig(planner.getExecutionEnvironment, planner.getConfig)
val pythonOperator = getPythonTableFunctionOperator(
- getConfig(planner.getExecutionEnvironment, planner.getConfig),
+ config,
pythonOperatorInputRowType,
pythonOperatorOutputRowType,
pythonFunctionInfo,
@@ -114,7 +115,7 @@ class DataStreamPythonCorrelate(
// keep parallelism to ensure order of accumulate and retract messages
.setParallelism(inputDataStream.getParallelism)
- if (isPythonWorkerUsingManagedMemory(planner.getConfig.getConfiguration)) {
+ if (isPythonWorkerUsingManagedMemory(config)) {
ret.getTransformation.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON)
}
ret