You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by di...@apache.org on 2022/04/27 12:59:04 UTC
[flink] 01/02: [FLINK-27282][python] Fix the bug of wrong positions mapping in RowCoder
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch release-1.15
in repository https://gitbox.apache.org/repos/asf/flink.git
commit ef4500c3d38e54a673302f7d36c2fad521e97eff
Author: huangxingbo <hx...@gmail.com>
AuthorDate: Mon Apr 18 16:34:39 2022 +0800
[FLINK-27282][python] Fix the bug of wrong positions mapping in RowCoder
This closes #19505.
---
.../docs/dev/python/table/intro_to_table_api.md | 2 +-
.../table/operations/row_based_operations.md | 108 +++++++++++++--------
.../docs/dev/python/table/intro_to_table_api.md | 2 +-
.../table/operations/row_based_operations.md | 108 +++++++++++++--------
flink-python/pyflink/common/types.py | 28 ++++--
.../pyflink/examples/table/basic_operations.py | 2 +-
.../pyflink/fn_execution/coder_impl_fast.pyx | 2 +-
.../pyflink/fn_execution/coder_impl_slow.py | 5 +-
.../pyflink/fn_execution/tests/test_coders.py | 4 +
.../operations/utils/OperationTreeBuilder.java | 17 +++-
.../plan/rules/logical/PythonMapMergeRule.java | 72 +++-----------
.../plan/rules/logical/PythonMapRenameRule.java | 103 ++++++++++++++++++++
.../planner/plan/rules/FlinkBatchRuleSets.scala | 1 +
.../planner/plan/rules/FlinkStreamRuleSets.scala | 1 +
.../table/planner/plan/utils/PythonUtil.scala | 66 +++++++++++++
.../plan/rules/logical/PythonMapMergeRuleTest.xml | 8 +-
.../table/planner/plan/stream/table/CalcTest.xml | 12 +--
.../table/planner/plan/stream/table/CalcTest.scala | 2 +-
18 files changed, 376 insertions(+), 167 deletions(-)
diff --git a/docs/content.zh/docs/dev/python/table/intro_to_table_api.md b/docs/content.zh/docs/dev/python/table/intro_to_table_api.md
index 6e36a879b59..64b795b012f 100644
--- a/docs/content.zh/docs/dev/python/table/intro_to_table_api.md
+++ b/docs/content.zh/docs/dev/python/table/intro_to_table_api.md
@@ -318,7 +318,7 @@ map_function = udf(lambda x: pd.concat([x.name, x.revenue * 10], axis=1),
DataTypes.FIELD("revenue", DataTypes.BIGINT())]),
func_type="pandas")
-orders.map(map_function).alias('name', 'revenue').to_pandas()
+orders.map(map_function).to_pandas()
```
结果为:
diff --git a/docs/content.zh/docs/dev/python/table/operations/row_based_operations.md b/docs/content.zh/docs/dev/python/table/operations/row_based_operations.md
index 723e292dddc..59f3a61de5b 100644
--- a/docs/content.zh/docs/dev/python/table/operations/row_based_operations.md
+++ b/docs/content.zh/docs/dev/python/table/operations/row_based_operations.md
@@ -36,6 +36,7 @@ from pyflink.common import Row
from pyflink.table import EnvironmentSettings, TableEnvironment
from pyflink.table.expressions import col
from pyflink.table.types import DataTypes
+from pyflink.table.udf import udf
env_settings = EnvironmentSettings.in_batch_mode()
table_env = TableEnvironment.create(env_settings)
@@ -48,11 +49,14 @@ def func1(id: int, data: str) -> Row:
return Row(id, data * 2)
# the input columns are specified as the inputs
-table.map(func1(col('id'), col('data'))).to_pandas()
+table.map(func1(col('id'), col('data'))).execute().print()
# result is
-# _c0 _c1
-# 0 1 HiHi
-# 1 2 HelloHello
+#+----------------------+--------------------------------+
+#| id | data |
+#+----------------------+--------------------------------+
+#| 1 | HiHi |
+#| 2 | HelloHello |
+#+----------------------+--------------------------------+
```
It also supports to take a Row object (containing all the columns of the input table) as input.
@@ -64,11 +68,14 @@ def func2(data: Row) -> Row:
return Row(data.id, data.data * 2)
# specify the function without the input columns
-table.map(func2).alias('id', 'data').to_pandas()
+table.map(func2).execute().print()
# result is
-# id data
-# 0 1 HiHi
-# 1 2 HelloHello
+#+----------------------+--------------------------------+
+#| id | data |
+#+----------------------+--------------------------------+
+#| 1 | HiHi |
+#| 2 | HelloHello |
+#+----------------------+--------------------------------+
```
<span class="label label-info">Note</span> The input columns should not be specified when using func2 in the map operation.
@@ -80,16 +87,19 @@ It should be noted that the input type and output type should be pandas.DataFram
import pandas as pd
@udf(result_type=DataTypes.ROW([DataTypes.FIELD("id", DataTypes.BIGINT()),
DataTypes.FIELD("data", DataTypes.STRING())]),
- func_type='pandas')
+ func_type='pandas')
def func3(data: pd.DataFrame) -> pd.DataFrame:
res = pd.concat([data.id, data.data * 2], axis=1)
return res
-table.map(func3).alias('id', 'data').to_pandas()
+table.map(func3).execute().print()
# result is
-# id data
-# 0 1 HiHi
-# 1 2 HelloHello
+#+----------------------+--------------------------------+
+#| id | data |
+#+----------------------+--------------------------------+
+#| 1 | HiHi |
+#| 2 | HelloHello |
+#+----------------------+--------------------------------+
```
## FlatMap
@@ -101,7 +111,7 @@ from pyflink.common import Row
from pyflink.table.udf import udtf
from pyflink.table import DataTypes, EnvironmentSettings, TableEnvironment
-env_settings = EnvironmentSettings.in_streaming_mode()
+env_settings = EnvironmentSettings.in_batch_mode()
table_env = TableEnvironment.create(env_settings)
table = table_env.from_elements([(1, 'Hi,Flink'), (2, 'Hello')], ['id', 'data'])
@@ -112,24 +122,30 @@ def split(x: Row) -> Row:
yield x.id, s
# use split in `flat_map`
-table.flat_map(split).to_pandas()
+table.flat_map(split).execute().print()
# result is
-# f0 f1
-# 0 1 Hi
-# 1 1 Flink
-# 2 2 Hello
+#+-------------+--------------------------------+
+#| f0 | f1 |
+#+-------------+--------------------------------+
+#| 1 | Hi |
+#| 1 | Flink |
+#| 2 | Hello |
+#+-------------+--------------------------------+
```
The python [table function]({{< ref "docs/dev/python/table/udfs/python_udfs" >}}#table-functions) could also be used in `join_lateral` and `left_outer_join_lateral`.
```python
# use table function in `join_lateral` or `left_outer_join_lateral`
-table.join_lateral(split.alias('a', 'b')).to_pandas()
+table.join_lateral(split.alias('a', 'b')).execute().print()
# result is
-# id data a b
-# 0 1 Hi,Flink 1 Hi
-# 1 1 Hi,Flink 1 Flink
-# 2 2 Hello 2 Hello
+#+----------------------+--------------------------------+-------------+--------------------------------+
+#| id | data | a | b |
+#+----------------------+--------------------------------+-------------+--------------------------------+
+#| 1 | Hi,Flink | 1 | Hi |
+#| 1 | Hi,Flink | 1 | Flink |
+#| 2 | Hello | 2 | Hello |
+#+----------------------+--------------------------------+-------------+--------------------------------+
```
## Aggregate
@@ -188,12 +204,15 @@ t = table_env.from_elements([(1, 2), (2, 1), (1, 3)], ['a', 'b'])
result = t.group_by(col('a')) \
.aggregate(agg.alias("c", "d")) \
.select(col('a'), col('c'), col('d'))
-result.to_pandas()
+result.execute().print()
# the result is
-# a c d
-# 0 1 2 5
-# 1 2 1 1
+#+----+----------------------+----------------------+----------------------+
+#| op | a | c | d |
+#+----+----------------------+----------------------+----------------------+
+#| +I | 1 | 2 | 5 |
+#| +I | 2 | 1 | 1 |
+#+----+----------------------+----------------------+----------------------+
# aggregate with a python vectorized aggregate function
env_settings = EnvironmentSettings.in_batch_mode()
@@ -207,11 +226,14 @@ pandas_udaf = udaf(lambda pd: (pd.b.mean(), pd.b.max()),
DataTypes.FIELD("b", DataTypes.INT())]),
func_type="pandas")
t.aggregate(pandas_udaf.alias("a", "b")) \
- .select(col('a'), col('b')).to_pandas()
+ .select(col('a'), col('b')).execute().print()
# the result is
-# a b
-# 0 2.0 3
+#+--------------------------------+-------------+
+#| a | b |
+#+--------------------------------+-------------+
+#| 2.0 | 3 |
+#+--------------------------------+-------------+
```
<span class="label label-info">Note</span> Similar to `map` operation, if you specify the aggregate function without the input columns in `aggregate` operation, it will take Row or Pandas.DataFrame as input which contains all the columns of the input table including the grouping keys.
@@ -229,6 +251,7 @@ Similar to `aggregate`, you have to close the `flat_aggregate` with a select sta
```python
from pyflink.common import Row
from pyflink.table import DataTypes, TableEnvironment, EnvironmentSettings
+from pyflink.table.expressions import col
from pyflink.table.udf import udtaf, TableAggregateFunction
class Top2(TableAggregateFunction):
@@ -262,18 +285,21 @@ table_env = TableEnvironment.create(env_settings)
# top2 = udtaf(Top2(), result_type=DataTypes.ROW([DataTypes.FIELD("a", DataTypes.BIGINT())]), accumulator_type=DataTypes.ARRAY(DataTypes.BIGINT()))
top2 = udtaf(Top2())
t = table_env.from_elements([(1, 'Hi', 'Hello'),
- (3, 'Hi', 'hi'),
- (5, 'Hi2', 'hi'),
- (7, 'Hi', 'Hello'),
- (2, 'Hi', 'Hello')], ['a', 'b', 'c'])
+ (3, 'Hi', 'hi'),
+ (5, 'Hi2', 'hi'),
+ (7, 'Hi', 'Hello'),
+ (2, 'Hi', 'Hello')], ['a', 'b', 'c'])
# call function "inline" without registration in Table API
-result = t.group_by(t.b).flat_aggregate(top2).select('*').to_pandas()
+result = t.group_by(t.b).flat_aggregate(top2).select(col('*')).to_pandas()
# the result is:
-# b a
-# 0 Hi2 5.0
-# 1 Hi2 NaN
-# 2 Hi 7.0
-# 3 Hi 3.0
+#+----+--------------------------------+----------------------+
+#| op | b | a |
+#+----+--------------------------------+----------------------+
+#| +I | Hi2 | 5 |
+#| +I | Hi2 | <NULL> |
+#| +I | Hi | 7 |
+#| +I | Hi | 3 |
+#+----+--------------------------------+----------------------+
```
diff --git a/docs/content/docs/dev/python/table/intro_to_table_api.md b/docs/content/docs/dev/python/table/intro_to_table_api.md
index 2b1e172b9f8..35d570d39aa 100644
--- a/docs/content/docs/dev/python/table/intro_to_table_api.md
+++ b/docs/content/docs/dev/python/table/intro_to_table_api.md
@@ -318,7 +318,7 @@ map_function = udf(lambda x: pd.concat([x.name, x.revenue * 10], axis=1),
DataTypes.FIELD("revenue", DataTypes.BIGINT())]),
func_type="pandas")
-orders.map(map_function).alias('name', 'revenue').to_pandas()
+orders.map(map_function).to_pandas()
```
The result is:
diff --git a/docs/content/docs/dev/python/table/operations/row_based_operations.md b/docs/content/docs/dev/python/table/operations/row_based_operations.md
index 723e292dddc..3e313b274bc 100644
--- a/docs/content/docs/dev/python/table/operations/row_based_operations.md
+++ b/docs/content/docs/dev/python/table/operations/row_based_operations.md
@@ -36,6 +36,7 @@ from pyflink.common import Row
from pyflink.table import EnvironmentSettings, TableEnvironment
from pyflink.table.expressions import col
from pyflink.table.types import DataTypes
+from pyflink.table.udf import udf
env_settings = EnvironmentSettings.in_batch_mode()
table_env = TableEnvironment.create(env_settings)
@@ -48,11 +49,14 @@ def func1(id: int, data: str) -> Row:
return Row(id, data * 2)
# the input columns are specified as the inputs
-table.map(func1(col('id'), col('data'))).to_pandas()
+table.map(func1(col('id'), col('data'))).execute().print()
# result is
-# _c0 _c1
-# 0 1 HiHi
-# 1 2 HelloHello
+#+----------------------+--------------------------------+
+#| id | data |
+#+----------------------+--------------------------------+
+#| 1 | HiHi |
+#| 2 | HelloHello |
+#+----------------------+--------------------------------+
```
It also supports to take a Row object (containing all the columns of the input table) as input.
@@ -64,11 +68,14 @@ def func2(data: Row) -> Row:
return Row(data.id, data.data * 2)
# specify the function without the input columns
-table.map(func2).alias('id', 'data').to_pandas()
+table.map(func2).execute().print()
# result is
-# id data
-# 0 1 HiHi
-# 1 2 HelloHello
+#+----------------------+--------------------------------+
+#| id | data |
+#+----------------------+--------------------------------+
+#| 1 | HiHi |
+#| 2 | HelloHello |
+#+----------------------+--------------------------------+
```
<span class="label label-info">Note</span> The input columns should not be specified when using func2 in the map operation.
@@ -80,16 +87,19 @@ It should be noted that the input type and output type should be pandas.DataFram
import pandas as pd
@udf(result_type=DataTypes.ROW([DataTypes.FIELD("id", DataTypes.BIGINT()),
DataTypes.FIELD("data", DataTypes.STRING())]),
- func_type='pandas')
+ func_type='pandas')
def func3(data: pd.DataFrame) -> pd.DataFrame:
res = pd.concat([data.id, data.data * 2], axis=1)
return res
-table.map(func3).alias('id', 'data').to_pandas()
+table.map(func3).execute().print()
# result is
-# id data
-# 0 1 HiHi
-# 1 2 HelloHello
+#+----------------------+--------------------------------+
+#| id | data |
+#+----------------------+--------------------------------+
+#| 1 | HiHi |
+#| 2 | HelloHello |
+#+----------------------+--------------------------------+
```
## FlatMap
@@ -101,7 +111,7 @@ from pyflink.common import Row
from pyflink.table.udf import udtf
from pyflink.table import DataTypes, EnvironmentSettings, TableEnvironment
-env_settings = EnvironmentSettings.in_streaming_mode()
+env_settings = EnvironmentSettings.in_batch_mode()
table_env = TableEnvironment.create(env_settings)
table = table_env.from_elements([(1, 'Hi,Flink'), (2, 'Hello')], ['id', 'data'])
@@ -112,24 +122,30 @@ def split(x: Row) -> Row:
yield x.id, s
# use split in `flat_map`
-table.flat_map(split).to_pandas()
+table.flat_map(split).execute().print()
# result is
-# f0 f1
-# 0 1 Hi
-# 1 1 Flink
-# 2 2 Hello
+#+-------------+--------------------------------+
+#| f0 | f1 |
+#+-------------+--------------------------------+
+#| 1 | Hi |
+#| 1 | Flink |
+#| 2 | Hello |
+#+-------------+--------------------------------+
```
The python [table function]({{< ref "docs/dev/python/table/udfs/python_udfs" >}}#table-functions) could also be used in `join_lateral` and `left_outer_join_lateral`.
```python
# use table function in `join_lateral` or `left_outer_join_lateral`
-table.join_lateral(split.alias('a', 'b')).to_pandas()
+table.join_lateral(split.alias('a', 'b')).execute().print()
# result is
-# id data a b
-# 0 1 Hi,Flink 1 Hi
-# 1 1 Hi,Flink 1 Flink
-# 2 2 Hello 2 Hello
+#+----------------------+--------------------------------+-------------+--------------------------------+
+#| id | data | a | b |
+#+----------------------+--------------------------------+-------------+--------------------------------+
+#| 1 | Hi,Flink | 1 | Hi |
+#| 1 | Hi,Flink | 1 | Flink |
+#| 2 | Hello | 2 | Hello |
+#+----------------------+--------------------------------+-------------+--------------------------------+
```
## Aggregate
@@ -188,12 +204,15 @@ t = table_env.from_elements([(1, 2), (2, 1), (1, 3)], ['a', 'b'])
result = t.group_by(col('a')) \
.aggregate(agg.alias("c", "d")) \
.select(col('a'), col('c'), col('d'))
-result.to_pandas()
+result.execute().print()
# the result is
-# a c d
-# 0 1 2 5
-# 1 2 1 1
+#+----+----------------------+----------------------+----------------------+
+#| op | a | c | d |
+#+----+----------------------+----------------------+----------------------+
+#| +I | 1 | 2 | 5 |
+#| +I | 2 | 1 | 1 |
+#+----+----------------------+----------------------+----------------------+
# aggregate with a python vectorized aggregate function
env_settings = EnvironmentSettings.in_batch_mode()
@@ -207,11 +226,14 @@ pandas_udaf = udaf(lambda pd: (pd.b.mean(), pd.b.max()),
DataTypes.FIELD("b", DataTypes.INT())]),
func_type="pandas")
t.aggregate(pandas_udaf.alias("a", "b")) \
- .select(col('a'), col('b')).to_pandas()
+ .select(col('a'), col('b')).execute().print()
# the result is
-# a b
-# 0 2.0 3
+#+--------------------------------+-------------+
+#| a | b |
+#+--------------------------------+-------------+
+#| 2.0 | 3 |
+#+--------------------------------+-------------+
```
<span class="label label-info">Note</span> Similar to `map` operation, if you specify the aggregate function without the input columns in `aggregate` operation, it will take Row or Pandas.DataFrame as input which contains all the columns of the input table including the grouping keys.
@@ -229,6 +251,7 @@ Similar to `aggregate`, you have to close the `flat_aggregate` with a select sta
```python
from pyflink.common import Row
from pyflink.table import DataTypes, TableEnvironment, EnvironmentSettings
+from pyflink.table.expressions import col
from pyflink.table.udf import udtaf, TableAggregateFunction
class Top2(TableAggregateFunction):
@@ -262,18 +285,21 @@ table_env = TableEnvironment.create(env_settings)
# top2 = udtaf(Top2(), result_type=DataTypes.ROW([DataTypes.FIELD("a", DataTypes.BIGINT())]), accumulator_type=DataTypes.ARRAY(DataTypes.BIGINT()))
top2 = udtaf(Top2())
t = table_env.from_elements([(1, 'Hi', 'Hello'),
- (3, 'Hi', 'hi'),
- (5, 'Hi2', 'hi'),
- (7, 'Hi', 'Hello'),
- (2, 'Hi', 'Hello')], ['a', 'b', 'c'])
+ (3, 'Hi', 'hi'),
+ (5, 'Hi2', 'hi'),
+ (7, 'Hi', 'Hello'),
+ (2, 'Hi', 'Hello')], ['a', 'b', 'c'])
# call function "inline" without registration in Table API
-result = t.group_by(t.b).flat_aggregate(top2).select('*').to_pandas()
+result = t.group_by(t.b).flat_aggregate(top2).select(col('*')).execute().print()
# the result is:
-# b a
-# 0 Hi2 5.0
-# 1 Hi2 NaN
-# 2 Hi 7.0
-# 3 Hi 3.0
+#+----+--------------------------------+----------------------+
+#| op | b | a |
+#+----+--------------------------------+----------------------+
+#| +I | Hi2 | 5 |
+#| +I | Hi2 | <NULL> |
+#| +I | Hi | 7 |
+#| +I | Hi | 3 |
+#+----+--------------------------------+----------------------+
```
diff --git a/flink-python/pyflink/common/types.py b/flink-python/pyflink/common/types.py
index 0542fb0e9a7..1711d7d2b7a 100644
--- a/flink-python/pyflink/common/types.py
+++ b/flink-python/pyflink/common/types.py
@@ -99,9 +99,8 @@ class Row(object):
raise ValueError("Can not use both args "
"and kwargs to create Row")
if kwargs:
- names = sorted(kwargs.keys())
- self._fields = names
- self._values = [kwargs[n] for n in names]
+ self._fields = list(kwargs.keys())
+ self._values = [kwargs[n] for n in self._fields]
self._from_dict = True
else:
self._values = list(args)
@@ -151,6 +150,16 @@ class Row(object):
def set_field_names(self, field_names: List):
self._fields = field_names
+ def get_fields_by_names(self, names: List[str]):
+ if not hasattr(self, '_fields') or names == self._fields:
+ return self._values
+
+ difference = list(set(names).difference(set(self._fields)))
+ if difference:
+ raise Exception("Field names {0} not exist in {1}.".format(difference, self._fields))
+ else:
+ return [self._values[self._fields.index(name)] for name in names]
+
def _is_retract_msg(self):
return self._row_kind == RowKind.UPDATE_BEFORE or self._row_kind == RowKind.DELETE
@@ -246,14 +255,19 @@ class Row(object):
if hasattr(self, "_fields"):
if not hasattr(other, "_fields"):
return False
- if self._fields != other._fields:
+ if sorted(self._fields) != sorted(other._fields):
return False
+ sorted_fields = sorted(self._fields)
+ return (self.__class__ == other.__class__ and
+ self._row_kind == other._row_kind and
+ [self._values[self._fields.index(name)] for name in sorted_fields] ==
+ [other._values[other._fields.index(name)] for name in sorted_fields])
else:
if hasattr(other, "_fields"):
return False
- return self.__class__ == other.__class__ and \
- self._row_kind == other._row_kind and \
- self._values == other._values
+ return (self.__class__ == other.__class__ and
+ self._row_kind == other._row_kind and
+ self._values == other._values)
def __hash__(self):
return tuple(self).__hash__()
diff --git a/flink-python/pyflink/examples/table/basic_operations.py b/flink-python/pyflink/examples/table/basic_operations.py
index 6b9454b757d..48e7abe4052 100644
--- a/flink-python/pyflink/examples/table/basic_operations.py
+++ b/flink-python/pyflink/examples/table/basic_operations.py
@@ -319,7 +319,7 @@ def row_operations():
table.map(extract_country) \
.execute().print()
# +----+----------------------+--------------------------------+
- # | op | _c0 | _c1 |
+ # | op | id | country |
# +----+----------------------+--------------------------------+
# | +I | 1 | Germany |
# | +I | 2 | China |
diff --git a/flink-python/pyflink/fn_execution/coder_impl_fast.pyx b/flink-python/pyflink/fn_execution/coder_impl_fast.pyx
index 75c7706eeec..518fcf826c5 100644
--- a/flink-python/pyflink/fn_execution/coder_impl_fast.pyx
+++ b/flink-python/pyflink/fn_execution/coder_impl_fast.pyx
@@ -385,7 +385,7 @@ cdef class RowCoderImpl(FieldCoderImpl):
row_kind_value = <unsigned char> value.row_kind
else:
# the type is Row
- list_values = <list> value._values
+ list_values = <list> value.get_fields_by_names(self._field_names)
row_kind_value = <unsigned char> value.get_row_kind().value
# encode mask value
self._mask_utils.write_mask(list_values, row_kind_value, out_stream)
diff --git a/flink-python/pyflink/fn_execution/coder_impl_slow.py b/flink-python/pyflink/fn_execution/coder_impl_slow.py
index dc3c186f92d..75fd258d94f 100644
--- a/flink-python/pyflink/fn_execution/coder_impl_slow.py
+++ b/flink-python/pyflink/fn_execution/coder_impl_slow.py
@@ -238,11 +238,12 @@ class RowCoderImpl(FieldCoderImpl):
def encode_to_stream(self, value: Row, out_stream: OutputStream):
# encode mask value
- self._mask_utils.write_mask(value._values, value.get_row_kind().value, out_stream)
+ values = value.get_fields_by_names(self._field_names)
+ self._mask_utils.write_mask(values, value.get_row_kind().value, out_stream)
# encode every field value
for i in range(self._field_count):
- item = value[i]
+ item = values[i]
if item is not None:
self._field_coders[i].encode_to_stream(item, out_stream)
diff --git a/flink-python/pyflink/fn_execution/tests/test_coders.py b/flink-python/pyflink/fn_execution/tests/test_coders.py
index e72bfaa249b..4d11ca18e5f 100644
--- a/flink-python/pyflink/fn_execution/tests/test_coders.py
+++ b/flink-python/pyflink/fn_execution/tests/test_coders.py
@@ -168,6 +168,10 @@ class CodersTest(PyFlinkTestCase):
v.set_row_kind(RowKind.DELETE)
self.check_coder(coder, v)
+ coder = RowCoder([BigIntCoder(), CharCoder()], ['f1', 'f0'])
+ v = Row(f0="flink", f1=11)
+ self.check_coder(coder, v)
+
def test_basic_decimal_coder(self):
basic_dec_coder = BigDecimalCoder()
value = decimal.Decimal(1.200)
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/utils/OperationTreeBuilder.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/utils/OperationTreeBuilder.java
index 14dee92e7b2..e01ecbbb69b 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/utils/OperationTreeBuilder.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/utils/OperationTreeBuilder.java
@@ -467,17 +467,24 @@ public final class OperationTreeBuilder {
}
public QueryOperation map(Expression mapFunction, QueryOperation child) {
+ final ExpressionResolver resolver = getResolverBuilder(child).build();
+ final ResolvedExpression resolvedCall = resolveSingleExpression(mapFunction, resolver);
- Expression resolvedMapFunction = mapFunction.accept(lookupResolver);
-
- if (!isFunctionOfKind(resolvedMapFunction, FunctionKind.SCALAR)) {
+ if (!isFunctionOfKind(resolvedCall, FunctionKind.SCALAR)) {
throw new ValidationException(
"Only a scalar function can be used in the map operator.");
}
+ final List<String> originFieldNames =
+ DataTypeUtils.flattenToNames(resolvedCall.getOutputDataType());
+
Expression expandedFields =
- unresolvedCall(BuiltInFunctionDefinitions.FLATTEN, resolvedMapFunction);
- return project(Collections.singletonList(expandedFields), child, false);
+ unresolvedCall(BuiltInFunctionDefinitions.FLATTEN, resolvedCall);
+ return alias(
+ originFieldNames.stream()
+ .map(ApiExpressionUtils::unresolvedRef)
+ .collect(Collectors.toList()),
+ project(Collections.singletonList(expandedFields), child, false));
}
public QueryOperation flatMap(Expression tableFunctionCall, QueryOperation child) {
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonMapMergeRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonMapMergeRule.java
index a9c6fbade15..57d2f2cea68 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonMapMergeRule.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonMapMergeRule.java
@@ -27,7 +27,6 @@ import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.rel.core.Calc;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexFieldAccess;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexProgram;
@@ -65,50 +64,37 @@ public class PythonMapMergeRule extends RelOptRule {
.map(topProgram::expandLocalRef)
.collect(Collectors.toList());
- if (topProjects.size() != 1
- || !PythonUtil.isPythonCall(topProjects.get(0), null)
- || !PythonUtil.takesRowAsInput((RexCall) topProjects.get(0))) {
- return false;
- }
-
RexProgram bottomProgram = bottomCalc.getProgram();
List<RexNode> bottomProjects =
bottomProgram.getProjectList().stream()
.map(bottomProgram::expandLocalRef)
.collect(Collectors.toList());
- if (bottomProjects.size() != 1 || !PythonUtil.isPythonCall(bottomProjects.get(0), null)) {
+
+ // both the bottom calc and the top calc contain only one project which is a Python
+ // function
+ if (topProjects.size() != 1
+ || !PythonUtil.isPythonCall(topProjects.get(0), null)
+ || topProgram.getCondition() != null
+ || bottomProjects.size() != 1
+ || !PythonUtil.isPythonCall(bottomProjects.get(0), null)
+ || bottomProgram.getCondition() != null) {
return false;
}
- // Only Python Functions with same Python function kind can be merged together.
- if (PythonUtil.isPythonCall(topProjects.get(0), PythonFunctionKind.GENERAL)
- ^ PythonUtil.isPythonCall(bottomProjects.get(0), PythonFunctionKind.GENERAL)) {
+ // top project is a map function which takes row as input
+ if (!PythonUtil.takesRowAsInput((RexCall) topProjects.get(0))) {
return false;
}
- RexProgram middleProgram = middleCalc.getProgram();
- if (topProgram.getCondition() != null
- || middleProgram.getCondition() != null
- || bottomProgram.getCondition() != null) {
+ // only Python functions with same Python function kind can be merged together.
+ if (PythonUtil.isPythonCall(topProjects.get(0), PythonFunctionKind.GENERAL)
+ ^ PythonUtil.isPythonCall(bottomProjects.get(0), PythonFunctionKind.GENERAL)) {
return false;
}
- List<RexNode> middleProjects =
- middleProgram.getProjectList().stream()
- .map(middleProgram::expandLocalRef)
- .collect(Collectors.toList());
- int inputRowFieldCount =
- middleProgram
- .getInputRowType()
- .getFieldList()
- .get(0)
- .getValue()
- .getFieldList()
- .size();
-
- return isFlattenCalc(middleProjects, inputRowFieldCount)
+ return PythonUtil.isFlattenCalc(middleCalc)
&& isTopCalcTakesWholeMiddleCalcAsInputs(
- (RexCall) topProjects.get(0), middleProjects.size());
+ (RexCall) topProjects.get(0), middleCalc.getRowType().getFieldCount());
}
private boolean isTopCalcTakesWholeMiddleCalcAsInputs(
@@ -130,32 +116,6 @@ public class PythonMapMergeRule extends RelOptRule {
return true;
}
- private boolean isFlattenCalc(List<RexNode> middleProjects, int inputRowFieldCount) {
- if (inputRowFieldCount != middleProjects.size()) {
- return false;
- }
- for (int i = 0; i < inputRowFieldCount; i++) {
- RexNode middleProject = middleProjects.get(i);
- if (middleProject instanceof RexFieldAccess) {
- RexFieldAccess rexField = ((RexFieldAccess) middleProject);
- if (rexField.getField().getIndex() != i) {
- return false;
- }
- RexNode expr = rexField.getReferenceExpr();
- if (expr instanceof RexInputRef) {
- if (((RexInputRef) expr).getIndex() != 0) {
- return false;
- }
- } else {
- return false;
- }
- } else {
- return false;
- }
- }
- return true;
- }
-
@Override
public void onMatch(RelOptRuleCall call) {
FlinkLogicalCalc topCalc = call.rel(0);
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonMapRenameRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonMapRenameRule.java
new file mode 100644
index 00000000000..3345c636054
--- /dev/null
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonMapRenameRule.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.plan.rules.logical;
+
+import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc;
+import org.apache.flink.table.planner.plan.utils.PythonUtil;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexProgram;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Rule which renames the field names of the Flatten {@link FlinkLogicalCalc} which is right after a
+ * {@link FlinkLogicalCalc} representing a Python Map operation to the output names of the map
+ * function.
+ */
+public class PythonMapRenameRule extends RelOptRule {
+
+ public static final PythonMapRenameRule INSTANCE = new PythonMapRenameRule();
+
+ private PythonMapRenameRule() {
+ super(
+ operand(FlinkLogicalCalc.class, operand(FlinkLogicalCalc.class, none())),
+ "PythonMapRenameRule");
+ }
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ FlinkLogicalCalc topCalc = call.rel(0);
+ FlinkLogicalCalc bottomCalc = call.rel(1);
+
+ // the bottom node is a Python map function
+ List<RexNode> bottomProjects =
+ bottomCalc.getProgram().getProjectList().stream()
+ .map(bottomCalc.getProgram()::expandLocalRef)
+ .collect(Collectors.toList());
+ if (bottomProjects.size() != 1
+ || !PythonUtil.isPythonCall(bottomProjects.get(0), null)
+ || bottomCalc.getProgram().getCondition() != null) {
+ return false;
+ }
+
+ // the top calc is a flatten operation and the field names of the flatten are not equal to
+ // the field names of the bottom calc
+ return PythonUtil.isFlattenCalc(topCalc)
+ && !bottomCalc
+ .getRowType()
+ .getFieldList()
+ .get(0)
+ .getValue()
+ .getFieldNames()
+ .equals(topCalc.getProgram().getOutputRowType().getFieldNames());
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ FlinkLogicalCalc topCalc = call.rel(0);
+ FlinkLogicalCalc bottomCalc = call.rel(1);
+
+ List<RexNode> topProjects =
+ topCalc.getProgram().getProjectList().stream()
+ .map(topCalc.getProgram()::expandLocalRef)
+ .collect(Collectors.toList());
+
+ FlinkLogicalCalc newCalc =
+ new FlinkLogicalCalc(
+ topCalc.getCluster(),
+ topCalc.getTraitSet(),
+ bottomCalc,
+ RexProgram.create(
+ bottomCalc.getRowType(),
+ topProjects,
+ null,
+ bottomCalc
+ .getRowType()
+ .getFieldList()
+ .get(0)
+ .getValue()
+ .getFieldNames(),
+ call.builder().getRexBuilder()));
+ call.transformTo(newCalc);
+ }
+}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala
index 171616b5a20..6aa07fc2a20 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala
@@ -391,6 +391,7 @@ object FlinkBatchRuleSets {
PythonCalcSplitRule.EXPAND_PROJECT,
PythonCalcSplitRule.PUSH_CONDITION,
PythonCalcSplitRule.REWRITE_PROJECT,
+ PythonMapRenameRule.INSTANCE,
PythonMapMergeRule.INSTANCE,
// remove output of rank number when it is not used by successor calc
RedundantRankNumberColumnRemoveRule.INSTANCE
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala
index ec2efe2ffba..a146bd647a5 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala
@@ -410,6 +410,7 @@ object FlinkStreamRuleSets {
PythonCalcSplitRule.EXPAND_PROJECT,
PythonCalcSplitRule.PUSH_CONDITION,
PythonCalcSplitRule.REWRITE_PROJECT,
+ PythonMapRenameRule.INSTANCE,
PythonMapMergeRule.INSTANCE
)
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/PythonUtil.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/PythonUtil.scala
index 4a1ecad5c2f..4ba865a44f4 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/PythonUtil.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/PythonUtil.scala
@@ -25,8 +25,19 @@ import org.apache.flink.table.functions.python.{PythonFunction, PythonFunctionKi
import org.apache.flink.table.planner.functions.aggfunctions.DeclarativeAggregateFunction
import org.apache.flink.table.planner.functions.bridging.{BridgingSqlAggFunction, BridgingSqlFunction}
import org.apache.flink.table.planner.functions.utils.{AggSqlFunction, ScalarSqlFunction, TableSqlFunction}
+import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc
import org.apache.flink.table.runtime.functions.aggregate.BuiltInAggregateFunction
+import org.apache.calcite.plan.hep.HepRelVertex
+import org.apache.calcite.plan.volcano.RelSubset
+import org.apache.calcite.rel.core.AggregateCall
+import org.apache.calcite.rex.{RexCall, RexFieldAccess, RexInputRef, RexNode, RexProgram}
+import org.apache.calcite.sql.SqlKind
+
+import java.util
+import java.util.List
+import java.util.stream.IntStream
+
import scala.collection.JavaConversions._
object PythonUtil {
@@ -121,6 +132,35 @@ object PythonUtil {
}
}
+ def isFlattenCalc(calc: FlinkLogicalCalc): Boolean = {
+ val child = calc.getInput match {
+ case relSubset: RelSubset => relSubset.getOriginal
+ case hepRelVertex: HepRelVertex => hepRelVertex.getCurrentRel
+ }
+ if (!child.isInstanceOf[FlinkLogicalCalc]) {
+ return false
+ }
+
+ if (calc.getProgram.getCondition != null) {
+ return false
+ }
+
+ val inputFields = calc.getProgram.getInputRowType.getFieldList
+ if (inputFields.size != 1 || !inputFields.get(0).getValue.isStruct) {
+ return false
+ }
+
+ val projects = calc.getProgram.getProjectList.map(calc.getProgram.expandLocalRef)
+
+ if (inputFields.get(0).getValue.getFieldList.size() != projects.size) {
+ return false
+ }
+
+ projects.zipWithIndex.forall {
+ case (project: RexNode, idx: Int) => project.accept(new FieldReferenceDetector(idx))
+ }
+ }
+
/**
* Checks whether it contains the specified kind of function in a RexNode.
*
@@ -166,4 +206,30 @@ object PythonUtil {
override def visitNode(rexNode: RexNode): Boolean = false
}
+
+ /** Checks whether a rexNode is only a field reference of the given index. */
+ private class FieldReferenceDetector(idx: Int) extends RexDefaultVisitor[Boolean] {
+
+ override def visitNode(rexNode: RexNode): Boolean = false
+
+ override def visitFieldAccess(fieldAccess: RexFieldAccess): Boolean = {
+ if (fieldAccess.getField.getIndex != idx) {
+ return false
+ }
+
+ val expr: RexNode = fieldAccess.getReferenceExpr
+ expr match {
+ case ref: RexInputRef => ref.getIndex == 0
+ case _ => false
+ }
+ }
+
+ override def visitCall(call: RexCall): Boolean = {
+ if (call.getKind == SqlKind.AS) {
+ call.getOperands.get(0).accept(this)
+ } else {
+ false
+ }
+ }
+ }
}
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PythonMapMergeRuleTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PythonMapMergeRuleTest.xml
index fcd6225fe06..54e0a85e09c 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PythonMapMergeRuleTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PythonMapMergeRuleTest.xml
@@ -19,13 +19,13 @@ limitations under the License.
<TestCase name="testMapOperationsChained">
<Resource name="ast">
<![CDATA[
-LogicalProject(_c0=[*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$4548a2f7c9e382d11c82a28716e0fd02*(*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$4548a2f7c9e382d11c82a28716e0fd02*(*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$4548a2f7c9e382d11c82a28716e0fd02*($0, $1, $2).f0, *org.apache.flink.table.planner.runtime.utils.JavaU [...]
+LogicalProject(f0=[AS(*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$4548a2f7c9e382d11c82a28716e0fd02*(AS(*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$4548a2f7c9e382d11c82a28716e0fd02*(AS(*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$4548a2f7c9e382d11c82a28716e0fd02*($0, $1, $2).f0, _UTF-16LE'f0'), AS(*org.apache.flink.table. [...]
+- LogicalTableScan(table=[[default_catalog, default_database, source, source: [TestTableSource(a, b, c)]]])
]]>
</Resource>
<Resource name="optimized rel plan">
<![CDATA[
-FlinkLogicalCalc(select=[f0.f0 AS _c0, f0.f1 AS _c1])
+FlinkLogicalCalc(select=[f0.f0 AS f0, f0.f1 AS f1])
+- FlinkLogicalCalc(select=[*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$4548a2f7c9e382d11c82a28716e0fd02*(*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$4548a2f7c9e382d11c82a28716e0fd02*(*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$4548a2f7c9e382d11c82a28716e0fd02*(a, b, c))) AS f0])
+- FlinkLogicalLegacyTableSourceScan(table=[[default_catalog, default_database, source, source: [TestTableSource(a, b, c)]]], fields=[a, b, c])
]]>
@@ -34,13 +34,13 @@ FlinkLogicalCalc(select=[f0.f0 AS _c0, f0.f1 AS _c1])
<TestCase name="testMapOperationMixedWithPandasUDFAndGeneralUDF">
<Resource name="ast">
<![CDATA[
-LogicalProject(_c0=[*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPandasScalarFunction$fc2ab005eb7004743f2fd6dcc3fab1ca*(*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$6d0a5b48d8e66954803fdd965425adec*($0, $1, $2).f0, *org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$6d0a5b48d8e66954803fdd965425adec*($0, $1, $2).f1).f0], _c1=[*org.apache.flink.table.p [...]
+LogicalProject(f0=[AS(*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPandasScalarFunction$fc2ab005eb7004743f2fd6dcc3fab1ca*(AS(*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$6d0a5b48d8e66954803fdd965425adec*($0, $1, $2).f0, _UTF-16LE'f0'), AS(*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$6d0a5b48d8e66954803fdd965425adec*($0, $1, $2).f1, _UTF-16LE'f [...]
+- LogicalTableScan(table=[[default_catalog, default_database, source, source: [TestTableSource(a, b, c)]]])
]]>
</Resource>
<Resource name="optimized rel plan">
<![CDATA[
-FlinkLogicalCalc(select=[f0.f0 AS _c0, f0.f1 AS _c1])
+FlinkLogicalCalc(select=[f0.f0 AS f0, f0.f1 AS f1])
+- FlinkLogicalCalc(select=[*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPandasScalarFunction$fc2ab005eb7004743f2fd6dcc3fab1ca*(f0, f1) AS f0])
+- FlinkLogicalCalc(select=[f0.f0 AS f0, f0.f1 AS f1])
+- FlinkLogicalCalc(select=[*org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions$RowPythonScalarFunction$6d0a5b48d8e66954803fdd965425adec*(a, b, c) AS f0])
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/table/CalcTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/table/CalcTest.xml
index b6ba6cdd7e6..862b5ee5095 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/table/CalcTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/table/CalcTest.xml
@@ -62,13 +62,13 @@ Calc(select=[a, b, c], where=[(SEARCH(b, Sarg[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
<TestCase name="testSimpleMap">
<Resource name="ast">
<![CDATA[
-LogicalProject(_c0=[*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*($0, $1, $2).f0], _c1=[*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*($0, $1, $2).f1], _c2=[*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*($0, $1, $2).f2], _c3=[*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*($0, $1, $2).f3])
+LogicalProject(f0=[AS(*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*($0, $1, $2).f0, _UTF-16LE'f0')], f1=[AS(*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*($0, $1, $2).f1, _UTF-16LE'f1')], f2=[AS(*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*($0, $1, $2).f2, _UTF-16LE'f2')], f3=[AS(*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78ed [...]
+- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [TestTableSource(a, b, c)]]])
]]>
</Resource>
<Resource name="optimized exec plan">
<![CDATA[
-Calc(select=[*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f0 AS _c0, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f1 AS _c1, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f2 AS _c2, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f3 AS _c3])
+Calc(select=[*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f0 AS f0, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f1 AS f1, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f2 AS f2, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f3 AS f3])
+- LegacyTableSourceScan(table=[[default_catalog, default_database, MyTable, source: [TestTableSource(a, b, c)]]], fields=[a, b, c])
]]>
</Resource>
@@ -93,13 +93,13 @@ Calc(select=[a, b], where=[((a > 0) AND (b < 2) AND (MOD(a, 2) = 1))])
<TestCase name="testMultiMap">
<Resource name="ast">
<![CDATA[
-LogicalProject(_c0=[*org.apache.flink.table.planner.expressions.utils.Func24$$2da7dcb3c9bedfd621ee539995571a90*(*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*($0, $1, $2).f0, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*($0, $1, $2).f1, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*($0, $1, $2).f2, *org.apache.flink.table.planner.expressions.utils.Func23 [...]
+LogicalProject(f0=[AS(*org.apache.flink.table.planner.expressions.utils.Func24$$2da7dcb3c9bedfd621ee539995571a90*(AS(*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*($0, $1, $2).f0, _UTF-16LE'f0'), AS(*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*($0, $1, $2).f1, _UTF-16LE'f1'), AS(*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*($0, $1, $2).f2, _UTF-16LE'f2' [...]
+- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [TestTableSource(a, b, c)]]])
]]>
</Resource>
<Resource name="optimized exec plan">
<![CDATA[
-Calc(select=[*org.apache.flink.table.planner.expressions.utils.Func24$$2da7dcb3c9bedfd621ee539995571a90*(*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f0, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f1, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f2, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78 [...]
+Calc(select=[*org.apache.flink.table.planner.expressions.utils.Func24$$2da7dcb3c9bedfd621ee539995571a90*(*org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f0, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f1, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78edece166661f30bbd0*(a, b, c).f2, *org.apache.flink.table.planner.expressions.utils.Func23$$79c21fb5a52b78 [...]
+- LegacyTableSourceScan(table=[[default_catalog, default_database, MyTable, source: [TestTableSource(a, b, c)]]], fields=[a, b, c])
]]>
</Resource>
@@ -122,13 +122,13 @@ Calc(select=[a, b, c], where=[(SEARCH(b, Sarg[(-∞..1), (1..2), (2..3), (3..4),
<TestCase name="testScalarResult">
<Resource name="ast">
<![CDATA[
-LogicalProject(_c0=[*org.apache.flink.table.planner.expressions.utils.Func1$$879c8537562dbe74f3349fa0e6502755*($0)])
+LogicalProject(f0=[AS(*org.apache.flink.table.planner.expressions.utils.Func1$$879c8537562dbe74f3349fa0e6502755*($0), _UTF-16LE'f0')])
+- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [TestTableSource(a, b, c)]]])
]]>
</Resource>
<Resource name="optimized exec plan">
<![CDATA[
-Calc(select=[*org.apache.flink.table.planner.expressions.utils.Func1$$879c8537562dbe74f3349fa0e6502755*(a) AS _c0])
+Calc(select=[*org.apache.flink.table.planner.expressions.utils.Func1$$879c8537562dbe74f3349fa0e6502755*(a) AS f0])
+- LegacyTableSourceScan(table=[[default_catalog, default_database, MyTable, source: [TestTableSource(a, b, c)]]], fields=[a, b, c])
]]>
</Resource>
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/table/CalcTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/table/CalcTest.scala
index 14bbeccfb57..6e544bd489c 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/table/CalcTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/table/CalcTest.scala
@@ -151,7 +151,7 @@ class CalcTest extends TableTestBase {
val sourceTable = util.addTableSource[(Int, Long, String)]("MyTable", 'a, 'b, 'c)
val resultTable = sourceTable
.map(Func23('a, 'b, 'c))
- .map(Func24('_c0, '_c1, '_c2, '_c3))
+ .map(Func24('f0, 'f1, 'f2, 'f3))
util.verifyExecPlan(resultTable)
}