You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by di...@apache.org on 2020/12/09 01:53:43 UTC
[flink] branch release-1.12 updated: [FLINK-20525][python] Fix
StreamArrowPythonGroupWindowAggregateFunctionOperator incorrect handling of
rowtime and proctime fields
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch release-1.12
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.12 by this push:
new cac9c1a [FLINK-20525][python] Fix StreamArrowPythonGroupWindowAggregateFunctionOperator incorrect handling of rowtime and proctime fields
cac9c1a is described below
commit cac9c1a5197e602774fb2cbe4a19361eea13a717
Author: huangxingbo <hx...@gmail.com>
AuthorDate: Tue Dec 8 12:53:13 2020 +0800
[FLINK-20525][python] Fix StreamArrowPythonGroupWindowAggregateFunctionOperator incorrect handling of rowtime and proctime fields
This closes #14327.
---
flink-python/pyflink/table/tests/test_pandas_udaf.py | 16 +++++++++-------
...mArrowPythonGroupWindowAggregateFunctionOperator.java | 4 +++-
.../stream/StreamExecPythonGroupWindowAggregate.scala | 3 ++-
3 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/flink-python/pyflink/table/tests/test_pandas_udaf.py b/flink-python/pyflink/table/tests/test_pandas_udaf.py
index b104325..e360f62 100644
--- a/flink-python/pyflink/table/tests/test_pandas_udaf.py
+++ b/flink-python/pyflink/table/tests/test_pandas_udaf.py
@@ -442,24 +442,26 @@ class StreamPandasUDAFITTests(PyFlinkBlinkStreamTableTestCase):
t = self.t_env.from_path("source_table")
table_sink = source_sink_utils.TestAppendSink(
- ['a', 'b', 'c', 'd'],
+ ['a', 'b', 'c', 'd', 'e'],
[
DataTypes.TINYINT(),
DataTypes.TIMESTAMP(3),
DataTypes.TIMESTAMP(3),
+ DataTypes.TIMESTAMP(3),
DataTypes.FLOAT()])
self.t_env.register_table_sink("Results", table_sink)
t.window(Tumble.over("1.hours").on("rowtime").alias("w")) \
.group_by("a, b, w") \
- .select("a, w.start, w.end, mean_udaf(c) as b") \
+ .select("a, w.start, w.end, w.rowtime, mean_udaf(c) as b") \
.execute_insert("Results") \
.wait()
actual = source_sink_utils.results()
- self.assert_equals(actual,
- ["1,2018-03-11 03:00:00.0,2018-03-11 04:00:00.0,2.5",
- "1,2018-03-11 04:00:00.0,2018-03-11 05:00:00.0,8.0",
- "2,2018-03-11 03:00:00.0,2018-03-11 04:00:00.0,2.0",
- "3,2018-03-11 03:00:00.0,2018-03-11 04:00:00.0,2.0"])
+ self.assert_equals(actual, [
+ "1,2018-03-11 03:00:00.0,2018-03-11 04:00:00.0,2018-03-11 03:59:59.999,2.5",
+ "1,2018-03-11 04:00:00.0,2018-03-11 05:00:00.0,2018-03-11 04:59:59.999,8.0",
+ "2,2018-03-11 03:00:00.0,2018-03-11 04:00:00.0,2018-03-11 03:59:59.999,2.0",
+ "3,2018-03-11 03:00:00.0,2018-03-11 04:00:00.0,2018-03-11 03:59:59.999,2.0",
+ ])
os.remove(source_path)
def test_tumbling_group_window_over_count(self):
diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/aggregate/arrow/stream/StreamArrowPythonGroupWindowAggregateFunctionOperator.java b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/aggregate/arrow/stream/StreamArrowPythonGroupWindowAggregateFunctionOperator.java
index 6c458c1..29a77a6 100644
--- a/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/aggregate/arrow/stream/StreamArrowPythonGroupWindowAggregateFunctionOperator.java
+++ b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/aggregate/arrow/stream/StreamArrowPythonGroupWindowAggregateFunctionOperator.java
@@ -380,8 +380,10 @@ public class StreamArrowPythonGroupWindowAggregateFunctionOperator<K, W extends
windowProperty.setField(i, TimestampData.fromEpochMillis(((TimeWindow) currentWindow).getEnd()));
break;
case 2:
- windowProperty.setField(i, TimestampData.fromEpochMillis(currentWindow.maxTimestamp()));
+ windowProperty.setField(i, TimestampData.fromEpochMillis(((TimeWindow) currentWindow).getEnd() - 1));
break;
+ case 3:
+ windowProperty.setField(i, TimestampData.fromEpochMillis(-1));
}
}
}
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamExecPythonGroupWindowAggregate.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamExecPythonGroupWindowAggregate.scala
index 7de00c9..54479ba 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamExecPythonGroupWindowAggregate.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamExecPythonGroupWindowAggregate.scala
@@ -29,7 +29,7 @@ import org.apache.flink.table.functions.python.PythonFunctionInfo
import org.apache.flink.table.planner.calcite.FlinkRelBuilder.PlannerNamedWindowProperty
import org.apache.flink.table.planner.calcite.FlinkTypeFactory
import org.apache.flink.table.planner.delegation.StreamPlanner
-import org.apache.flink.table.planner.expressions.{PlannerRowtimeAttribute, PlannerWindowEnd, PlannerWindowStart}
+import org.apache.flink.table.planner.expressions.{PlannerProctimeAttribute, PlannerRowtimeAttribute, PlannerWindowEnd, PlannerWindowStart}
import org.apache.flink.table.planner.plan.logical.{LogicalWindow, SlidingGroupWindow, TumblingGroupWindow}
import org.apache.flink.table.planner.plan.nodes.common.CommonPythonAggregate
import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamExecPythonGroupWindowAggregate.ARROW_STREAM_PYTHON_GROUP_WINDOW_AGGREGATE_FUNCTION_OPERATOR_NAME
@@ -219,6 +219,7 @@ class StreamExecPythonGroupWindowAggregate(
case PlannerWindowStart(_) => 0
case PlannerWindowEnd(_) => 1
case PlannerRowtimeAttribute(_) => 2
+ case PlannerProctimeAttribute(_) => 3
}
}.toArray