You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tw...@apache.org on 2022/03/26 06:25:14 UTC
[flink] branch release-1.15 updated (608f5e1 -> ec8e43d)
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a change to branch release-1.15
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 608f5e1 [FLINK-26847][python] Ensure command line option '-py' works in YARN application mode
add 1d8e2d5 [hotfix][core] Add method to get all ConfigOptions from an "Options" class
add de77172 [hotfix][table-planner] Remove consumed options not affecting the topology.
add 00c1439 [FLINK-26075][table-planner] Persist node configuration to JSON plan
add ec8e43d [FLINK-26075][table-planner][test] Regenerate JSON plans
No new revisions were added by this update.
Summary of changes:
.../apache/flink/configuration/ConfigUtils.java | 23 ++
.../table/planner/plan/nodes/exec/ExecNode.java | 1 +
.../planner/plan/nodes/exec/ExecNodeBase.java | 21 +-
.../planner/plan/nodes/exec/ExecNodeConfig.java | 8 +-
.../planner/plan/nodes/exec/ExecNodeContext.java | 30 ++-
.../exec/batch/BatchExecBoundedStreamScan.java | 3 +
.../plan/nodes/exec/batch/BatchExecCalc.java | 3 +
.../plan/nodes/exec/batch/BatchExecCorrelate.java | 3 +
.../plan/nodes/exec/batch/BatchExecExchange.java | 8 +-
.../plan/nodes/exec/batch/BatchExecExpand.java | 3 +
.../nodes/exec/batch/BatchExecHashAggregate.java | 3 +
.../plan/nodes/exec/batch/BatchExecHashJoin.java | 3 +
.../exec/batch/BatchExecHashWindowAggregate.java | 3 +
.../plan/nodes/exec/batch/BatchExecLegacySink.java | 3 +
.../exec/batch/BatchExecLegacyTableSourceScan.java | 4 +
.../plan/nodes/exec/batch/BatchExecLimit.java | 3 +
.../plan/nodes/exec/batch/BatchExecLookupJoin.java | 3 +
.../nodes/exec/batch/BatchExecMultipleInput.java | 3 +
.../nodes/exec/batch/BatchExecNestedLoopJoin.java | 3 +
.../nodes/exec/batch/BatchExecOverAggregate.java | 3 +
.../exec/batch/BatchExecOverAggregateBase.java | 10 +-
.../plan/nodes/exec/batch/BatchExecPythonCalc.java | 3 +
.../nodes/exec/batch/BatchExecPythonCorrelate.java | 3 +
.../exec/batch/BatchExecPythonGroupAggregate.java | 4 +
.../batch/BatchExecPythonGroupWindowAggregate.java | 4 +
.../exec/batch/BatchExecPythonOverAggregate.java | 3 +
.../plan/nodes/exec/batch/BatchExecRank.java | 3 +
.../plan/nodes/exec/batch/BatchExecSink.java | 3 +
.../plan/nodes/exec/batch/BatchExecSort.java | 3 +
.../nodes/exec/batch/BatchExecSortAggregate.java | 3 +
.../plan/nodes/exec/batch/BatchExecSortLimit.java | 3 +
.../nodes/exec/batch/BatchExecSortMergeJoin.java | 3 +
.../exec/batch/BatchExecSortWindowAggregate.java | 3 +
.../nodes/exec/batch/BatchExecTableSourceScan.java | 7 +-
.../plan/nodes/exec/batch/BatchExecUnion.java | 7 +-
.../plan/nodes/exec/batch/BatchExecValues.java | 8 +-
.../exec/batch/BatchExecWindowTableFunction.java | 3 +
.../plan/nodes/exec/common/CommonExecCalc.java | 4 +-
.../nodes/exec/common/CommonExecCorrelate.java | 4 +-
.../plan/nodes/exec/common/CommonExecExchange.java | 4 +-
.../plan/nodes/exec/common/CommonExecExpand.java | 4 +-
.../nodes/exec/common/CommonExecLegacySink.java | 10 +-
.../common/CommonExecLegacyTableSourceScan.java | 4 +-
.../nodes/exec/common/CommonExecLookupJoin.java | 4 +-
.../nodes/exec/common/CommonExecPythonCalc.java | 4 +-
.../exec/common/CommonExecPythonCorrelate.java | 4 +-
.../plan/nodes/exec/common/CommonExecSink.java | 3 +-
.../exec/common/CommonExecTableSourceScan.java | 3 +-
.../plan/nodes/exec/common/CommonExecUnion.java | 4 +-
.../plan/nodes/exec/common/CommonExecValues.java | 4 +-
.../exec/common/CommonExecWindowTableFunction.java | 4 +-
.../processor/ForwardHashExchangeProcessor.java | 32 ++-
.../MultipleInputNodeCreationProcessor.java | 38 ++-
.../utils/InputPriorityConflictResolver.java | 19 +-
...zer.java => ConfigurationJsonDeserializer.java} | 27 ++-
...lizer.java => ConfigurationJsonSerializer.java} | 21 +-
.../serde/ConfigurationJsonSerializerFilter.java | 28 ++-
.../plan/nodes/exec/serde/JsonSerdeUtil.java | 3 +
.../nodes/exec/stream/StreamExecAggregateBase.java | 4 +-
.../plan/nodes/exec/stream/StreamExecCalc.java | 5 +
.../exec/stream/StreamExecChangelogNormalize.java | 7 +-
.../nodes/exec/stream/StreamExecCorrelate.java | 5 +
.../exec/stream/StreamExecDataStreamScan.java | 3 +
.../nodes/exec/stream/StreamExecDeduplicate.java | 6 +-
.../exec/stream/StreamExecDropUpdateBefore.java | 10 +-
.../plan/nodes/exec/stream/StreamExecExchange.java | 11 +-
.../plan/nodes/exec/stream/StreamExecExpand.java | 5 +
.../stream/StreamExecGlobalGroupAggregate.java | 13 +-
.../stream/StreamExecGlobalWindowAggregate.java | 7 +-
.../exec/stream/StreamExecGroupAggregate.java | 12 +-
.../exec/stream/StreamExecGroupTableAggregate.java | 4 +
.../stream/StreamExecGroupWindowAggregate.java | 7 +-
.../StreamExecIncrementalGroupAggregate.java | 13 +-
.../nodes/exec/stream/StreamExecIntervalJoin.java | 5 +-
.../plan/nodes/exec/stream/StreamExecJoin.java | 6 +-
.../nodes/exec/stream/StreamExecLegacySink.java | 3 +
.../stream/StreamExecLegacyTableSourceScan.java | 4 +
.../plan/nodes/exec/stream/StreamExecLimit.java | 7 +-
.../exec/stream/StreamExecLocalGroupAggregate.java | 7 +-
.../stream/StreamExecLocalWindowAggregate.java | 7 +-
.../nodes/exec/stream/StreamExecLookupJoin.java | 5 +
.../plan/nodes/exec/stream/StreamExecMatch.java | 5 +-
.../exec/stream/StreamExecMiniBatchAssigner.java | 6 +-
.../nodes/exec/stream/StreamExecMultipleInput.java | 7 +-
.../nodes/exec/stream/StreamExecOverAggregate.java | 7 +-
.../nodes/exec/stream/StreamExecPythonCalc.java | 6 +-
.../exec/stream/StreamExecPythonCorrelate.java | 5 +
.../stream/StreamExecPythonGroupAggregate.java | 7 +-
.../StreamExecPythonGroupTableAggregate.java | 4 +
.../StreamExecPythonGroupWindowAggregate.java | 10 +-
.../exec/stream/StreamExecPythonOverAggregate.java | 10 +-
.../plan/nodes/exec/stream/StreamExecRank.java | 8 +-
.../plan/nodes/exec/stream/StreamExecSink.java | 6 +-
.../plan/nodes/exec/stream/StreamExecSort.java | 3 +
.../nodes/exec/stream/StreamExecSortLimit.java | 7 +-
.../exec/stream/StreamExecTableSourceScan.java | 10 +-
.../nodes/exec/stream/StreamExecTemporalJoin.java | 7 +-
.../nodes/exec/stream/StreamExecTemporalSort.java | 6 +-
.../plan/nodes/exec/stream/StreamExecUnion.java | 10 +-
.../plan/nodes/exec/stream/StreamExecValues.java | 11 +-
.../exec/stream/StreamExecWatermarkAssigner.java | 7 +-
.../exec/stream/StreamExecWindowAggregate.java | 6 +-
.../exec/stream/StreamExecWindowAggregateBase.java | 4 +-
.../exec/stream/StreamExecWindowDeduplicate.java | 6 +-
.../nodes/exec/stream/StreamExecWindowJoin.java | 6 +-
.../nodes/exec/stream/StreamExecWindowRank.java | 6 +-
.../exec/stream/StreamExecWindowTableFunction.java | 14 +-
.../planner/plan/utils/ExecNodeMetadataUtil.java | 79 +++++++
.../batch/BatchPhysicalBoundedStreamScan.scala | 2 +
.../nodes/physical/batch/BatchPhysicalCalc.scala | 4 +-
.../physical/batch/BatchPhysicalCorrelate.scala | 4 +-
.../physical/batch/BatchPhysicalExchange.scala | 1 +
.../nodes/physical/batch/BatchPhysicalExpand.scala | 7 +-
.../batch/BatchPhysicalHashAggregate.scala | 7 +-
.../physical/batch/BatchPhysicalHashJoin.scala | 25 +-
.../batch/BatchPhysicalHashWindowAggregate.scala | 5 +-
.../physical/batch/BatchPhysicalLegacySink.scala | 5 +-
.../batch/BatchPhysicalLegacyTableSourceScan.scala | 2 +
.../nodes/physical/batch/BatchPhysicalLimit.scala | 16 +-
.../batch/BatchPhysicalLocalHashAggregate.scala | 7 +-
.../BatchPhysicalLocalHashWindowAggregate.scala | 5 +-
.../batch/BatchPhysicalLocalSortAggregate.scala | 7 +-
.../BatchPhysicalLocalSortWindowAggregate.scala | 5 +-
.../physical/batch/BatchPhysicalLookupJoin.scala | 2 +
.../batch/BatchPhysicalNestedLoopJoin.scala | 5 +-
.../batch/BatchPhysicalOverAggregate.scala | 5 +-
.../physical/batch/BatchPhysicalPythonCalc.scala | 2 +
.../batch/BatchPhysicalPythonCorrelate.scala | 7 +-
.../batch/BatchPhysicalPythonGroupAggregate.scala | 7 +-
.../BatchPhysicalPythonGroupWindowAggregate.scala | 5 +-
.../batch/BatchPhysicalPythonOverAggregate.scala | 5 +-
.../nodes/physical/batch/BatchPhysicalRank.scala | 5 +-
.../nodes/physical/batch/BatchPhysicalSink.scala | 8 +-
.../nodes/physical/batch/BatchPhysicalSort.scala | 7 +-
.../batch/BatchPhysicalSortAggregate.scala | 5 +-
.../physical/batch/BatchPhysicalSortLimit.scala | 7 +-
.../batch/BatchPhysicalSortMergeJoin.scala | 5 +-
.../batch/BatchPhysicalSortWindowAggregate.scala | 5 +-
.../batch/BatchPhysicalTableSourceScan.scala | 2 +
.../nodes/physical/batch/BatchPhysicalUnion.scala | 7 +-
.../nodes/physical/batch/BatchPhysicalValues.scala | 5 +-
.../batch/BatchPhysicalWindowTableFunction.scala | 5 +-
.../nodes/physical/stream/StreamPhysicalCalc.scala | 2 +
.../stream/StreamPhysicalChangelogNormalize.scala | 2 +
.../physical/stream/StreamPhysicalCorrelate.scala | 4 +-
.../stream/StreamPhysicalDataStreamScan.scala | 2 +
.../stream/StreamPhysicalDeduplicate.scala | 7 +-
.../stream/StreamPhysicalDropUpdateBefore.scala | 7 +-
.../physical/stream/StreamPhysicalExchange.scala | 4 +-
.../physical/stream/StreamPhysicalExpand.scala | 7 +-
.../StreamPhysicalGlobalGroupAggregate.scala | 7 +-
.../StreamPhysicalGlobalWindowAggregate.scala | 5 +-
.../stream/StreamPhysicalGroupAggregate.scala | 7 +-
.../stream/StreamPhysicalGroupTableAggregate.scala | 7 +-
.../StreamPhysicalGroupWindowAggregate.scala | 5 +-
.../StreamPhysicalGroupWindowTableAggregate.scala | 5 +-
.../StreamPhysicalIncrementalGroupAggregate.scala | 7 +-
.../stream/StreamPhysicalIntervalJoin.scala | 12 +-
.../nodes/physical/stream/StreamPhysicalJoin.scala | 18 +-
.../physical/stream/StreamPhysicalLegacySink.scala | 5 +-
.../StreamPhysicalLegacyTableSourceScan.scala | 2 +
.../physical/stream/StreamPhysicalLimit.scala | 19 +-
.../stream/StreamPhysicalLocalGroupAggregate.scala | 7 +-
.../StreamPhysicalLocalWindowAggregate.scala | 5 +-
.../physical/stream/StreamPhysicalLookupJoin.scala | 2 +
.../physical/stream/StreamPhysicalMatch.scala | 7 +-
.../stream/StreamPhysicalMiniBatchAssigner.scala | 7 +-
.../stream/StreamPhysicalOverAggregate.scala | 7 +-
.../physical/stream/StreamPhysicalPythonCalc.scala | 4 +-
.../stream/StreamPhysicalPythonCorrelate.scala | 7 +-
.../StreamPhysicalPythonGroupAggregate.scala | 7 +-
.../StreamPhysicalPythonGroupTableAggregate.scala | 7 +-
.../StreamPhysicalPythonGroupWindowAggregate.scala | 5 +-
.../stream/StreamPhysicalPythonOverAggregate.scala | 7 +-
.../nodes/physical/stream/StreamPhysicalRank.scala | 5 +-
.../nodes/physical/stream/StreamPhysicalSink.scala | 5 +-
.../nodes/physical/stream/StreamPhysicalSort.scala | 5 +-
.../physical/stream/StreamPhysicalSortLimit.scala | 7 +-
.../stream/StreamPhysicalTableSourceScan.scala | 2 +
.../stream/StreamPhysicalTemporalJoin.scala | 2 +
.../stream/StreamPhysicalTemporalSort.scala | 5 +-
.../physical/stream/StreamPhysicalUnion.scala | 7 +-
.../physical/stream/StreamPhysicalValues.scala | 5 +-
.../stream/StreamPhysicalWatermarkAssigner.scala | 2 +
.../stream/StreamPhysicalWindowAggregate.scala | 5 +-
.../stream/StreamPhysicalWindowDeduplicate.scala | 5 +-
.../physical/stream/StreamPhysicalWindowJoin.scala | 2 +
.../physical/stream/StreamPhysicalWindowRank.scala | 5 +-
.../stream/StreamPhysicalWindowTableFunction.scala | 5 +-
.../apache/flink/table/api/CompiledPlanITCase.java | 51 +++-
.../utils/InputPriorityConflictResolverTest.java | 1 +
.../plan/utils/ExecNodeMetadataUtilTest.java | 256 ++++++++++++++++++---
.../test/resources/jsonplan/testGetJsonPlan.out | 6 +
.../CalcJsonPlanTest_jsonplan/testComplexCalc.out | 6 +
.../CalcJsonPlanTest_jsonplan/testSimpleFilter.out | 6 +
.../testSimpleProject.out | 6 +
.../testChangelogSource.out | 10 +
.../testUpsertSource.out | 10 +
.../testCrossJoin.out | 6 +
.../testCrossJoinOverrideParameters.out | 6 +
.../testJoinWithFilter.out | 6 +
.../testLeftOuterJoinWithLiteralTrue.out | 6 +
.../testDeduplication.out | 12 +
.../ExpandJsonPlanTest_jsonplan/testExpand.out | 14 ++
...tDistinctAggCalls[isMiniBatchEnabled=false].out | 10 +
...stDistinctAggCalls[isMiniBatchEnabled=true].out | 14 ++
...gCallsWithGroupBy[isMiniBatchEnabled=false].out | 10 +
...ggCallsWithGroupBy[isMiniBatchEnabled=true].out | 14 ++
...AggWithoutGroupBy[isMiniBatchEnabled=false].out | 10 +
...eAggWithoutGroupBy[isMiniBatchEnabled=true].out | 14 ++
...erDefinedAggCalls[isMiniBatchEnabled=false].out | 10 +
...serDefinedAggCalls[isMiniBatchEnabled=true].out | 10 +
.../testEventTimeHopWindow.out | 11 +
.../testEventTimeSessionWindow.out | 11 +
.../testEventTimeTumbleWindow.out | 11 +
.../testProcTimeHopWindow.out | 11 +
.../testProcTimeSessionWindow.out | 11 +
.../testProcTimeTumbleWindow.out | 11 +
.../testIncrementalAggregate.out | 18 ++
...lAggregateWithSumCountDistinctAndRetraction.out | 26 +++
.../testProcessingTimeInnerJoinWithOnClause.out | 6 +
.../testRowTimeInnerJoinWithOnClause.out | 6 +
.../JoinJsonPlanTest_jsonplan/testInnerJoin.out | 6 +
.../testInnerJoinWithEqualPk.out | 14 ++
.../testInnerJoinWithPk.out | 14 ++
.../testLeftJoinNonEqui.out | 6 +
.../LimitJsonPlanTest_jsonplan/testLimit.out | 9 +
.../testJoinTemporalTable.out | 6 +
...testJoinTemporalTableWithProjectionPushDown.out | 6 +
.../testMatch.out | 6 +
.../testProcTimeBoundedNonPartitionedRangeOver.out | 6 +
.../testProcTimeBoundedPartitionedRangeOver.out | 6 +
...undedPartitionedRowsOverWithBuiltinProctime.out | 6 +
.../testProcTimeUnboundedPartitionedRangeOver.out | 6 +
...stProctimeBoundedDistinctPartitionedRowOver.out | 6 +
...edDistinctWithNonDistinctPartitionedRowOver.out | 6 +
.../testRowTimeBoundedPartitionedRowsOver.out | 6 +
.../stream/RankJsonPlanTest_jsonplan/testRank.out | 9 +
.../testSortLimit.out | 9 +
.../testOverwrite.out | 6 +
.../testPartitioning.out | 6 +
.../testWritingMetadata.out | 6 +
.../testFilterPushDown.out | 6 +
.../testLimitPushDown.out | 9 +
.../testPartitionPushDown.out | 6 +
.../testProjectPushDown.out | 6 +
.../testReadingMetadata.out | 6 +
.../testWatermarkPushDown.out | 6 +
.../testJoinTemporalFunction.out | 6 +
.../testTemporalTableJoin.out | 6 +
.../testSortProcessingTime.out | 6 +
.../testSortRowTime.out | 6 +
.../UnionJsonPlanTest_jsonplan/testUnion.out | 6 +
.../ValuesJsonPlanTest_jsonplan/testValues.out | 6 +
.../testWatermarkAssigner.out | 6 +
.../testDistinctSplitEnabled.out | 18 ++
.../testEventTimeCumulateWindow.out | 12 +
.../testEventTimeCumulateWindowWithOffset.out | 12 +
.../testEventTimeHopWindow.out | 12 +
.../testEventTimeHopWindowWithOffset.out | 12 +
.../testEventTimeTumbleWindow.out | 12 +
.../testEventTimeTumbleWindowWithOffset.out | 12 +
.../testProcTimeCumulateWindow.out | 9 +
.../testProcTimeHopWindow.out | 9 +
.../testProcTimeTumbleWindow.out | 9 +
.../testEventTimeTumbleWindow.out | 21 ++
.../testFollowedByWindowDeduplicate.out | 12 +
.../testFollowedByWindowJoin.out | 15 ++
.../testFollowedByWindowRank.out | 12 +
.../testIndividualWindowTVF.out | 9 +
.../testIndividualWindowTVFProcessingTime.out | 9 +
271 files changed, 1972 insertions(+), 393 deletions(-)
copy flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/{ExecNodeGraphJsonDeserializer.java => ConfigurationJsonDeserializer.java} (65%)
copy flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/{ObjectIdentifierJsonSerializer.java => ConfigurationJsonSerializer.java} (68%)
copy flink-yarn/src/main/java/org/apache/flink/yarn/cli/FallbackYarnSessionCli.java => flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/ConfigurationJsonSerializerFilter.java (59%)