You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gu...@apache.org on 2025/01/08 02:22:37 UTC
(flink) branch master updated (9720dcccc9e -> 75c26e8c9fd)
This is an automated email from the ASF dual-hosted git repository.
guoweijie pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from 9720dcccc9e [FLINK-37010][Runtime] Unify KeyedProcessFunction and the async one (#25901)
add 473aff1fb3a [FLINK-36960][API] Introduce Watermark and Combine Function for DSv2 Generalized Watermark
add ad68d76b597 [FLINK-36960][API] Introduce WatermarkDeclaration for Generalized Watermark in DSv2
add 5c03fffe856 [FLINK-36960][API] Enable PartitionedContext to access NonPartitionedContext
add 7915cb67dca [FLINK-36960][API] Introduce WatermarkDeclaration and onWatermark callback in ProcessFunction
add 3462cf3a220 [FLINK-36960][runtime] Introduce InternalWatermarkDeclaration, WatermarkCombiner and Alignable interface
add 472eca8d071 [FLINK-36960][API] Introduce WatermarkManager in NonPartitionedContext
add 75d4230ca6d [FLINK-36960][runtime] Introduce WatermarkEvent to propagate Generalized Watermarks within Stream
add c5031d7bb33 [FLINK-36960][API] Make Source is able to declare and output Generalized Watermarks
add 84e86b24538 [FLINK-36960][runtime] Write and load the declared WatermarkDeclarations with StreamGraph and StreamConfig
add 75fca0d7169 [FLINK-36960][runtime] Process Generalized Watermarks in ProcessOperators
add 52ef4ea7136 [FLINK-36960][runtime] Combine Generalized Watermarks in StreamTaskNetworkInput for input channels
add 75c26e8c9fd [FLINK-36960][test] Add WatermarkITCase for Generalized Watermark
No new revisions were added by this update.
Summary of changes:
.../18509c9e-3250-4c52-91b9-11ccefc85db1 | 2 +
.../e5126cae-f3fe-48aa-b6fb-60ae6cc3fcd5 | 8 +-
.../base/source/reader/SourceReaderBaseTest.java | 4 +
.../flink/api/common/watermark/BoolWatermark.java | 73 ++
.../common/watermark/BoolWatermarkDeclaration.java | 97 ++
.../flink/api/common/watermark/LongWatermark.java | 72 ++
.../common/watermark/LongWatermarkDeclaration.java | 99 ++
.../flink/api/common/watermark/Watermark.java | 34 +
.../watermark/WatermarkCombinationFunction.java | 55 +
.../watermark/WatermarkCombinationPolicy.java | 79 ++
.../api/common/watermark/WatermarkDeclaration.java | 22 +-
.../common/watermark/WatermarkDeclarations.java | 185 ++++
.../common/watermark/WatermarkHandlingResult.java | 23 +-
.../watermark/WatermarkHandlingStrategy.java | 16 +-
.../api/common/watermark/WatermarkManager.java | 18 +-
.../watermark/WatermarkDeclarationsTest.java | 91 ++
.../apache/flink/api/connector/source/Source.java | 14 +
.../api/connector/source/SourceReaderContext.java | 10 +
...nedContext.java => BasePartitionedContext.java} | 4 +-
.../api/context/NonPartitionedContext.java | 8 +
.../datastream/api/context/PartitionedContext.java | 14 +-
.../context/TwoOutputNonPartitionedContext.java | 8 +
...ntext.java => TwoOutputPartitionedContext.java} | 13 +-
.../function/OneInputStreamProcessFunction.java | 8 +
.../datastream/api/function/ProcessFunction.java | 11 +
.../TwoInputBroadcastStreamProcessFunction.java | 26 +
.../TwoInputNonBroadcastStreamProcessFunction.java | 26 +
.../function/TwoOutputApplyPartitionFunction.java | 7 +-
.../function/TwoOutputStreamProcessFunction.java | 27 +-
.../datastream/impl/ExecutionEnvironmentImpl.java | 2 +-
...ontext.java => AbstractPartitionedContext.java} | 14 +-
.../impl/context/DefaultNonPartitionedContext.java | 22 +-
.../impl/context/DefaultPartitionedContext.java | 55 +-
.../DefaultTwoOutputNonPartitionedContext.java | 22 +-
...ava => DefaultTwoOutputPartitionedContext.java} | 62 +-
.../impl/operators/KeyedProcessOperator.java | 8 +-
.../KeyedTwoInputBroadcastProcessOperator.java | 8 +-
.../KeyedTwoInputNonBroadcastProcessOperator.java | 8 +-
.../operators/KeyedTwoOutputProcessOperator.java | 9 +-
.../datastream/impl/operators/ProcessOperator.java | 46 +-
.../TwoInputBroadcastProcessOperator.java | 58 +-
.../TwoInputNonBroadcastProcessOperator.java | 58 +-
.../impl/operators/TwoOutputProcessOperator.java | 50 +-
.../impl/watermark/DefaultWatermarkManager.java | 57 +
...treamingJobGraphGeneratorWithAttributeTest.java | 3 +-
.../context/DefaultNonPartitionedContextTest.java | 61 +-
.../DefaultTwoOutputNonPartitionedContextTest.java | 61 +-
.../impl/functions/ProcessFunctionTest.java | 3 +-
.../KeyedTwoOutputProcessOperatorTest.java | 8 +-
.../operators/TwoOutputProcessOperatorTest.java | 6 +-
.../datastream/impl/stream/StreamTestUtils.java | 3 +-
.../datastream/impl/utils/StreamUtilsTest.java | 3 +-
.../datastream/impl/utils/WatermarkUtilsTest.java | 182 ++++
.../src/test/resources/log4j2-test.properties | 0
.../operators/StateBootstrapWrapperOperator.java | 4 +
.../StateBootstrapWrapperOperatorFactory.java | 4 +
flink-runtime/pom.xml | 6 +
.../AbstractAsyncStateStreamOperator.java | 43 +
.../TimestampedCollectorWithDeclaredVariable.java | 6 +
.../apache/flink/runtime/event/WatermarkEvent.java | 127 +++
.../network/api/serialization/EventSerializer.java | 17 +
.../flink/runtime/io/network/buffer/Buffer.java | 21 +-
.../io/network/partition/consumer/InputGate.java | 2 +
.../partition/consumer/SingleInputGate.java | 7 +
.../network/partition/consumer/UnionInputGate.java | 6 +
.../runtime/taskmanager/InputGateWithMetrics.java | 4 +
.../flink/streaming/api/graph/StreamConfig.java | 33 +
.../flink/streaming/api/graph/StreamGraph.java | 25 +
.../streaming/api/graph/StreamGraphGenerator.java | 2 +
.../flink/streaming/api/graph/StreamNode.java | 1 -
.../api/graph/StreamingJobGraphGenerator.java | 1 +
.../api/operators/AbstractStreamOperator.java | 16 +
.../streaming/api/operators/CountingOutput.java | 6 +
.../flink/streaming/api/operators/Input.java | 8 +
.../api/operators/KeyedProcessOperator.java | 6 +
.../flink/streaming/api/operators/Output.java | 8 +
.../streaming/api/operators/SourceOperator.java | 18 +-
.../api/operators/SourceOperatorFactory.java | 30 +-
.../api/operators/TimestampedCollector.java | 6 +
.../api/operators/TwoInputStreamOperator.java | 15 +
.../operators/sort/MultiInputSortingDataInput.java | 4 +
.../api/operators/sort/SortingDataInput.java | 4 +
.../runtime/io/AbstractStreamTaskNetworkInput.java | 73 +-
.../streaming/runtime/io/FinishedDataOutput.java | 6 +
.../runtime/io/PushingAsyncDataInput.java | 3 +
.../streaming/runtime/io/RecordWriterOutput.java | 18 +
.../io/StreamMultipleInputProcessorFactory.java | 9 +-
.../runtime/io/StreamTaskNetworkInput.java | 24 +-
.../runtime/io/StreamTaskNetworkInputFactory.java | 8 +-
.../runtime/io/StreamTwoInputProcessorFactory.java | 16 +-
.../io/checkpointing/CheckpointedInputGate.java | 4 +
.../recovery/RescalingStreamTaskNetworkInput.java | 4 +-
.../runtime/tasks/BroadcastingOutputCollector.java | 8 +
.../streaming/runtime/tasks/ChainingOutput.java | 10 +
.../tasks/FinishedOnRestoreMainOperatorOutput.java | 8 +
.../runtime/tasks/OneInputStreamTask.java | 14 +-
.../runtime/tasks/SourceOperatorStreamTask.java | 6 +
.../runtime/tasks/StreamIterationTail.java | 4 +
.../AbstractInternalWatermarkDeclaration.java | 97 ++
.../streaming/runtime/watermark/Alignable.java | 20 +-
.../AlignableBoolWatermarkDeclaration.java | 46 +
.../AlignableLongWatermarkDeclaration.java | 48 +
.../watermark/AlignedWatermarkCombiner.java | 70 ++
.../runtime/watermark/BoolWatermarkCombiner.java | 125 +++
.../InternalBoolWatermarkDeclaration.java | 69 ++
.../InternalLongWatermarkDeclaration.java | 68 ++
.../runtime/watermark/LongWatermarkCombiner.java | 184 ++++
.../runtime/watermark/WatermarkCombiner.java | 23 +-
.../runtime/watermarkstatus/HeapPriorityQueue.java | 4 +-
.../streaming/util/watermark/WatermarkUtils.java | 98 ++
.../api/serialization/EventSerializerTest.java | 23 +-
.../api/operators/SourceOperatorTest.java | 6 +
.../api/operators/sort/CollectingDataOutput.java | 6 +
.../sort/LargeSortingDataInputITCase.java | 4 +
.../api/operators/source/CollectingDataOutput.java | 6 +
.../operators/source/TestingSourceOperator.java | 3 +-
.../generalized/WatermarkCombinerTest.java | 405 ++++++++
.../streaming/runtime/io/MockIndexedInputGate.java | 3 +
.../flink/streaming/runtime/io/MockInputGate.java | 6 +
.../AlignedCheckpointsMassiveRandomTest.java | 3 +
.../watermarkstatus/StatusWatermarkValveTest.java | 6 +
.../util/AbstractStreamOperatorTestHarness.java | 6 +
.../flink/streaming/util/CollectorOutput.java | 32 +-
.../apache/flink/streaming/util/MockOutput.java | 6 +
.../streaming/util/SourceOperatorTestHarness.java | 6 +
.../runtime/io/StreamTaskNetworkInputTest.java | 4 +
.../table/planner/runtime/utils/TimeTestUtil.scala | 4 +
.../multipleinput/output/BroadcastingOutput.java | 8 +
...gSecondInputOfTwoInputStreamOperatorOutput.java | 10 +
.../FirstInputOfTwoInputStreamOperatorOutput.java | 10 +
.../output/OneInputStreamOperatorOutput.java | 10 +
.../SecondInputOfTwoInputStreamOperatorOutput.java | 10 +
.../multipleinput/output/BlackHoleOutput.java | 6 +
.../over/NonBufferOverWindowOperatorTest.java | 6 +
.../streaming/api/datastream/WatermarkITCase.java | 1097 ++++++++++++++++++++
pom.xml | 3 +
136 files changed, 4769 insertions(+), 280 deletions(-)
create mode 100644 flink-core-api/src/main/java/org/apache/flink/api/common/watermark/BoolWatermark.java
create mode 100644 flink-core-api/src/main/java/org/apache/flink/api/common/watermark/BoolWatermarkDeclaration.java
create mode 100644 flink-core-api/src/main/java/org/apache/flink/api/common/watermark/LongWatermark.java
create mode 100644 flink-core-api/src/main/java/org/apache/flink/api/common/watermark/LongWatermarkDeclaration.java
create mode 100644 flink-core-api/src/main/java/org/apache/flink/api/common/watermark/Watermark.java
create mode 100644 flink-core-api/src/main/java/org/apache/flink/api/common/watermark/WatermarkCombinationFunction.java
create mode 100644 flink-core-api/src/main/java/org/apache/flink/api/common/watermark/WatermarkCombinationPolicy.java
copy flink-datastream-api/src/main/java/org/apache/flink/datastream/api/context/PartitionedContext.java => flink-core-api/src/main/java/org/apache/flink/api/common/watermark/WatermarkDeclaration.java (62%)
create mode 100644 flink-core-api/src/main/java/org/apache/flink/api/common/watermark/WatermarkDeclarations.java
copy flink-datastream-api/src/main/java/org/apache/flink/datastream/api/context/PartitionedContext.java => flink-core-api/src/main/java/org/apache/flink/api/common/watermark/WatermarkHandlingResult.java (62%)
copy flink-datastream-api/src/main/java/org/apache/flink/datastream/api/context/PartitionedContext.java => flink-core-api/src/main/java/org/apache/flink/api/common/watermark/WatermarkHandlingStrategy.java (63%)
copy flink-datastream-api/src/main/java/org/apache/flink/datastream/api/context/PartitionedContext.java => flink-core-api/src/main/java/org/apache/flink/api/common/watermark/WatermarkManager.java (63%)
create mode 100644 flink-core-api/src/test/java/org/apache/flink/api/common/watermark/WatermarkDeclarationsTest.java
copy flink-datastream-api/src/main/java/org/apache/flink/datastream/api/context/{PartitionedContext.java => BasePartitionedContext.java} (88%)
copy flink-datastream-api/src/main/java/org/apache/flink/datastream/api/context/{PartitionedContext.java => TwoOutputPartitionedContext.java} (65%)
copy flink-datastream/src/main/java/org/apache/flink/datastream/impl/context/{DefaultPartitionedContext.java => AbstractPartitionedContext.java} (85%)
copy flink-datastream/src/main/java/org/apache/flink/datastream/impl/context/{DefaultPartitionedContext.java => DefaultTwoOutputPartitionedContext.java} (51%)
create mode 100644 flink-datastream/src/main/java/org/apache/flink/datastream/impl/watermark/DefaultWatermarkManager.java
create mode 100644 flink-datastream/src/test/java/org/apache/flink/datastream/impl/utils/WatermarkUtilsTest.java
copy {flink-connectors/flink-hadoop-compatibility => flink-datastream}/src/test/resources/log4j2-test.properties (100%)
create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/event/WatermarkEvent.java
create mode 100644 flink-runtime/src/main/java/org/apache/flink/streaming/runtime/watermark/AbstractInternalWatermarkDeclaration.java
copy flink-datastream-api/src/main/java/org/apache/flink/datastream/api/context/PartitionedContext.java => flink-runtime/src/main/java/org/apache/flink/streaming/runtime/watermark/Alignable.java (58%)
create mode 100644 flink-runtime/src/main/java/org/apache/flink/streaming/runtime/watermark/AlignableBoolWatermarkDeclaration.java
create mode 100644 flink-runtime/src/main/java/org/apache/flink/streaming/runtime/watermark/AlignableLongWatermarkDeclaration.java
create mode 100644 flink-runtime/src/main/java/org/apache/flink/streaming/runtime/watermark/AlignedWatermarkCombiner.java
create mode 100644 flink-runtime/src/main/java/org/apache/flink/streaming/runtime/watermark/BoolWatermarkCombiner.java
create mode 100644 flink-runtime/src/main/java/org/apache/flink/streaming/runtime/watermark/InternalBoolWatermarkDeclaration.java
create mode 100644 flink-runtime/src/main/java/org/apache/flink/streaming/runtime/watermark/InternalLongWatermarkDeclaration.java
create mode 100644 flink-runtime/src/main/java/org/apache/flink/streaming/runtime/watermark/LongWatermarkCombiner.java
copy flink-datastream-api/src/main/java/org/apache/flink/datastream/api/context/PartitionedContext.java => flink-runtime/src/main/java/org/apache/flink/streaming/runtime/watermark/WatermarkCombiner.java (56%)
create mode 100644 flink-runtime/src/main/java/org/apache/flink/streaming/util/watermark/WatermarkUtils.java
create mode 100644 flink-runtime/src/test/java/org/apache/flink/streaming/api/watermark/generalized/WatermarkCombinerTest.java
create mode 100644 flink-tests/src/test/java/org/apache/flink/test/streaming/api/datastream/WatermarkITCase.java