You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by di...@apache.org on 2022/06/17 01:22:52 UTC
[flink] branch master updated: [FLINK-27932][python] Introduce WatermarkStrategy.no_watermarks and WatermarkStrategy.no_watermarks to align with Java API
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new d0f0071b767 [FLINK-27932][python] Introduce WatermarkStrategy.no_watermarks and WatermarkStrategy.no_watermarks to align with Java API
d0f0071b767 is described below
commit d0f0071b767b8dbe2e811e9b6b5e1e9027a20e2b
Author: Juntao Hu <ma...@gmail.com>
AuthorDate: Tue Jun 7 16:21:03 2022 +0800
[FLINK-27932][python] Introduce WatermarkStrategy.no_watermarks and WatermarkStrategy.no_watermarks to align with Java API
This closes #19896.
---
.../docs/dev/datastream/event-time/built_in.md | 10 +++
.../datastream/event-time/generating_watermarks.md | 47 +++++++++++++
.../docs/dev/datastream/event-time/built_in.md | 10 +++
.../datastream/event-time/generating_watermarks.md | 47 +++++++++++++
.../common/tests/test_watermark_strategy.py | 79 ++++++++++++++++++++++
flink-python/pyflink/common/watermark_strategy.py | 74 +++++++++++++++++---
6 files changed, 259 insertions(+), 8 deletions(-)
diff --git a/docs/content.zh/docs/dev/datastream/event-time/built_in.md b/docs/content.zh/docs/dev/datastream/event-time/built_in.md
index 4b4452827fe..792dfad43c7 100644
--- a/docs/content.zh/docs/dev/datastream/event-time/built_in.md
+++ b/docs/content.zh/docs/dev/datastream/event-time/built_in.md
@@ -50,6 +50,11 @@ WatermarkStrategy.forMonotonousTimestamps();
WatermarkStrategy.forMonotonousTimestamps()
```
{{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy.for_monotonous_timestamps()
+```
+{{< /tab >}}
{{< /tabs >}}
<a name="fixed-amount-of-lateness"></a>
@@ -70,6 +75,11 @@ WatermarkStrategy
.forBoundedOutOfOrderness(Duration.ofSeconds(10))
```
{{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy.for_bounded_out_of_orderness(Duration.of_seconds(10))
+```
+{{< /tab >}}
{{< /tabs >}}
{{< top >}}
diff --git a/docs/content.zh/docs/dev/datastream/event-time/generating_watermarks.md b/docs/content.zh/docs/dev/datastream/event-time/generating_watermarks.md
index 01b1c869473..e4dbc627480 100644
--- a/docs/content.zh/docs/dev/datastream/event-time/generating_watermarks.md
+++ b/docs/content.zh/docs/dev/datastream/event-time/generating_watermarks.md
@@ -77,6 +77,19 @@ WatermarkStrategy
})
```
{{< /tab >}}
+{{< tab "Python" >}}
+```python
+class FirstElementTimestampAssigner(TimestampAssigner):
+
+ def extract_timestamp(self, value, record_timestamp):
+ return value[0]
+
+
+WatermarkStrategy \
+ .for_bounded_out_of_orderness(Duration.of_seconds(20)) \
+ .with_timestamp_assigner(FirstElementTimestampAssigner())
+```
+{{< /tab >}}
{{< /tabs >}}
其中 `TimestampAssigner` 的设置与否是可选的,大多数情况下,可以不用去特别指定。例如,当使用 Kafka 或 Kinesis 数据源时,你可以直接从 Kafka/Kinesis 数据源记录中获取到时间戳。
@@ -136,6 +149,26 @@ withTimestampsAndWatermarks
.addSink(...)
```
{{< /tab >}}
+{{< tab "Python" >}}
+```python
+env = StreamExecutionEnvironment.get_execution_environment()
+
+# currently read_file is not supported in PyFlink
+stream = env \
+ .read_text_file(my_file_path, charset) \
+ .map(lambda s: MyEvent.from_string(s))
+
+with_timestamp_and_watermarks = stream \
+ .filter(lambda e: e.severity() == WARNING) \
+ .assign_timestamp_and_watermarks(<watermark strategy>)
+
+with_timestamp_and_watermarks \
+ .key_by(lambda e: e.get_group()) \
+ .window(TumblingEventTimeWindows.of(Time.seconds(10))) \
+ .reduce(lambda a, b: a.add(b)) \
+ .add_sink(...)
+```
+{{< /tab >}}
{{< /tabs >}}
使用 `WatermarkStrategy` 去获取流并生成带有时间戳的元素和 watermark 的新流时,如果原始流已经具有时间戳或 watermark,则新指定的时间戳分配器将覆盖原有的时间戳和 watermark。
@@ -163,6 +196,13 @@ WatermarkStrategy
.withIdleness(Duration.ofMinutes(1))
```
{{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy \
+ .for_bounded_out_of_orderness(Duration.of_seconds(20)) \
+ .with_idleness(Duration.of_minutes(1))
+```
+{{< /tab >}}
{{< /tabs >}}
## Watermark alignment _`Beta`_
@@ -200,6 +240,13 @@ WatermarkStrategy
.withWatermarkAlignment("alignment-group-1", Duration.ofSeconds(20), Duration.ofSeconds(1))
```
{{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy \
+ .for_bounded_out_of_orderness(Duration.of_seconds(20)) \
+ .with_watermark_alignment("alignment-group-1", Duration.of_seconds(20), Duration.of_seconds(1))
+```
+{{< /tab >}}
{{< /tabs >}}
{{< hint warning >}}
diff --git a/docs/content/docs/dev/datastream/event-time/built_in.md b/docs/content/docs/dev/datastream/event-time/built_in.md
index 836a84ba332..bf74eacc097 100644
--- a/docs/content/docs/dev/datastream/event-time/built_in.md
+++ b/docs/content/docs/dev/datastream/event-time/built_in.md
@@ -62,6 +62,11 @@ WatermarkStrategy.forMonotonousTimestamps();
WatermarkStrategy.forMonotonousTimestamps()
```
{{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy.for_monotonous_timestamps()
+```
+{{< /tab >}}
{{< /tabs >}}
## Fixed Amount of Lateness
@@ -93,6 +98,11 @@ WatermarkStrategy.forBoundedOutOfOrderness(Duration.ofSeconds(10));
WatermarkStrategy.forBoundedOutOfOrderness(Duration.ofSeconds(10))
```
{{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy.for_bounded_out_of_orderness(Duration.of_seconds(10))
+```
+{{< /tab >}}
{{< /tabs >}}
{{< top >}}
diff --git a/docs/content/docs/dev/datastream/event-time/generating_watermarks.md b/docs/content/docs/dev/datastream/event-time/generating_watermarks.md
index 1ae08ec1f9f..2cc5ed1d4d6 100644
--- a/docs/content/docs/dev/datastream/event-time/generating_watermarks.md
+++ b/docs/content/docs/dev/datastream/event-time/generating_watermarks.md
@@ -94,6 +94,19 @@ WatermarkStrategy
})
```
{{< /tab >}}
+{{< tab "Python" >}}
+```python
+class FirstElementTimestampAssigner(TimestampAssigner):
+
+ def extract_timestamp(self, value, record_timestamp):
+ return value[0]
+
+
+WatermarkStrategy \
+ .for_bounded_out_of_orderness(Duration.of_seconds(20)) \
+ .with_timestamp_assigner(FirstElementTimestampAssigner())
+```
+{{< /tab >}}
{{< /tabs >}}
Specifying a `TimestampAssigner` is optional and in most cases you don't
@@ -165,6 +178,26 @@ withTimestampsAndWatermarks
.addSink(...)
```
{{< /tab >}}
+{{< tab "Python" >}}
+```python
+env = StreamExecutionEnvironment.get_execution_environment()
+
+# currently read_file is not supported in PyFlink
+stream = env \
+ .read_text_file(my_file_path, charset) \
+ .map(lambda s: MyEvent.from_string(s))
+
+with_timestamp_and_watermarks = stream \
+ .filter(lambda e: e.severity() == WARNING) \
+ .assign_timestamp_and_watermarks(<watermark strategy>)
+
+with_timestamp_and_watermarks \
+ .key_by(lambda e: e.get_group()) \
+ .window(TumblingEventTimeWindows.of(Time.seconds(10))) \
+ .reduce(lambda a, b: a.add(b)) \
+ .add_sink(...)
+```
+{{< /tab >}}
{{< /tabs >}}
Using a `WatermarkStrategy` this way takes a stream and produce a new stream
@@ -199,6 +232,13 @@ WatermarkStrategy
.withIdleness(Duration.ofMinutes(1))
```
{{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy \
+ .for_bounded_out_of_orderness(Duration.of_seconds(20)) \
+ .with_idleness(Duration.of_minutes(1))
+```
+{{< /tab >}}
{{< /tabs >}}
## Watermark alignment _`Beta`_
@@ -236,6 +276,13 @@ WatermarkStrategy
.withWatermarkAlignment("alignment-group-1", Duration.ofSeconds(20), Duration.ofSeconds(1))
```
{{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy \
+ .for_bounded_out_of_orderness(Duration.of_seconds(20)) \
+ .with_watermark_alignment("alignment-group-1", Duration.of_seconds(20), Duration.of_seconds(1))
+```
+{{< /tab >}}
{{< /tabs >}}
{{< hint warning >}}
diff --git a/flink-python/pyflink/common/tests/test_watermark_strategy.py b/flink-python/pyflink/common/tests/test_watermark_strategy.py
new file mode 100644
index 00000000000..7853605d17c
--- /dev/null
+++ b/flink-python/pyflink/common/tests/test_watermark_strategy.py
@@ -0,0 +1,79 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from pyflink.common import Duration
+from pyflink.common.watermark_strategy import WatermarkStrategy
+from pyflink.java_gateway import get_gateway
+from pyflink.util.java_utils import is_instance_of, get_field_value
+
+from pyflink.testing.test_case_utils import PyFlinkTestCase
+
+
+class WatermarkStrategyTests(PyFlinkTestCase):
+
+ def test_with_idleness(self):
+ jvm = get_gateway().jvm
+ j_watermark_strategy = WatermarkStrategy.no_watermarks().with_idleness(
+ Duration.of_seconds(5)
+ )._j_watermark_strategy
+ self.assertTrue(is_instance_of(
+ j_watermark_strategy,
+ jvm.org.apache.flink.api.common.eventtime.WatermarkStrategyWithIdleness
+ ))
+ self.assertEqual(get_field_value(j_watermark_strategy, "idlenessTimeout").toMillis(), 5000)
+
+ def test_with_watermark_alignment(self):
+ jvm = get_gateway().jvm
+ j_watermark_strategy = WatermarkStrategy.no_watermarks().with_watermark_alignment(
+ "alignment-group-1", Duration.of_seconds(20), Duration.of_seconds(10)
+ )._j_watermark_strategy
+ self.assertTrue(is_instance_of(
+ j_watermark_strategy,
+ jvm.org.apache.flink.api.common.eventtime.WatermarksWithWatermarkAlignment
+ ))
+ alignment_parameters = j_watermark_strategy.getAlignmentParameters()
+ self.assertEqual(alignment_parameters.getWatermarkGroup(), "alignment-group-1")
+ self.assertEqual(alignment_parameters.getMaxAllowedWatermarkDrift(), 20000)
+ self.assertEqual(alignment_parameters.getUpdateInterval(), 10000)
+
+ def test_for_monotonous_timestamps(self):
+ jvm = get_gateway().jvm
+ j_watermark_strategy = WatermarkStrategy.for_monotonous_timestamps()._j_watermark_strategy
+ self.assertTrue(is_instance_of(
+ j_watermark_strategy.createWatermarkGenerator(None),
+ jvm.org.apache.flink.api.common.eventtime.AscendingTimestampsWatermarks
+ ))
+
+ def test_for_bounded_out_of_orderness(self):
+ jvm = get_gateway().jvm
+ j_watermark_strategy = WatermarkStrategy.for_bounded_out_of_orderness(
+ Duration.of_seconds(3)
+ )._j_watermark_strategy
+ j_watermark_generator = j_watermark_strategy.createWatermarkGenerator(None)
+ self.assertTrue(is_instance_of(
+ j_watermark_generator,
+ jvm.org.apache.flink.api.common.eventtime.BoundedOutOfOrdernessWatermarks
+ ))
+ self.assertEqual(get_field_value(j_watermark_generator, "outOfOrdernessMillis"), 3000)
+
+ def test_no_watermarks(self):
+ jvm = get_gateway().jvm
+ j_watermark_strategy = WatermarkStrategy.no_watermarks()._j_watermark_strategy
+ self.assertTrue(is_instance_of(
+ j_watermark_strategy.createWatermarkGenerator(None),
+ jvm.org.apache.flink.api.common.eventtime.NoWatermarksGenerator
+ ))
diff --git a/flink-python/pyflink/common/watermark_strategy.py b/flink-python/pyflink/common/watermark_strategy.py
index aa763320aad..532e962b06a 100644
--- a/flink-python/pyflink/common/watermark_strategy.py
+++ b/flink-python/pyflink/common/watermark_strategy.py
@@ -17,7 +17,7 @@
################################################################################
import abc
-from typing import Any
+from typing import Any, Optional
from pyflink.common.time import Duration
from pyflink.java_gateway import get_gateway
@@ -42,9 +42,11 @@ class WatermarkStrategy(object):
Creates a new WatermarkStrategy that wraps this strategy but instead uses the given a
TimestampAssigner by implementing TimestampAssigner interface.
+ Example:
::
- >>> watermark_strategy = WatermarkStrategy.for_monotonous_timestamps()
- >>> .with_timestamp_assigner(MyTimestampAssigner())
+
+ >>> watermark_strategy = WatermarkStrategy.for_monotonous_timestamps() \\
+ >>> .with_timestamp_assigner(MyTimestampAssigner())
:param timestamp_assigner: The given TimestampAssigner.
:return: A WaterMarkStrategy that wraps a TimestampAssigner.
@@ -57,15 +59,59 @@ class WatermarkStrategy(object):
Creates a new enriched WatermarkStrategy that also does idleness detection in the created
WatermarkGenerator.
+ Example:
+ ::
+
+ >>> WatermarkStrategy \\
+ ... .for_bounded_out_of_orderness(Duration.of_seconds(20)) \\
+ ... .with_idleness(Duration.of_minutes(1))
+
:param idle_timeout: The idle timeout.
:return: A new WatermarkStrategy with idle detection configured.
"""
- self._j_watermark_strategy = self._j_watermark_strategy\
- .withIdleness(idle_timeout._j_duration)
- return self
+ return WatermarkStrategy(self._j_watermark_strategy.withIdleness(idle_timeout._j_duration))
+
+ def with_watermark_alignment(self, watermark_group: str, max_allowed_watermark_drift: Duration,
+ update_interval: Optional[Duration] = None) -> 'WatermarkStrategy':
+ """
+ Creates a new :class:`WatermarkStrategy` that configures the maximum watermark drift from
+ other sources/tasks/partitions in the same watermark group. The group may contain completely
+ independent sources (e.g. File and Kafka).
+
+ Once configured Flink will "pause" consuming from a source/task/partition that is ahead
+ of the emitted watermark in the group by more than the maxAllowedWatermarkDrift.
+
+ Example:
+ ::
+
+ >>> WatermarkStrategy \\
+ ... .for_bounded_out_of_orderness(Duration.of_seconds(20)) \\
+ ... .with_watermark_alignment("alignment-group-1", Duration.of_seconds(20),
+ ... Duration.of_seconds(1))
+
+ :param watermark_group: A group of sources to align watermarks
+ :param max_allowed_watermark_drift: Maximal drift, before we pause consuming from the
+ source/task/partition
+ :param update_interval: How often tasks should notify coordinator about the current
+ watermark and how often the coordinator should announce the maximal aligned watermark.
+ If is None, default update interval (1000ms) is used.
+ :return: A new WatermarkStrategy with watermark alignment configured.
+
+ .. versionadded:: 1.16.0
+ """
+ if update_interval is None:
+ return WatermarkStrategy(self._j_watermark_strategy.withWatermarkAlignment(
+ watermark_group, max_allowed_watermark_drift._j_duration
+ ))
+ else:
+ return WatermarkStrategy(self._j_watermark_strategy.withWatermarkAlignment(
+ watermark_group,
+ max_allowed_watermark_drift._j_duration,
+ update_interval._j_duration,
+ ))
@staticmethod
- def for_monotonous_timestamps():
+ def for_monotonous_timestamps() -> 'WatermarkStrategy':
"""
Creates a watermark strategy for situations with monotonously ascending timestamps.
@@ -78,7 +124,7 @@ class WatermarkStrategy(object):
return WatermarkStrategy(JWaterMarkStrategy.forMonotonousTimestamps())
@staticmethod
- def for_bounded_out_of_orderness(max_out_of_orderness: Duration):
+ def for_bounded_out_of_orderness(max_out_of_orderness: Duration) -> 'WatermarkStrategy':
"""
Creates a watermark strategy for situations where records are out of order, but you can
place an upper bound on how far the events are out of order. An out-of-order bound B means
@@ -90,6 +136,18 @@ class WatermarkStrategy(object):
return WatermarkStrategy(
JWaterMarkStrategy.forBoundedOutOfOrderness(max_out_of_orderness._j_duration))
+ @staticmethod
+ def no_watermarks() -> 'WatermarkStrategy':
+ """
+ Creates a watermark strategy that generates no watermarks at all. This may be useful in
+ scenarios that do pure processing-time based stream processing.
+
+ .. versionadded:: 1.16.0
+ """
+ JWaterMarkStrategy = get_gateway().jvm \
+ .org.apache.flink.api.common.eventtime.WatermarkStrategy
+ return WatermarkStrategy(JWaterMarkStrategy.noWatermarks())
+
class TimestampAssigner(abc.ABC):
"""