You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by di...@apache.org on 2022/06/17 01:22:52 UTC

[flink] branch master updated: [FLINK-27932][python] Introduce WatermarkStrategy.no_watermarks and WatermarkStrategy.no_watermarks to align with Java API

This is an automated email from the ASF dual-hosted git repository.

dianfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new d0f0071b767 [FLINK-27932][python] Introduce WatermarkStrategy.no_watermarks and WatermarkStrategy.no_watermarks to align with Java API
d0f0071b767 is described below

commit d0f0071b767b8dbe2e811e9b6b5e1e9027a20e2b
Author: Juntao Hu <ma...@gmail.com>
AuthorDate: Tue Jun 7 16:21:03 2022 +0800

    [FLINK-27932][python] Introduce WatermarkStrategy.no_watermarks and WatermarkStrategy.no_watermarks to align with Java API
    
    This closes #19896.
---
 .../docs/dev/datastream/event-time/built_in.md     | 10 +++
 .../datastream/event-time/generating_watermarks.md | 47 +++++++++++++
 .../docs/dev/datastream/event-time/built_in.md     | 10 +++
 .../datastream/event-time/generating_watermarks.md | 47 +++++++++++++
 .../common/tests/test_watermark_strategy.py        | 79 ++++++++++++++++++++++
 flink-python/pyflink/common/watermark_strategy.py  | 74 +++++++++++++++++---
 6 files changed, 259 insertions(+), 8 deletions(-)

diff --git a/docs/content.zh/docs/dev/datastream/event-time/built_in.md b/docs/content.zh/docs/dev/datastream/event-time/built_in.md
index 4b4452827fe..792dfad43c7 100644
--- a/docs/content.zh/docs/dev/datastream/event-time/built_in.md
+++ b/docs/content.zh/docs/dev/datastream/event-time/built_in.md
@@ -50,6 +50,11 @@ WatermarkStrategy.forMonotonousTimestamps();
 WatermarkStrategy.forMonotonousTimestamps()
 ```
 {{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy.for_monotonous_timestamps()
+```
+{{< /tab >}}
 {{< /tabs >}}
 
 <a name="fixed-amount-of-lateness"></a>
@@ -70,6 +75,11 @@ WatermarkStrategy
   .forBoundedOutOfOrderness(Duration.ofSeconds(10))
 ```
 {{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy.for_bounded_out_of_orderness(Duration.of_seconds(10))
+```
+{{< /tab >}}
 {{< /tabs >}}
 
 {{< top >}}
diff --git a/docs/content.zh/docs/dev/datastream/event-time/generating_watermarks.md b/docs/content.zh/docs/dev/datastream/event-time/generating_watermarks.md
index 01b1c869473..e4dbc627480 100644
--- a/docs/content.zh/docs/dev/datastream/event-time/generating_watermarks.md
+++ b/docs/content.zh/docs/dev/datastream/event-time/generating_watermarks.md
@@ -77,6 +77,19 @@ WatermarkStrategy
   })
 ```
 {{< /tab >}}
+{{< tab "Python" >}}
+```python
+class FirstElementTimestampAssigner(TimestampAssigner):
+   
+    def extract_timestamp(self, value, record_timestamp):
+       return value[0]
+
+
+WatermarkStrategy \
+    .for_bounded_out_of_orderness(Duration.of_seconds(20)) \
+    .with_timestamp_assigner(FirstElementTimestampAssigner())
+```
+{{< /tab >}}
 {{< /tabs >}}
 
 其中 `TimestampAssigner` 的设置与否是可选的,大多数情况下,可以不用去特别指定。例如,当使用 Kafka 或 Kinesis 数据源时,你可以直接从 Kafka/Kinesis 数据源记录中获取到时间戳。
@@ -136,6 +149,26 @@ withTimestampsAndWatermarks
         .addSink(...)
 ```
 {{< /tab >}}
+{{< tab "Python" >}}
+```python
+env = StreamExecutionEnvironment.get_execution_environment()
+
+# currently read_file is not supported in PyFlink
+stream = env \
+    .read_text_file(my_file_path, charset) \
+    .map(lambda s: MyEvent.from_string(s))
+
+with_timestamp_and_watermarks = stream \
+    .filter(lambda e: e.severity() == WARNING) \
+    .assign_timestamp_and_watermarks(<watermark strategy>)
+
+with_timestamp_and_watermarks \
+    .key_by(lambda e: e.get_group()) \
+    .window(TumblingEventTimeWindows.of(Time.seconds(10))) \
+    .reduce(lambda a, b: a.add(b)) \
+    .add_sink(...)
+```
+{{< /tab >}}
 {{< /tabs >}}
 
 使用 `WatermarkStrategy` 去获取流并生成带有时间戳的元素和 watermark 的新流时,如果原始流已经具有时间戳或 watermark,则新指定的时间戳分配器将覆盖原有的时间戳和 watermark。
@@ -163,6 +196,13 @@ WatermarkStrategy
   .withIdleness(Duration.ofMinutes(1))
 ```
 {{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy \
+    .for_bounded_out_of_orderness(Duration.of_seconds(20)) \
+    .with_idleness(Duration.of_minutes(1))
+```
+{{< /tab >}}
 {{< /tabs >}}
 
 ## Watermark alignment _`Beta`_
@@ -200,6 +240,13 @@ WatermarkStrategy
   .withWatermarkAlignment("alignment-group-1", Duration.ofSeconds(20), Duration.ofSeconds(1))
 ```
 {{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy \
+    .for_bounded_out_of_orderness(Duration.of_seconds(20)) \
+    .with_watermark_alignment("alignment-group-1", Duration.of_seconds(20), Duration.of_seconds(1))
+```
+{{< /tab >}}
 {{< /tabs >}}
 
 {{< hint warning >}}
diff --git a/docs/content/docs/dev/datastream/event-time/built_in.md b/docs/content/docs/dev/datastream/event-time/built_in.md
index 836a84ba332..bf74eacc097 100644
--- a/docs/content/docs/dev/datastream/event-time/built_in.md
+++ b/docs/content/docs/dev/datastream/event-time/built_in.md
@@ -62,6 +62,11 @@ WatermarkStrategy.forMonotonousTimestamps();
 WatermarkStrategy.forMonotonousTimestamps()
 ```
 {{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy.for_monotonous_timestamps()
+```
+{{< /tab >}}
 {{< /tabs >}}
 
 ## Fixed Amount of Lateness
@@ -93,6 +98,11 @@ WatermarkStrategy.forBoundedOutOfOrderness(Duration.ofSeconds(10));
 WatermarkStrategy.forBoundedOutOfOrderness(Duration.ofSeconds(10))
 ```
 {{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy.for_bounded_out_of_orderness(Duration.of_seconds(10))
+```
+{{< /tab >}}
 {{< /tabs >}}
 
 {{< top >}}
diff --git a/docs/content/docs/dev/datastream/event-time/generating_watermarks.md b/docs/content/docs/dev/datastream/event-time/generating_watermarks.md
index 1ae08ec1f9f..2cc5ed1d4d6 100644
--- a/docs/content/docs/dev/datastream/event-time/generating_watermarks.md
+++ b/docs/content/docs/dev/datastream/event-time/generating_watermarks.md
@@ -94,6 +94,19 @@ WatermarkStrategy
   })
 ```
 {{< /tab >}}
+{{< tab "Python" >}}
+```python
+class FirstElementTimestampAssigner(TimestampAssigner):
+   
+    def extract_timestamp(self, value, record_timestamp):
+        return value[0]
+
+
+WatermarkStrategy \
+    .for_bounded_out_of_orderness(Duration.of_seconds(20)) \
+    .with_timestamp_assigner(FirstElementTimestampAssigner())
+```
+{{< /tab >}}
 {{< /tabs >}}
 
 Specifying a `TimestampAssigner` is optional and in most cases you don't
@@ -165,6 +178,26 @@ withTimestampsAndWatermarks
         .addSink(...)
 ```
 {{< /tab >}}
+{{< tab "Python" >}}
+```python
+env = StreamExecutionEnvironment.get_execution_environment()
+
+# currently read_file is not supported in PyFlink
+stream = env \
+    .read_text_file(my_file_path, charset) \
+    .map(lambda s: MyEvent.from_string(s))
+
+with_timestamp_and_watermarks = stream \
+    .filter(lambda e: e.severity() == WARNING) \
+    .assign_timestamp_and_watermarks(<watermark strategy>)
+
+with_timestamp_and_watermarks \
+    .key_by(lambda e: e.get_group()) \
+    .window(TumblingEventTimeWindows.of(Time.seconds(10))) \
+    .reduce(lambda a, b: a.add(b)) \
+    .add_sink(...)
+```
+{{< /tab >}}
 {{< /tabs >}}
 
 Using a `WatermarkStrategy` this way takes a stream and produce a new stream
@@ -199,6 +232,13 @@ WatermarkStrategy
   .withIdleness(Duration.ofMinutes(1))
 ```
 {{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy \
+    .for_bounded_out_of_orderness(Duration.of_seconds(20)) \
+    .with_idleness(Duration.of_minutes(1))
+```
+{{< /tab >}}
 {{< /tabs >}}
 
 ## Watermark alignment _`Beta`_
@@ -236,6 +276,13 @@ WatermarkStrategy
   .withWatermarkAlignment("alignment-group-1", Duration.ofSeconds(20), Duration.ofSeconds(1))
 ```
 {{< /tab >}}
+{{< tab "Python" >}}
+```python
+WatermarkStrategy \
+    .for_bounded_out_of_orderness(Duration.of_seconds(20)) \
+    .with_watermark_alignment("alignment-group-1", Duration.of_seconds(20), Duration.of_seconds(1))
+```
+{{< /tab >}}
 {{< /tabs >}}
 
 {{< hint warning >}}
diff --git a/flink-python/pyflink/common/tests/test_watermark_strategy.py b/flink-python/pyflink/common/tests/test_watermark_strategy.py
new file mode 100644
index 00000000000..7853605d17c
--- /dev/null
+++ b/flink-python/pyflink/common/tests/test_watermark_strategy.py
@@ -0,0 +1,79 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from pyflink.common import Duration
+from pyflink.common.watermark_strategy import WatermarkStrategy
+from pyflink.java_gateway import get_gateway
+from pyflink.util.java_utils import is_instance_of, get_field_value
+
+from pyflink.testing.test_case_utils import PyFlinkTestCase
+
+
+class WatermarkStrategyTests(PyFlinkTestCase):
+
+    def test_with_idleness(self):
+        jvm = get_gateway().jvm
+        j_watermark_strategy = WatermarkStrategy.no_watermarks().with_idleness(
+            Duration.of_seconds(5)
+        )._j_watermark_strategy
+        self.assertTrue(is_instance_of(
+            j_watermark_strategy,
+            jvm.org.apache.flink.api.common.eventtime.WatermarkStrategyWithIdleness
+        ))
+        self.assertEqual(get_field_value(j_watermark_strategy, "idlenessTimeout").toMillis(), 5000)
+
+    def test_with_watermark_alignment(self):
+        jvm = get_gateway().jvm
+        j_watermark_strategy = WatermarkStrategy.no_watermarks().with_watermark_alignment(
+            "alignment-group-1", Duration.of_seconds(20), Duration.of_seconds(10)
+        )._j_watermark_strategy
+        self.assertTrue(is_instance_of(
+            j_watermark_strategy,
+            jvm.org.apache.flink.api.common.eventtime.WatermarksWithWatermarkAlignment
+        ))
+        alignment_parameters = j_watermark_strategy.getAlignmentParameters()
+        self.assertEqual(alignment_parameters.getWatermarkGroup(), "alignment-group-1")
+        self.assertEqual(alignment_parameters.getMaxAllowedWatermarkDrift(), 20000)
+        self.assertEqual(alignment_parameters.getUpdateInterval(), 10000)
+
+    def test_for_monotonous_timestamps(self):
+        jvm = get_gateway().jvm
+        j_watermark_strategy = WatermarkStrategy.for_monotonous_timestamps()._j_watermark_strategy
+        self.assertTrue(is_instance_of(
+            j_watermark_strategy.createWatermarkGenerator(None),
+            jvm.org.apache.flink.api.common.eventtime.AscendingTimestampsWatermarks
+        ))
+
+    def test_for_bounded_out_of_orderness(self):
+        jvm = get_gateway().jvm
+        j_watermark_strategy = WatermarkStrategy.for_bounded_out_of_orderness(
+            Duration.of_seconds(3)
+        )._j_watermark_strategy
+        j_watermark_generator = j_watermark_strategy.createWatermarkGenerator(None)
+        self.assertTrue(is_instance_of(
+            j_watermark_generator,
+            jvm.org.apache.flink.api.common.eventtime.BoundedOutOfOrdernessWatermarks
+        ))
+        self.assertEqual(get_field_value(j_watermark_generator, "outOfOrdernessMillis"), 3000)
+
+    def test_no_watermarks(self):
+        jvm = get_gateway().jvm
+        j_watermark_strategy = WatermarkStrategy.no_watermarks()._j_watermark_strategy
+        self.assertTrue(is_instance_of(
+            j_watermark_strategy.createWatermarkGenerator(None),
+            jvm.org.apache.flink.api.common.eventtime.NoWatermarksGenerator
+        ))
diff --git a/flink-python/pyflink/common/watermark_strategy.py b/flink-python/pyflink/common/watermark_strategy.py
index aa763320aad..532e962b06a 100644
--- a/flink-python/pyflink/common/watermark_strategy.py
+++ b/flink-python/pyflink/common/watermark_strategy.py
@@ -17,7 +17,7 @@
 ################################################################################
 import abc
 
-from typing import Any
+from typing import Any, Optional
 
 from pyflink.common.time import Duration
 from pyflink.java_gateway import get_gateway
@@ -42,9 +42,11 @@ class WatermarkStrategy(object):
         Creates a new WatermarkStrategy that wraps this strategy but instead uses the given a
         TimestampAssigner by implementing TimestampAssigner interface.
 
+        Example:
         ::
-            >>> watermark_strategy = WatermarkStrategy.for_monotonous_timestamps()
-            >>>   .with_timestamp_assigner(MyTimestampAssigner())
+
+            >>> watermark_strategy = WatermarkStrategy.for_monotonous_timestamps() \\
+            >>>     .with_timestamp_assigner(MyTimestampAssigner())
 
         :param timestamp_assigner: The given TimestampAssigner.
         :return: A WaterMarkStrategy that wraps a TimestampAssigner.
@@ -57,15 +59,59 @@ class WatermarkStrategy(object):
         Creates a new enriched WatermarkStrategy that also does idleness detection in the created
         WatermarkGenerator.
 
+        Example:
+        ::
+
+            >>> WatermarkStrategy \\
+            ...     .for_bounded_out_of_orderness(Duration.of_seconds(20)) \\
+            ...     .with_idleness(Duration.of_minutes(1))
+
         :param idle_timeout: The idle timeout.
         :return: A new WatermarkStrategy with idle detection configured.
         """
-        self._j_watermark_strategy = self._j_watermark_strategy\
-            .withIdleness(idle_timeout._j_duration)
-        return self
+        return WatermarkStrategy(self._j_watermark_strategy.withIdleness(idle_timeout._j_duration))
+
+    def with_watermark_alignment(self, watermark_group: str, max_allowed_watermark_drift: Duration,
+                                 update_interval: Optional[Duration] = None) -> 'WatermarkStrategy':
+        """
+        Creates a new :class:`WatermarkStrategy` that configures the maximum watermark drift from
+        other sources/tasks/partitions in the same watermark group. The group may contain completely
+        independent sources (e.g. File and Kafka).
+
+        Once configured Flink will "pause" consuming from a source/task/partition that is ahead
+        of the emitted watermark in the group by more than the maxAllowedWatermarkDrift.
+
+        Example:
+        ::
+
+            >>> WatermarkStrategy \\
+            ...     .for_bounded_out_of_orderness(Duration.of_seconds(20)) \\
+            ...     .with_watermark_alignment("alignment-group-1", Duration.of_seconds(20),
+            ...         Duration.of_seconds(1))
+
+        :param watermark_group: A group of sources to align watermarks
+        :param max_allowed_watermark_drift: Maximal drift, before we pause consuming from the
+            source/task/partition
+        :param update_interval: How often tasks should notify coordinator about the current
+            watermark and how often the coordinator should announce the maximal aligned watermark.
+            If is None, default update interval (1000ms) is used.
+        :return: A new WatermarkStrategy with watermark alignment configured.
+
+        .. versionadded:: 1.16.0
+        """
+        if update_interval is None:
+            return WatermarkStrategy(self._j_watermark_strategy.withWatermarkAlignment(
+                watermark_group, max_allowed_watermark_drift._j_duration
+            ))
+        else:
+            return WatermarkStrategy(self._j_watermark_strategy.withWatermarkAlignment(
+                watermark_group,
+                max_allowed_watermark_drift._j_duration,
+                update_interval._j_duration,
+            ))
 
     @staticmethod
-    def for_monotonous_timestamps():
+    def for_monotonous_timestamps() -> 'WatermarkStrategy':
         """
         Creates a watermark strategy for situations with monotonously ascending timestamps.
 
@@ -78,7 +124,7 @@ class WatermarkStrategy(object):
         return WatermarkStrategy(JWaterMarkStrategy.forMonotonousTimestamps())
 
     @staticmethod
-    def for_bounded_out_of_orderness(max_out_of_orderness: Duration):
+    def for_bounded_out_of_orderness(max_out_of_orderness: Duration) -> 'WatermarkStrategy':
         """
         Creates a watermark strategy for situations where records are out of order, but you can
         place an upper bound on how far the events are out of order. An out-of-order bound B means
@@ -90,6 +136,18 @@ class WatermarkStrategy(object):
         return WatermarkStrategy(
             JWaterMarkStrategy.forBoundedOutOfOrderness(max_out_of_orderness._j_duration))
 
+    @staticmethod
+    def no_watermarks() -> 'WatermarkStrategy':
+        """
+        Creates a watermark strategy that generates no watermarks at all. This may be useful in
+        scenarios that do pure processing-time based stream processing.
+
+        .. versionadded:: 1.16.0
+        """
+        JWaterMarkStrategy = get_gateway().jvm \
+            .org.apache.flink.api.common.eventtime.WatermarkStrategy
+        return WatermarkStrategy(JWaterMarkStrategy.noWatermarks())
+
 
 class TimestampAssigner(abc.ABC):
     """