You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by xt...@apache.org on 2023/12/19 11:00:49 UTC

(flink) 03/09: [FLINK-33810][runtime] Introduce RecordAttributes

This is an automated email from the ASF dual-hosted git repository.

xtsong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 42df7ebc8c1d7d589a9e9416d9000fd07015599d
Author: sxnan <su...@gmail.com>
AuthorDate: Thu Nov 16 15:15:26 2023 +0800

    [FLINK-33810][runtime] Introduce RecordAttributes
---
 .../runtime/streamrecord/RecordAttributes.java     | 71 ++++++++++++++++++++
 .../streamrecord/RecordAttributesBuilder.java      | 78 ++++++++++++++++++++++
 .../runtime/streamrecord/StreamElement.java        | 22 +++++-
 .../streamrecord/StreamElementSerializer.java      | 10 +++
 .../streamrecord/StreamElementSerializerTest.java  |  5 ++
 5 files changed, 185 insertions(+), 1 deletion(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributes.java
new file mode 100644
index 00000000000..7c0335ea4fe
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributes.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.runtime.streamrecord;
+
+import org.apache.flink.annotation.Experimental;
+
+import java.util.Collections;
+import java.util.Objects;
+
+/**
+ * A RecordAttributes describes the attributes of records from the current RecordAttributes until
+ * the next one is received. It provides stream task with information that can be used to optimize
+ * the stream task's performance.
+ */
+@Experimental
+public class RecordAttributes extends StreamElement {
+
+    public static final RecordAttributes EMPTY_RECORD_ATTRIBUTES =
+            new RecordAttributesBuilder(Collections.emptyList()).build();
+    private final boolean isBacklog;
+
+    public RecordAttributes(boolean isBacklog) {
+        this.isBacklog = isBacklog;
+    }
+
+    /**
+     * If it returns true, then the records received after this element are stale and an operator
+     * can optionally buffer records until isBacklog=false. This allows an operator to optimize
+     * throughput at the cost of processing latency.
+     */
+    public boolean isBacklog() {
+        return isBacklog;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        RecordAttributes that = (RecordAttributes) o;
+        return isBacklog == that.isBacklog;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(isBacklog);
+    }
+
+    @Override
+    public String toString() {
+        return "RecordAttributes{" + "backlog=" + isBacklog + '}';
+    }
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributesBuilder.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributesBuilder.java
new file mode 100644
index 00000000000..74281a48fbf
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributesBuilder.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.runtime.streamrecord;
+
+import org.apache.flink.annotation.Experimental;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.util.List;
+
+/** The builder class for {@link RecordAttributes}. */
+@Experimental
+public class RecordAttributesBuilder {
+    private static final Logger LOG = LoggerFactory.getLogger(RecordAttributesBuilder.class);
+    private final List<RecordAttributes> lastRecordAttributesOfInputs;
+    @Nullable private Boolean isBacklog = null;
+
+    /**
+     * This constructor takes a list of the last RecordAttributes received from each of the
+     * operator's inputs. Each input is corresponding to an input edge of the job graph. When this
+     * list is not empty, it will be used to determine the default values for those attributes that
+     * have not been explicitly set by caller.
+     */
+    public RecordAttributesBuilder(List<RecordAttributes> lastRecordAttributesOfInputs) {
+        this.lastRecordAttributesOfInputs = lastRecordAttributesOfInputs;
+    }
+
+    public RecordAttributesBuilder setBacklog(boolean isBacklog) {
+        this.isBacklog = isBacklog;
+        return this;
+    }
+
+    /**
+     * If any operator attribute is null, we will log it at DEBUG level and determine a non-null
+     * default value as described below.
+     *
+     * <p>Default value for backlog: if any element in lastRecordAttributesOfInputs has
+     * backlog=true, use true. Otherwise, use false.
+     */
+    public RecordAttributes build() {
+        if (isBacklog == null) {
+            final boolean defaultBacklog = getDefaultBacklog();
+            LOG.debug(
+                    "backlog is not set, set to {} from the last record attributes {}.",
+                    defaultBacklog,
+                    lastRecordAttributesOfInputs);
+            isBacklog = defaultBacklog;
+        }
+        return new RecordAttributes(isBacklog);
+    }
+
+    private boolean getDefaultBacklog() {
+        for (RecordAttributes lastAttributes : lastRecordAttributesOfInputs) {
+            if (lastAttributes.isBacklog()) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElement.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElement.java
index 3fbcf2e42dc..c65b0b5bf7b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElement.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElement.java
@@ -22,7 +22,7 @@ import org.apache.flink.annotation.Internal;
 import org.apache.flink.streaming.api.watermark.Watermark;
 import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;
 
-/** An element in a data stream. Can be a record or a Watermark. */
+/** An element in a data stream. Can be a record, a Watermark, or a RecordAttributes. */
 @Internal
 public abstract class StreamElement {
 
@@ -62,6 +62,15 @@ public abstract class StreamElement {
         return getClass() == LatencyMarker.class;
     }
 
+    /**
+     * Check whether this element is record attributes.
+     *
+     * @return True, if this element is record attributes, false otherwise.
+     */
+    public final boolean isRecordAttributes() {
+        return getClass() == RecordAttributes.class;
+    }
+
     /**
      * Casts this element into a StreamRecord.
      *
@@ -103,4 +112,15 @@ public abstract class StreamElement {
     public final LatencyMarker asLatencyMarker() {
         return (LatencyMarker) this;
     }
+
+    /**
+     * Casts this element into a RecordAttributes.
+     *
+     * @return This element as a RecordAttributes.
+     * @throws java.lang.ClassCastException Thrown, if this element is actually not a
+     *     RecordAttributes.
+     */
+    public final RecordAttributes asRecordAttributes() {
+        return (RecordAttributes) this;
+    }
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
index b1052131769..4140c52a9ec 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
@@ -50,6 +50,7 @@ public final class StreamElementSerializer<T> extends TypeSerializer<StreamEleme
     private static final int TAG_WATERMARK = 2;
     private static final int TAG_LATENCY_MARKER = 3;
     private static final int TAG_STREAM_STATUS = 4;
+    private static final int TAG_RECORD_ATTRIBUTES = 5;
 
     private final TypeSerializer<T> typeSerializer;
 
@@ -146,6 +147,8 @@ public final class StreamElementSerializer<T> extends TypeSerializer<StreamEleme
             target.writeLong(source.readLong());
             target.writeLong(source.readLong());
             target.writeInt(source.readInt());
+        } else if (tag == TAG_RECORD_ATTRIBUTES) {
+            target.writeBoolean(source.readBoolean());
         } else {
             throw new IOException("Corrupt stream, found tag: " + tag);
         }
@@ -175,6 +178,9 @@ public final class StreamElementSerializer<T> extends TypeSerializer<StreamEleme
             target.writeLong(value.asLatencyMarker().getOperatorId().getLowerPart());
             target.writeLong(value.asLatencyMarker().getOperatorId().getUpperPart());
             target.writeInt(value.asLatencyMarker().getSubtaskIndex());
+        } else if (value.isRecordAttributes()) {
+            target.write(TAG_RECORD_ATTRIBUTES);
+            target.writeBoolean(value.asRecordAttributes().isBacklog());
         } else {
             throw new RuntimeException();
         }
@@ -197,6 +203,8 @@ public final class StreamElementSerializer<T> extends TypeSerializer<StreamEleme
                     source.readLong(),
                     new OperatorID(source.readLong(), source.readLong()),
                     source.readInt());
+        } else if (tag == TAG_RECORD_ATTRIBUTES) {
+            return new RecordAttributes(source.readBoolean());
         } else {
             throw new IOException("Corrupt stream, found tag: " + tag);
         }
@@ -223,6 +231,8 @@ public final class StreamElementSerializer<T> extends TypeSerializer<StreamEleme
                     source.readLong(),
                     new OperatorID(source.readLong(), source.readLong()),
                     source.readInt());
+        } else if (tag == TAG_RECORD_ATTRIBUTES) {
+            return new RecordAttributes(source.readBoolean());
         } else {
             throw new IOException("Corrupt stream, found tag: " + tag);
         }
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializerTest.java
index dc441328856..a328f5e9d03 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializerTest.java
@@ -29,6 +29,7 @@ import org.apache.flink.streaming.api.watermark.Watermark;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.util.Collections;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -92,6 +93,10 @@ public class StreamElementSerializerTest {
         LatencyMarker latencyMarker =
                 new LatencyMarker(System.currentTimeMillis(), new OperatorID(-1, -1), 1);
         assertEquals(latencyMarker, serializeAndDeserialize(latencyMarker, serializer));
+
+        RecordAttributes recordAttributes =
+                new RecordAttributesBuilder(Collections.emptyList()).setBacklog(true).build();
+        assertEquals(recordAttributes, serializeAndDeserialize(recordAttributes, serializer));
     }
 
     @SuppressWarnings("unchecked")