You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by xt...@apache.org on 2023/12/19 11:00:49 UTC
(flink) 03/09: [FLINK-33810][runtime] Introduce RecordAttributes
This is an automated email from the ASF dual-hosted git repository.
xtsong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 42df7ebc8c1d7d589a9e9416d9000fd07015599d
Author: sxnan <su...@gmail.com>
AuthorDate: Thu Nov 16 15:15:26 2023 +0800
[FLINK-33810][runtime] Introduce RecordAttributes
---
.../runtime/streamrecord/RecordAttributes.java | 71 ++++++++++++++++++++
.../streamrecord/RecordAttributesBuilder.java | 78 ++++++++++++++++++++++
.../runtime/streamrecord/StreamElement.java | 22 +++++-
.../streamrecord/StreamElementSerializer.java | 10 +++
.../streamrecord/StreamElementSerializerTest.java | 5 ++
5 files changed, 185 insertions(+), 1 deletion(-)
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributes.java
new file mode 100644
index 00000000000..7c0335ea4fe
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributes.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.runtime.streamrecord;
+
+import org.apache.flink.annotation.Experimental;
+
+import java.util.Collections;
+import java.util.Objects;
+
+/**
+ * A RecordAttributes describes the attributes of records from the current RecordAttributes until
+ * the next one is received. It provides stream task with information that can be used to optimize
+ * the stream task's performance.
+ */
+@Experimental
+public class RecordAttributes extends StreamElement {
+
+ public static final RecordAttributes EMPTY_RECORD_ATTRIBUTES =
+ new RecordAttributesBuilder(Collections.emptyList()).build();
+ private final boolean isBacklog;
+
+ public RecordAttributes(boolean isBacklog) {
+ this.isBacklog = isBacklog;
+ }
+
+ /**
+ * If it returns true, then the records received after this element are stale and an operator
+ * can optionally buffer records until isBacklog=false. This allows an operator to optimize
+ * throughput at the cost of processing latency.
+ */
+ public boolean isBacklog() {
+ return isBacklog;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ RecordAttributes that = (RecordAttributes) o;
+ return isBacklog == that.isBacklog;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(isBacklog);
+ }
+
+ @Override
+ public String toString() {
+ return "RecordAttributes{" + "backlog=" + isBacklog + '}';
+ }
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributesBuilder.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributesBuilder.java
new file mode 100644
index 00000000000..74281a48fbf
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributesBuilder.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.runtime.streamrecord;
+
+import org.apache.flink.annotation.Experimental;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.util.List;
+
+/** The builder class for {@link RecordAttributes}. */
+@Experimental
+public class RecordAttributesBuilder {
+ private static final Logger LOG = LoggerFactory.getLogger(RecordAttributesBuilder.class);
+ private final List<RecordAttributes> lastRecordAttributesOfInputs;
+ @Nullable private Boolean isBacklog = null;
+
+ /**
+ * This constructor takes a list of the last RecordAttributes received from each of the
+ * operator's inputs. Each input is corresponding to an input edge of the job graph. When this
+ * list is not empty, it will be used to determine the default values for those attributes that
+ * have not been explicitly set by caller.
+ */
+ public RecordAttributesBuilder(List<RecordAttributes> lastRecordAttributesOfInputs) {
+ this.lastRecordAttributesOfInputs = lastRecordAttributesOfInputs;
+ }
+
+ public RecordAttributesBuilder setBacklog(boolean isBacklog) {
+ this.isBacklog = isBacklog;
+ return this;
+ }
+
+ /**
+ * If any operator attribute is null, we will log it at DEBUG level and determine a non-null
+ * default value as described below.
+ *
+ * <p>Default value for backlog: if any element in lastRecordAttributesOfInputs has
+ * backlog=true, use true. Otherwise, use false.
+ */
+ public RecordAttributes build() {
+ if (isBacklog == null) {
+ final boolean defaultBacklog = getDefaultBacklog();
+ LOG.debug(
+ "backlog is not set, set to {} from the last record attributes {}.",
+ defaultBacklog,
+ lastRecordAttributesOfInputs);
+ isBacklog = defaultBacklog;
+ }
+ return new RecordAttributes(isBacklog);
+ }
+
+ private boolean getDefaultBacklog() {
+ for (RecordAttributes lastAttributes : lastRecordAttributesOfInputs) {
+ if (lastAttributes.isBacklog()) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElement.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElement.java
index 3fbcf2e42dc..c65b0b5bf7b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElement.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElement.java
@@ -22,7 +22,7 @@ import org.apache.flink.annotation.Internal;
import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;
-/** An element in a data stream. Can be a record or a Watermark. */
+/** An element in a data stream. Can be a record, a Watermark, or a RecordAttributes. */
@Internal
public abstract class StreamElement {
@@ -62,6 +62,15 @@ public abstract class StreamElement {
return getClass() == LatencyMarker.class;
}
+ /**
+ * Check whether this element is record attributes.
+ *
+ * @return True, if this element is record attributes, false otherwise.
+ */
+ public final boolean isRecordAttributes() {
+ return getClass() == RecordAttributes.class;
+ }
+
/**
* Casts this element into a StreamRecord.
*
@@ -103,4 +112,15 @@ public abstract class StreamElement {
public final LatencyMarker asLatencyMarker() {
return (LatencyMarker) this;
}
+
+ /**
+ * Casts this element into a RecordAttributes.
+ *
+ * @return This element as a RecordAttributes.
+ * @throws java.lang.ClassCastException Thrown, if this element is actually not a
+ * RecordAttributes.
+ */
+ public final RecordAttributes asRecordAttributes() {
+ return (RecordAttributes) this;
+ }
}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
index b1052131769..4140c52a9ec 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
@@ -50,6 +50,7 @@ public final class StreamElementSerializer<T> extends TypeSerializer<StreamEleme
private static final int TAG_WATERMARK = 2;
private static final int TAG_LATENCY_MARKER = 3;
private static final int TAG_STREAM_STATUS = 4;
+ private static final int TAG_RECORD_ATTRIBUTES = 5;
private final TypeSerializer<T> typeSerializer;
@@ -146,6 +147,8 @@ public final class StreamElementSerializer<T> extends TypeSerializer<StreamEleme
target.writeLong(source.readLong());
target.writeLong(source.readLong());
target.writeInt(source.readInt());
+ } else if (tag == TAG_RECORD_ATTRIBUTES) {
+ target.writeBoolean(source.readBoolean());
} else {
throw new IOException("Corrupt stream, found tag: " + tag);
}
@@ -175,6 +178,9 @@ public final class StreamElementSerializer<T> extends TypeSerializer<StreamEleme
target.writeLong(value.asLatencyMarker().getOperatorId().getLowerPart());
target.writeLong(value.asLatencyMarker().getOperatorId().getUpperPart());
target.writeInt(value.asLatencyMarker().getSubtaskIndex());
+ } else if (value.isRecordAttributes()) {
+ target.write(TAG_RECORD_ATTRIBUTES);
+ target.writeBoolean(value.asRecordAttributes().isBacklog());
} else {
throw new RuntimeException();
}
@@ -197,6 +203,8 @@ public final class StreamElementSerializer<T> extends TypeSerializer<StreamEleme
source.readLong(),
new OperatorID(source.readLong(), source.readLong()),
source.readInt());
+ } else if (tag == TAG_RECORD_ATTRIBUTES) {
+ return new RecordAttributes(source.readBoolean());
} else {
throw new IOException("Corrupt stream, found tag: " + tag);
}
@@ -223,6 +231,8 @@ public final class StreamElementSerializer<T> extends TypeSerializer<StreamEleme
source.readLong(),
new OperatorID(source.readLong(), source.readLong()),
source.readInt());
+ } else if (tag == TAG_RECORD_ATTRIBUTES) {
+ return new RecordAttributes(source.readBoolean());
} else {
throw new IOException("Corrupt stream, found tag: " + tag);
}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializerTest.java
index dc441328856..a328f5e9d03 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializerTest.java
@@ -29,6 +29,7 @@ import org.apache.flink.streaming.api.watermark.Watermark;
import org.junit.Test;
import java.io.IOException;
+import java.util.Collections;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -92,6 +93,10 @@ public class StreamElementSerializerTest {
LatencyMarker latencyMarker =
new LatencyMarker(System.currentTimeMillis(), new OperatorID(-1, -1), 1);
assertEquals(latencyMarker, serializeAndDeserialize(latencyMarker, serializer));
+
+ RecordAttributes recordAttributes =
+ new RecordAttributesBuilder(Collections.emptyList()).setBacklog(true).build();
+ assertEquals(recordAttributes, serializeAndDeserialize(recordAttributes, serializer));
}
@SuppressWarnings("unchecked")