You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ab...@apache.org on 2026/10/12 07:29:41 UTC
(fluss) branch main updated: [flink] Persist lake split state for restore (#4589) (#4594)
This is an automated email from the ASF dual-hosted git repository.
fresh-borzoni pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
The following commit(s) were added to refs/heads/main by this push:
new 0a46dc544 [flink] Persist lake split state for restore (#4589) (#4594)
0a46dc544 is described below
commit 0a46dc544a5568729588e1ff8c710dffcaeac31a
Author: Giannis Polyzos <ip...@gmail.com>
AuthorDate: Mon Oct 12 08:29:37 2026 +0100
[flink] Persist lake split state for restore (#4589) (#4594)
---
.../fluss/flink/lake/LakeSplitSerializer.java | 31 +++++-
.../flink/source/split/SourceSplitSerializer.java | 31 +++++-
.../source/TableBucketWriteResultSerializer.java | 28 ++++-
.../fluss/flink/lake/LakeSplitSerializerTest.java | 33 +++++-
.../flink/source/reader/FlinkSourceReaderTest.java | 78 ++++++++++++++
.../source/split/SourceSplitSerializerTest.java | 92 ++++++++++++++++
.../TableBucketWriteResultSerializerTest.java | 120 +++++++++++++++++++++
7 files changed, 395 insertions(+), 18 deletions(-)
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/lake/LakeSplitSerializer.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/lake/LakeSplitSerializer.java
index f297f7c32..3ff6deb39 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/lake/LakeSplitSerializer.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/lake/LakeSplitSerializer.java
@@ -21,6 +21,7 @@ import org.apache.fluss.flink.lake.split.LakeSnapshotAndFlussLogSplit;
import org.apache.fluss.flink.lake.split.LakeSnapshotSplit;
import org.apache.fluss.flink.source.split.LogSplit;
import org.apache.fluss.flink.source.split.SourceSplitBase;
+import org.apache.fluss.flink.source.split.SourceSplitSerializer;
import org.apache.fluss.lake.serializer.SimpleVersionedSerializer;
import org.apache.fluss.lake.source.LakeSplit;
import org.apache.fluss.metadata.TableBucket;
@@ -37,7 +38,12 @@ import java.util.List;
import static org.apache.fluss.flink.lake.split.LakeSnapshotAndFlussLogSplit.LAKE_SNAPSHOT_FLUSS_LOG_SPLIT_KIND;
import static org.apache.fluss.flink.lake.split.LakeSnapshotSplit.LAKE_SNAPSHOT_SPLIT_KIND;
-/** A serializer for lake split. */
+/**
+ * A serializer for lake split.
+ *
+ * <p>The layout of the lake split section is versioned by the enclosing {@link
+ * SourceSplitSerializer}, see its version evolution for details.
+ */
public class LakeSplitSerializer {
private final SimpleVersionedSerializer<LakeSplit> sourceSplitSerializer;
@@ -54,6 +60,7 @@ public class LakeSplitSerializer {
byte[] serializeBytes = sourceSplitSerializer.serialize(lakeSplit.getLakeSplit());
out.writeInt(serializeBytes.length);
out.write(serializeBytes);
+ out.writeLong(lakeSplit.getRecordsToSkip());
} else if (split instanceof LakeSnapshotAndFlussLogSplit) {
// writing file store source split
LakeSnapshotAndFlussLogSplit lakeSnapshotAndFlussLogSplit =
@@ -86,19 +93,33 @@ public class LakeSplitSerializer {
}
}
+ /**
+ * Deserializes a lake split.
+ *
+ * @param splitKind the kind of the split
+ * @param tableBucket the table bucket of the split
+ * @param partition the partition name of the split, null for non-partitioned table
+ * @param input the input to read the lake split section from
+ * @param readRecordsToSkip whether the records to skip of a {@link LakeSnapshotSplit} are
+ * persisted, which depends on the version of the enclosing {@link SourceSplitSerializer}.
+ * If not, the split is read from the beginning.
+ */
public SourceSplitBase deserialize(
byte splitKind,
TableBucket tableBucket,
@Nullable String partition,
- DataInputDeserializer input)
+ DataInputDeserializer input,
+ boolean readRecordsToSkip)
throws IOException {
int version = input.readInt();
if (splitKind == LAKE_SNAPSHOT_SPLIT_KIND) {
int splitIndex = input.readInt();
byte[] serializeBytes = new byte[input.readInt()];
- input.read(serializeBytes);
+ input.readFully(serializeBytes);
LakeSplit lakeSplit = sourceSplitSerializer.deserialize(version, serializeBytes);
- return new LakeSnapshotSplit(tableBucket, partition, lakeSplit, splitIndex);
+ long recordsToSkip = readRecordsToSkip ? input.readLong() : 0;
+ return new LakeSnapshotSplit(
+ tableBucket, partition, lakeSplit, splitIndex, recordsToSkip);
} else if (splitKind == LAKE_SNAPSHOT_FLUSS_LOG_SPLIT_KIND) {
List<LakeSplit> lakeSplits = null;
if (input.readBoolean()) {
@@ -106,7 +127,7 @@ public class LakeSplitSerializer {
lakeSplits = new ArrayList<>(lakeSplitSize);
for (int i = 0; i < lakeSplitSize; i++) {
byte[] serializeBytes = new byte[input.readInt()];
- input.read(serializeBytes);
+ input.readFully(serializeBytes);
lakeSplits.add(sourceSplitSerializer.deserialize(version, serializeBytes));
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/split/SourceSplitSerializer.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/split/SourceSplitSerializer.java
index e6cbdaeb4..0f5bc2b60 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/split/SourceSplitSerializer.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/split/SourceSplitSerializer.java
@@ -32,11 +32,29 @@ import java.io.IOException;
import static org.apache.fluss.utils.Preconditions.checkNotNull;
-/** A serializer for the {@link SourceSplitBase}. */
+/**
+ * A serializer for the {@link SourceSplitBase}.
+ *
+ * <h3>Version Evolution:</h3>
+ *
+ * <ul>
+ * <li><b>Version 0:</b> Initial version.
+ * <li><b>Version 1:</b> Adds the log stopping offset and the batch flag to {@link
+ * HybridSnapshotLogSplit}.
+ * <li><b>Version 2 (Current):</b> Adds {@code recordsToSkip} to {@link
+ * org.apache.fluss.flink.lake.split.LakeSnapshotSplit}, so that a partially consumed lake
+ * snapshot split resumes from where it stopped instead of re-emitting already emitted
+ * records. Lake snapshot splits restored from older versions start from the beginning.
+ * </ul>
+ *
+ * <p>This serializer can deserialize splits of all previous versions, but always produces the
+ * current version during serialization.
+ */
public class SourceSplitSerializer implements SimpleVersionedSerializer<SourceSplitBase> {
private static final int VERSION_0 = 0;
private static final int VERSION_1 = 1;
+ private static final int VERSION_2 = 2;
private static final ThreadLocal<DataOutputSerializer> SERIALIZER_CACHE =
ThreadLocal.withInitial(() -> new DataOutputSerializer(64));
@@ -45,7 +63,7 @@ public class SourceSplitSerializer implements SimpleVersionedSerializer<SourceSp
private static final byte LOG_SPLIT_FLAG = 2;
private static final byte KV_BATCH_SPLIT_FLAG = 3;
- private static final int CURRENT_VERSION = VERSION_1;
+ private static final int CURRENT_VERSION = VERSION_2;
@Nullable private final LakeSource<LakeSplit> lakeSource;
@@ -122,7 +140,7 @@ public class SourceSplitSerializer implements SimpleVersionedSerializer<SourceSp
@Override
public SourceSplitBase deserialize(int version, byte[] serialized) throws IOException {
- if (version != VERSION_0 && version != VERSION_1) {
+ if (version < VERSION_0 || version > CURRENT_VERSION) {
throw new IOException("Unknown version " + version);
}
final DataInputDeserializer in = new DataInputDeserializer(serialized);
@@ -146,7 +164,7 @@ public class SourceSplitSerializer implements SimpleVersionedSerializer<SourceSp
long logStartingOffset = in.readLong();
long logStoppingOffset = LogSplit.NO_STOPPING_OFFSET;
boolean isBatch = false;
- if (version == VERSION_1) {
+ if (version >= VERSION_1) {
logStoppingOffset = in.readLong();
isBatch = in.readBoolean();
}
@@ -168,7 +186,10 @@ public class SourceSplitSerializer implements SimpleVersionedSerializer<SourceSp
} else {
LakeSplitSerializer lakeSplitSerializer =
new LakeSplitSerializer(checkNotNull(lakeSource).getSplitSerializer());
- return lakeSplitSerializer.deserialize(splitKind, tableBucket, partitionName, in);
+ // records to skip of LakeSnapshotSplit are persisted since VERSION_2
+ boolean readRecordsToSkip = version >= VERSION_2;
+ return lakeSplitSerializer.deserialize(
+ splitKind, tableBucket, partitionName, in, readRecordsToSkip);
}
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TableBucketWriteResultSerializer.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TableBucketWriteResultSerializer.java
index 365176095..25a6fea6f 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TableBucketWriteResultSerializer.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TableBucketWriteResultSerializer.java
@@ -26,14 +26,29 @@ import org.apache.flink.core.memory.DataOutputSerializer;
import java.io.IOException;
-/** The serializer for {@link TableBucketWriteResult}. */
+/**
+ * The serializer for {@link TableBucketWriteResult}.
+ *
+ * <h3>Version Evolution:</h3>
+ *
+ * <ul>
+ * <li><b>Version 1:</b> Initial version. The version of the lake write result serializer was not
+ * persisted; the write result was deserialized with this serializer's version instead.
+ * <li><b>Version 2 (Current):</b> Persists the version of the lake write result serializer before
+ * the write result bytes and passes it to the lake serializer on deserialization, so that
+ * lake formats can evolve their write result format independently.
+ * </ul>
+ */
public class TableBucketWriteResultSerializer<WriteResult>
implements SimpleVersionedSerializer<TableBucketWriteResult<WriteResult>> {
private static final ThreadLocal<DataOutputSerializer> SERIALIZER_CACHE =
ThreadLocal.withInitial(() -> new DataOutputSerializer(64));
- private static final int CURRENT_VERSION = 1;
+ private static final int VERSION_1 = 1;
+ private static final int VERSION_2 = 2;
+
+ private static final int CURRENT_VERSION = VERSION_2;
private final org.apache.fluss.lake.serializer.SimpleVersionedSerializer<WriteResult>
writeResultSerializer;
@@ -79,6 +94,8 @@ public class TableBucketWriteResultSerializer<WriteResult>
} else {
byte[] serializeBytes = writeResultSerializer.serialize(writeResult);
out.writeInt(serializeBytes.length);
+ // since VERSION_2
+ out.writeInt(writeResultSerializer.getVersion());
out.write(serializeBytes);
}
@@ -99,7 +116,7 @@ public class TableBucketWriteResultSerializer<WriteResult>
@Override
public TableBucketWriteResult<WriteResult> deserialize(int version, byte[] serialized)
throws IOException {
- if (version != CURRENT_VERSION) {
+ if (version != VERSION_1 && version != VERSION_2) {
throw new IOException("Unknown version " + version);
}
final DataInputDeserializer in = new DataInputDeserializer(serialized);
@@ -123,9 +140,12 @@ public class TableBucketWriteResultSerializer<WriteResult>
int writeResultLength = in.readInt();
WriteResult writeResult;
if (writeResultLength >= 0) {
+ // VERSION_1 did not persist the lake serializer version and passed its own version
+ // to the lake serializer, keep that behavior for data written with VERSION_1
+ int writeResultVersion = version >= VERSION_2 ? in.readInt() : VERSION_1;
byte[] writeResultBytes = new byte[writeResultLength];
in.readFully(writeResultBytes);
- writeResult = writeResultSerializer.deserialize(version, writeResultBytes);
+ writeResult = writeResultSerializer.deserialize(writeResultVersion, writeResultBytes);
} else {
writeResult = null;
}
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/lake/LakeSplitSerializerTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/lake/LakeSplitSerializerTest.java
index ddc3e9f1f..1bd75ce1a 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/lake/LakeSplitSerializerTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/lake/LakeSplitSerializerTest.java
@@ -71,7 +71,8 @@ class LakeSplitSerializerTest {
LAKE_SNAPSHOT_SPLIT_KIND,
tableBucket,
"2025-08-18",
- new DataInputDeserializer(output.getCopyOfBuffer()));
+ new DataInputDeserializer(output.getCopyOfBuffer()),
+ true);
assertThat(deserializedSplit instanceof LakeSnapshotSplit).isTrue();
LakeSnapshotSplit result = (LakeSnapshotSplit) deserializedSplit;
@@ -82,6 +83,27 @@ class LakeSplitSerializerTest {
assertThat(splitIndex).isEqualTo(result.getSplitIndex());
}
+ @Test
+ void testSerializeAndDeserializeLakeSnapshotSplitWithRecordsToSkip() throws IOException {
+ LakeSnapshotSplit originalSplit =
+ new LakeSnapshotSplit(tableBucket, "2025-08-18", LAKE_SPLIT, 3, 42L);
+
+ DataOutputSerializer output = new DataOutputSerializer(STOPPING_OFFSET);
+ serializer.serialize(output, originalSplit);
+ DataInputDeserializer input = new DataInputDeserializer(output.getCopyOfBuffer());
+
+ LakeSnapshotSplit result =
+ (LakeSnapshotSplit)
+ serializer.deserialize(
+ LAKE_SNAPSHOT_SPLIT_KIND, tableBucket, "2025-08-18", input, true);
+
+ assertThat(result.getSplitIndex()).isEqualTo(3);
+ assertThat(result.getRecordsToSkip()).isEqualTo(42L);
+ assertThat(result.getLakeSplit()).isEqualTo(LAKE_SPLIT);
+ // the whole lake split section is consumed
+ assertThat(input.available()).isZero();
+ }
+
@Test
void testSerializeAndDeserializeLakeSnapshotSplitBackwardCompatibility() throws IOException {
SimpleVersionedSerializer<LakeSplit> sourceSplitSerializerV1 =
@@ -104,7 +126,8 @@ class LakeSplitSerializerTest {
LAKE_SNAPSHOT_SPLIT_KIND,
tableBucket,
"2025-08-18",
- new DataInputDeserializer(output.getCopyOfBuffer()));
+ new DataInputDeserializer(output.getCopyOfBuffer()),
+ true);
assertThat(deserializedSplit instanceof LakeSnapshotSplit).isTrue();
LakeSnapshotSplit result = (LakeSnapshotSplit) deserializedSplit;
@@ -136,7 +159,8 @@ class LakeSplitSerializerTest {
LAKE_SNAPSHOT_FLUSS_LOG_SPLIT_KIND,
tableBucket,
"2025-08-18",
- new DataInputDeserializer(output.getCopyOfBuffer()));
+ new DataInputDeserializer(output.getCopyOfBuffer()),
+ true);
assertThat(deserializedSplit instanceof LakeSnapshotAndFlussLogSplit).isTrue();
LakeSnapshotAndFlussLogSplit result = (LakeSnapshotAndFlussLogSplit) deserializedSplit;
@@ -162,7 +186,8 @@ class LakeSplitSerializerTest {
(byte) 99,
tableBucket,
"2023-10-01",
- new DataInputDeserializer(output.getCopyOfBuffer())))
+ new DataInputDeserializer(output.getCopyOfBuffer()),
+ true))
.withFailMessage(() -> "Unsupported split kind: ")
.isInstanceOf(UnsupportedOperationException.class);
}
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
index 7ae9a32c7..4187cf19c 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
@@ -24,6 +24,7 @@ import org.apache.fluss.client.table.writer.UpsertWriter;
import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.flink.lake.split.LakeSnapshotAndFlussLogSplit;
+import org.apache.fluss.flink.lake.split.LakeSnapshotSplit;
import org.apache.fluss.flink.source.deserializer.DeserializerInitContextImpl;
import org.apache.fluss.flink.source.deserializer.RowDataDeserializationSchema;
import org.apache.fluss.flink.source.emitter.FlinkRecordEmitter;
@@ -34,6 +35,7 @@ import org.apache.fluss.flink.source.metrics.FlinkSourceReaderMetrics;
import org.apache.fluss.flink.source.split.HybridSnapshotLogSplit;
import org.apache.fluss.flink.source.split.LogSplit;
import org.apache.fluss.flink.source.split.SourceSplitBase;
+import org.apache.fluss.flink.source.split.SourceSplitSerializer;
import org.apache.fluss.flink.utils.FlinkTestBase;
import org.apache.fluss.lake.source.LakeSource;
import org.apache.fluss.lake.source.LakeSplit;
@@ -317,6 +319,82 @@ class FlinkSourceReaderTest extends FlinkTestBase {
}
}
+ @Test
+ void testRestorePartiallyConsumedLakeSnapshotSplit() throws Exception {
+ TablePath tablePath =
+ TablePath.of(DEFAULT_DB, "test_restore_partially_consumed_lake_snapshot_split");
+ TableDescriptor tableDescriptor =
+ TableDescriptor.builder().schema(DEFAULT_LOG_TABLE_SCHEMA).distributedBy(1).build();
+ long tableId = createTable(tablePath, tableDescriptor);
+ RowType rowType = tableDescriptor.getSchema().getRowType();
+
+ List<LogRecord> lakeRecords = new ArrayList<>();
+ for (int i = 1; i <= 3; i++) {
+ lakeRecords.add(new GenericRecord(-1L, -1L, ChangeType.APPEND_ONLY, row(i, "v" + i)));
+ }
+ LakeSnapshotSplit split =
+ new LakeSnapshotSplit(
+ new TableBucket(tableId, 0),
+ null,
+ new TestingLakeSplit(0, Collections.emptyList()),
+ 0);
+
+ // consume the first record of the lake split, then checkpoint
+ TrackingLakeSource lakeSource = new TrackingLakeSource(lakeRecords);
+ SourceSplitSerializer splitSerializer = new SourceSplitSerializer(lakeSource);
+ TestingReaderOutput<RowData> output = new TestingReaderOutput<>();
+ SourceSplitBase restoredSplit;
+ try (FlinkSourceReader<RowData> reader =
+ createReader(
+ clientConf, tablePath, rowType, new TestingReaderContext(), lakeSource)) {
+ reader.addSplits(Collections.singletonList(split));
+ // each poll emits at most one record
+ retry(
+ Duration.ofMinutes(1),
+ () -> {
+ reader.pollNext(output);
+ assertThat(output.getEmittedRecords()).hasSize(1);
+ });
+
+ List<SourceSplitBase> checkpoint = reader.snapshotState(1L);
+ assertThat(checkpoint).hasSize(1);
+ LakeSnapshotSplit checkpointSplit = (LakeSnapshotSplit) checkpoint.get(0);
+ assertThat(checkpointSplit.getRecordsToSkip()).isEqualTo(1L);
+
+ // round trip through the split serializer, as Flink does when checkpointing
+ restoredSplit =
+ splitSerializer.deserialize(
+ splitSerializer.getVersion(),
+ splitSerializer.serialize(checkpointSplit));
+ }
+ assertThat(((LakeSnapshotSplit) restoredSplit).getRecordsToSkip()).isEqualTo(1L);
+
+ // the restored reader must continue with the remaining records only
+ TestingReaderOutput<RowData> restoredOutput = new TestingReaderOutput<>();
+ try (FlinkSourceReader<RowData> restoredReader =
+ createReader(
+ clientConf,
+ tablePath,
+ rowType,
+ new TestingReaderContext(),
+ new TrackingLakeSource(lakeRecords))) {
+ restoredReader.addSplits(Collections.singletonList(restoredSplit));
+ retry(
+ Duration.ofMinutes(1),
+ () -> {
+ restoredReader.pollNext(restoredOutput);
+ assertThat(restoredOutput.getEmittedRecords()).hasSize(2);
+ });
+ }
+
+ assertThat(output.getEmittedRecords())
+ .extracting(record -> record.getInt(0))
+ .containsExactly(1);
+ assertThat(restoredOutput.getEmittedRecords())
+ .extracting(record -> record.getInt(0))
+ .containsExactly(2, 3);
+ }
+
@Test
void testHandlePartitionsRemovedEvent() throws Exception {
TablePath tablePath = TablePath.of(DEFAULT_DB, "test_partitioned_table");
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/split/SourceSplitSerializerTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/split/SourceSplitSerializerTest.java
index b3e0b53f1..4a7684e90 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/split/SourceSplitSerializerTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/split/SourceSplitSerializerTest.java
@@ -17,13 +17,20 @@
package org.apache.fluss.flink.source.split;
+import org.apache.fluss.flink.lake.split.LakeSnapshotSplit;
+import org.apache.fluss.lake.source.TestingLakeSource;
+import org.apache.fluss.lake.source.TestingLakeSplit;
import org.apache.fluss.metadata.TableBucket;
import org.apache.flink.core.memory.DataOutputSerializer;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import java.io.IOException;
+import java.util.Collections;
+
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/**
* Unit tests for {@link org.apache.fluss.flink.source.split.SourceSplitSerializer} of serializing
@@ -34,6 +41,9 @@ import static org.assertj.core.api.Assertions.assertThat;
class SourceSplitSerializerTest {
private static final SourceSplitSerializer serializer = new SourceSplitSerializer(null);
+ private static final TestingLakeSource lakeSource = new TestingLakeSource();
+ private static final SourceSplitSerializer lakeSerializer =
+ new SourceSplitSerializer(lakeSource);
private static final TableBucket tableBucket = new TableBucket(1, 2);
private static final TableBucket partitionedTableBucket = new TableBucket(1, 100L, 2);
@@ -142,6 +152,76 @@ class SourceSplitSerializerTest {
assertThat(deserializedSplit.asHybridSnapshotLogSplit().getLogStoppingOffset()).isEmpty();
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testDeserializeVersionOneHybridSnapshotLogSplit(boolean isPartitioned) throws Exception {
+ // the layout of HybridSnapshotLogSplit is unchanged since VERSION_1
+ TableBucket bucket = isPartitioned ? partitionedTableBucket : tableBucket;
+ String partitionName = isPartitioned ? "2024" : null;
+ HybridSnapshotLogSplit split =
+ new HybridSnapshotLogSplit(bucket, partitionName, 100L, 3L, true, 20L, 30L, true);
+
+ SourceSplitBase deserializedSplit = serializer.deserialize(1, serializer.serialize(split));
+
+ assertThat(deserializedSplit).isEqualTo(split);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testLakeSnapshotSplitSerde(boolean isPartitioned) throws Exception {
+ TableBucket bucket = isPartitioned ? partitionedTableBucket : tableBucket;
+ String partitionName = isPartitioned ? "2024" : null;
+ TestingLakeSplit lakeSplit =
+ new TestingLakeSplit(
+ bucket.getBucket(),
+ isPartitioned ? Collections.singletonList(partitionName) : null);
+ LakeSnapshotSplit split = new LakeSnapshotSplit(bucket, partitionName, lakeSplit, 5, 7L);
+
+ LakeSnapshotSplit deserializedSplit =
+ (LakeSnapshotSplit)
+ lakeSerializer.deserialize(
+ lakeSerializer.getVersion(), lakeSerializer.serialize(split));
+
+ assertLakeSnapshotSplit(deserializedSplit, split, 7L);
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {0, 1})
+ void testDeserializeLakeSnapshotSplitBeforeVersionTwo(int version) throws Exception {
+ TestingLakeSplit lakeSplit = new TestingLakeSplit(2, null);
+ LakeSnapshotSplit split = new LakeSnapshotSplit(tableBucket, null, lakeSplit, 5, 7L);
+ byte[] lakeSplitBytes = lakeSource.getSplitSerializer().serialize(lakeSplit);
+
+ // the layout of LakeSnapshotSplit before VERSION_2, which has no records to skip
+ DataOutputSerializer out = new DataOutputSerializer(64);
+ out.writeByte(LakeSnapshotSplit.LAKE_SNAPSHOT_SPLIT_KIND);
+ out.writeLong(tableBucket.getTableId());
+ out.writeBoolean(false);
+ out.writeInt(tableBucket.getBucket());
+ out.writeInt(lakeSource.getSplitSerializer().getVersion());
+ out.writeInt(5);
+ out.writeInt(lakeSplitBytes.length);
+ out.write(lakeSplitBytes);
+
+ LakeSnapshotSplit deserializedSplit =
+ (LakeSnapshotSplit) lakeSerializer.deserialize(version, out.getCopyOfBuffer());
+
+ // restored from the beginning of the lake split
+ assertLakeSnapshotSplit(deserializedSplit, split, 0L);
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {-1, 3})
+ void testDeserializeUnknownVersion(int version) {
+ assertThatThrownBy(
+ () ->
+ serializer.deserialize(
+ version,
+ serializer.serialize(new LogSplit(tableBucket, null, 0))))
+ .isInstanceOf(IOException.class)
+ .hasMessage("Unknown version " + version);
+ }
+
@ParameterizedTest
@ValueSource(booleans = {true, false})
void testKvBatchSplitSerde(boolean isPartitioned) throws Exception {
@@ -156,4 +236,16 @@ class SourceSplitSerializerTest {
assertThat(deserializedSplit.isKvBatchSplit()).isTrue();
assertThat(deserializedSplit.asKvBatchSplit().splitId()).isEqualTo(split.splitId());
}
+
+ private static void assertLakeSnapshotSplit(
+ LakeSnapshotSplit actual, LakeSnapshotSplit expected, long expectedRecordsToSkip) {
+ assertThat(actual.getTableBucket()).isEqualTo(expected.getTableBucket());
+ assertThat(actual.getPartitionName()).isEqualTo(expected.getPartitionName());
+ assertThat(actual.getSplitIndex()).isEqualTo(expected.getSplitIndex());
+ assertThat(actual.getRecordsToSkip()).isEqualTo(expectedRecordsToSkip);
+ assertThat(actual.getLakeSplit().bucket()).isEqualTo(expected.getLakeSplit().bucket());
+ assertThat(actual.getLakeSplit().partition())
+ .isEqualTo(expected.getLakeSplit().partition());
+ assertThat(actual.splitId()).isEqualTo(expected.splitId());
+ }
}
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TableBucketWriteResultSerializerTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TableBucketWriteResultSerializerTest.java
index dbb40eae1..b07c6667c 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TableBucketWriteResultSerializerTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TableBucketWriteResultSerializerTest.java
@@ -18,13 +18,22 @@
package org.apache.fluss.flink.tiering.source;
import org.apache.fluss.flink.tiering.TestingWriteResult;
+import org.apache.fluss.lake.serializer.SimpleVersionedSerializer;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TablePath;
+import org.apache.flink.core.memory.DataInputDeserializer;
+import org.apache.flink.core.memory.DataOutputSerializer;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Test for {@link TableBucketWriteResultSerializer}. */
class TableBucketWriteResultSerializerTest {
@@ -75,4 +84,115 @@ class TableBucketWriteResultSerializerTest {
assertThat(deserialized.writeResult()).isNull();
assertThat(deserialized.numberOfWriteResults()).isEqualTo(30);
}
+
+ @Test
+ void testLakeWriteResultSerializerVersionIsPersisted() throws Exception {
+ VersionRecordingWriteResultSerializer lakeSerializer =
+ new VersionRecordingWriteResultSerializer(7);
+ TableBucketWriteResultSerializer<TestingWriteResult> serializer =
+ new TableBucketWriteResultSerializer<>(lakeSerializer);
+
+ TableBucketWriteResult<TestingWriteResult> deserialized =
+ serializer.deserialize(
+ serializer.getVersion(),
+ serializer.serialize(
+ new TableBucketWriteResult<>(
+ TablePath.of("db1", "tb1"),
+ new TableBucket(1, 2),
+ null,
+ new TestingWriteResult(5),
+ 10,
+ 30L,
+ 1)));
+
+ assertThat(lakeSerializer.deserializedVersions).containsExactly(7);
+ assertThat(deserialized.writeResult().getWriteResult()).isEqualTo(5);
+ assertThat(deserialized.logEndOffset()).isEqualTo(10);
+ assertThat(deserialized.maxTimestamp()).isEqualTo(30L);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testDeserializeVersionOne(boolean withWriteResult) throws Exception {
+ VersionRecordingWriteResultSerializer lakeSerializer =
+ new VersionRecordingWriteResultSerializer(7);
+ TableBucketWriteResultSerializer<TestingWriteResult> serializer =
+ new TableBucketWriteResultSerializer<>(lakeSerializer);
+
+ // the layout of VERSION_1, which doesn't persist the lake write result serializer version
+ DataOutputSerializer out = new DataOutputSerializer(64);
+ out.writeUTF("db1");
+ out.writeUTF("tb1");
+ out.writeLong(1L);
+ out.writeBoolean(true);
+ out.writeLong(1000L);
+ out.writeUTF("partition1");
+ out.writeInt(2);
+ if (withWriteResult) {
+ byte[] writeResultBytes = lakeSerializer.serialize(new TestingWriteResult(5));
+ out.writeInt(writeResultBytes.length);
+ out.write(writeResultBytes);
+ } else {
+ out.writeInt(-1);
+ }
+ out.writeLong(10L);
+ out.writeLong(30L);
+ out.writeInt(3);
+
+ TableBucketWriteResult<TestingWriteResult> deserialized =
+ serializer.deserialize(1, out.getCopyOfBuffer());
+
+ assertThat(deserialized.tablePath()).isEqualTo(TablePath.of("db1", "tb1"));
+ assertThat(deserialized.tableBucket()).isEqualTo(new TableBucket(1L, 1000L, 2));
+ assertThat(deserialized.partitionName()).isEqualTo("partition1");
+ assertThat(deserialized.logEndOffset()).isEqualTo(10L);
+ assertThat(deserialized.maxTimestamp()).isEqualTo(30L);
+ assertThat(deserialized.numberOfWriteResults()).isEqualTo(3);
+ if (withWriteResult) {
+ // VERSION_1 passed its own version to the lake serializer
+ assertThat(lakeSerializer.deserializedVersions).containsExactly(1);
+ assertThat(deserialized.writeResult().getWriteResult()).isEqualTo(5);
+ } else {
+ assertThat(lakeSerializer.deserializedVersions).isEmpty();
+ assertThat(deserialized.writeResult()).isNull();
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {0, 3})
+ void testDeserializeUnknownVersion(int version) {
+ assertThatThrownBy(() -> tableBucketWriteResultSerializer.deserialize(version, new byte[0]))
+ .isInstanceOf(IOException.class)
+ .hasMessage("Unknown version " + version);
+ }
+
+ /** A lake write result serializer which records the versions passed for deserialization. */
+ private static class VersionRecordingWriteResultSerializer
+ implements SimpleVersionedSerializer<TestingWriteResult> {
+
+ private final int version;
+ private final List<Integer> deserializedVersions = new ArrayList<>();
+
+ private VersionRecordingWriteResultSerializer(int version) {
+ this.version = version;
+ }
+
+ @Override
+ public int getVersion() {
+ return version;
+ }
+
+ @Override
+ public byte[] serialize(TestingWriteResult writeResult) throws IOException {
+ DataOutputSerializer out = new DataOutputSerializer(4);
+ out.writeInt(writeResult.getWriteResult());
+ return out.getCopyOfBuffer();
+ }
+
+ @Override
+ public TestingWriteResult deserialize(int version, byte[] serialized) throws IOException {
+ deserializedVersions.add(version);
+ return new TestingWriteResult(new DataInputDeserializer(serialized).readInt());
+ }
+ }
}