You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ho...@apache.org on 2026/07/30 09:28:30 UTC
(fluss) branch main updated: [FIP-43][client] Support log scanner and writer for multiple tables (#3140)
This is an automated email from the ASF dual-hosted git repository.
loserwang1024 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
The following commit(s) were added to refs/heads/main by this push:
new d951e4d6b [FIP-43][client] Support log scanner and writer for multiple tables (#3140)
d951e4d6b is described below
commit d951e4d6b40e8802131a9c0bd499adfff9f3d647
Author: Hongshun Wang <12...@users.noreply.github.com>
AuthorDate: Thu Jul 30 17:28:25 2026 +0800
[FIP-43][client] Support log scanner and writer for multiple tables (#3140)
---
.../java/org/apache/fluss/client/Connection.java | 11 +
.../org/apache/fluss/client/FlussConnection.java | 7 +
.../metrics/MultiTableWriterMetricGroup.java | 45 +
.../fluss/client/metrics/WriterMetricGroup.java | 38 +-
.../org/apache/fluss/client/table/MultiTable.java | 48 ++
.../apache/fluss/client/table/MultiTableImpl.java | 56 ++
...ndingFetch.java => MultiTableBatchScanner.java} | 22 +-
.../client/table/scanner/MultiTableRecord.java | 117 +++
.../fluss/client/table/scanner/MultiTableScan.java | 41 +
.../client/table/scanner/MultiTableScanImpl.java | 55 ++
.../fluss/client/table/scanner/ScanRecord.java | 25 +
.../table/scanner/batch/LimitBatchScanner.java | 7 +-
.../scanner/log/AbstractLogFetchCollector.java | 11 +-
.../table/scanner/log/AbstractLogScanner.java | 207 +++++
.../table/scanner/log/ArrowLogFetchCollector.java | 13 +-
.../client/table/scanner/log/CompletedFetch.java | 16 +-
.../table/scanner/log/CompletedPendingFetch.java | 5 +
.../table/scanner/log/DefaultCompletedFetch.java | 3 +
.../client/table/scanner/log/LogFetchBuffer.java | 11 +-
.../table/scanner/log/LogFetchCollector.java | 4 +-
.../fluss/client/table/scanner/log/LogFetcher.java | 431 +++++++---
.../client/table/scanner/log/LogScannerImpl.java | 290 +++----
.../table/scanner/log/MultiTableLogScanner.java | 99 +++
.../scanner/log/MultiTableLogScannerImpl.java | 387 +++++++++
.../table/scanner/log/MultiTableRecords.java | 223 +++++
.../client/table/scanner/log/PendingFetch.java | 3 +
.../table/scanner/log/RemoteCompletedFetch.java | 3 +
.../table/scanner/log/RemoteLogDownloadFuture.java | 17 +
.../table/scanner/log/RemoteLogDownloader.java | 44 +-
.../table/scanner/log/RemotePendingFetch.java | 10 +
.../client/table/scanner/log/ScanRecords.java | 31 +-
.../client/table/scanner/log/TableScanSpec.java | 64 ++
.../MultiTableWrite.java} | 21 +-
.../client/table/writer/MultiTableWriteImpl.java | 60 ++
.../client/table/writer/MultiTableWriteRecord.java | 138 +++
.../client/table/writer/MultiTableWriter.java | 58 ++
.../client/table/writer/MultiTableWriterImpl.java | 521 +++++++++++
.../PendingFetch.java => writer/WriteResult.java} | 22 +-
.../client/write/AbstractRowLogWriteBatch.java | 6 +-
.../fluss/client/write/ArrowLogWriteBatch.java | 4 +-
.../fluss/client/write/CompactedLogWriteBatch.java | 2 +
.../fluss/client/write/IndexedLogWriteBatch.java | 2 +
.../apache/fluss/client/write/KvWriteBatch.java | 19 +-
.../fluss/client/write/RecordAccumulator.java | 3 +
.../java/org/apache/fluss/client/write/Sender.java | 27 +-
.../org/apache/fluss/client/write/WriteBatch.java | 45 +-
.../apache/fluss/client/write/WriterClient.java | 13 +-
.../client/metrics/WriterMetricGroupTest.java | 122 +++
.../DefaultCompletedFetchBufferLifecycleTest.java | 2 +
.../scanner/log/DefaultCompletedFetchTest.java | 11 +-
.../table/scanner/log/LogFetchBufferTest.java | 37 +
.../table/scanner/log/LogFetchCollectorTest.java | 18 +-
.../table/scanner/log/LogFetcherFilterITCase.java | 54 +-
.../client/table/scanner/log/LogFetcherITCase.java | 51 +-
.../client/table/scanner/log/LogFetcherTest.java | 99 ++-
.../client/table/scanner/log/LogScannerITCase.java | 61 ++
.../scanner/log/MultiTableLogScannerITCase.java | 793 +++++++++++++++++
.../scanner/log/RemoteCompletedFetchTest.java | 10 +-
.../table/scanner/log/RemoteLogDownloaderTest.java | 136 ++-
.../client/table/scanner/log/ScanRecordsTest.java | 31 +
.../table/writer/MultiTableWriteRecordTest.java | 88 ++
.../table/writer/MultiTableWriterITCase.java | 949 +++++++++++++++++++++
.../client/write/CompactedLogWriteBatchTest.java | 66 +-
.../fluss/client/write/RecordAccumulatorTest.java | 50 +-
.../apache/fluss/record/DefaultLogRecordBatch.java | 20 +-
.../apache/fluss/record/LogRecordReadContext.java | 336 ++++----
.../fluss/record/DefaultLogRecordBatchTest.java | 5 +-
.../apache/fluss/record/FileLogProjectionTest.java | 21 +-
.../apache/fluss/record/FileLogRecordsTest.java | 8 +-
.../fluss/record/LogRecordBatchStatisticsTest.java | 31 +-
.../java/org/apache/fluss/record/LogTestBase.java | 9 +-
.../record/MemoryLogRecordsArrowBuilderTest.java | 13 +-
.../MemoryLogRecordsCompactedBuilderTest.java | 7 +-
.../apache/fluss/record/MemoryLogRecordsTest.java | 2 +
.../java/org/apache/fluss/record/TestData.java | 2 +-
.../apache/fluss/record/TestingSchemaGetter.java | 11 +-
.../org/apache/fluss/testutils/DataTestUtils.java | 16 +-
.../apache/fluss/testutils/LogRecordsAssert.java | 3 +-
.../apache/fluss/server/kv/KvRecoverHelper.java | 2 +-
.../fluss/server/kv/KvTabletMergeModeTest.java | 9 +
.../server/kv/KvTabletSchemaEvolutionTest.java | 2 +
.../org/apache/fluss/server/kv/KvTabletTest.java | 19 +-
.../kv/rowmerger/AggregateRowMergerTest.java | 3 +-
.../apache/fluss/server/log/LogSegmentTest.java | 2 +-
.../server/log/PredicateSchemaResolverTest.java | 2 +-
.../apache/fluss/server/replica/ReplicaTest.java | 1 +
.../replica/fetcher/ReplicaFetcherITCase.java | 3 +-
.../fluss/server/tablet/TabletServiceITCase.java | 6 +-
.../server/testutils/FlussClusterExtension.java | 1 +
89 files changed, 5705 insertions(+), 692 deletions(-)
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/Connection.java b/fluss-client/src/main/java/org/apache/fluss/client/Connection.java
index b4c81e41b..05213ffd9 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/Connection.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/Connection.java
@@ -19,6 +19,7 @@ package org.apache.fluss.client;
import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.client.admin.Admin;
+import org.apache.fluss.client.table.MultiTable;
import org.apache.fluss.client.table.Table;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.metadata.TablePath;
@@ -56,6 +57,16 @@ public interface Connection extends AutoCloseable {
/** Retrieve a new Table client to operate data in table. */
Table getTable(TablePath tablePath);
+ /**
+ * Retrieve a {@link MultiTable} client to scan and write data across multiple tables in a
+ * single unified API. Unlike {@link #getTable(TablePath)} which is bound to one table, {@code
+ * MultiTable} is table-agnostic: tables are identified per scan subscription and per write
+ * record.
+ *
+ * <p>{@link MultiTable} instances are light-weight and NOT thread-safe; obtain per-thread.
+ */
+ MultiTable getMultiTable();
+
/** Close the connection and release all resources. */
@Override
void close() throws Exception;
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java b/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java
index 0d800b3c9..9327ad0ac 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java
@@ -22,6 +22,8 @@ import org.apache.fluss.client.admin.FlussAdmin;
import org.apache.fluss.client.lookup.LookupClient;
import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.client.table.FlussTable;
+import org.apache.fluss.client.table.MultiTable;
+import org.apache.fluss.client.table.MultiTableImpl;
import org.apache.fluss.client.table.Table;
import org.apache.fluss.client.table.scanner.RemoteFileDownloader;
import org.apache.fluss.client.token.DefaultSecurityTokenManager;
@@ -108,6 +110,11 @@ public final class FlussConnection implements Connection {
return new FlussTable(this, tablePath, admin.getTableInfo(tablePath).join());
}
+ @Override
+ public MultiTable getMultiTable() {
+ return new MultiTableImpl(this);
+ }
+
public MetadataUpdater getMetadataUpdater() {
return metadataUpdater;
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/metrics/MultiTableWriterMetricGroup.java b/fluss-client/src/main/java/org/apache/fluss/client/metrics/MultiTableWriterMetricGroup.java
new file mode 100644
index 000000000..850f11975
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/metrics/MultiTableWriterMetricGroup.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.metrics;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.client.write.WriterClient;
+import org.apache.fluss.rpc.metrics.ClientMetricGroup;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Metrics for {@link WriterClient}s owned by multi-table writers.
+ *
+ * <p>Unlike the connection-level shared writer client which keeps the {@code client.writer} metric
+ * identity, each multi-table writer owns a dedicated {@link WriterClient}, so multiple instances
+ * may coexist under the same {@link ClientMetricGroup}. This group therefore uses the dedicated
+ * {@code multi_table_writer} scope plus a unique {@code writer_instance_id} variable so that
+ * reporters derive distinct metric identities (e.g. JMX ObjectNames, Prometheus label sets) instead
+ * of colliding.
+ */
+@Internal
+public class MultiTableWriterMetricGroup extends WriterMetricGroup {
+
+ private static final String NAME = "multi_table_writer";
+ private static final AtomicLong NEXT_WRITER_INSTANCE_ID = new AtomicLong();
+
+ public MultiTableWriterMetricGroup(ClientMetricGroup parent) {
+ super(parent, NAME, String.valueOf(NEXT_WRITER_INSTANCE_ID.incrementAndGet()));
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/metrics/WriterMetricGroup.java b/fluss-client/src/main/java/org/apache/fluss/client/metrics/WriterMetricGroup.java
index c717a780e..c5ccc20ca 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/metrics/WriterMetricGroup.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/metrics/WriterMetricGroup.java
@@ -29,14 +29,30 @@ import org.apache.fluss.metrics.ThreadSafeSimpleCounter;
import org.apache.fluss.metrics.groups.AbstractMetricGroup;
import org.apache.fluss.rpc.metrics.ClientMetricGroup;
+import javax.annotation.Nullable;
+
+import java.util.Map;
+
import static org.apache.fluss.metrics.utils.MetricGroupUtils.makeScope;
-/** Metrics for {@link WriterClient}. */
+/**
+ * Metrics for {@link WriterClient}.
+ *
+ * <p>The connection-level shared writer client uses the {@code client.writer} scope without any
+ * extra variables. Writer clients that may have multiple instances under the same {@link
+ * ClientMetricGroup} should use a subclass with a dedicated scope and a disambiguating {@code
+ * writer_instance_id} variable, see {@link MultiTableWriterMetricGroup}.
+ */
@Internal
public class WriterMetricGroup extends AbstractMetricGroup {
private static final String name = "writer";
private static final int WINDOW_SIZE = 1024;
+ private final String groupName;
+
+ /** Only set for subclasses whose instances may coexist under the same parent. */
+ @Nullable private final String writerInstanceId;
+
private final Counter recordsRetryTotal;
private final Counter recordsSendTotal;
private final Counter bytesSendTotal;
@@ -47,7 +63,16 @@ public class WriterMetricGroup extends AbstractMetricGroup {
private volatile long batchQueueTimeMs = -1;
public WriterMetricGroup(ClientMetricGroup parent) {
- super(parent.getMetricRegistry(), makeScope(parent, name), parent);
+ this(parent, name, null);
+ }
+
+ protected WriterMetricGroup(
+ ClientMetricGroup parent, String groupName, @Nullable String writerInstanceId) {
+ super(parent.getMetricRegistry(), makeScope(parent, groupName), parent);
+ this.groupName = groupName;
+ // must be assigned before the first metric registration below, as reporters resolve
+ // getAllVariables() (which reads writerInstanceId) when a metric is added
+ this.writerInstanceId = writerInstanceId;
gauge(MetricNames.WRITER_BATCH_QUEUE_TIME_MS, () -> batchQueueTimeMs);
@@ -99,6 +124,13 @@ public class WriterMetricGroup extends AbstractMetricGroup {
@Override
protected String getGroupName(CharacterFilter filter) {
- return name;
+ return groupName;
+ }
+
+ @Override
+ protected final void putVariables(Map<String, String> variables) {
+ if (writerInstanceId != null) {
+ variables.put("writer_instance_id", writerInstanceId);
+ }
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/MultiTable.java b/fluss-client/src/main/java/org/apache/fluss/client/table/MultiTable.java
new file mode 100644
index 000000000..ba90f1fa6
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/MultiTable.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table;
+
+import org.apache.fluss.annotation.PublicEvolving;
+import org.apache.fluss.client.Connection;
+import org.apache.fluss.client.table.scanner.MultiTableScan;
+import org.apache.fluss.client.table.writer.MultiTableWrite;
+import org.apache.fluss.metadata.TablePath;
+
+/**
+ * Used to communicate with multiple Fluss tables through a single client. Obtain an instance from
+ * {@link Connection#getMultiTable()}.
+ *
+ * <p>{@code MultiTable} is the cross-table counterpart of {@link Table}. Useful for CDC ingestion,
+ * cross-table streaming pipelines, and multi-table catalog sinks.
+ *
+ * <p>Unlike {@link Connection#getTable(TablePath)} which is bound to one table, {@code MultiTable}
+ * is table-agnostic: tables are identified per scan subscription and per write record.
+ *
+ * <p>{@code MultiTable} instances are light-weight and NOT thread-safe; obtain per-thread.
+ *
+ * @since 0.7
+ */
+@PublicEvolving
+public interface MultiTable {
+
+ /** Build a scanner that reads data from multiple tables simultaneously. */
+ MultiTableScan newMultiTableScan();
+
+ /** Build a writer that writes data (with change types) to multiple tables. */
+ MultiTableWrite newMultiTableWrite();
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/MultiTableImpl.java b/fluss-client/src/main/java/org/apache/fluss/client/table/MultiTableImpl.java
new file mode 100644
index 000000000..201a938aa
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/MultiTableImpl.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.client.FlussConnection;
+import org.apache.fluss.client.table.scanner.MultiTableScan;
+import org.apache.fluss.client.table.scanner.MultiTableScanImpl;
+import org.apache.fluss.client.table.writer.MultiTableWrite;
+import org.apache.fluss.client.table.writer.MultiTableWriteImpl;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/**
+ * Default implementation of {@link MultiTable}.
+ *
+ * <p>{@code MultiTableImpl} is light-weight and NOT thread-safe; the underlying {@link
+ * FlussConnection} is shared and thread-safe, but each {@code MultiTableImpl} instance should be
+ * obtained per-thread.
+ *
+ * @since 0.7
+ */
+@Internal
+public class MultiTableImpl implements MultiTable {
+
+ private final FlussConnection connection;
+
+ public MultiTableImpl(FlussConnection connection) {
+ this.connection = checkNotNull(connection, "connection");
+ }
+
+ @Override
+ public MultiTableScan newMultiTableScan() {
+ return new MultiTableScanImpl(connection);
+ }
+
+ @Override
+ public MultiTableWrite newMultiTableWrite() {
+ return new MultiTableWriteImpl(connection);
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/PendingFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/MultiTableBatchScanner.java
similarity index 66%
copy from fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/PendingFetch.java
copy to fluss-client/src/main/java/org/apache/fluss/client/table/scanner/MultiTableBatchScanner.java
index c3fa29936..5149c6417 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/PendingFetch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/MultiTableBatchScanner.java
@@ -15,19 +15,17 @@
* limitations under the License.
*/
-package org.apache.fluss.client.table.scanner.log;
+package org.apache.fluss.client.table.scanner;
-import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.annotation.PublicEvolving;
/**
- * A pending fetch is a fetch that requested to download remote log segment file to local but not
- * yet been completed.
+ * Reads current data from multiple tables through a single client.
+ *
+ * <p>NOTE: planned for a future release; declared as a placeholder for forward compatibility. There
+ * is no concrete implementation today.
+ *
+ * @since 0.7
*/
-interface PendingFetch {
-
- TableBucket tableBucket();
-
- boolean isCompleted();
-
- CompletedFetch toCompletedFetch();
-}
+@PublicEvolving
+public interface MultiTableBatchScanner extends AutoCloseable {}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/MultiTableRecord.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/MultiTableRecord.java
new file mode 100644
index 000000000..e05662f6d
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/MultiTableRecord.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.scanner;
+
+import org.apache.fluss.annotation.PublicEvolving;
+import org.apache.fluss.client.table.scanner.log.MultiTableLogScanner;
+import org.apache.fluss.client.table.writer.MultiTableWriter;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.record.ChangeType;
+import org.apache.fluss.row.InternalRow;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/**
+ * A record produced by a {@link MultiTableLogScanner}. Wraps an unchanged {@link ScanRecord} and
+ * enriches it with the source table identity (path, id) and schema (schema object, schema id).
+ *
+ * <p>The underlying {@link ScanRecord} structure is NOT modified.
+ *
+ * <p>Read-side only. For writes via {@link MultiTableWriter}, use {@link
+ * org.apache.fluss.client.table.writer.MultiTableWriteRecord} which carries only the write-relevant
+ * fields.
+ *
+ * @since 0.7
+ */
+@PublicEvolving
+public final class MultiTableRecord {
+
+ private final TablePath tablePath;
+ private final long tableId;
+ private final Schema schema;
+ private final int schemaId;
+ private final ScanRecord scanRecord;
+
+ public MultiTableRecord(
+ TablePath tablePath, long tableId, int schemaId, Schema schema, ScanRecord scanRecord) {
+ this.tablePath = checkNotNull(tablePath, "tablePath");
+ this.tableId = tableId;
+ this.schema = checkNotNull(schema, "schema");
+ this.schemaId = schemaId;
+ this.scanRecord = checkNotNull(scanRecord, "scanRecord");
+ }
+
+ // ---- table identity ----
+
+ public TablePath getTablePath() {
+ return tablePath;
+ }
+
+ public long getTableId() {
+ return tableId;
+ }
+
+ // ---- schema info ----
+
+ public Schema getSchema() {
+ return schema;
+ }
+
+ public int getSchemaId() {
+ return schemaId;
+ }
+
+ // ---- underlying record (unchanged) ----
+
+ public ScanRecord getScanRecord() {
+ return scanRecord;
+ }
+
+ // ---- convenience delegates ----
+
+ public long logOffset() {
+ return scanRecord.logOffset();
+ }
+
+ public long timestamp() {
+ return scanRecord.timestamp();
+ }
+
+ public ChangeType getChangeType() {
+ return scanRecord.getChangeType();
+ }
+
+ public InternalRow getRow() {
+ return scanRecord.getRow();
+ }
+
+ @Override
+ public String toString() {
+ return "MultiTableRecord{"
+ + "tablePath="
+ + tablePath
+ + ", tableId="
+ + tableId
+ + ", schemaId="
+ + schemaId
+ + ", scanRecord="
+ + scanRecord
+ + '}';
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/MultiTableScan.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/MultiTableScan.java
new file mode 100644
index 000000000..a533753cf
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/MultiTableScan.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.scanner;
+
+import org.apache.fluss.annotation.PublicEvolving;
+import org.apache.fluss.client.table.scanner.log.MultiTableLogScanner;
+
+/**
+ * Builder for a multi-table scanner.
+ *
+ * <p>Phase 1 surface intentionally exposes no per-table builder configuration: tables are
+ * registered dynamically on the created {@link MultiTableLogScanner} when {@code subscribe(...)} is
+ * first called for a {@code TablePath}. Future phases may add {@code withTable(...)} overloads to
+ * pre-declare tables together with projection / filter pushdown.
+ *
+ * @since 0.7
+ */
+@PublicEvolving
+public interface MultiTableScan {
+
+ /**
+ * Creates a {@link MultiTableLogScanner} to continuously read log data from multiple tables.
+ * Tables are registered dynamically on first {@code subscribe(...)} call.
+ */
+ MultiTableLogScanner createLogScanner();
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/MultiTableScanImpl.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/MultiTableScanImpl.java
new file mode 100644
index 000000000..53fa823a7
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/MultiTableScanImpl.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.scanner;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.client.FlussConnection;
+import org.apache.fluss.client.table.scanner.log.MultiTableLogScanner;
+import org.apache.fluss.client.table.scanner.log.MultiTableLogScannerImpl;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/**
+ * Default implementation of {@link MultiTableScan}.
+ *
+ * <p>Phase 1: a thin pass-through that creates a {@link MultiTableLogScanner} bound to the {@link
+ * FlussConnection}. Tables are registered dynamically on the scanner via {@code subscribe(...)} —
+ * no upfront declaration is required.
+ *
+ * @since 0.7
+ */
+@Internal
+public class MultiTableScanImpl implements MultiTableScan {
+
+ private final FlussConnection connection;
+
+ public MultiTableScanImpl(FlussConnection connection) {
+ this.connection = checkNotNull(connection, "connection");
+ }
+
+ @Override
+ public MultiTableLogScanner createLogScanner() {
+ return new MultiTableLogScannerImpl(
+ "multi-table-log-scanner",
+ connection.getMetadataUpdater(),
+ connection.getConfiguration(),
+ connection.getClientMetricGroup(),
+ connection.getOrCreateRemoteFileDownloader(),
+ connection.getOrCreateAdmin());
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/ScanRecord.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/ScanRecord.java
index 829c56a00..3c34a7f1d 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/ScanRecord.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/ScanRecord.java
@@ -33,6 +33,8 @@ public class ScanRecord implements LogRecord {
/** Indicates that the size in bytes is unknown for this record. */
public static final int UNKNOWN_SIZE_IN_BYTES = -1;
+ private final long tableId;
+ private final int schemaId;
private final long offset;
private final long timestamp;
private final ChangeType changeType;
@@ -53,6 +55,19 @@ public class ScanRecord implements LogRecord {
public ScanRecord(
long offset, long timestamp, ChangeType changeType, InternalRow row, int sizeInBytes) {
+ this(INVALID, -1, offset, timestamp, changeType, row, sizeInBytes);
+ }
+
+ public ScanRecord(
+ long tableId,
+ int schemaId,
+ long offset,
+ long timestamp,
+ ChangeType changeType,
+ InternalRow row,
+ int sizeInBytes) {
+ this.tableId = tableId;
+ this.schemaId = schemaId;
this.offset = offset;
this.timestamp = timestamp;
this.changeType = changeType;
@@ -85,6 +100,16 @@ public class ScanRecord implements LogRecord {
return sizeInBytes;
}
+ /** The table id this record belongs to. */
+ public long getTableId() {
+ return tableId;
+ }
+
+ /** The schema id of the batch this record was produced with. */
+ public int getSchemaId() {
+ return schemaId;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
index 26e097fef..b5a381ad1 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
@@ -181,7 +181,12 @@ public class LimitBatchScanner implements BatchScanner {
} else {
LogRecordReadContext readContext =
LogRecordReadContext.createReadContext(
- tableInfo, false, null, schemaGetter, chunkedFactory);
+ tableInfo,
+ false,
+ LogRecordReadContext.SchemaResolution.TARGET,
+ null,
+ schemaGetter,
+ chunkedFactory);
LogRecords records = MemoryLogRecords.pointToByteBuffer(recordsBuffer);
for (LogRecordBatch logRecordBatch : records.batches()) {
// A batch of log record maybe little more than limit, thus we need slice the
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogFetchCollector.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogFetchCollector.java
index e058e0e2d..3ffa640f9 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogFetchCollector.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogFetchCollector.java
@@ -44,19 +44,16 @@ import java.util.Map;
@Internal
abstract class AbstractLogFetchCollector<T, R> {
protected final Logger log;
- protected final TablePath tablePath;
protected final LogScannerStatus logScannerStatus;
private final int maxPollRecords;
private final MetadataUpdater metadataUpdater;
protected AbstractLogFetchCollector(
Logger log,
- TablePath tablePath,
LogScannerStatus logScannerStatus,
Configuration conf,
MetadataUpdater metadataUpdater) {
this.log = log;
- this.tablePath = tablePath;
this.logScannerStatus = logScannerStatus;
this.maxPollRecords = conf.getInt(ConfigOptions.CLIENT_SCANNER_LOG_MAX_POLL_RECORDS);
this.metadataUpdater = metadataUpdater;
@@ -159,7 +156,11 @@ abstract class AbstractLogFetchCollector<T, R> {
if (error.isSuccess()) {
return handleInitializeSuccess(completedFetch);
} else {
- handleInitializeErrors(completedFetch, error.error(), error.messageWithFallback());
+ handleInitializeErrors(
+ completedFetch,
+ error.error(),
+ error.messageWithFallback(),
+ completedFetch.tablePath);
return null;
}
} finally {
@@ -205,7 +206,7 @@ abstract class AbstractLogFetchCollector<T, R> {
}
private void handleInitializeErrors(
- CompletedFetch completedFetch, Errors error, String errorMessage) {
+ CompletedFetch completedFetch, Errors error, String errorMessage, TablePath tablePath) {
TableBucket tb = completedFetch.tableBucket;
long fetchOffset = completedFetch.nextFetchOffset();
if (error == Errors.NOT_LEADER_OR_FOLLOWER
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogScanner.java
new file mode 100644
index 000000000..8e16b35ef
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogScanner.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.scanner.log;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.client.metrics.ScannerMetricGroup;
+import org.apache.fluss.exception.WakeupException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.ConcurrentModificationException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Base class that centralizes the shared poll loop, lightweight single-thread guard, wakeup and
+ * close lifecycle for log scanners. Concrete scanners supply the result type {@code R} and an
+ * enrichment step from raw {@link ScanRecords} to that type.
+ *
+ * <p>This class is NOT thread-safe. The lightweight guard only detects accidental concurrent use
+ * and throws {@link ConcurrentModificationException}; it does not serialize callers.
+ */
+@Internal
+abstract class AbstractLogScanner<R> implements AutoCloseable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractLogScanner.class);
+
+ private static final long NO_CURRENT_THREAD = -1L;
+
+ protected final String name;
+ protected final LogScannerStatus logScannerStatus;
+ protected final LogFetcher logFetcher;
+ protected final ScannerMetricGroup scannerMetricGroup;
+
+ private volatile boolean closed = false;
+
+ // currentThread holds the threadId of the current thread accessing the scanner
+ // and is used to prevent multithreaded access.
+ private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD);
+ // refCount is used to allow reentrant access by the thread that has acquired currentThread.
+ private final AtomicInteger refCount = new AtomicInteger(0);
+
+ protected AbstractLogScanner(
+ String name,
+ LogScannerStatus logScannerStatus,
+ LogFetcher logFetcher,
+ ScannerMetricGroup scannerMetricGroup) {
+ this.name = name;
+ this.logScannerStatus = logScannerStatus;
+ this.logFetcher = logFetcher;
+ this.scannerMetricGroup = scannerMetricGroup;
+ }
+
+ /**
+ * Template poll loop shared by all concrete scanners.
+ *
+ * <ul>
+ * <li>If {@link #pollForFetches()} returns a batch with scanner progress, {@link
+ * #toResult(ScanRecords)} is invoked to produce the concrete result type.
+ * <li>If the batch has no progress, we wait on {@link LogFetcher#awaitNotEmpty(long)} until
+ * either data arrives, the timeout elapses, or {@link #wakeup()} is called.
+ * <li>Any empty/timeout/wakeup path returns {@link #emptyResult()}.
+ * </ul>
+ */
+ public final R poll(Duration timeout) {
+ acquireAndEnsureOpen();
+ try {
+ if (!logScannerStatus.prepareToPoll()) {
+ throw new IllegalStateException(notSubscribedMessage());
+ }
+
+ scannerMetricGroup.recordPollStart(System.currentTimeMillis());
+ long timeoutNanos = timeout.toNanos();
+ long startNanos = System.nanoTime();
+ do {
+ ScanRecords scanRecords = pollForFetches();
+ if (!scanRecords.hasProgress()) {
+ try {
+ if (!logFetcher.awaitNotEmpty(startNanos + timeoutNanos)) {
+ // no data in buffer within the timeout
+ return emptyResult();
+ }
+ } catch (WakeupException e) {
+ // wakeup() was called, return empty
+ return emptyResult();
+ }
+ } else {
+ // before returning the fetched records, send off the next round of fetches
+ // to enable pipelining while the user is handling the current batch.
+ logFetcher.sendFetches();
+ return toResult(scanRecords);
+ }
+ } while (System.nanoTime() - startNanos < timeoutNanos);
+
+ return emptyResult();
+ } finally {
+ release();
+ scannerMetricGroup.recordPollEnd(System.currentTimeMillis());
+ }
+ }
+
+ private ScanRecords pollForFetches() {
+ ScanRecords scanRecords = logFetcher.collectFetch();
+ if (scanRecords.hasProgress()) {
+ return scanRecords;
+ }
+
+ // send any new fetches (won't resend pending fetches).
+ logFetcher.sendFetches();
+
+ return logFetcher.collectFetch();
+ }
+
+ /** The empty sentinel for the concrete result type. */
+ protected abstract R emptyResult();
+
+ /** Convert a non-empty batch of {@link ScanRecords} into the concrete result type. */
+ protected abstract R toResult(ScanRecords scanRecords);
+
+ /** Error message for {@link IllegalStateException} when no buckets are subscribed. */
+ protected abstract String notSubscribedMessage();
+
+ public final void wakeup() {
+ logFetcher.wakeup();
+ }
+
+ /**
+ * Acquire the light lock and ensure that the scanner hasn't been closed.
+ *
+ * @throws IllegalStateException if the scanner has been closed
+ */
+ protected final void acquireAndEnsureOpen() {
+ acquire();
+ if (closed) {
+ release();
+ throw new IllegalStateException("This scanner has already been closed.");
+ }
+ }
+
+ /**
+ * Acquire the light lock protecting this scanner from multithreaded access. Instead of blocking
+ * when the lock is not available, we throw an exception (multithreaded usage is not supported).
+ *
+ * @throws ConcurrentModificationException if another thread already has the lock
+ */
+ private void acquire() {
+ final Thread thread = Thread.currentThread();
+ final long threadId = thread.getId();
+ if (threadId != currentThread.get()
+ && !currentThread.compareAndSet(NO_CURRENT_THREAD, threadId)) {
+ throw new ConcurrentModificationException(
+ "Scanner is not safe for multithreaded access. "
+ + "currentThread(name: "
+ + thread.getName()
+ + ", id: "
+ + threadId
+ + ")"
+ + " otherThread(id: "
+ + currentThread.get()
+ + ")");
+ }
+ refCount.incrementAndGet();
+ }
+
+ /** Release the light lock protecting the scanner from multithreaded access. */
+ protected final void release() {
+ if (refCount.decrementAndGet() == 0) {
+ currentThread.set(NO_CURRENT_THREAD);
+ }
+ }
+
+ @Override
+ public final void close() {
+ acquire();
+ try {
+ if (!closed) {
+ LOG.trace("Closing log scanner: {}", name);
+ scannerMetricGroup.close();
+ logFetcher.close();
+ }
+ LOG.debug("Log scanner: {} has been closed", name);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to close log scanner: " + name, e);
+ } finally {
+ closed = true;
+ release();
+ }
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowLogFetchCollector.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowLogFetchCollector.java
index ef53b464e..cbe56b094 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowLogFetchCollector.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowLogFetchCollector.java
@@ -21,7 +21,6 @@ import org.apache.fluss.annotation.Internal;
import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.metadata.TableBucket;
-import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.record.ArrowBatchData;
import org.slf4j.Logger;
@@ -40,11 +39,10 @@ public class ArrowLogFetchCollector
private static final Logger LOG = LoggerFactory.getLogger(ArrowLogFetchCollector.class);
public ArrowLogFetchCollector(
- TablePath tablePath,
LogScannerStatus logScannerStatus,
Configuration conf,
MetadataUpdater metadataUpdater) {
- super(LOG, tablePath, logScannerStatus, conf, metadataUpdater);
+ super(LOG, logScannerStatus, conf, metadataUpdater);
}
@Override
@@ -70,14 +68,15 @@ public class ArrowLogFetchCollector
@Override
protected void closeFetchedRecords(Map<TableBucket, List<ArrowBatchData>> fetched) {
- for (List<ArrowBatchData> batches : fetched.values()) {
- for (ArrowBatchData batch : batches) {
+ for (Map.Entry<TableBucket, List<ArrowBatchData>> entry : fetched.entrySet()) {
+
+ for (ArrowBatchData batch : entry.getValue()) {
try {
batch.close();
} catch (Exception e) {
LOG.warn(
- "Failed to close Arrow batch during cleanup for table {}",
- tablePath,
+ "Failed to close Arrow batch during cleanup for bucket {}",
+ entry.getKey(),
e);
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java
index f22a65213..de9cb1d3f 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java
@@ -22,6 +22,7 @@ import org.apache.fluss.client.table.scanner.ScanRecord;
import org.apache.fluss.exception.CorruptRecordException;
import org.apache.fluss.exception.FetchException;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.record.ArrowBatchData;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.record.CompactedLogRecord;
@@ -56,6 +57,7 @@ public abstract class CompletedFetch {
static final long NO_FILTERED_END_OFFSET = -1L;
final TableBucket tableBucket;
+ final TablePath tablePath;
final ApiError error;
final int sizeInBytes;
final long highWatermark;
@@ -66,9 +68,9 @@ public abstract class CompletedFetch {
private final Iterator<LogRecordBatch> batches;
private final LogScannerStatus logScannerStatus;
protected final LogRecordReadContext readContext;
- protected final InternalRow.FieldGetter[] selectedFieldGetters;
private LogRecordBatch currentBatch;
+ private int currentBatchSchemaId = -1;
private LogRecord lastRecord;
private CloseableIterator<LogRecord> records;
private int recordsRead = 0;
@@ -80,6 +82,7 @@ public abstract class CompletedFetch {
public CompletedFetch(
TableBucket tableBucket,
+ TablePath tablePath,
ApiError error,
int sizeInBytes,
long highWatermark,
@@ -90,6 +93,7 @@ public abstract class CompletedFetch {
long fetchOffset,
long filteredEndOffset) {
this.tableBucket = tableBucket;
+ this.tablePath = tablePath;
this.error = error;
this.sizeInBytes = sizeInBytes;
this.highWatermark = highWatermark;
@@ -97,7 +101,6 @@ public abstract class CompletedFetch {
this.readContext = readContext;
this.isCheckCrcs = isCheckCrcs;
this.logScannerStatus = logScannerStatus;
- this.selectedFieldGetters = readContext.getSelectedFieldGetters();
this.fetchOffset = fetchOffset;
checkArgument(
filteredEndOffset == NO_FILTERED_END_OFFSET || filteredEndOffset >= fetchOffset,
@@ -114,12 +117,20 @@ public abstract class CompletedFetch {
// refactor #fetchRecords to return an iterator which lazily deserialize
// from underlying record stream and arrow buffer.
ScanRecord toScanRecord(LogRecord record) {
+ long tableId = tableBucket.getTableId();
+ int schemaId = currentBatchSchemaId;
+ InternalRow.FieldGetter[] selectedFieldGetters =
+ readContext.getSelectedFieldGetters(schemaId);
+
GenericRow newRow = new GenericRow(selectedFieldGetters.length);
InternalRow internalRow = record.getRow();
for (int i = 0; i < selectedFieldGetters.length; i++) {
newRow.setField(i, selectedFieldGetters[i].getFieldOrNull(internalRow));
}
+
return new ScanRecord(
+ tableId,
+ schemaId,
record.logOffset(),
record.timestamp(),
record.getChangeType(),
@@ -345,6 +356,7 @@ public abstract class CompletedFetch {
return null;
}
+ currentBatchSchemaId = batch.schemaId();
records = batch.records(readContext);
} else {
LogRecord record = records.next();
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedPendingFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedPendingFetch.java
index 2ca237875..c713cd076 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedPendingFetch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedPendingFetch.java
@@ -45,4 +45,9 @@ class CompletedPendingFetch implements PendingFetch {
public CompletedFetch toCompletedFetch() {
return completedFetch;
}
+
+ @Override
+ public void discard() {
+ completedFetch.drain();
+ }
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetch.java
index 595f18dd2..0384389c2 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetch.java
@@ -19,6 +19,7 @@ package org.apache.fluss.client.table.scanner.log;
import org.apache.fluss.annotation.Internal;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.rpc.entity.FetchLogResultForBucket;
import org.apache.fluss.rpc.messages.FetchLogRequest;
@@ -42,6 +43,7 @@ class DefaultCompletedFetch extends CompletedFetch {
public DefaultCompletedFetch(
TableBucket tableBucket,
+ TablePath tablePath,
FetchLogResultForBucket fetchLogResultForBucket,
LogRecordReadContext readContext,
LogScannerStatus logScannerStatus,
@@ -50,6 +52,7 @@ class DefaultCompletedFetch extends CompletedFetch {
@Nullable ByteBuf parsedByteBuf) {
super(
tableBucket,
+ tablePath,
fetchLogResultForBucket.getError(),
fetchLogResultForBucket.recordsOrEmpty().sizeInBytes(),
fetchLogResultForBucket.getHighWatermark(),
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchBuffer.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchBuffer.java
index 12928eda4..b6b6e6144 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchBuffer.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchBuffer.java
@@ -227,7 +227,16 @@ public class LogFetchBuffer implements AutoCloseable {
}
// remove entries that not matches the buckets from pendingFetches
- pendingFetches.entrySet().removeIf(entry -> !buckets.contains(entry.getKey()));
+ pendingFetches
+ .entrySet()
+ .removeIf(
+ entry -> {
+ if (!buckets.contains(entry.getKey())) {
+ entry.getValue().forEach(PendingFetch::discard);
+ return true;
+ }
+ return false;
+ });
});
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchCollector.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchCollector.java
index 799e74d2e..abdbede39 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchCollector.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchCollector.java
@@ -22,7 +22,6 @@ import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.client.table.scanner.ScanRecord;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.metadata.TableBucket;
-import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.record.LogRecord;
import org.apache.fluss.record.LogRecordBatch;
@@ -49,11 +48,10 @@ public class LogFetchCollector extends AbstractLogFetchCollector<ScanRecord, Sca
private static final Logger LOG = LoggerFactory.getLogger(LogFetchCollector.class);
public LogFetchCollector(
- TablePath tablePath,
LogScannerStatus logScannerStatus,
Configuration conf,
MetadataUpdater metadataUpdater) {
- super(LOG, tablePath, logScannerStatus, conf, metadataUpdater);
+ super(LOG, logScannerStatus, conf, metadataUpdater);
}
@Override
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
index a5b8ba483..a178ab206 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
@@ -75,7 +75,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.stream.Collectors;
import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.getFetchLogResultForBucket;
import static org.apache.fluss.utils.Preconditions.checkNotNull;
@@ -89,17 +88,11 @@ import static org.apache.fluss.utils.Preconditions.checkNotNull;
public class LogFetcher implements Closeable {
private static final Logger LOG = LoggerFactory.getLogger(LogFetcher.class);
- private final TablePath tablePath;
- private final boolean isPartitioned;
- private final LogRecordReadContext readContext;
- // TODO this context can be merge with readContext. Introduce it only because log remote read
- // currently can only do project when generate scanRecord instead of doing project while read
- // bytes from remote file.
- private final LogRecordReadContext remoteReadContext;
+ /** Per-table read contexts keyed by tableId. */
+ @GuardedBy("this")
+ private final Map<Long, TableReadContext> tableReadContexts;
+
private final ChunkedAllocationManager.ChunkedFactory chunkedFactory;
- @Nullable private final Projection projection;
- @Nullable private final org.apache.fluss.rpc.messages.PbPredicate cachedPbPredicate;
- private final int filterSchemaId;
private final int maxFetchBytes;
private final int maxBucketFetchBytes;
private final int minFetchBytes;
@@ -120,33 +113,19 @@ public class LogFetcher implements Closeable {
private final MetadataUpdater metadataUpdater;
private final ScannerMetricGroup scannerMetricGroup;
+ private final LogRecordReadContext.SchemaResolution schemaResolution;
+ /** Create a multi-table LogFetcher without an initial table. */
public LogFetcher(
- TableInfo tableInfo,
- @Nullable Projection projection,
- @Nullable Predicate recordBatchFilter,
+ String scannerName,
LogScannerStatus logScannerStatus,
Configuration conf,
MetadataUpdater metadataUpdater,
ScannerMetricGroup scannerMetricGroup,
RemoteFileDownloader remoteFileDownloader,
- SchemaGetter schemaGetter) {
- this.tablePath = tableInfo.getTablePath();
- this.isPartitioned = tableInfo.isPartitioned();
+ LogRecordReadContext.SchemaResolution schemaResolution) {
+ this.tableReadContexts = new HashMap<>();
this.chunkedFactory = new ChunkedAllocationManager.ChunkedFactory();
- this.readContext =
- LogRecordReadContext.createReadContext(
- tableInfo, false, projection, schemaGetter, chunkedFactory);
- this.remoteReadContext =
- LogRecordReadContext.createReadContext(
- tableInfo, true, projection, schemaGetter, chunkedFactory);
- this.projection = projection;
- this.cachedPbPredicate =
- recordBatchFilter != null
- ? PredicateMessageUtils.toPbPredicate(
- recordBatchFilter, tableInfo.getRowType())
- : null;
- this.filterSchemaId = tableInfo.getSchemaId();
this.logScannerStatus = logScannerStatus;
this.maxFetchBytes =
(int) conf.get(ConfigOptions.CLIENT_SCANNER_LOG_FETCH_MAX_BYTES).getBytes();
@@ -163,15 +142,49 @@ public class LogFetcher implements Closeable {
this.logFetchBuffer = new LogFetchBuffer();
this.nodesWithPendingFetchRequests = new HashSet<>();
this.metadataUpdater = metadataUpdater;
- this.logFetchCollector =
- new LogFetchCollector(tablePath, logScannerStatus, conf, metadataUpdater);
+ this.logFetchCollector = new LogFetchCollector(logScannerStatus, conf, metadataUpdater);
this.arrowLogFetchCollector =
- new ArrowLogFetchCollector(tablePath, logScannerStatus, conf, metadataUpdater);
+ new ArrowLogFetchCollector(logScannerStatus, conf, metadataUpdater);
this.scannerMetricGroup = scannerMetricGroup;
this.readPreference = conf.get(ConfigOptions.CLIENT_SCANNER_LOG_READ_PREFERENCE);
this.remoteLogDownloader =
- new RemoteLogDownloader(tablePath, conf, remoteFileDownloader, scannerMetricGroup);
+ new RemoteLogDownloader(
+ scannerName, conf, remoteFileDownloader, scannerMetricGroup);
remoteLogDownloader.start();
+ this.schemaResolution = schemaResolution;
+ }
+
+ /**
+ * Register a new table for multi-table fetching. If the table is already registered, this is a
+ * no-op.
+ *
+ * <p>The fetcher monitor protects registration against asynchronous fetch response callbacks.
+ */
+ public synchronized void registerTable(TableScanSpec spec, SchemaGetter schemaGetter) {
+ TableInfo tableInfo = spec.getTableInfo();
+ long tableId = tableInfo.getTableId();
+ if (!tableReadContexts.containsKey(tableId)) {
+ TableReadContext tableReadContext =
+ new TableReadContext(
+ tableInfo,
+ schemaResolution,
+ spec.getProjection(),
+ spec.getFilter(),
+ schemaGetter,
+ chunkedFactory);
+ tableReadContexts.put(tableId, tableReadContext);
+ }
+ }
+
+ synchronized void unregisterTable(long tableId) {
+ TableReadContext tableReadContext = tableReadContexts.remove(tableId);
+ if (tableReadContext == null) {
+ return;
+ }
+ Set<TableBucket> retainedBuckets =
+ new HashSet<>(logScannerStatus.fetchableBuckets(ignored -> true));
+ logFetchBuffer.retainAll(retainedBuckets);
+ tableReadContext.close();
}
/**
@@ -196,11 +209,12 @@ public class LogFetcher implements Closeable {
* have an in-flight fetch or pending fetch data.
*/
public void sendFetches() {
- checkAndUpdateMetadata(fetchableBuckets());
+ List<TableBucket> fetchable = fetchableBuckets();
+ checkAndUpdateMetadata(fetchable);
synchronized (this) {
// NOTE: Don't perform heavy I/O operations or synchronous waits inside this lock to
// avoid blocking the future complete of FetchLogResponse.
- Map<Integer, FetchLogRequest> fetchRequestMap = prepareFetchLogRequests();
+ Map<Integer, FetchLogRequest> fetchRequestMap = prepareFetchLogRequests(fetchable);
fetchRequestMap.forEach(
(nodeId, fetchLogRequest) -> {
LOG.debug("Adding pending request for node id {}", nodeId);
@@ -229,94 +243,118 @@ public class LogFetcher implements Closeable {
}
private void checkAndUpdateMetadata(List<TableBucket> tableBuckets) {
- // If the table is partitioned table, check if we need update partition metadata.
- List<Long> partitionIds = isPartitioned ? new ArrayList<>() : null;
- // If the table is none-partitioned table, check if we need update table metadata.
- boolean needUpdate = false;
+ // Group buckets (without a leader) by tableId to support multi-table fetching.
+ Map<Long, List<TableBucket>> bucketsByTable = new HashMap<>();
for (TableBucket tb : tableBuckets) {
if (getTableBucketLeader(tb) != null) {
continue;
}
-
- if (isPartitioned) {
- partitionIds.add(tb.getPartitionId());
- } else {
- needUpdate = true;
- break;
- }
+ bucketsByTable.computeIfAbsent(tb.getTableId(), k -> new ArrayList<>()).add(tb);
}
- try {
- if (isPartitioned && !partitionIds.isEmpty()) {
- metadataUpdater.updateMetadata(
- Collections.singleton(tablePath), null, partitionIds);
- } else if (needUpdate) {
- metadataUpdater.updateTableOrPartitionMetadata(tablePath, null);
+ for (Map.Entry<Long, List<TableBucket>> entry : bucketsByTable.entrySet()) {
+ long tableId = entry.getKey();
+ List<TableBucket> buckets = entry.getValue();
+ TableReadContext tableReadContext;
+ synchronized (this) {
+ tableReadContext = tableReadContexts.get(tableId);
}
- } catch (Exception e) {
- if (e instanceof PartitionNotExistException) {
- // ignore this exception, this is probably happen because the partition is deleted.
- // The fetcher can also work fine. The caller like flink can remove the partition
- // from fetch list when receive exception.
- LOG.warn("Receive PartitionNotExistException when update metadata, ignore it", e);
- } else {
- throw e;
+ checkNotNull(tableReadContext, "Table read context for table id " + tableId);
+ boolean isPartitioned = tableReadContext.isPartitioned;
+ TablePath tablePath = tableReadContext.tablePath;
+ try {
+ if (isPartitioned) {
+ List<Long> partitionIds = new ArrayList<>();
+ for (TableBucket tb : buckets) {
+ partitionIds.add(tb.getPartitionId());
+ }
+ if (!partitionIds.isEmpty()) {
+ metadataUpdater.updateMetadata(
+ Collections.singleton(tablePath), null, partitionIds);
+ }
+ } else {
+ metadataUpdater.updateTableOrPartitionMetadata(tablePath, null);
+ }
+ } catch (Exception e) {
+ if (e instanceof PartitionNotExistException) {
+ LOG.warn(
+ "Receive PartitionNotExistException when update metadata, ignore it",
+ e);
+ } else {
+ throw e;
+ }
}
}
}
@VisibleForTesting
- void sendFetchRequest(int destination, FetchLogRequest fetchLogRequest) {
- TableOrPartitions tableOrPartitionsInFetchRequest =
- getTableOrPartitionsInFetchRequest(fetchLogRequest);
+ synchronized void sendFetchRequest(int destination, FetchLogRequest fetchLogRequest) {
+ FetchRequestContext fetchRequestContext = createFetchRequestContext(fetchLogRequest);
// TODO cache the tablet server gateway.
TabletServerGateway gateway = metadataUpdater.newTabletServerClientForNode(destination);
if (gateway == null) {
handleFetchLogException(
destination,
- tableOrPartitionsInFetchRequest,
+ fetchRequestContext,
new LeaderNotAvailableException(
"Server " + destination + " is not found in metadata cache."));
} else {
final long requestStartTime = System.currentTimeMillis();
scannerMetricGroup.fetchRequestCount().inc();
- gateway.fetchLog(fetchLogRequest)
- .whenComplete(
- (fetchLogResponse, e) -> {
- if (e != null) {
- handleFetchLogException(
- destination, tableOrPartitionsInFetchRequest, e);
- } else {
- handleFetchLogResponse(
- destination, requestStartTime, fetchLogResponse);
- }
- });
+ try {
+ gateway.fetchLog(fetchLogRequest)
+ .whenComplete(
+ (fetchLogResponse, e) -> {
+ if (e != null) {
+ handleFetchLogException(
+ destination, fetchRequestContext, e);
+ } else {
+ handleFetchLogResponse(
+ destination,
+ requestStartTime,
+ fetchLogResponse,
+ fetchRequestContext);
+ }
+ });
+ } catch (Throwable t) {
+ handleFetchLogException(destination, fetchRequestContext, t);
+ }
}
}
- private TableOrPartitions getTableOrPartitionsInFetchRequest(FetchLogRequest fetchLogRequest) {
- Set<Long> tableIdsInFetchRequest = null;
- Set<TablePartition> tablePartitionsInFetchRequest = null;
- if (!isPartitioned) {
- tableIdsInFetchRequest =
- fetchLogRequest.getTablesReqsList().stream()
- .map(PbFetchLogReqForTable::getTableId)
- .collect(Collectors.toSet());
- } else {
- tablePartitionsInFetchRequest = new HashSet<>();
- // iterate over table requests
- for (PbFetchLogReqForTable fetchTableRequest : fetchLogRequest.getTablesReqsList()) {
+ private FetchRequestContext createFetchRequestContext(FetchLogRequest fetchLogRequest) {
+ Map<Long, TableReadContext> requestedTableContexts = new HashMap<>();
+ Set<Long> tableIdsInFetchRequest = new HashSet<>();
+ Set<TablePartition> tablePartitionsInFetchRequest = new HashSet<>();
+ for (PbFetchLogReqForTable fetchTableRequest : fetchLogRequest.getTablesReqsList()) {
+ long tableId = fetchTableRequest.getTableId();
+ TableReadContext tableReadContext = tableReadContexts.get(tableId);
+ if (tableReadContext == null) {
+ LOG.warn(
+ "Skip fetch log request for table id {} because its read context is not "
+ + "registered. The table may have already been unregistered.",
+ tableId);
+ continue;
+ }
+ requestedTableContexts.put(tableId, tableReadContext);
+ if (!tableReadContext.isPartitioned) {
+ tableIdsInFetchRequest.add(tableId);
+ } else {
for (PbFetchLogReqForBucket fetchLogReqForBucket :
fetchTableRequest.getBucketsReqsList()) {
tablePartitionsInFetchRequest.add(
- new TablePartition(
- fetchTableRequest.getTableId(),
- fetchLogReqForBucket.getPartitionId()));
+ new TablePartition(tableId, fetchLogReqForBucket.getPartitionId()));
}
}
}
- return new TableOrPartitions(tableIdsInFetchRequest, tablePartitionsInFetchRequest);
+ return new FetchRequestContext(
+ requestedTableContexts,
+ new TableOrPartitions(
+ tableIdsInFetchRequest.isEmpty() ? null : tableIdsInFetchRequest,
+ tablePartitionsInFetchRequest.isEmpty()
+ ? null
+ : tablePartitionsInFetchRequest));
}
/** A helper class to hold table ids or table partitions. */
@@ -340,8 +378,8 @@ public class LogFetcher implements Closeable {
metadataUpdater.invalidPhysicalTableBucketMeta(physicalTablePaths);
}
- private void handleFetchLogException(
- int destination, TableOrPartitions tableOrPartitionsInFetchRequest, Throwable e) {
+ private synchronized void handleFetchLogException(
+ int destination, FetchRequestContext fetchRequestContext, Throwable e) {
try {
if (isClosed) {
return;
@@ -355,7 +393,7 @@ public class LogFetcher implements Closeable {
"Invalid metadata error in fetch log request. "
+ "Going to request metadata update.",
e);
- invalidTableOrPartitions(tableOrPartitionsInFetchRequest);
+ invalidTableOrPartitions(fetchRequestContext.tableOrPartitions);
}
} finally {
LOG.debug("Removing pending request for node: {}", destination);
@@ -365,7 +403,10 @@ public class LogFetcher implements Closeable {
/** Implements the core logic for a successful fetch log response. */
private synchronized void handleFetchLogResponse(
- int destination, long requestStartTime, FetchLogResponse fetchLogResponse) {
+ int destination,
+ long requestStartTime,
+ FetchLogResponse fetchLogResponse,
+ FetchRequestContext fetchRequestContext) {
// Capture the parsed ByteBuf for buffer lifecycle management. The response may
// have been lazily parsed from the network buffer. Each DefaultCompletedFetch
// that references the buffer's records data must retain it. We release the base
@@ -382,6 +423,14 @@ public class LogFetcher implements Closeable {
for (PbFetchLogRespForTable respForTable : fetchLogResponse.getTablesRespsList()) {
long tableId = respForTable.getTableId();
+ TableReadContext trc = fetchRequestContext.requestedTableContexts.get(tableId);
+ if (trc == null || tableReadContexts.get(tableId) != trc) {
+ LOG.debug(
+ "Ignoring stale or unsolicited fetch log response for table id {}.",
+ tableId);
+ continue;
+ }
+ TablePath tablePathForResp = trc.tablePath;
for (PbFetchLogRespForBucket respForBucket : respForTable.getBucketsRespsList()) {
TableBucket tb =
new TableBucket(
@@ -391,7 +440,7 @@ public class LogFetcher implements Closeable {
: null,
respForBucket.getBucketId());
FetchLogResultForBucket fetchResultForBucket =
- getFetchLogResultForBucket(tb, tablePath, respForBucket);
+ getFetchLogResultForBucket(tb, tablePathForResp, respForBucket);
// if error code is not NONE, it means the fetch log request failed, we need to
// clear table bucket meta for InvalidMetadataException.
@@ -411,6 +460,7 @@ public class LogFetcher implements Closeable {
} else {
if (fetchResultForBucket.fetchFromRemote()) {
pendRemoteFetches(
+ trc,
fetchResultForBucket.remoteLogFetchInfo(),
fetchOffset,
fetchResultForBucket.getHighWatermark());
@@ -428,8 +478,9 @@ public class LogFetcher implements Closeable {
logFetchBuffer.add(
new DefaultCompletedFetch(
tb,
+ tablePathForResp,
fetchResultForBucket,
- readContext,
+ trc.readContext,
logScannerStatus,
// skipping CRC check if projection push downed as
// the data is pruned
@@ -477,7 +528,10 @@ public class LogFetcher implements Closeable {
}
private void pendRemoteFetches(
- RemoteLogFetchInfo remoteLogFetchInfo, long firstFetchOffset, long highWatermark) {
+ TableReadContext tableReadContext,
+ RemoteLogFetchInfo remoteLogFetchInfo,
+ long firstFetchOffset,
+ long highWatermark) {
checkNotNull(remoteLogFetchInfo);
FsPath remoteLogTabletDir = new FsPath(remoteLogFetchInfo.remoteLogTabletDir());
List<RemoteLogSegment> remoteLogSegments = remoteLogFetchInfo.remoteLogSegmentList();
@@ -495,10 +549,11 @@ public class LogFetcher implements Closeable {
new RemotePendingFetch(
segment,
downloadFuture,
+ tableReadContext.tablePath,
posInLogSegment,
fetchOffset,
highWatermark,
- remoteReadContext,
+ tableReadContext.remoteReadContext,
logScannerStatus,
isCheckCrcs);
logFetchBuffer.pend(pendingFetch);
@@ -507,14 +562,12 @@ public class LogFetcher implements Closeable {
}
@VisibleForTesting
- Map<Integer, FetchLogRequest> prepareFetchLogRequests() {
- Map<Integer, List<PbFetchLogReqForBucket>> fetchLogReqForBuckets = new HashMap<>();
+ Map<Integer, FetchLogRequest> prepareFetchLogRequests(List<TableBucket> fetchableBuckets) {
+ // Outer key: leaderId, inner key: tableId -> list of bucket requests
+ Map<Integer, Map<Long, List<PbFetchLogReqForBucket>>> fetchReqsByLeaderAndTable =
+ new HashMap<>();
int readyForFetchCount = 0;
- Long tableId = null;
- for (TableBucket tb : fetchableBuckets()) {
- if (tableId == null) {
- tableId = tb.getTableId();
- }
+ for (TableBucket tb : fetchableBuckets) {
Long offset = logScannerStatus.getBucketOffset(tb);
if (offset == null) {
LOG.debug(
@@ -546,8 +599,9 @@ public class LogFetcher implements Closeable {
if (tb.getPartitionId() != null) {
fetchLogReqForBucket.setPartitionId(tb.getPartitionId());
}
- fetchLogReqForBuckets
- .computeIfAbsent(leader, key -> new ArrayList<>())
+ fetchReqsByLeaderAndTable
+ .computeIfAbsent(leader, k -> new HashMap<>())
+ .computeIfAbsent(tb.getTableId(), k -> new ArrayList<>())
.add(fetchLogReqForBucket);
readyForFetchCount++;
}
@@ -557,34 +611,57 @@ public class LogFetcher implements Closeable {
return Collections.emptyMap();
} else {
Map<Integer, FetchLogRequest> fetchLogRequests = new HashMap<>();
- long finalTableId = tableId;
- fetchLogReqForBuckets.forEach(
- (leaderId, reqForBuckets) -> {
- FetchLogRequest fetchLogRequest =
- new FetchLogRequest()
- .setFollowerServerId(-1)
- .setMaxBytes(maxFetchBytes)
- .setMinBytes(minFetchBytes)
- .setMaxWaitMs(maxFetchWaitMs)
- .setReadPreference(readPreference.value());
- PbFetchLogReqForTable reqForTable =
- new PbFetchLogReqForTable().setTableId(finalTableId);
- if (readContext.isProjectionPushDowned()) {
- assert projection != null;
- reqForTable
- .setProjectionPushdownEnabled(true)
- .setProjectedFields(projection.getProjectionInOrder());
- } else {
- reqForTable.setProjectionPushdownEnabled(false);
- }
- if (cachedPbPredicate != null) {
- reqForTable.setFilterPredicate(cachedPbPredicate);
- reqForTable.setFilterSchemaId(filterSchemaId);
- }
- reqForTable.addAllBucketsReqs(reqForBuckets);
- fetchLogRequest.addAllTablesReqs(Collections.singletonList(reqForTable));
- fetchLogRequests.put(leaderId, fetchLogRequest);
- });
+ for (Map.Entry<Integer, Map<Long, List<PbFetchLogReqForBucket>>> leaderEntry :
+ fetchReqsByLeaderAndTable.entrySet()) {
+ int leaderId = leaderEntry.getKey();
+ Map<Long, List<PbFetchLogReqForBucket>> tableReqs = leaderEntry.getValue();
+
+ FetchLogRequest fetchLogRequest =
+ new FetchLogRequest()
+ .setFollowerServerId(-1)
+ .setMaxBytes(maxFetchBytes)
+ .setMinBytes(minFetchBytes)
+ .setMaxWaitMs(maxFetchWaitMs)
+ .setReadPreference(readPreference.value());
+
+ List<PbFetchLogReqForTable> tablesReqs = new ArrayList<>();
+ for (Map.Entry<Long, List<PbFetchLogReqForBucket>> tableEntry :
+ tableReqs.entrySet()) {
+ long tableId = tableEntry.getKey();
+ List<PbFetchLogReqForBucket> bucketReqs = tableEntry.getValue();
+ TableReadContext trc = tableReadContexts.get(tableId);
+ if (trc == null) {
+ LOG.warn(
+ "Skip fetch log request for table id {} because its read context "
+ + "is not registered. The table may have already been "
+ + "unregistered.",
+ tableId);
+ continue;
+ }
+
+ PbFetchLogReqForTable reqForTable =
+ new PbFetchLogReqForTable().setTableId(tableId);
+ if (trc.readContext.isProjectionPushDowned()) {
+ assert trc.projection != null;
+ reqForTable
+ .setProjectionPushdownEnabled(true)
+ .setProjectedFields(trc.projection.getProjectionInOrder());
+ } else {
+ reqForTable.setProjectionPushdownEnabled(false);
+ }
+ if (trc.cachedPbPredicate != null) {
+ reqForTable.setFilterPredicate(trc.cachedPbPredicate);
+ reqForTable.setFilterSchemaId(trc.filterSchemaId);
+ }
+ reqForTable.addAllBucketsReqs(bucketReqs);
+ tablesReqs.add(reqForTable);
+ }
+ if (tablesReqs.isEmpty()) {
+ continue;
+ }
+ fetchLogRequest.addAllTablesReqs(tablesReqs);
+ fetchLogRequests.put(leaderId, fetchLogRequest);
+ }
return fetchLogRequests;
}
}
@@ -615,13 +692,14 @@ public class LogFetcher implements Closeable {
@Override
public synchronized void close() throws IOException {
if (!isClosed) {
+ isClosed = true;
IOUtils.closeQuietly(logFetchBuffer, "logFetchBuffer");
IOUtils.closeQuietly(remoteLogDownloader, "remoteLogDownloader");
- readContext.close();
- remoteReadContext.close();
+ for (TableReadContext tableReadContext : tableReadContexts.values()) {
+ tableReadContext.close();
+ }
chunkedFactory.close();
- isClosed = true;
- LOG.info("Fetcher for {} is closed.", tablePath);
+ LOG.info("LogFetcher is closed.");
}
}
@@ -634,4 +712,75 @@ public class LogFetcher implements Closeable {
int getCompletedFetchesSize() {
return logFetchBuffer.bufferedBuckets().size();
}
+
+ @VisibleForTesting
+ synchronized int getRegisteredTableCount() {
+ return tableReadContexts.size();
+ }
+
+ /** Request-scoped context snapshot. */
+ private static class FetchRequestContext {
+ private final Map<Long, TableReadContext> requestedTableContexts;
+ private final TableOrPartitions tableOrPartitions;
+
+ private FetchRequestContext(
+ Map<Long, TableReadContext> requestedTableContexts,
+ TableOrPartitions tableOrPartitions) {
+ this.requestedTableContexts = requestedTableContexts;
+ this.tableOrPartitions = tableOrPartitions;
+ }
+ }
+
+ /** Per-table context holding read contexts, projection, and predicate info. */
+ @VisibleForTesting
+ static class TableReadContext {
+ final TablePath tablePath;
+ final boolean isPartitioned;
+ final LogRecordReadContext readContext;
+ final LogRecordReadContext remoteReadContext;
+ @Nullable final Projection projection;
+ @Nullable final org.apache.fluss.rpc.messages.PbPredicate cachedPbPredicate;
+ final int filterSchemaId;
+
+ TableReadContext(
+ TableInfo tableInfo,
+ LogRecordReadContext.SchemaResolution schemaResolution,
+ @Nullable Projection projection,
+ @Nullable Predicate recordBatchFilter,
+ SchemaGetter schemaGetter,
+ ChunkedAllocationManager.ChunkedFactory chunkedFactory) {
+ this.tablePath = tableInfo.getTablePath();
+ this.isPartitioned = tableInfo.isPartitioned();
+ // Share the LogFetcher-owned chunked factory so all tables reuse one chunk
+ // pool and LogFetcher.close() releases the chunks after contexts are closed.
+ this.readContext =
+ LogRecordReadContext.createReadContext(
+ tableInfo,
+ false,
+ schemaResolution,
+ projection,
+ schemaGetter,
+ chunkedFactory);
+ this.remoteReadContext =
+ LogRecordReadContext.createReadContext(
+ tableInfo,
+ true,
+ schemaResolution,
+ projection,
+ schemaGetter,
+ chunkedFactory);
+ this.projection = projection;
+ this.cachedPbPredicate =
+ recordBatchFilter != null
+ ? PredicateMessageUtils.toPbPredicate(
+ recordBatchFilter, tableInfo.getRowType())
+ : null;
+ this.filterSchemaId = tableInfo.getSchemaId();
+ }
+
+ void close() {
+ readContext.close();
+ remoteReadContext.close();
+ }
+ }
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java
index 9e9316069..4637f87f6 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java
@@ -31,22 +31,16 @@ import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.predicate.Predicate;
+import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.rpc.metrics.ClientMetricGroup;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.Projection;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import javax.annotation.Nullable;
-import java.io.IOException;
import java.time.Duration;
import java.util.Collections;
import java.util.ConcurrentModificationException;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Supplier;
/**
* The default impl of {@link LogScanner}.
@@ -58,29 +52,20 @@ import java.util.function.Supplier;
* @since 0.1
*/
@PublicEvolving
-public class LogScannerImpl implements LogScanner {
- private static final Logger LOG = LoggerFactory.getLogger(LogScannerImpl.class);
- private static final long NO_CURRENT_THREAD = -1L;
+public class LogScannerImpl extends AbstractLogScanner<ScanRecords> implements LogScanner {
private final TablePath tablePath;
- private final LogScannerStatus logScannerStatus;
private final MetadataUpdater metadataUpdater;
- private final LogFetcher logFetcher;
private final long tableId;
private final boolean isPartitionedTable;
private final boolean isArrowLogFormat;
private final boolean isLogTable;
private final boolean hasProjection;
- // metrics
- private final ScannerMetricGroup scannerMetricGroup;
-
- // currentThread holds the threadId of the current thread accessing FlussLogScanner
- // and is used to prevent multithreaded access
- private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD);
- // refCount is used to allow reentrant access by the thread who has acquired currentThread.
- private final AtomicInteger refCount = new AtomicInteger(0);
- private volatile boolean closed = false;
+ private final TableInfo tableInfo;
+ private final @Nullable Projection projection;
+ private final @Nullable Predicate recordBatchFilter;
+ private final SchemaGetter schemaGetter;
public LogScannerImpl(
Configuration conf,
@@ -91,6 +76,40 @@ public class LogScannerImpl implements LogScanner {
@Nullable int[] projectedFields,
SchemaGetter schemaGetter,
@Nullable Predicate recordBatchFilter) {
+ this(
+ conf,
+ tableInfo,
+ metadataUpdater,
+ new LogScannerStatus(),
+ new ScannerMetricGroup(clientMetricGroup, tableInfo.getTablePath()),
+ remoteFileDownloader,
+ projectedFields,
+ schemaGetter,
+ recordBatchFilter);
+ }
+
+ private LogScannerImpl(
+ Configuration conf,
+ TableInfo tableInfo,
+ MetadataUpdater metadataUpdater,
+ LogScannerStatus logScannerStatus,
+ ScannerMetricGroup scannerMetricGroup,
+ RemoteFileDownloader remoteFileDownloader,
+ @Nullable int[] projectedFields,
+ SchemaGetter schemaGetter,
+ @Nullable Predicate recordBatchFilter) {
+ super(
+ tableInfo.getTablePath().toString(),
+ logScannerStatus,
+ new LogFetcher(
+ tableInfo.getTablePath().toString(),
+ logScannerStatus,
+ conf,
+ metadataUpdater,
+ scannerMetricGroup,
+ remoteFileDownloader,
+ LogRecordReadContext.SchemaResolution.TARGET),
+ scannerMetricGroup);
this.tablePath = tableInfo.getTablePath();
this.tableId = tableInfo.getTableId();
this.isPartitionedTable = tableInfo.isPartitioned();
@@ -99,21 +118,11 @@ public class LogScannerImpl implements LogScanner {
this.hasProjection = projectedFields != null;
// add this table to metadata updater.
metadataUpdater.checkAndUpdateTableMetadata(Collections.singleton(tablePath));
- this.logScannerStatus = new LogScannerStatus();
this.metadataUpdater = metadataUpdater;
- Projection projection = sanityProjection(projectedFields, tableInfo);
- this.scannerMetricGroup = new ScannerMetricGroup(clientMetricGroup, tablePath);
- this.logFetcher =
- new LogFetcher(
- tableInfo,
- projection,
- recordBatchFilter,
- logScannerStatus,
- conf,
- metadataUpdater,
- scannerMetricGroup,
- remoteFileDownloader,
- schemaGetter);
+ this.projection = sanityProjection(projectedFields, tableInfo);
+ this.tableInfo = tableInfo;
+ this.recordBatchFilter = recordBatchFilter;
+ this.schemaGetter = schemaGetter;
}
/**
@@ -121,7 +130,8 @@ public class LogScannerImpl implements LogScanner {
* not null.
*/
@Nullable
- private Projection sanityProjection(@Nullable int[] projectedFields, TableInfo tableInfo) {
+ private static Projection sanityProjection(
+ @Nullable int[] projectedFields, TableInfo tableInfo) {
RowType tableRowType = tableInfo.getRowType();
if (projectedFields != null) {
for (int projectedField : projectedFields) {
@@ -140,12 +150,18 @@ public class LogScannerImpl implements LogScanner {
}
@Override
- public ScanRecords poll(Duration timeout) {
- return doPoll(
- timeout,
- this::pollForFetches,
- scanRecords -> scanRecords.buckets().isEmpty(),
- () -> ScanRecords.EMPTY);
+ protected ScanRecords emptyResult() {
+ return ScanRecords.EMPTY;
+ }
+
+ @Override
+ protected ScanRecords toResult(ScanRecords scanRecords) {
+ return scanRecords;
+ }
+
+ @Override
+ protected String notSubscribedMessage() {
+ return "LogScanner is not subscribed any buckets.";
}
/**
@@ -168,11 +184,47 @@ public class LogScannerImpl implements LogScanner {
"Arrow record batch polling does not support projection. Please create the scanner without projection.");
}
- return doPoll(
- timeout,
- this::pollForRecordBatches,
- ArrowScanRecords::isEmpty,
- () -> ArrowScanRecords.EMPTY);
+ acquireAndEnsureOpen();
+ try {
+ if (!logScannerStatus.prepareToPoll()) {
+ throw new IllegalStateException("LogScanner is not subscribed any buckets.");
+ }
+
+ scannerMetricGroup.recordPollStart(System.currentTimeMillis());
+ long timeoutNanos = timeout.toNanos();
+ long startNanos = System.nanoTime();
+ do {
+ ArrowScanRecords scanRecords = pollForRecordBatches();
+ if (scanRecords.isEmpty()) {
+ try {
+ if (!logFetcher.awaitNotEmpty(startNanos + timeoutNanos)) {
+ return scanRecords;
+ }
+ } catch (WakeupException e) {
+ return scanRecords;
+ }
+ } else {
+ logFetcher.sendFetches();
+ return scanRecords;
+ }
+ } while (System.nanoTime() - startNanos < timeoutNanos);
+
+ return ArrowScanRecords.EMPTY;
+ } finally {
+ release();
+ scannerMetricGroup.recordPollEnd(System.currentTimeMillis());
+ }
+ }
+
+ private ArrowScanRecords pollForRecordBatches() {
+ ArrowScanRecords scanRecords = logFetcher.collectArrowFetch();
+ if (!scanRecords.isEmpty()) {
+ return scanRecords;
+ }
+
+ // send any new fetches (won't resend pending fetches).
+ logFetcher.sendFetches();
+ return logFetcher.collectArrowFetch();
}
@Override
@@ -186,8 +238,10 @@ public class LogScannerImpl implements LogScanner {
acquireAndEnsureOpen();
try {
TableBucket tableBucket = new TableBucket(tableId, bucket);
- metadataUpdater.checkAndUpdateTableMetadata(Collections.singleton(tablePath));
- logScannerStatus.assignScanBuckets(Collections.singletonMap(tableBucket, offset));
+ this.logFetcher.registerTable(
+ new TableScanSpec(tableInfo, projection, recordBatchFilter), schemaGetter);
+ this.metadataUpdater.checkAndUpdateTableMetadata(Collections.singleton(tablePath));
+ this.logScannerStatus.assignScanBuckets(Collections.singletonMap(tableBucket, offset));
} finally {
release();
}
@@ -204,6 +258,8 @@ public class LogScannerImpl implements LogScanner {
acquireAndEnsureOpen();
try {
TableBucket tableBucket = new TableBucket(tableId, partitionId, bucket);
+ this.logFetcher.registerTable(
+ new TableScanSpec(tableInfo, projection, recordBatchFilter), schemaGetter);
// we make assumption that the partition id must belong to the current table
// if we can't find the partition id from the table path, we'll consider the table
// is not exist
@@ -246,140 +302,4 @@ public class LogScannerImpl implements LogScanner {
release();
}
}
-
- @Override
- public void wakeup() {
- logFetcher.wakeup();
- }
-
- private ScanRecords pollForFetches() {
- ScanRecords scanRecords = logFetcher.collectFetch();
- // Check buckets() (includes progress-only buckets).
- if (!scanRecords.buckets().isEmpty()) {
- return scanRecords;
- }
-
- // send any new fetches (won't resend pending fetches).
- logFetcher.sendFetches();
- return logFetcher.collectFetch();
- }
-
- private ArrowScanRecords pollForRecordBatches() {
- ArrowScanRecords scanRecords = logFetcher.collectArrowFetch();
- if (!scanRecords.isEmpty()) {
- return scanRecords;
- }
-
- // send any new fetches (won't resend pending fetches).
- logFetcher.sendFetches();
- return logFetcher.collectArrowFetch();
- }
-
- /** Shared polling loop for row and Arrow scan results. */
- private <T> T doPoll(
- Duration timeout,
- Supplier<T> pollForFetches,
- java.util.function.Predicate<T> isEmpty,
- Supplier<T> emptyResult) {
- acquireAndEnsureOpen();
- try {
- if (!logScannerStatus.prepareToPoll()) {
- throw new IllegalStateException("LogScanner is not subscribed any buckets.");
- }
-
- scannerMetricGroup.recordPollStart(System.currentTimeMillis());
- long timeoutNanos = timeout.toNanos();
- long startNanos = System.nanoTime();
- do {
- T scanRecords = pollForFetches.get();
- if (isEmpty.test(scanRecords)) {
- try {
- if (!logFetcher.awaitNotEmpty(startNanos + timeoutNanos)) {
- // logFetcher waits for the timeout and no data in buffer,
- // so we return empty
- return scanRecords;
- }
- } catch (WakeupException e) {
- // wakeup() is called, we need to return empty
- return scanRecords;
- }
- } else {
- // before returning the fetched records, we can send off the next round of
- // fetches and avoid block waiting for their responses to enable pipelining
- // while the user is handling the fetched records.
- logFetcher.sendFetches();
- return scanRecords;
- }
- } while (System.nanoTime() - startNanos < timeoutNanos);
-
- return emptyResult.get();
- } finally {
- release();
- scannerMetricGroup.recordPollEnd(System.currentTimeMillis());
- }
- }
-
- /**
- * Acquire the light lock and ensure that the scanner hasn't been closed.
- *
- * @throws IllegalStateException If the scanner has been closed
- */
- private void acquireAndEnsureOpen() {
- acquire();
- if (closed) {
- release();
- throw new IllegalStateException("This scanner has already been closed.");
- }
- }
-
- /**
- * Acquire the light lock protecting this scanner from multithreaded access. Instead of blocking
- * when the lock is not available, however, we just throw an exception (since multithreaded
- * usage is not supported).
- *
- * @throws ConcurrentModificationException if another thread already has the lock
- */
- private void acquire() {
- Thread thread = Thread.currentThread();
- long threadId = thread.getId();
- if (threadId != currentThread.get()
- && !currentThread.compareAndSet(NO_CURRENT_THREAD, threadId)) {
- throw new ConcurrentModificationException(
- "Scanner is not safe for multithreaded access. "
- + "currentThread(name: "
- + thread.getName()
- + ", id: "
- + threadId
- + ")"
- + " otherThread(id: "
- + currentThread.get()
- + ")");
- }
- refCount.incrementAndGet();
- }
-
- /** Release the light lock protecting the scanner from multithreaded access. */
- private void release() {
- if (refCount.decrementAndGet() == 0) {
- currentThread.set(NO_CURRENT_THREAD);
- }
- }
-
- @Override
- public void close() {
- acquire();
- try {
- if (!closed) {
- LOG.trace("Closing log scanner for table: {}", tablePath);
- scannerMetricGroup.close();
- logFetcher.close();
- }
- LOG.debug("Log scanner for table: {} has been closed", tablePath);
- } catch (IOException e) {
- throw new RuntimeException("Failed to close log scanner for table " + tablePath, e);
- } finally {
- closed = true;
- release();
- }
- }
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/MultiTableLogScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/MultiTableLogScanner.java
new file mode 100644
index 000000000..9055fd750
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/MultiTableLogScanner.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.scanner.log;
+
+import org.apache.fluss.annotation.PublicEvolving;
+import org.apache.fluss.metadata.TablePath;
+
+import java.time.Duration;
+
+/**
+ * Reads log data from multiple tables through a single client. All subscribed buckets (across
+ * different tables) are polled fairly and returned in one {@link MultiTableRecords} result.
+ *
+ * <p>Unlike {@link LogScanner} which is bound to a single table, {@code MultiTableLogScanner}
+ * allows subscribing to buckets across different tables. Per-table projection and filter pushdown
+ * are configured on the {@link org.apache.fluss.client.table.scanner.MultiTableScan} builder, not
+ * on {@code subscribe}.
+ *
+ * <p>{@code MultiTableLogScanner} is NOT thread-safe (consistent with {@link LogScanner}). It is
+ * the responsibility of the user to ensure that multithreaded access is properly synchronized.
+ *
+ * @since 0.7
+ */
+@PublicEvolving
+public interface MultiTableLogScanner extends AutoCloseable {
+
+ /**
+ * The earliest offset to fetch from. Fluss uses "-2" to indicate fetching from log start
+ * offset.
+ */
+ long EARLIEST_OFFSET = LogScanner.EARLIEST_OFFSET;
+
+ /**
+ * The latest offset to fetch to. Fluss uses this to indicate the default stopping offset for
+ * unbounded Fluss sources.
+ */
+ long NO_STOPPING_OFFSET = LogScanner.NO_STOPPING_OFFSET;
+
+ /**
+ * Poll log data from all subscribed table buckets.
+ *
+ * <p>On each poll, the scanner will try to use the last scanned offset as the starting offset
+ * and fetch sequentially for each subscribed bucket. The result may contain records from
+ * multiple tables.
+ *
+ * @param timeout the timeout to poll.
+ * @return the result of poll, organized by {@link TablePath} and {@link
+ * org.apache.fluss.metadata.TableBucket}.
+ * @throws IllegalStateException if the scanner is not subscribed to any buckets.
+ */
+ MultiTableRecords poll(Duration timeout);
+
+ // ---- non-partitioned tables ----
+
+ /** Subscribe to the given non-partitioned table bucket starting at {@code offset}. */
+ void subscribe(TablePath tablePath, int bucket, long offset);
+
+ /** Subscribe to the given non-partitioned table bucket starting from the earliest offset. */
+ default void subscribeFromBeginning(TablePath tablePath, int bucket) {
+ subscribe(tablePath, bucket, EARLIEST_OFFSET);
+ }
+
+ /** Unsubscribe from the given non-partitioned table bucket. */
+ void unsubscribe(TablePath tablePath, int bucket);
+
+ // ---- partitioned tables ----
+
+ /** Subscribe to the given partitioned table bucket starting at {@code offset}. */
+ void subscribe(TablePath tablePath, long partitionId, int bucket, long offset);
+
+ /** Subscribe to the given partitioned table bucket starting from the earliest offset. */
+ default void subscribeFromBeginning(TablePath tablePath, long partitionId, int bucket) {
+ subscribe(tablePath, partitionId, bucket, EARLIEST_OFFSET);
+ }
+
+ /** Unsubscribe from the given partitioned table bucket. */
+ void unsubscribe(TablePath tablePath, long partitionId, int bucket);
+
+ /**
+ * Wake up the log scanner in case the fetcher thread in the log scanner is blocking in {@link
+ * #poll(Duration)}.
+ */
+ void wakeup();
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/MultiTableLogScannerImpl.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/MultiTableLogScannerImpl.java
new file mode 100644
index 000000000..170be2947
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/MultiTableLogScannerImpl.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.scanner.log;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.annotation.VisibleForTesting;
+import org.apache.fluss.client.admin.Admin;
+import org.apache.fluss.client.metadata.ClientSchemaGetter;
+import org.apache.fluss.client.metadata.MetadataUpdater;
+import org.apache.fluss.client.metrics.ScannerMetricGroup;
+import org.apache.fluss.client.table.scanner.MultiTableRecord;
+import org.apache.fluss.client.table.scanner.RemoteFileDownloader;
+import org.apache.fluss.client.table.scanner.ScanRecord;
+import org.apache.fluss.config.Configuration;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TableInfo;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.record.LogRecordReadContext;
+import org.apache.fluss.rpc.metrics.ClientMetricGroup;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.ConcurrentModificationException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Default implementation of {@link MultiTableLogScanner}.
+ *
+ * <p>This scanner subscribes to log data from multiple tables simultaneously. Tables are registered
+ * dynamically on the first {@code subscribe(...)} call to a given {@link TablePath}: the scanner
+ * resolves the {@link TableInfo} via the admin client and registers the table on the underlying
+ * {@link LogFetcher} with no projection / filter (phase 1 surface).
+ *
+ * <p>Internally, it delegates to a single shared {@link LogFetcher} that manages per-table read
+ * contexts.
+ *
+ * <p>{@code MultiTableLogScannerImpl} is NOT thread-safe. It is the responsibility of the user to
+ * ensure that multithreaded access is properly synchronized. Un-synchronized access will result in
+ * {@link ConcurrentModificationException}.
+ *
+ * @since 0.7
+ */
+@Internal
+public class MultiTableLogScannerImpl extends AbstractLogScanner<MultiTableRecords>
+ implements MultiTableLogScanner {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MultiTableLogScannerImpl.class);
+
+ private final MetadataUpdater metadataUpdater;
+ private final Admin admin;
+
+ /** Per-table {@link TableInfo}, populated lazily on first subscribe to the table. */
+ private final Map<TablePath, TableInfo> registeredByPath = new HashMap<>();
+
+ /** tableId -> tablePath, for poll-time record enrichment. */
+ private final Map<Long, TablePath> pathByTableId = new HashMap<>();
+
+ private final Map<Long, SchemaGetter> schemaGetters = new HashMap<>();
+
+ /**
+ * Currently subscribed buckets per table. When the last bucket of a table is unsubscribed, the
+ * table registration is evicted so that a subsequent {@code subscribe(...)} re-resolves a fresh
+ * {@link TableInfo}. This is what lets callers recover from a drop+recreate (tableId change) by
+ * unsubscribing all buckets of the stale table and subscribing again.
+ */
+ private final Map<TablePath, Set<TableBucket>> subscribedBucketsByPath = new HashMap<>();
+
+ public MultiTableLogScannerImpl(
+ String scannerName,
+ MetadataUpdater metadataUpdater,
+ Configuration conf,
+ ClientMetricGroup clientMetricGroup,
+ RemoteFileDownloader remoteFileDownloader,
+ Admin admin) {
+ this(
+ scannerName,
+ metadataUpdater,
+ conf,
+ remoteFileDownloader,
+ admin,
+ new LogScannerStatus(),
+ // Use a synthetic TablePath for scanner-level metrics scoping.
+ new ScannerMetricGroup(
+ clientMetricGroup, TablePath.of("multi-table", scannerName)));
+ }
+
+ private MultiTableLogScannerImpl(
+ String scannerName,
+ MetadataUpdater metadataUpdater,
+ Configuration conf,
+ RemoteFileDownloader remoteFileDownloader,
+ Admin admin,
+ LogScannerStatus logScannerStatus,
+ ScannerMetricGroup scannerMetricGroup) {
+ this(
+ scannerName,
+ metadataUpdater,
+ admin,
+ logScannerStatus,
+ new LogFetcher(
+ scannerName,
+ logScannerStatus,
+ conf,
+ metadataUpdater,
+ scannerMetricGroup,
+ remoteFileDownloader,
+ LogRecordReadContext.SchemaResolution.DYNAMIC),
+ scannerMetricGroup);
+ }
+
+ @VisibleForTesting
+ MultiTableLogScannerImpl(
+ String scannerName,
+ MetadataUpdater metadataUpdater,
+ Admin admin,
+ LogScannerStatus logScannerStatus,
+ LogFetcher logFetcher,
+ ScannerMetricGroup scannerMetricGroup) {
+ super(scannerName, logScannerStatus, logFetcher, scannerMetricGroup);
+ this.metadataUpdater = metadataUpdater;
+ this.admin = admin;
+ }
+
+ @Override
+ protected MultiTableRecords emptyResult() {
+ return MultiTableRecords.EMPTY;
+ }
+
+ @Override
+ protected MultiTableRecords toResult(ScanRecords scanRecords) {
+ return enrich(scanRecords);
+ }
+
+ @Override
+ protected String notSubscribedMessage() {
+ return "MultiTableLogScanner is not subscribed to any buckets.";
+ }
+
+ @Override
+ public void subscribe(TablePath tablePath, int bucket, long offset) {
+ acquireAndEnsureOpen();
+ try {
+ TableInfo tableInfo = getTableInfo(tablePath);
+ if (tableInfo.isPartitioned()) {
+ throw new IllegalStateException(
+ "Table "
+ + tablePath
+ + " is a partitioned table, please use "
+ + "\"subscribe(TablePath, long partitionId, int bucket, long offset)\" "
+ + "to subscribe a partitioned bucket instead.");
+ }
+ // Validate metadata before mutating any scanner/fetcher state, so a failure
+ // here doesn't leave a stale registration without tracked buckets.
+ metadataUpdater.checkAndUpdateTableMetadata(Collections.singleton(tablePath));
+ registerIfAbsent(tablePath, tableInfo);
+ TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), bucket);
+ logScannerStatus.assignScanBuckets(Collections.singletonMap(tableBucket, offset));
+ trackSubscription(tablePath, tableBucket);
+ } finally {
+ release();
+ }
+ }
+
+ @Override
+ public void unsubscribe(TablePath tablePath, int bucket) {
+ acquireAndEnsureOpen();
+ try {
+ TableInfo tableInfo = requireRegistered(tablePath);
+ if (tableInfo.isPartitioned()) {
+ throw new IllegalStateException(
+ "Table "
+ + tablePath
+ + " is a partitioned table, please use "
+ + "\"unsubscribe(TablePath, long partitionId, int bucket)\" "
+ + "to unsubscribe a partitioned bucket instead.");
+ }
+ TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), bucket);
+ logScannerStatus.unassignScanBuckets(Collections.singletonList(tableBucket));
+ untrackSubscription(tablePath, tableBucket);
+ } finally {
+ release();
+ }
+ }
+
+ @Override
+ public void subscribe(TablePath tablePath, long partitionId, int bucket, long offset) {
+ acquireAndEnsureOpen();
+ try {
+ TableInfo tableInfo = getTableInfo(tablePath);
+ if (!tableInfo.isPartitioned()) {
+ throw new IllegalStateException(
+ "Table "
+ + tablePath
+ + " is not a partitioned table, please use "
+ + "\"subscribe(TablePath, int bucket, long offset)\" "
+ + "to subscribe a non-partitioned bucket instead.");
+ }
+ // Validate table and partition metadata (e.g. a nonexistent partition) before
+ // mutating any scanner/fetcher state, so a failure here doesn't leave a stale
+ // registration without tracked buckets.
+ metadataUpdater.checkAndUpdateTableMetadata(Collections.singleton(tablePath));
+ metadataUpdater.checkAndUpdatePartitionMetadata(
+ tablePath, Collections.singleton(partitionId));
+ registerIfAbsent(tablePath, tableInfo);
+ TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), partitionId, bucket);
+ logScannerStatus.assignScanBuckets(Collections.singletonMap(tableBucket, offset));
+ trackSubscription(tablePath, tableBucket);
+ } finally {
+ release();
+ }
+ }
+
+ @Override
+ public void unsubscribe(TablePath tablePath, long partitionId, int bucket) {
+ acquireAndEnsureOpen();
+ try {
+ TableInfo tableInfo = requireRegistered(tablePath);
+ if (!tableInfo.isPartitioned()) {
+ throw new IllegalStateException(
+ "Table " + tablePath + " is not a partitioned table.");
+ }
+ TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), partitionId, bucket);
+ logScannerStatus.unassignScanBuckets(Collections.singletonList(tableBucket));
+ untrackSubscription(tablePath, tableBucket);
+ } finally {
+ release();
+ }
+ }
+
+ /**
+ * Register the table on the underlying {@link LogFetcher} if not yet registered. Must be called
+ * under the acquire-lock. Phase 1 always registers with no projection / filter.
+ */
+ private void registerIfAbsent(TablePath tablePath, TableInfo tableInfo) {
+ TableInfo cached = registeredByPath.get(tablePath);
+ if (cached != null) {
+ checkTableIdNotChanged(tablePath, cached);
+ return;
+ }
+
+ TableScanSpec spec = new TableScanSpec(tableInfo, null, null);
+ SchemaGetter schemaGetter =
+ new ClientSchemaGetter(tablePath, tableInfo.getSchemaInfo(), admin);
+ logFetcher.registerTable(spec, schemaGetter);
+ registeredByPath.put(tablePath, tableInfo);
+ pathByTableId.put(tableInfo.getTableId(), tablePath);
+ schemaGetters.put(tableInfo.getTableId(), schemaGetter);
+ }
+
+ private TableInfo getTableInfo(TablePath tablePath) {
+ TableInfo cached = registeredByPath.get(tablePath);
+ if (cached != null) {
+ checkTableIdNotChanged(tablePath, cached);
+ return cached;
+ }
+ return resolveTableInfo(tablePath);
+ }
+
+ /**
+ * Record a freshly assigned bucket against its table. Must be called under the acquire-lock.
+ */
+ private void trackSubscription(TablePath tablePath, TableBucket tableBucket) {
+ subscribedBucketsByPath.computeIfAbsent(tablePath, k -> new HashSet<>()).add(tableBucket);
+ }
+
+ /**
+ * Drop a bucket from its table's subscription set. When the table has no remaining subscribed
+ * buckets, evict its cached registration so a later {@code subscribe(...)} re-resolves the
+ * table (picking up a new tableId after a drop+recreate). Must be called under the
+ * acquire-lock.
+ */
+ private void untrackSubscription(TablePath tablePath, TableBucket tableBucket) {
+ Set<TableBucket> buckets = subscribedBucketsByPath.get(tablePath);
+ if (buckets == null) {
+ return;
+ }
+ buckets.remove(tableBucket);
+ if (buckets.isEmpty()) {
+ subscribedBucketsByPath.remove(tablePath);
+ TableInfo evicted = registeredByPath.remove(tablePath);
+ if (evicted != null) {
+ pathByTableId.remove(evicted.getTableId());
+ schemaGetters.remove(evicted.getTableId());
+ logFetcher.unregisterTable(evicted.getTableId());
+ }
+ }
+ }
+
+ private void checkTableIdNotChanged(TablePath tablePath, TableInfo cached) {
+ Optional<Long> currentTableId = metadataUpdater.getCluster().getTableId(tablePath);
+ if (currentTableId.isPresent() && currentTableId.get() != cached.getTableId()) {
+ throw new IllegalStateException(
+ "Table "
+ + tablePath
+ + " has been recreated (tableId changed from "
+ + cached.getTableId()
+ + " to "
+ + currentTableId.get()
+ + "). Please unsubscribe the stale table and subscribe again.");
+ }
+ }
+
+ private TableInfo requireRegistered(TablePath tablePath) {
+ TableInfo tableInfo = registeredByPath.get(tablePath);
+ if (tableInfo == null) {
+ throw new IllegalArgumentException(
+ "Table "
+ + tablePath
+ + " has not been subscribed on this MultiTableLogScanner.");
+ }
+ return tableInfo;
+ }
+
+ private TableInfo resolveTableInfo(TablePath tablePath) {
+ try {
+ return admin.getTableInfo(tablePath).get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while resolving table info: " + tablePath, e);
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause() != null ? e.getCause() : e;
+ if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ }
+ throw new RuntimeException("Failed to resolve table info: " + tablePath, cause);
+ }
+ }
+
+ private MultiTableRecords enrich(ScanRecords scanRecords) {
+ MultiTableRecords.Builder builder = new MultiTableRecords.Builder();
+ for (TableBucket bucket : scanRecords.buckets()) {
+ long tableId = bucket.getTableId();
+ TablePath tablePath = pathByTableId.get(tableId);
+ SchemaGetter schemaGetter = schemaGetters.get(tableId);
+ if (tablePath == null || schemaGetter == null) {
+ // Should not happen: only registered tables can produce records.
+ throw new IllegalStateException(
+ String.format(
+ "Dropping records for unknown tableId %s in bucket %s",
+ tableId, bucket));
+ }
+ builder.addConsumedUpToOffset(
+ tablePath, bucket, scanRecords.consumedUpToOffset(bucket));
+ List<ScanRecord> records = scanRecords.records(bucket);
+ for (ScanRecord scanRecord : records) {
+ // The SchemaGetter is consumed by LogFetcher while decoding records: it resolves
+ // the schema for each record's schemaId. ClientSchemaGetter is seeded with the
+ // latest
+ // SchemaInfo and caches every schema it ever resolves (by schemaId), so during the
+ // records loading path lookups are served from the in-memory cache; The per-record
+ // overhead is therefore negligible.
+ int schemaId = scanRecord.getSchemaId();
+ Schema schema = schemaGetter.getSchema(schemaId);
+ builder.add(
+ tablePath,
+ bucket,
+ new MultiTableRecord(tablePath, tableId, schemaId, schema, scanRecord));
+ }
+ }
+ return builder.build();
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/MultiTableRecords.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/MultiTableRecords.java
new file mode 100644
index 000000000..c99221232
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/MultiTableRecords.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.scanner.log;
+
+import org.apache.fluss.annotation.PublicEvolving;
+import org.apache.fluss.client.table.scanner.MultiTableRecord;
+import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.utils.AbstractIterator;
+
+import javax.annotation.Nullable;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Container for {@link MultiTableRecord}s produced by {@link MultiTableLogScanner#poll(Duration)}.
+ * Records are organized by {@link TablePath} and {@link TableBucket}.
+ *
+ * @since 0.7
+ */
+@PublicEvolving
+public class MultiTableRecords implements Iterable<MultiTableRecord> {
+
+ public static final MultiTableRecords EMPTY =
+ new MultiTableRecords(Collections.emptyMap(), Collections.emptyMap());
+
+ private final Map<TablePath, Map<TableBucket, List<MultiTableRecord>>> records;
+
+ private final Map<TableBucket, Long> consumedUpToOffsets;
+
+ MultiTableRecords(
+ Map<TablePath, Map<TableBucket, List<MultiTableRecord>>> records,
+ Map<TableBucket, Long> consumedUpToOffsets) {
+ this.records = records;
+ this.consumedUpToOffsets = consumedUpToOffsets;
+ }
+
+ /** All tables with records or scanner progress in this result set. */
+ public Set<TablePath> tablePaths() {
+ return Collections.unmodifiableSet(records.keySet());
+ }
+
+ /** All buckets with records or scanner progress for a given table. */
+ public Set<TableBucket> buckets(TablePath tablePath) {
+ Map<TableBucket, List<MultiTableRecord>> byBucket = records.get(tablePath);
+ if (byBucket == null) {
+ return Collections.emptySet();
+ }
+ return Collections.unmodifiableSet(byBucket.keySet());
+ }
+
+ /** Records for all buckets of a given table. */
+ public List<MultiTableRecord> records(TablePath tablePath) {
+ Map<TableBucket, List<MultiTableRecord>> byBucket = records.get(tablePath);
+ if (byBucket == null || byBucket.isEmpty()) {
+ return Collections.emptyList();
+ }
+ // Preserve insertion order across buckets.
+ java.util.ArrayList<MultiTableRecord> all = new java.util.ArrayList<>();
+ for (List<MultiTableRecord> list : byBucket.values()) {
+ all.addAll(list);
+ }
+ return Collections.unmodifiableList(all);
+ }
+
+ /** Records for a specific bucket of a given table. */
+ public List<MultiTableRecord> records(TablePath tablePath, TableBucket bucket) {
+ Map<TableBucket, List<MultiTableRecord>> byBucket = records.get(tablePath);
+ if (byBucket == null) {
+ return Collections.emptyList();
+ }
+ List<MultiTableRecord> list = byBucket.get(bucket);
+ if (list == null) {
+ return Collections.emptyList();
+ }
+ return Collections.unmodifiableList(list);
+ }
+
+ /** Total number of records across all tables. */
+ public int count() {
+ int count = 0;
+ for (Map<TableBucket, List<MultiTableRecord>> byBucket : records.values()) {
+ for (List<MultiTableRecord> list : byBucket.values()) {
+ count += list.size();
+ }
+ }
+ return count;
+ }
+
+ /** Returns {@code true} if this {@code MultiTableRecords} contains no materialized records. */
+ public boolean isEmpty() {
+ return count() == 0;
+ }
+
+ /**
+ * Returns {@code true} if this {@code MultiTableRecords} carries any scanner progress.
+ *
+ * <p>A result may have progress even when {@link #isEmpty()} is {@code true}, for example when
+ * a bucket returns no materialized records but advances its consumed offset.
+ */
+ public boolean hasProgress() {
+ return count() > 0 || !consumedUpToOffsets.isEmpty();
+ }
+
+ /**
+ * Get the exclusive upper bound of offsets consumed for the given bucket of a given table in
+ * this poll round.
+ *
+ * @param tablePath the table the bucket belongs to
+ * @param bucket the bucket to query
+ * @return the exclusive upper bound offset, or {@code null} if the bucket of the table was not
+ * polled in this round
+ */
+ @Nullable
+ public Long consumedUpToOffset(TablePath tablePath, TableBucket bucket) {
+ Map<TableBucket, List<MultiTableRecord>> byBucket = records.get(tablePath);
+ if (byBucket == null || !byBucket.containsKey(bucket)) {
+ return null;
+ }
+ return consumedUpToOffsets.get(bucket);
+ }
+
+ /** Iterate over every record across all tables. */
+ @Override
+ public Iterator<MultiTableRecord> iterator() {
+ // Flatten the nested structure for iteration while keeping insertion order stable.
+ Set<List<MultiTableRecord>> flat = new LinkedHashSet<>();
+ for (Map<TableBucket, List<MultiTableRecord>> byBucket : records.values()) {
+ flat.addAll(byBucket.values());
+ }
+ return new ConcatenatedIterable(flat).iterator();
+ }
+
+ /**
+ * A mutable builder for {@link MultiTableRecords}; not part of the public API surface, used
+ * internally by the scanner.
+ */
+ public static final class Builder {
+ private final Map<TablePath, Map<TableBucket, List<MultiTableRecord>>> records =
+ new LinkedHashMap<>();
+ private final Map<TableBucket, Long> consumedUpToOffsets = new LinkedHashMap<>();
+
+ public Builder add(TablePath tablePath, TableBucket bucket, MultiTableRecord record) {
+ records.computeIfAbsent(tablePath, k -> new LinkedHashMap<>())
+ .computeIfAbsent(bucket, k -> new ArrayList<>())
+ .add(record);
+ return this;
+ }
+
+ /**
+ * Record the consumed offset for a bucket. Also creates an (empty) table/bucket entry in
+ * the result so that {@link MultiTableRecords#tablePaths()} and {@link
+ * MultiTableRecords#buckets(TablePath)} surface progress-only buckets, as documented.
+ */
+ public Builder addConsumedUpToOffset(
+ TablePath tablePath, TableBucket bucket, @Nullable Long consumedUpToOffset) {
+ if (consumedUpToOffset != null) {
+ consumedUpToOffsets.put(bucket, consumedUpToOffset);
+ records.computeIfAbsent(tablePath, k -> new LinkedHashMap<>())
+ .computeIfAbsent(bucket, k -> new ArrayList<>());
+ }
+ return this;
+ }
+
+ public MultiTableRecords build() {
+ if (records.isEmpty() && consumedUpToOffsets.isEmpty()) {
+ return EMPTY;
+ }
+ return new MultiTableRecords(records, consumedUpToOffsets);
+ }
+ }
+
+ private static class ConcatenatedIterable implements Iterable<MultiTableRecord> {
+
+ private final Iterable<? extends Iterable<MultiTableRecord>> iterables;
+
+ ConcatenatedIterable(Iterable<? extends Iterable<MultiTableRecord>> iterables) {
+ this.iterables = iterables;
+ }
+
+ @Override
+ public Iterator<MultiTableRecord> iterator() {
+ return new AbstractIterator<MultiTableRecord>() {
+ final Iterator<? extends Iterable<MultiTableRecord>> iters = iterables.iterator();
+ Iterator<MultiTableRecord> current;
+
+ public MultiTableRecord makeNext() {
+ while (current == null || !current.hasNext()) {
+ if (iters.hasNext()) {
+ current = iters.next().iterator();
+ } else {
+ return allDone();
+ }
+ }
+ return current.next();
+ }
+ };
+ }
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/PendingFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/PendingFetch.java
index c3fa29936..3f49bc5e7 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/PendingFetch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/PendingFetch.java
@@ -30,4 +30,7 @@ interface PendingFetch {
boolean isCompleted();
CompletedFetch toCompletedFetch();
+
+ /** Release resources when this pending fetch is removed before completion. */
+ default void discard() {}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetch.java
index 0ac064307..9d7d1cbc0 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetch.java
@@ -19,6 +19,7 @@ package org.apache.fluss.client.table.scanner.log;
import org.apache.fluss.annotation.Internal;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.record.FileLogRecords;
import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.rpc.protocol.ApiError;
@@ -39,6 +40,7 @@ class RemoteCompletedFetch extends CompletedFetch {
RemoteCompletedFetch(
TableBucket tableBucket,
+ TablePath tablePath,
FileLogRecords fileLogRecords,
long highWatermark,
LogRecordReadContext readContext,
@@ -48,6 +50,7 @@ class RemoteCompletedFetch extends CompletedFetch {
Runnable recycleCallback) {
super(
tableBucket,
+ tablePath,
ApiError.NONE,
fileLogRecords.sizeInBytes(),
highWatermark,
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloadFuture.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloadFuture.java
index 339df422c..57e16989c 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloadFuture.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloadFuture.java
@@ -23,17 +23,28 @@ import org.apache.fluss.record.FileLogRecords;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicBoolean;
/** Represents the future of a remote log download request. */
public class RemoteLogDownloadFuture {
private final CompletableFuture<File> logFileFuture;
private final Runnable recycleCallback;
+ private final Runnable discardCallback;
+ private final AtomicBoolean discarded = new AtomicBoolean(false);
public RemoteLogDownloadFuture(
CompletableFuture<File> logFileFuture, Runnable recycleCallback) {
+ this(logFileFuture, recycleCallback, () -> {});
+ }
+
+ RemoteLogDownloadFuture(
+ CompletableFuture<File> logFileFuture,
+ Runnable recycleCallback,
+ Runnable discardCallback) {
this.logFileFuture = logFileFuture;
this.recycleCallback = recycleCallback;
+ this.discardCallback = discardCallback;
}
public boolean isDone() {
@@ -57,6 +68,12 @@ public class RemoteLogDownloadFuture {
return recycleCallback;
}
+ public void discard() {
+ if (discarded.compareAndSet(false, true)) {
+ discardCallback.run();
+ }
+ }
+
public void onComplete(Runnable callback) {
logFileFuture.thenRun(callback);
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloader.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloader.java
index 15bc1f631..04c03eb70 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloader.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloader.java
@@ -26,7 +26,6 @@ import org.apache.fluss.config.Configuration;
import org.apache.fluss.fs.FsPath;
import org.apache.fluss.fs.FsPathAndFileName;
import org.apache.fluss.metadata.TableBucket;
-import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.remote.RemoteLogSegment;
import org.apache.fluss.utils.ExceptionUtils;
import org.apache.fluss.utils.FlussPaths;
@@ -88,17 +87,17 @@ public class RemoteLogDownloader implements Closeable {
private volatile boolean closed = false;
public RemoteLogDownloader(
- TablePath tablePath,
+ String scannerName,
Configuration conf,
RemoteFileDownloader remoteFileDownloader,
ScannerMetricGroup scannerMetricGroup) {
// default we give a 5s long interval to avoid frequent loop
- this(tablePath, conf, remoteFileDownloader, scannerMetricGroup, POLL_TIMEOUT);
+ this(scannerName, conf, remoteFileDownloader, scannerMetricGroup, POLL_TIMEOUT);
}
@VisibleForTesting
RemoteLogDownloader(
- TablePath tablePath,
+ String scannerName,
Configuration conf,
RemoteFileDownloader remoteFileDownloader,
ScannerMetricGroup scannerMetricGroup,
@@ -116,7 +115,7 @@ public class RemoteLogDownloader implements Closeable {
Paths.get(
conf.get(ConfigOptions.CLIENT_SCANNER_IO_TMP_DIR),
"remote-logs-" + UUID.randomUUID());
- this.downloadThread = new DownloadRemoteLogThread(tablePath);
+ this.downloadThread = new DownloadRemoteLogThread(scannerName);
}
public void start() {
@@ -127,7 +126,8 @@ public class RemoteLogDownloader implements Closeable {
public RemoteLogDownloadFuture requestRemoteLog(FsPath logTabletDir, RemoteLogSegment segment) {
RemoteLogDownloadRequest request = new RemoteLogDownloadRequest(segment, logTabletDir);
segmentsToFetch.add(request);
- return new RemoteLogDownloadFuture(request.future, () -> recycleRemoteLog(segment));
+ return new RemoteLogDownloadFuture(
+ request.future, () -> recycleRemoteLog(segment), () -> discardRemoteLog(request));
}
/**
@@ -139,6 +139,14 @@ public class RemoteLogDownloader implements Closeable {
prefetchSemaphore.release();
}
+ private void discardRemoteLog(RemoteLogDownloadRequest request) {
+ if (request.future.cancel(false)) {
+ segmentsToFetch.remove(request);
+ } else if (!request.future.isCompletedExceptionally()) {
+ recycleRemoteLog(request.segment);
+ }
+ }
+
/**
* Fetch a remote log segment file to local. This method will block until there is a log segment
* to fetch.
@@ -158,6 +166,10 @@ public class RemoteLogDownloader implements Closeable {
prefetchSemaphore.release();
return;
}
+ if (request.future.isCancelled()) {
+ prefetchSemaphore.release();
+ return;
+ }
TableBucket tableBucket = request.getTableBucket();
try {
@@ -191,6 +203,13 @@ public class RemoteLogDownloader implements Closeable {
deleteDirectoryQuietly(localLogDir.toFile());
return;
}
+ if (request.future.isCancelled()) {
+ prefetchSemaphore.release();
+ if (throwable == null) {
+ cleanupFinishedRemoteLog(request.segment);
+ }
+ return;
+ }
if (throwable != null) {
LOG.error(
"Failed to download remote log segment file {} for table bucket {}.",
@@ -210,15 +229,18 @@ public class RemoteLogDownloader implements Closeable {
File localFile =
new File(localLogDir.toFile(), fsPathAndFileName.getFileName());
scannerMetricGroup.remoteFetchBytes().inc(bytes);
- request.future.complete(localFile);
+ if (!request.future.complete(localFile)) {
+ prefetchSemaphore.release();
+ cleanupFinishedRemoteLog(request.segment);
+ }
}
});
} catch (Throwable t) {
prefetchSemaphore.release();
// only re-queue the request if the downloader is still active
- if (!closed) {
+ if (!closed && !request.future.isCancelled()) {
segmentsToFetch.add(request);
- } else {
+ } else if (!request.future.isDone()) {
request.future.cancel(false);
}
scannerMetricGroup.remoteFetchErrorCount().inc();
@@ -315,8 +337,8 @@ public class RemoteLogDownloader implements Closeable {
* until it is interrupted.
*/
private class DownloadRemoteLogThread extends ShutdownableThread {
- public DownloadRemoteLogThread(TablePath tablePath) {
- super(String.format("DownloadRemoteLog-[%s]", tablePath.toString()), true);
+ public DownloadRemoteLogThread(String scannerName) {
+ super(String.format("DownloadRemoteLog-[%s]", scannerName), true);
}
@Override
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemotePendingFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemotePendingFetch.java
index ba4ffd8c8..5bb6d3568 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemotePendingFetch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemotePendingFetch.java
@@ -18,6 +18,7 @@
package org.apache.fluss.client.table.scanner.log;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.record.FileLogRecords;
import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.remote.RemoteLogSegment;
@@ -30,6 +31,7 @@ class RemotePendingFetch implements PendingFetch {
final RemoteLogSegment remoteLogSegment;
private final RemoteLogDownloadFuture downloadFuture;
+ private final TablePath tablePath;
private final int posInLogSegment;
private final long fetchOffset;
@@ -41,6 +43,7 @@ class RemotePendingFetch implements PendingFetch {
RemotePendingFetch(
RemoteLogSegment remoteLogSegment,
RemoteLogDownloadFuture downloadFuture,
+ TablePath tablePath,
int posInLogSegment,
long fetchOffset,
long highWatermark,
@@ -49,6 +52,7 @@ class RemotePendingFetch implements PendingFetch {
boolean isCheckCrc) {
this.remoteLogSegment = remoteLogSegment;
this.downloadFuture = downloadFuture;
+ this.tablePath = tablePath;
this.posInLogSegment = posInLogSegment;
this.fetchOffset = fetchOffset;
this.highWatermark = highWatermark;
@@ -72,6 +76,7 @@ class RemotePendingFetch implements PendingFetch {
FileLogRecords fileLogRecords = downloadFuture.getFileLogRecords(posInLogSegment);
return new RemoteCompletedFetch(
remoteLogSegment.tableBucket(),
+ tablePath,
fileLogRecords,
highWatermark,
readContext,
@@ -81,6 +86,11 @@ class RemotePendingFetch implements PendingFetch {
downloadFuture.getRecycleCallback());
}
+ @Override
+ public void discard() {
+ downloadFuture.discard();
+ }
+
@Override
public String toString() {
return "RemotePendingFetch{"
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ScanRecords.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ScanRecords.java
index eb3e157d1..ebeab20c2 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ScanRecords.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ScanRecords.java
@@ -26,6 +26,7 @@ import javax.annotation.Nullable;
import java.util.Collections;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -53,10 +54,28 @@ public class ScanRecords implements Iterable<ScanRecord> {
public ScanRecords(
Map<TableBucket, List<ScanRecord>> records,
Map<TableBucket, Long> consumedUpToOffsets) {
- this.records = records;
+ this.records = withProgressOnlyBuckets(records, consumedUpToOffsets);
this.consumedUpToOffsets = consumedUpToOffsets;
}
+ /**
+ * Ensures every bucket with a consumed offset has a (possibly empty) record list entry, so that
+ * {@link #buckets()} surfaces progress-only buckets as documented. Returns the original map
+ * when the invariant already holds (the common case for collector-produced results).
+ */
+ private static Map<TableBucket, List<ScanRecord>> withProgressOnlyBuckets(
+ Map<TableBucket, List<ScanRecord>> records,
+ Map<TableBucket, Long> consumedUpToOffsets) {
+ if (records.keySet().containsAll(consumedUpToOffsets.keySet())) {
+ return records;
+ }
+ Map<TableBucket, List<ScanRecord>> merged = new LinkedHashMap<>(records);
+ for (TableBucket bucket : consumedUpToOffsets.keySet()) {
+ merged.putIfAbsent(bucket, Collections.emptyList());
+ }
+ return merged;
+ }
+
/**
* Get just the records for the given bucketId.
*
@@ -104,6 +123,16 @@ public class ScanRecords implements Iterable<ScanRecord> {
return count() == 0;
}
+ /**
+ * Returns {@code true} if this {@code ScanRecords} carries any scanner progress.
+ *
+ * <p>A result may have progress even when {@link #isEmpty()} is {@code true}, for example when
+ * a bucket returns no materialized records but advances its consumed offset.
+ */
+ public boolean hasProgress() {
+ return count() > 0 || !consumedUpToOffsets.isEmpty();
+ }
+
@Override
public Iterator<ScanRecord> iterator() {
return new ConcatenatedIterable(records.values()).iterator();
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/TableScanSpec.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/TableScanSpec.java
new file mode 100644
index 000000000..28346a496
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/TableScanSpec.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.scanner.log;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.metadata.TableInfo;
+import org.apache.fluss.predicate.Predicate;
+import org.apache.fluss.utils.Projection;
+
+import javax.annotation.Nullable;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/**
+ * Per-table scan specification consumed by {@link LogFetcher#registerTable}.
+ *
+ * <p>Captures the {@link TableInfo} together with optional projection and filter pushdown to be
+ * applied while reading log records for the table.
+ *
+ * @since 0.7
+ */
+@Internal
+public final class TableScanSpec {
+
+ private final TableInfo tableInfo;
+ @Nullable private final Projection projection;
+ @Nullable private final Predicate filter;
+
+ public TableScanSpec(
+ TableInfo tableInfo, @Nullable Projection projection, @Nullable Predicate filter) {
+ this.tableInfo = checkNotNull(tableInfo, "tableInfo");
+ this.projection = projection;
+ this.filter = filter;
+ }
+
+ public TableInfo getTableInfo() {
+ return tableInfo;
+ }
+
+ @Nullable
+ public Projection getProjection() {
+ return projection;
+ }
+
+ @Nullable
+ public Predicate getFilter() {
+ return filter;
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/PendingFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWrite.java
similarity index 62%
copy from fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/PendingFetch.java
copy to fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWrite.java
index c3fa29936..c1b589e82 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/PendingFetch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWrite.java
@@ -15,19 +15,20 @@
* limitations under the License.
*/
-package org.apache.fluss.client.table.scanner.log;
+package org.apache.fluss.client.table.writer;
-import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.annotation.PublicEvolving;
/**
- * A pending fetch is a fetch that requested to download remote log segment file to local but not
- * yet been completed.
+ * Builder for a multi-table writer. Unlike {@link Append} / {@link Upsert}, no table is bound at
+ * builder time; the target table is carried on each {@link MultiTableWriteRecord} passed to the
+ * writer.
+ *
+ * @since 0.7
*/
-interface PendingFetch {
-
- TableBucket tableBucket();
-
- boolean isCompleted();
+@PublicEvolving
+public interface MultiTableWrite {
- CompletedFetch toCompletedFetch();
+ /** Creates a {@link MultiTableWriter} that can write append/upsert/delete records. */
+ MultiTableWriter createWriter();
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWriteImpl.java b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWriteImpl.java
new file mode 100644
index 000000000..12fca0ec6
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWriteImpl.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.writer;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.client.FlussConnection;
+import org.apache.fluss.client.metrics.MultiTableWriterMetricGroup;
+import org.apache.fluss.client.write.WriterClient;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/**
+ * Default implementation of {@link MultiTableWrite}.
+ *
+ * <p>Thin pass-through builder that constructs a {@link MultiTableWriterImpl} owning a dedicated
+ * {@link WriterClient}, so that flushes and schema-scoped batching of one writer never interfere
+ * with other writers on the same connection. Tables are registered dynamically on the writer on
+ * first {@code write(...)} call — no upfront declaration is required.
+ *
+ * @since 0.7
+ */
+@Internal
+public class MultiTableWriteImpl implements MultiTableWrite {
+
+ private final FlussConnection connection;
+
+ public MultiTableWriteImpl(FlussConnection connection) {
+ this.connection = checkNotNull(connection, "connection");
+ }
+
+ @Override
+ public MultiTableWriter createWriter() {
+ // Each writer owns a dedicated WriterClient (buffer pool + sender thread) instead of the
+ // connection-level shared one, so MultiTableWriter#flush() only affects this writer and
+ // schema-id batching of different writers cannot interfere with each other.
+ return new MultiTableWriterImpl(
+ connection.getMetadataUpdater(),
+ connection.getOrCreateAdmin(),
+ new WriterClient(
+ connection.getConfiguration(),
+ connection.getMetadataUpdater(),
+ new MultiTableWriterMetricGroup(connection.getClientMetricGroup()),
+ connection.getOrCreateAdmin()));
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWriteRecord.java b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWriteRecord.java
new file mode 100644
index 000000000..103ef5ea5
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWriteRecord.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.writer;
+
+import org.apache.fluss.annotation.PublicEvolving;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.row.InternalRow;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/**
+ * A write-side record for {@link MultiTableWriter}. Carries only the fields a writer needs:
+ *
+ * <ul>
+ * <li>{@link #getTablePath()} — target table
+ * <li>{@link #getOperation()} — APPEND / UPSERT / DELETE
+ * <li>{@link #getRow()} — the row payload encoded against the table's schema
+ * <li>{@link #getSchemaId()} — the schema id the row was encoded against; must be a valid
+ * (positive) schema id obtained from table metadata
+ * </ul>
+ *
+ * <p>Server-assigned identity ({@code tableId}) and read-only metadata ({@code schema} object,
+ * {@code logOffset}, {@code timestamp}) are intentionally absent here — they belong to the
+ * read-side {@link org.apache.fluss.client.table.scanner.MultiTableRecord}.
+ *
+ * <p>Instances are immutable.
+ *
+ * @since 0.7
+ */
+@PublicEvolving
+public final class MultiTableWriteRecord {
+
+ /**
+ * The write operation understood by {@link MultiTableWriter}.
+ *
+ * <p>Log tables accept {@link #APPEND}; primary-key tables accept {@link #UPSERT} and {@link
+ * #DELETE}.
+ */
+ @PublicEvolving
+ public enum Operation {
+ APPEND,
+ UPSERT,
+ DELETE
+ }
+
+ private final TablePath tablePath;
+ private final Operation operation;
+ private final InternalRow row;
+ private final int schemaId;
+
+ private MultiTableWriteRecord(
+ TablePath tablePath, Operation operation, InternalRow row, int schemaId) {
+ this.tablePath = checkNotNull(tablePath, "tablePath");
+ this.operation = checkNotNull(operation, "operation");
+ this.row = checkNotNull(row, "row");
+ this.schemaId = schemaId;
+ }
+
+ /**
+ * Build an append record for a log table.
+ *
+ * @param tablePath target log table
+ * @param row the row payload
+ * @param schemaId the schema id the row was encoded against; must be a valid positive id
+ */
+ public static MultiTableWriteRecord forAppend(
+ TablePath tablePath, InternalRow row, int schemaId) {
+ return new MultiTableWriteRecord(tablePath, Operation.APPEND, row, schemaId);
+ }
+
+ /**
+ * Build an upsert record for a primary-key table.
+ *
+ * @param tablePath target primary-key table
+ * @param row the row payload
+ * @param schemaId the schema id the row was encoded against
+ */
+ public static MultiTableWriteRecord forUpsert(
+ TablePath tablePath, InternalRow row, int schemaId) {
+ return new MultiTableWriteRecord(tablePath, Operation.UPSERT, row, schemaId);
+ }
+
+ /**
+ * Build a delete record for a primary-key table.
+ *
+ * @param tablePath target primary-key table
+ * @param row the row payload
+ * @param schemaId the schema id the row was encoded against
+ */
+ public static MultiTableWriteRecord forDelete(
+ TablePath tablePath, InternalRow row, int schemaId) {
+ return new MultiTableWriteRecord(tablePath, Operation.DELETE, row, schemaId);
+ }
+
+ public TablePath getTablePath() {
+ return tablePath;
+ }
+
+ public Operation getOperation() {
+ return operation;
+ }
+
+ public InternalRow getRow() {
+ return row;
+ }
+
+ /** Returns the schema id the row was encoded against. */
+ public int getSchemaId() {
+ return schemaId;
+ }
+
+ @Override
+ public String toString() {
+ return "MultiTableWriteRecord{"
+ + "tablePath="
+ + tablePath
+ + ", operation="
+ + operation
+ + ", schemaId="
+ + schemaId
+ + '}';
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWriter.java b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWriter.java
new file mode 100644
index 000000000..8edf038d8
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWriter.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.writer;
+
+import org.apache.fluss.annotation.PublicEvolving;
+import org.apache.fluss.exception.TableNotExistException;
+
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * A writer that sends records to multiple tables through a writer-owned underlying writer client.
+ * The target table, operation, row, and schema id are carried by the {@link MultiTableWriteRecord}
+ * itself.
+ *
+ * <p>NOT thread-safe.
+ *
+ * @since 0.7
+ */
+@PublicEvolving
+public interface MultiTableWriter extends AutoCloseable {
+
+ /**
+ * Write a record to the table identified by {@link MultiTableWriteRecord#getTablePath()}.
+ *
+ * <p>Asynchronous; the returned future completes when the server has acknowledged the write (or
+ * has failed to write it).
+ *
+ * @throws TableNotExistException wrapped in the future, if the table is unknown
+ */
+ CompletableFuture<WriteResult> write(MultiTableWriteRecord record);
+
+ /**
+ * Flush all buffered records and block until acknowledged by the servers (or failed).
+ * Consistent with {@link TableWriter#flush()}.
+ */
+ void flush();
+
+ /**
+ * Close the writer and release its writer-owned resources; pending records are flushed first.
+ */
+ @Override
+ void close() throws Exception;
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWriterImpl.java b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWriterImpl.java
new file mode 100644
index 000000000..5702188d3
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/MultiTableWriterImpl.java
@@ -0,0 +1,521 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.writer;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.client.admin.Admin;
+import org.apache.fluss.client.metadata.MetadataUpdater;
+import org.apache.fluss.client.table.getter.PartitionGetter;
+import org.apache.fluss.client.write.WriteFormat;
+import org.apache.fluss.client.write.WriteRecord;
+import org.apache.fluss.client.write.WriterClient;
+import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.PhysicalTablePath;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableInfo;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.InternalRow.FieldGetter;
+import org.apache.fluss.row.compacted.CompactedRow;
+import org.apache.fluss.row.encode.CompactedRowEncoder;
+import org.apache.fluss.row.encode.IndexedRowEncoder;
+import org.apache.fluss.row.encode.KeyEncoder;
+import org.apache.fluss.row.encode.RowEncoder;
+import org.apache.fluss.row.indexed.IndexedRow;
+import org.apache.fluss.rpc.protocol.MergeMode;
+import org.apache.fluss.types.DataType;
+import org.apache.fluss.types.RowType;
+
+import javax.annotation.Nullable;
+
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+import static org.apache.fluss.utils.Preconditions.checkState;
+
+/** Default implementation of {@link MultiTableWriter}. */
+@Internal
+class MultiTableWriterImpl implements MultiTableWriter {
+
+ private final MetadataUpdater metadataUpdater;
+ private final Admin admin;
+ /** Owned by this writer; created per writer instance and closed in {@link #close()}. */
+ private final WriterClient writerClient;
+
+ /** Per-table encoder states keyed by schemaId, supporting writes against historical schemas. */
+ private final Map<TablePath, Map<Integer, TableWriteState>> states = new HashMap<>();
+
+ /** The latest known schemaId per table, for fast-path lookup of the current encoder state. */
+ private final Map<TablePath, Integer> latestSchemaIdByPath = new HashMap<>();
+
+ private volatile boolean closed;
+
+ MultiTableWriterImpl(MetadataUpdater metadataUpdater, Admin admin, WriterClient writerClient) {
+ this.metadataUpdater = checkNotNull(metadataUpdater, "metadataUpdater");
+ this.admin = checkNotNull(admin, "admin");
+ this.writerClient = checkNotNull(writerClient, "writerClient");
+ }
+
+ @Override
+ public CompletableFuture<WriteResult> write(MultiTableWriteRecord record) {
+ checkNotNull(record, "record");
+ checkState(!closed, "MultiTableWriter is already closed.");
+
+ TablePath tablePath = record.getTablePath();
+ MultiTableWriteRecord.Operation operation = record.getOperation();
+ InternalRow row = record.getRow();
+
+ final WriteRecord writeRecord;
+ try {
+ TableWriteState state = resolveWriteState(tablePath, record.getSchemaId());
+ state.checkFieldCount(row);
+ writeRecord = state.build(operation, row);
+ } catch (Exception e) {
+ CompletableFuture<WriteResult> failed = new CompletableFuture<>();
+ failed.completeExceptionally(e);
+ return failed;
+ }
+
+ CompletableFuture<WriteResult> future = new CompletableFuture<>();
+ writerClient.send(
+ writeRecord,
+ (bucket, logEndOffset, exception) -> {
+ if (exception == null) {
+ future.complete(new WriteResult());
+ } else {
+ future.completeExceptionally(exception);
+ }
+ });
+ return future;
+ }
+
+ @Override
+ public void flush() {
+ writerClient.flush();
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (closed) {
+ return;
+ }
+
+ try {
+ try {
+ writerClient.flush();
+ } finally {
+ // The writer owns its WriterClient: release its sender thread and buffers even
+ // when the final flush fails.
+ writerClient.close(Duration.ofMillis(Long.MAX_VALUE));
+ }
+ } finally {
+ // Drop per-table encoder state to help GC on long-lived connections that may
+ // create and close many MultiTableWriter instances.
+ states.clear();
+ latestSchemaIdByPath.clear();
+ closed = true;
+ }
+ }
+
+ /**
+ * Resolve the encoding state for the schemaId carried by the record: return the cached state
+ * (latest or historical) if present, so consecutive records carrying the same schemaId never
+ * re-trigger a refresh; otherwise refresh metadata (force-refresh once if the record's schemaId
+ * is ahead of the known latest), fail fast if it is still ahead, and otherwise encode against
+ * the latest or the matching historical schema.
+ */
+ private TableWriteState resolveWriteState(TablePath tablePath, int recordSchemaId) {
+ TableWriteState cached = getCachedState(tablePath, recordSchemaId);
+ if (cached != null) {
+ return cached;
+ }
+
+ TableWriteState latest = resolveLatestState(tablePath, false);
+ if (recordSchemaId > latest.tableInfo.getSchemaId()) {
+ // record may reference a newer schema; force-refresh metadata once.
+ latest = resolveLatestState(tablePath, true);
+ }
+ int latestSchemaId = latest.tableInfo.getSchemaId();
+ if (recordSchemaId > latestSchemaId) {
+ throw new IllegalArgumentException(
+ "SchemaId mismatch for table "
+ + tablePath
+ + ": record schemaId="
+ + recordSchemaId
+ + " is ahead of the current table schemaId="
+ + latestSchemaId
+ + " (after metadata refresh).");
+ }
+ // encode against the latest schema directly.
+ if (recordSchemaId == latestSchemaId) {
+ return latest;
+ }
+ return resolveHistoricalState(tablePath, latest.tableInfo, recordSchemaId);
+ }
+
+ @Nullable
+ private TableWriteState getCachedState(TablePath tablePath, int schemaId) {
+ Map<Integer, TableWriteState> bySchema = states.get(tablePath);
+ return bySchema == null ? null : bySchema.get(schemaId);
+ }
+
+ /**
+ * Resolve the encoding state for the table's latest schema. When {@code forceUpdate} is true,
+ * force-refresh metadata to avoid stale schema, matching {@code FlussConnection.getTable(...)}.
+ */
+ private TableWriteState resolveLatestState(TablePath tablePath, boolean forceUpdate) {
+ Integer latestSchemaId = latestSchemaIdByPath.get(tablePath);
+ if (latestSchemaId != null && !forceUpdate) {
+ return states.get(tablePath).get(latestSchemaId);
+ }
+ metadataUpdater.updateTableOrPartitionMetadata(tablePath, null);
+ TableInfo info = getTableInfo(tablePath);
+ TableWriteState state = TableWriteState.forTable(tablePath, info);
+ cacheState(tablePath, info.getSchemaId(), state);
+ latestSchemaIdByPath.put(tablePath, info.getSchemaId());
+ return state;
+ }
+
+ /**
+ * Resolve (or build and cache) the encoding state for a historical {@code schemaId}. The
+ * historical {@link TableInfo} reuses every non-schema attribute of the latest table info and
+ * only swaps in the historical schema / schemaId, so the produced {@link WriteRecord} carries
+ * the record's original schemaId (see {@link WriteRecord#getSchemaId()}).
+ */
+ private TableWriteState resolveHistoricalState(
+ TablePath tablePath, TableInfo latestTableInfo, int schemaId) {
+ Map<Integer, TableWriteState> bySchema = states.get(tablePath);
+ if (bySchema != null) {
+ TableWriteState cached = bySchema.get(schemaId);
+ if (cached != null) {
+ return cached;
+ }
+ }
+ Schema historicalSchema = getTableSchema(tablePath, schemaId);
+ TableInfo historicalTableInfo = withSchema(latestTableInfo, schemaId, historicalSchema);
+ TableWriteState state = TableWriteState.forTable(tablePath, historicalTableInfo);
+ cacheState(tablePath, schemaId, state);
+ return state;
+ }
+
+ private void cacheState(TablePath tablePath, int schemaId, TableWriteState state) {
+ states.computeIfAbsent(tablePath, k -> new HashMap<>()).put(schemaId, state);
+ }
+
+ /**
+ * Build a {@link TableInfo} for a historical schema by reusing every non-schema attribute of
+ * {@code base}. This assumes schema evolution does not change bucket keys / partition keys /
+ * bucket count (compatible evolution such as adding columns); otherwise encoding would fail
+ * fast downstream.
+ */
+ private static TableInfo withSchema(TableInfo base, int schemaId, Schema schema) {
+ return new TableInfo(
+ base.getTablePath(),
+ base.getTableId(),
+ schemaId,
+ schema,
+ base.getBucketKeys(),
+ base.getPartitionKeys(),
+ base.getNumBuckets(),
+ base.getProperties(),
+ base.getCustomProperties(),
+ base.getRemoteDataDir(),
+ base.getComment().orElse(null),
+ base.getCreatedTime(),
+ base.getModifiedTime());
+ }
+
+ private TableInfo getTableInfo(TablePath tablePath) {
+ try {
+ return admin.getTableInfo(tablePath).get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while resolving table info: " + tablePath, e);
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause() != null ? e.getCause() : e;
+ if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ }
+ throw new RuntimeException("Failed to resolve table info: " + tablePath, cause);
+ }
+ }
+
+ private Schema getTableSchema(TablePath tablePath, int schemaId) {
+ try {
+ return admin.getTableSchema(tablePath, schemaId).get().getSchema();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while resolving table schema: " + tablePath, e);
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause() != null ? e.getCause() : e;
+ if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ }
+ throw new RuntimeException("Failed to resolve table schema: " + tablePath, cause);
+ }
+ }
+
+ /**
+ * Per-table encoding state, lazily constructed on first write. Mirrors the encoder setup done
+ * in the constructors of {@link AppendWriterImpl} and {@link UpsertWriterImpl}. Sub-classed for
+ * log tables ({@link LogTableWriteState}) and primary-key tables ({@link
+ * PrimaryTableWriteState}) so that each path only carries the encoders it actually uses, and
+ * operation validation is owned by the table-shape that supports it.
+ */
+ private abstract static class TableWriteState {
+ final TablePath tablePath;
+ final TableInfo tableInfo;
+ final int fieldCount;
+ @Nullable final PartitionGetter partitionGetter;
+ final FieldGetter[] fieldGetters;
+
+ TableWriteState(TablePath tablePath, TableInfo tableInfo) {
+ this.tablePath = tablePath;
+ this.tableInfo = tableInfo;
+ RowType rowType = tableInfo.getRowType();
+ this.fieldCount = rowType.getFieldCount();
+ this.partitionGetter =
+ tableInfo.isPartitioned()
+ ? new PartitionGetter(rowType, tableInfo.getPartitionKeys())
+ : null;
+ this.fieldGetters = InternalRow.createFieldGetters(rowType);
+ }
+
+ /** Factory: pick the right shape based on whether the table has a primary key. */
+ static TableWriteState forTable(TablePath tablePath, TableInfo tableInfo) {
+ return tableInfo.hasPrimaryKey()
+ ? new PrimaryTableWriteState(tablePath, tableInfo)
+ : new LogTableWriteState(tablePath, tableInfo);
+ }
+
+ void checkFieldCount(InternalRow row) {
+ if (row.getFieldCount() != fieldCount) {
+ throw new IllegalArgumentException(
+ "The field count of the row does not match the table schema for "
+ + tablePath
+ + ". Expected: "
+ + fieldCount
+ + ", Actual: "
+ + row.getFieldCount());
+ }
+ }
+
+ PhysicalTablePath getPhysicalPath(InternalRow row) {
+ if (partitionGetter == null) {
+ return PhysicalTablePath.of(tablePath);
+ }
+ return PhysicalTablePath.of(tablePath, partitionGetter.getPartition(row));
+ }
+
+ /**
+ * Encode the row into a {@link WriteRecord} for the given operation. Throws {@link
+ * IllegalArgumentException} if the operation is not supported by the underlying table
+ * shape.
+ */
+ abstract WriteRecord build(MultiTableWriteRecord.Operation operation, InternalRow row);
+ }
+
+ /** Encoding state for a log (non-PK) table; only supports APPEND. */
+ private static final class LogTableWriteState extends TableWriteState {
+ final LogFormat logFormat;
+ /**
+ * Per-row encoder for the {@link #logFormat}. Only initialized for {@link
+ * LogFormat#INDEXED} and {@link LogFormat#COMPACTED}; {@code null} for {@link
+ * LogFormat#ARROW} which accepts a general {@link InternalRow} directly.
+ */
+ @Nullable final RowEncoder rowEncoder;
+
+ @Nullable final KeyEncoder logBucketKeyEncoder;
+
+ LogTableWriteState(TablePath tablePath, TableInfo tableInfo) {
+ super(tablePath, tableInfo);
+ RowType rowType = tableInfo.getRowType();
+ if (tableInfo.getBucketKeys().isEmpty()) {
+ this.logBucketKeyEncoder = null;
+ } else {
+ this.logBucketKeyEncoder =
+ KeyEncoder.ofBucketKeyEncoder(
+ rowType,
+ tableInfo.getBucketKeys(),
+ tableInfo.getTableConfig().getDataLakeFormat().orElse(null));
+ }
+ this.logFormat = tableInfo.getTableConfig().getLogFormat();
+ if (logFormat == LogFormat.INDEXED) {
+ this.rowEncoder = new IndexedRowEncoder(rowType);
+ } else if (logFormat == LogFormat.COMPACTED) {
+ DataType[] fieldDataTypes =
+ tableInfo.getSchema().getRowType().getChildren().toArray(new DataType[0]);
+ this.rowEncoder = new CompactedRowEncoder(fieldDataTypes);
+ } else {
+ // ARROW format accepts a general InternalRow; no per-row encoder needed.
+ this.rowEncoder = null;
+ }
+ }
+
+ @Override
+ WriteRecord build(MultiTableWriteRecord.Operation operation, InternalRow row) {
+ if (operation == MultiTableWriteRecord.Operation.APPEND) {
+ return buildAppendRecord(row);
+ }
+ throw new IllegalArgumentException(
+ "Operation "
+ + operation
+ + " is not supported for log table "
+ + tablePath
+ + "; please use MultiTableWriteRecord.forAppend(...).");
+ }
+
+ WriteRecord buildAppendRecord(InternalRow row) {
+ PhysicalTablePath physicalPath = getPhysicalPath(row);
+ byte[] bucketKey =
+ logBucketKeyEncoder != null ? logBucketKeyEncoder.encodeKey(row) : null;
+ if (logFormat == LogFormat.INDEXED) {
+ IndexedRow indexedRow =
+ row instanceof IndexedRow ? (IndexedRow) row : (IndexedRow) encodeRow(row);
+ return WriteRecord.forIndexedAppend(tableInfo, physicalPath, indexedRow, bucketKey);
+ } else if (logFormat == LogFormat.COMPACTED) {
+ CompactedRow compactedRow =
+ row instanceof CompactedRow
+ ? (CompactedRow) row
+ : (CompactedRow) encodeRow(row);
+ return WriteRecord.forCompactedAppend(
+ tableInfo, physicalPath, compactedRow, bucketKey);
+ } else {
+ // ARROW format accepts a general InternalRow.
+ return WriteRecord.forArrowAppend(tableInfo, physicalPath, row, bucketKey);
+ }
+ }
+
+ private BinaryRow encodeRow(InternalRow row) {
+ assert rowEncoder != null;
+ rowEncoder.startNewRow();
+ for (int i = 0; i < fieldCount; i++) {
+ rowEncoder.encodeField(i, fieldGetters[i].getFieldOrNull(row));
+ }
+ return rowEncoder.finishRow();
+ }
+ }
+
+ /** Encoding state for a primary-key (KV) table; only supports UPSERT / DELETE. */
+ private static final class PrimaryTableWriteState extends TableWriteState {
+ final KeyEncoder primaryKeyEncoder;
+ final KeyEncoder kvBucketKeyEncoder;
+ final KvFormat kvFormat;
+ final WriteFormat kvWriteFormat;
+ final RowEncoder kvRowEncoder;
+
+ PrimaryTableWriteState(TablePath tablePath, TableInfo tableInfo) {
+ super(tablePath, tableInfo);
+ RowType rowType = tableInfo.getRowType();
+ this.primaryKeyEncoder =
+ KeyEncoder.ofPrimaryKeyEncoder(
+ rowType,
+ tableInfo.getPhysicalPrimaryKeys(),
+ tableInfo.getTableConfig(),
+ tableInfo.isDefaultBucketKey());
+ this.kvBucketKeyEncoder =
+ tableInfo.isDefaultBucketKey()
+ ? this.primaryKeyEncoder
+ : KeyEncoder.ofBucketKeyEncoder(
+ rowType,
+ tableInfo.getBucketKeys(),
+ tableInfo.getTableConfig().getDataLakeFormat().orElse(null));
+ this.kvFormat = tableInfo.getTableConfig().getKvFormat();
+ this.kvWriteFormat = WriteFormat.fromKvFormat(this.kvFormat);
+ this.kvRowEncoder = RowEncoder.create(this.kvFormat, rowType);
+ }
+
+ @Override
+ WriteRecord build(MultiTableWriteRecord.Operation operation, InternalRow row) {
+ if (operation == MultiTableWriteRecord.Operation.UPSERT) {
+ return buildUpsertRecord(row);
+ }
+ if (operation == MultiTableWriteRecord.Operation.DELETE) {
+ return buildDeleteRecord(row);
+ }
+ throw new IllegalArgumentException(
+ "Operation "
+ + operation
+ + " is not supported for primary-key table "
+ + tablePath
+ + "; please use MultiTableWriteRecord.forUpsert(...) or "
+ + "MultiTableWriteRecord.forDelete(...).");
+ }
+
+ WriteRecord buildUpsertRecord(InternalRow row) {
+ byte[] key = primaryKeyEncoder.encodeKey(row);
+ return WriteRecord.forUpsert(
+ tableInfo,
+ getPhysicalPath(row),
+ encodeKvRow(row),
+ key,
+ encodeBucketKey(row, key),
+ kvWriteFormat,
+ null /*targetColumns*/,
+ MergeMode.DEFAULT);
+ }
+
+ WriteRecord buildDeleteRecord(InternalRow row) {
+ byte[] key = primaryKeyEncoder.encodeKey(row);
+ return WriteRecord.forDelete(
+ tableInfo,
+ getPhysicalPath(row),
+ key,
+ encodeBucketKey(row, key),
+ kvWriteFormat,
+ null /*targetColumns*/,
+ MergeMode.DEFAULT);
+ }
+
+ /**
+ * When the bucket key equals the primary key (default-bucket-key tables), reuse the already
+ * encoded primary key bytes to avoid a second encode pass.
+ */
+ private byte[] encodeBucketKey(InternalRow row, byte[] primaryKey) {
+ return kvBucketKeyEncoder == primaryKeyEncoder
+ ? primaryKey
+ : kvBucketKeyEncoder.encodeKey(row);
+ }
+
+ private BinaryRow encodeKvRow(InternalRow row) {
+ if (kvFormat == KvFormat.INDEXED && row instanceof IndexedRow) {
+ return (IndexedRow) row;
+ } else if (kvFormat == KvFormat.COMPACTED && row instanceof CompactedRow) {
+ return (CompactedRow) row;
+ }
+ kvRowEncoder.startNewRow();
+ for (int i = 0; i < fieldCount; i++) {
+ kvRowEncoder.encodeField(i, fieldGetters[i].getFieldOrNull(row));
+ }
+ return kvRowEncoder.finishRow();
+ }
+ }
+
+ // ----- test hooks -----
+
+ /** For tests: number of resolved tables (lazy registration count). */
+ int resolvedTableCount() {
+ return states.size();
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/PendingFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/WriteResult.java
similarity index 55%
copy from fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/PendingFetch.java
copy to fluss-client/src/main/java/org/apache/fluss/client/table/writer/WriteResult.java
index c3fa29936..5fa5579e1 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/PendingFetch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/WriteResult.java
@@ -15,19 +15,23 @@
* limitations under the License.
*/
-package org.apache.fluss.client.table.scanner.log;
+package org.apache.fluss.client.table.writer;
-import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.annotation.PublicEvolving;
/**
- * A pending fetch is a fetch that requested to download remote log segment file to local but not
- * yet been completed.
+ * Unified result of a {@link MultiTableWriter#write} call. Exposes the resolved table identity, the
+ * change type that was applied, and the bucket the record was routed to.
+ *
+ * <p>Avoids fan-out into {@link AppendResult} / {@link UpsertResult} / {@link DeleteResult} since
+ * the caller of {@code write(...)} does not know the table kind up-front.
+ *
+ * @since 0.7
*/
-interface PendingFetch {
-
- TableBucket tableBucket();
+@PublicEvolving
+public final class WriteResult {
- boolean isCompleted();
+ // currently, it's an empty class, it will be a compatible evolution if it extends
+ // to have offset, timestamp, etc. in the future
- CompletedFetch toCompletedFetch();
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java
index fcb71764a..104f8f29e 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java
@@ -47,18 +47,20 @@ abstract class AbstractRowLogWriteBatch<R> extends WriteBatch {
long tableId,
int bucketId,
PhysicalTablePath physicalTablePath,
+ int schemaId,
+ WriteFormat writeFormat,
long createdMs,
AbstractPagedOutputView outputView,
MemoryLogRecordsRowBuilder<R> recordsBuilder,
String buildErrorMessage) {
- super(tableId, bucketId, physicalTablePath, createdMs);
+ super(tableId, bucketId, physicalTablePath, schemaId, writeFormat, createdMs);
this.outputView = outputView;
this.recordsBuilder = recordsBuilder;
this.buildErrorMessage = buildErrorMessage;
}
@Override
- public boolean tryAppend(WriteRecord writeRecord, WriteCallback callback) throws Exception {
+ boolean tryAppendRecord(WriteRecord writeRecord, WriteCallback callback) throws Exception {
checkNotNull(callback, "write callback must be not null");
InternalRow rowObj = writeRecord.getRow();
checkNotNull(rowObj, "row must not be null for log record");
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java
index a800666bc..a6894b98b 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java
@@ -60,7 +60,7 @@ public class ArrowLogWriteBatch extends WriteBatch {
AbstractPagedOutputView outputView,
long createdMs,
@Nullable LogRecordBatchStatisticsCollector statisticsCollector) {
- super(tableId, bucketId, physicalTablePath, createdMs);
+ super(tableId, bucketId, physicalTablePath, schemaId, WriteFormat.ARROW_LOG, createdMs);
this.outputView = outputView;
this.recordsBuilder =
MemoryLogRecordsArrowBuilder.builder(
@@ -73,7 +73,7 @@ public class ArrowLogWriteBatch extends WriteBatch {
}
@Override
- public boolean tryAppend(WriteRecord writeRecord, WriteCallback callback) throws Exception {
+ boolean tryAppendRecord(WriteRecord writeRecord, WriteCallback callback) throws Exception {
InternalRow row = writeRecord.getRow();
checkArgument(
writeRecord.getTargetColumns() == null,
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java
index 9b7cee9a7..81bedff0f 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java
@@ -51,6 +51,8 @@ public final class CompactedLogWriteBatch extends AbstractRowLogWriteBatch<Compa
tableId,
bucketId,
physicalTablePath,
+ schemaId,
+ WriteFormat.COMPACTED_LOG,
createdMs,
outputView,
MemoryLogRecordsCompactedBuilder.builder(schemaId, writeLimit, outputView, true),
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java
index dc408a384..2bb496cbf 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java
@@ -50,6 +50,8 @@ public final class IndexedLogWriteBatch extends AbstractRowLogWriteBatch<Indexed
tableId,
bucketId,
physicalTablePath,
+ schemaId,
+ WriteFormat.INDEXED_LOG,
createdMs,
outputView,
MemoryLogRecordsIndexedBuilder.builder(schemaId, writeLimit, outputView, true),
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java
index 814df03f7..4ca01e133 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java
@@ -51,7 +51,6 @@ public class KvWriteBatch extends WriteBatch {
private final AbstractPagedOutputView outputView;
private final KvRecordBatchBuilder recordsBuilder;
private final @Nullable int[] targetColumns;
- private final int schemaId;
private final MergeMode mergeMode;
public KvWriteBatch(
@@ -65,12 +64,17 @@ public class KvWriteBatch extends WriteBatch {
@Nullable int[] targetColumns,
MergeMode mergeMode,
long createdMs) {
- super(tableId, bucketId, physicalTablePath, createdMs);
+ super(
+ tableId,
+ bucketId,
+ physicalTablePath,
+ schemaId,
+ WriteFormat.fromKvFormat(kvFormat),
+ createdMs);
this.outputView = outputView;
this.recordsBuilder =
KvRecordBatchBuilder.builder(schemaId, writeLimit, outputView, kvFormat);
this.targetColumns = targetColumns;
- this.schemaId = schemaId;
this.mergeMode = mergeMode;
}
@@ -80,14 +84,7 @@ public class KvWriteBatch extends WriteBatch {
}
@Override
- public boolean tryAppend(WriteRecord writeRecord, WriteCallback callback) throws Exception {
- if (schemaId != writeRecord.getSchemaId()) {
- throw new IllegalStateException(
- String.format(
- "schema id %d of the write record to append is not the same as the current schema id %d in the batch.",
- writeRecord.getSchemaId(), schemaId));
- }
-
+ boolean tryAppendRecord(WriteRecord writeRecord, WriteCallback callback) throws Exception {
// currently, we throw exception directly when the target columns of the write record is
// not the same as the current target columns in the batch.
// this should be quite fast as they should be the same objects.
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
index 7a0e0f5dd..8013f2ac8 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
@@ -695,6 +695,9 @@ public final class RecordAccumulator {
if (last != null) {
boolean success = last.tryAppend(writeRecord, callback);
if (!success) {
+ // The last batch is either full/closed or belongs to a different table/write
+ // format/schema. Close it so the incoming record rolls over to a compatible new
+ // batch.
// TODO For ArrowLogWriteBatch, close here is a heavy operation (including build
// logic), we need to avoid do that in an lock which locked dq. However, why we not
// remove build logic out of close for ArrowLogWriteBatch is that we want to release
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
index a8e62cd37..e16754763 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
@@ -91,6 +91,9 @@ public class Sender implements Runnable {
/** true when the caller wants to ignore all unsent/inflight messages and force close. */
private volatile boolean forceClose;
+ private final Object wakeupLock = new Object();
+ private boolean wakeup;
+
/**
* A per-bucket queue of batches ordered by creation time for tracking the in-flight batches.
*/
@@ -236,8 +239,7 @@ public class Sender implements Runnable {
// TODO The method sendWriteData is in a busy loop. If there is no data continuously, it
// will cause the CPU to be occupied.
// In the future, we need to introduce delay logic to deal with it.
- // TODO: condition waiter
- Thread.sleep(readyCheckResult.nextReadyCheckDelayMs);
+ awaitNextReadyCheck(readyCheckResult.nextReadyCheckDelayMs);
}
// get the list of batches prepare to send.
@@ -644,6 +646,27 @@ public class Sender implements Runnable {
// breaking from the sender loop. Otherwise, we may miss some callbacks when shutting down.
accumulator.close();
running = false;
+ wakeup();
+ }
+
+ /** Wake up the sender if it is waiting for the next ready check. */
+ public void wakeup() {
+ synchronized (wakeupLock) {
+ wakeup = true;
+ wakeupLock.notifyAll();
+ }
+ }
+
+ private void awaitNextReadyCheck(long delayMs) throws InterruptedException {
+ if (delayMs <= 0) {
+ return;
+ }
+ synchronized (wakeupLock) {
+ if (!wakeup) {
+ wakeupLock.wait(delayMs);
+ }
+ wakeup = false;
+ }
}
/**
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java
index 9702bb543..754f9caa7 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java
@@ -47,6 +47,8 @@ public abstract class WriteBatch {
private final PhysicalTablePath physicalTablePath;
private final RequestFuture requestFuture;
private final long tableId;
+ private final int schemaId;
+ private final WriteFormat writeFormat;
private final int bucketId;
protected final List<WriteCallback> callbacks = new ArrayList<>();
@@ -57,10 +59,17 @@ public abstract class WriteBatch {
private long drainedMs;
public WriteBatch(
- long tableId, int bucketId, PhysicalTablePath physicalTablePath, long createdMs) {
+ long tableId,
+ int bucketId,
+ PhysicalTablePath physicalTablePath,
+ int schemaId,
+ WriteFormat writeFormat,
+ long createdMs) {
this.physicalTablePath = physicalTablePath;
this.createdMs = createdMs;
this.tableId = tableId;
+ this.schemaId = schemaId;
+ this.writeFormat = checkNotNull(writeFormat, "write format must be not null");
this.bucketId = bucketId;
this.requestFuture = new RequestFuture();
this.recordCount = 0;
@@ -79,10 +88,18 @@ public abstract class WriteBatch {
*
* @param writeRecord the record to write
* @param callback the callback to send back to writer
- * @return true if append success, false if the batch is full.
+ * @return true if append success, false if the batch is full or belongs to a different table,
+ * write format or schema.
*/
- public abstract boolean tryAppend(WriteRecord writeRecord, WriteCallback callback)
- throws Exception;
+ public final boolean tryAppend(WriteRecord writeRecord, WriteCallback callback)
+ throws Exception {
+ checkNotNull(writeRecord, "write record must be not null");
+ checkNotNull(callback, "write callback must be not null");
+ if (!canAppend(writeRecord)) {
+ return false;
+ }
+ return tryAppendRecord(writeRecord, callback);
+ }
/**
* Gets the memory segment bytes view of the batch. This includes the latest updated {@link
@@ -164,6 +181,14 @@ public abstract class WriteBatch {
return tableId;
}
+ int schemaId() {
+ return schemaId;
+ }
+
+ WriteFormat writeFormat() {
+ return writeFormat;
+ }
+
public PhysicalTablePath physicalTablePath() {
return physicalTablePath;
}
@@ -243,6 +268,18 @@ public abstract class WriteBatch {
return finalState.get() != null;
}
+ abstract boolean tryAppendRecord(WriteRecord writeRecord, WriteCallback callback)
+ throws Exception;
+
+ private boolean canAppend(WriteRecord writeRecord) {
+ // a schema id mismatch is a batch-compatibility boundary like table id and write format:
+ // returning false makes the accumulator close this batch and roll a new one carrying the
+ // record's schema id, so a single batch never mixes schemas.
+ return tableId == writeRecord.getTableInfo().getTableId()
+ && writeFormat == writeRecord.getWriteFormat()
+ && schemaId == writeRecord.getSchemaId();
+ }
+
/**
* Finalize the state of a batch. Final state, once set, is immutable. This function may be
* called twice when an inflight batch expires before a response from the tablet server is
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java
index ff8b9232f..9d1a76c75 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java
@@ -92,6 +92,14 @@ public class WriterClient {
MetadataUpdater metadataUpdater,
ClientMetricGroup clientMetricGroup,
Admin admin) {
+ this(conf, metadataUpdater, new WriterMetricGroup(clientMetricGroup), admin);
+ }
+
+ public WriterClient(
+ Configuration conf,
+ MetadataUpdater metadataUpdater,
+ WriterMetricGroup writerMetricGroup,
+ Admin admin) {
int maxRequestSizeLocal = -1;
IdempotenceManager idempotenceManagerLocal = null;
try {
@@ -100,9 +108,9 @@ public class WriterClient {
maxRequestSizeLocal =
(int) conf.get(ConfigOptions.CLIENT_WRITER_REQUEST_MAX_SIZE).getBytes();
this.maxRequestSize = maxRequestSizeLocal;
+ this.writerMetricGroup = writerMetricGroup;
idempotenceManagerLocal = buildIdempotenceManager();
this.idempotenceManager = idempotenceManagerLocal;
- this.writerMetricGroup = new WriterMetricGroup(clientMetricGroup);
short acks = configureAcks(idempotenceManager.idempotenceEnabled());
int retries = configureRetries(idempotenceManager.idempotenceEnabled());
@@ -156,6 +164,7 @@ public class WriterClient {
LOG.trace("Flushing accumulated records in writer.");
long start = System.currentTimeMillis();
accumulator.beginFlush();
+ sender.wakeup();
try {
accumulator.awaitFlushCompletion();
} catch (InterruptedException e) {
@@ -212,7 +221,7 @@ public class WriterClient {
"Waking up the sender since table {} bucket {} is either full or getting a new batch",
record.getPhysicalTablePath(),
bucketId);
- // TODO add the wakeup logic refer to Kafka.
+ sender.wakeup();
}
} catch (Exception e) {
throw new FlussRuntimeException(
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/metrics/WriterMetricGroupTest.java b/fluss-client/src/test/java/org/apache/fluss/client/metrics/WriterMetricGroupTest.java
new file mode 100644
index 000000000..fc1c3d4ad
--- /dev/null
+++ b/fluss-client/src/test/java/org/apache/fluss/client/metrics/WriterMetricGroupTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.metrics;
+
+import org.apache.fluss.config.Configuration;
+import org.apache.fluss.metrics.CharacterFilter;
+import org.apache.fluss.metrics.Metric;
+import org.apache.fluss.metrics.groups.MetricGroup;
+import org.apache.fluss.metrics.registry.MetricRegistryImpl;
+import org.apache.fluss.metrics.reporter.MetricReporter;
+import org.apache.fluss.rpc.metrics.ClientMetricGroup;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.TreeMap;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Test for {@link WriterMetricGroup} and {@link MultiTableWriterMetricGroup}. */
+class WriterMetricGroupTest {
+
+ private static final String WRITER_INSTANCE_ID = "writer_instance_id";
+
+ @Test
+ void testSingleTableWriters() throws Exception {
+ try (MetricTestContext context = new MetricTestContext()) {
+ WriterMetricGroup writer = new WriterMetricGroup(context.clientMetricGroup);
+
+ assertThat(logicalScope(writer)).isEqualTo("client.writer");
+ assertThat(writer.getAllVariables()).containsOnlyKeys("client_id");
+ }
+ }
+
+ @Test
+ void testMultiTableWriter() throws Exception {
+ try (MetricTestContext context = new MetricTestContext()) {
+ WriterMetricGroup sharedWriter = new WriterMetricGroup(context.clientMetricGroup);
+ WriterMetricGroup multiWriter1 =
+ new MultiTableWriterMetricGroup(context.clientMetricGroup);
+ WriterMetricGroup multiWriter2 =
+ new MultiTableWriterMetricGroup(context.clientMetricGroup);
+ String instanceId1 = multiWriter1.getAllVariables().get(WRITER_INSTANCE_ID);
+ String instanceId2 = multiWriter2.getAllVariables().get(WRITER_INSTANCE_ID);
+
+ assertThat(logicalScope(multiWriter1)).isEqualTo("client.multi_table_writer");
+ assertThat(sharedWriter.getAllVariables()).doesNotContainKey(WRITER_INSTANCE_ID);
+ assertThat(instanceId1).isNotNull();
+ assertThat(instanceId2).isNotNull().isNotEqualTo(instanceId1);
+ assertThat(context.reporter.added).doesNotHaveDuplicates();
+
+ multiWriter1.close();
+
+ assertThat(context.reporter.removed).isNotEmpty();
+ assertThat(context.reporter.removed)
+ .allSatisfy(
+ identity ->
+ assertThat(identity)
+ .contains(WRITER_INSTANCE_ID + "=" + instanceId1));
+ }
+ }
+
+ private static String logicalScope(MetricGroup group) {
+ return group.getLogicalScope(CharacterFilter.NO_OP_FILTER, '.');
+ }
+
+ private static final class MetricTestContext implements AutoCloseable {
+ final RecordingReporter reporter = new RecordingReporter();
+ final MetricRegistryImpl registry =
+ new MetricRegistryImpl(Collections.singletonList(reporter));
+ final ClientMetricGroup clientMetricGroup = new ClientMetricGroup(registry, "test-client");
+
+ @Override
+ public void close() throws Exception {
+ registry.closeAsync().get();
+ }
+ }
+
+ /** Records reporter-facing metric identities: scoped metric name plus variables. */
+ private static final class RecordingReporter implements MetricReporter {
+
+ final List<String> added = new ArrayList<>();
+ final List<String> removed = new ArrayList<>();
+
+ @Override
+ public void open(Configuration config) {}
+
+ @Override
+ public void close() {}
+
+ @Override
+ public void notifyOfAddedMetric(Metric metric, String metricName, MetricGroup group) {
+ added.add(identityOf(metricName, group));
+ }
+
+ @Override
+ public void notifyOfRemovedMetric(Metric metric, String metricName, MetricGroup group) {
+ removed.add(identityOf(metricName, group));
+ }
+
+ private static String identityOf(String metricName, MetricGroup group) {
+ return logicalScope(group) + '.' + metricName + new TreeMap<>(group.getAllVariables());
+ }
+ }
+}
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchBufferLifecycleTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchBufferLifecycleTest.java
index 090bc932f..6a0943674 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchBufferLifecycleTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchBufferLifecycleTest.java
@@ -33,6 +33,7 @@ import java.util.Map;
import static org.apache.fluss.record.TestData.DATA1;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
+import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject;
@@ -155,6 +156,7 @@ class DefaultCompletedFetchBufferLifecycleTest {
throws Exception {
return new DefaultCompletedFetch(
tableBucket,
+ DATA1_TABLE_PATH,
new FetchLogResultForBucket(tableBucket, genMemoryLogRecordsByObject(DATA1), 10L),
readContext,
logScannerStatus,
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
index 427f59d4d..f8ccbbae5 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
@@ -36,6 +36,7 @@ import org.apache.fluss.row.GenericMap;
import org.apache.fluss.row.GenericRow;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.rpc.entity.FetchLogResultForBucket;
+import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.ChunkedAllocationManager;
import org.apache.fluss.testutils.InternalRowAssert;
import org.apache.fluss.types.DataTypes;
import org.apache.fluss.types.RowType;
@@ -336,13 +337,16 @@ public class DefaultCompletedFetchTest {
DefaultCompletedFetch defaultCompletedFetch =
new DefaultCompletedFetch(
tb,
+ DATA2_TABLE_PATH,
resultForBucket,
LogRecordReadContext.createReadContext(
tableInfo,
false,
+ LogRecordReadContext.SchemaResolution.DYNAMIC,
null,
new TestingSchemaGetter(
- tableInfo.getSchemaId(), tableInfo.getSchema())),
+ tableInfo.getSchemaId(), tableInfo.getSchema()),
+ new ChunkedAllocationManager.ChunkedFactory()),
logScannerStatus,
true,
fetchOffset,
@@ -391,12 +395,15 @@ public class DefaultCompletedFetchTest {
Projection projection) {
return new DefaultCompletedFetch(
tableBucket,
+ DATA2_TABLE_PATH,
resultForBucket,
LogRecordReadContext.createReadContext(
tableInfo,
false,
+ LogRecordReadContext.SchemaResolution.DYNAMIC,
projection,
- new TestingSchemaGetter(tableInfo.getSchemaId(), tableInfo.getSchema())),
+ new TestingSchemaGetter(tableInfo.getSchemaId(), tableInfo.getSchema()),
+ new ChunkedAllocationManager.ChunkedFactory()),
logScannerStatus,
true,
offset,
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchBufferTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchBufferTest.java
index abfd0f1d1..c65e9c06c 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchBufferTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchBufferTest.java
@@ -41,6 +41,7 @@ import java.util.concurrent.atomic.AtomicReference;
import static org.apache.fluss.record.TestData.DATA1;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
+import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject;
@@ -166,6 +167,41 @@ public class LogFetchBufferTest {
}
}
+ @Test
+ void testRetainAllDiscardsRemovedPendingFetches() throws Exception {
+ AtomicBoolean discarded = new AtomicBoolean(false);
+ PendingFetch pendingFetch =
+ new PendingFetch() {
+ @Override
+ public TableBucket tableBucket() {
+ return tableBucket1;
+ }
+
+ @Override
+ public boolean isCompleted() {
+ return false;
+ }
+
+ @Override
+ public CompletedFetch toCompletedFetch() {
+ throw new AssertionError("Pending fetch should be discarded.");
+ }
+
+ @Override
+ public void discard() {
+ discarded.set(true);
+ }
+ };
+
+ try (LogFetchBuffer logFetchBuffer = new LogFetchBuffer()) {
+ logFetchBuffer.pend(pendingFetch);
+ logFetchBuffer.retainAll(Collections.emptySet());
+
+ assertThat(discarded).isTrue();
+ assertThat(logFetchBuffer.pendedBuckets()).isEmpty();
+ }
+ }
+
@Test
void testWakeup() throws Exception {
try (LogFetchBuffer logFetchBuffer = new LogFetchBuffer()) {
@@ -260,6 +296,7 @@ public class LogFetchBufferTest {
private DefaultCompletedFetch makeCompletedFetch(TableBucket tableBucket) throws Exception {
return new DefaultCompletedFetch(
tableBucket,
+ DATA1_TABLE_PATH,
new FetchLogResultForBucket(tableBucket, genMemoryLogRecordsByObject(DATA1), 10L),
readContext,
logScannerStatus,
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
index 3b4d47fc1..f5752d74d 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
@@ -72,8 +72,7 @@ public class LogFetchCollectorTest {
logScannerStatus.assignScanBuckets(scanBuckets);
logFetchBuffer = new LogFetchBuffer();
logFetchCollector =
- new LogFetchCollector(
- DATA1_TABLE_PATH, logScannerStatus, new Configuration(), metadataUpdater);
+ new LogFetchCollector(logScannerStatus, new Configuration(), metadataUpdater);
readContext =
LogRecordReadContext.createArrowReadContext(
DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, TEST_SCHEMA_GETTER);
@@ -220,7 +219,7 @@ public class LogFetchCollectorTest {
new TestingMetadataUpdater(
Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO));
LogFetchCollector collector =
- new LogFetchCollector(DATA1_TABLE_PATH, logScannerStatus, conf, metadataUpdater);
+ new LogFetchCollector(logScannerStatus, conf, metadataUpdater);
TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0);
FetchLogResultForBucket result =
@@ -247,7 +246,7 @@ public class LogFetchCollectorTest {
new TestingMetadataUpdater(
Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO));
LogFetchCollector collector =
- new LogFetchCollector(DATA1_TABLE_PATH, logScannerStatus, conf, metadataUpdater);
+ new LogFetchCollector(logScannerStatus, conf, metadataUpdater);
TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1);
FetchLogResultForBucket filteredEmpty = new FetchLogResultForBucket(tb, 10L, 20L);
@@ -266,7 +265,14 @@ public class LogFetchCollectorTest {
private DefaultCompletedFetch makeCompletedFetch(
TableBucket tableBucket, FetchLogResultForBucket resultForBucket, long offset) {
return new DefaultCompletedFetch(
- tableBucket, resultForBucket, readContext, logScannerStatus, true, offset, null);
+ tableBucket,
+ DATA1_TABLE_PATH,
+ resultForBucket,
+ readContext,
+ logScannerStatus,
+ true,
+ offset,
+ null);
}
@Test
@@ -317,7 +323,7 @@ public class LogFetchCollectorTest {
new TestingMetadataUpdater(
Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO));
LogFetchCollector collector =
- new LogFetchCollector(DATA1_TABLE_PATH, logScannerStatus, conf, metadataUpdater);
+ new LogFetchCollector(logScannerStatus, conf, metadataUpdater);
TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0);
FetchLogResultForBucket result = new FetchLogResultForBucket(tb, records, 4L);
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherFilterITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherFilterITCase.java
index ac64fbdbd..df8e090db 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherFilterITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherFilterITCase.java
@@ -24,9 +24,11 @@ import org.apache.fluss.client.table.scanner.RemoteFileDownloader;
import org.apache.fluss.client.table.scanner.ScanRecord;
import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.predicate.Predicate;
import org.apache.fluss.predicate.PredicateBuilder;
import org.apache.fluss.record.LogRecordBatchStatisticsTestUtils;
+import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.record.MemoryLogRecords;
import org.apache.fluss.rpc.RpcClient;
import org.apache.fluss.rpc.gateway.TabletServerGateway;
@@ -46,8 +48,8 @@ import java.util.stream.Collectors;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
import static org.apache.fluss.record.TestData.DATA1_TABLE_DESCRIPTOR;
-import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO;
import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH;
+import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newProduceLogRequest;
@@ -109,15 +111,26 @@ public class LogFetcherFilterITCase extends ClientToServerITCaseBase {
TestingScannerMetricGroup scannerMetricGroup = TestingScannerMetricGroup.newInstance();
logFetcher =
new LogFetcher(
- DATA1_TABLE_INFO,
- null, // projection
- recordBatchFilter, // recordBatchFilter
+ "default-scanner",
logScannerStatus,
clientConf,
metadataUpdater,
scannerMetricGroup,
new RemoteFileDownloader(1),
- TEST_SCHEMA_GETTER);
+ LogRecordReadContext.SchemaResolution.TARGET);
+ logFetcher.registerTable(
+ new TableScanSpec(
+ TableInfo.of(
+ DATA1_TABLE_PATH,
+ tableId,
+ 1,
+ DATA1_TABLE_DESCRIPTOR,
+ DEFAULT_REMOTE_DATA_DIR,
+ System.currentTimeMillis(),
+ System.currentTimeMillis()),
+ null,
+ recordBatchFilter),
+ TEST_SCHEMA_GETTER);
}
@AfterEach
@@ -430,16 +443,29 @@ public class LogFetcherFilterITCase extends ClientToServerITCaseBase {
Predicate recordBatchFilter = builder.greaterThan(0, 5); // a > 5
TestingScannerMetricGroup scannerMetricGroup = TestingScannerMetricGroup.newInstance();
- return new LogFetcher(
- DATA1_TABLE_INFO,
- null,
- recordBatchFilter,
- scannerStatus,
- clientConf,
- metadataUpdater,
- scannerMetricGroup,
- new RemoteFileDownloader(1),
+ LogFetcher logFetcher =
+ new LogFetcher(
+ "default fetcher",
+ scannerStatus,
+ clientConf,
+ metadataUpdater,
+ scannerMetricGroup,
+ new RemoteFileDownloader(1),
+ LogRecordReadContext.SchemaResolution.TARGET);
+ logFetcher.registerTable(
+ new TableScanSpec(
+ TableInfo.of(
+ DATA1_TABLE_PATH,
+ tableId,
+ 1,
+ DATA1_TABLE_DESCRIPTOR,
+ DEFAULT_REMOTE_DATA_DIR,
+ System.currentTimeMillis(),
+ System.currentTimeMillis()),
+ null,
+ recordBatchFilter),
TEST_SCHEMA_GETTER);
+ return logFetcher;
}
private void addRecordsToBucket(TableBucket tableBucket, MemoryLogRecords logRecords)
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherITCase.java
index 147be7922..d45725776 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherITCase.java
@@ -30,6 +30,7 @@ import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TableInfo;
+import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.record.MemoryLogRecords;
import org.apache.fluss.row.GenericRow;
import org.apache.fluss.rpc.RpcClient;
@@ -62,6 +63,7 @@ import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH;
import static org.apache.fluss.record.TestData.DATA2;
import static org.apache.fluss.record.TestData.DATA2_ROW_TYPE;
import static org.apache.fluss.record.TestData.DATA2_SCHEMA;
+import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR;
import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newProduceLogRequest;
import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject;
import static org.apache.fluss.testutils.common.CommonTestUtils.retry;
@@ -71,6 +73,7 @@ import static org.assertj.core.api.Assertions.assertThat;
public class LogFetcherITCase extends ClientToServerITCaseBase {
private LogFetcher logFetcher;
private long tableId;
+ private TableInfo tableInfo;
private final int bucketId0 = 0;
private final int bucketId1 = 1;
private LogScannerStatus logScannerStatus;
@@ -101,15 +104,23 @@ public class LogFetcherITCase extends ClientToServerITCaseBase {
new ClientSchemaGetter(DATA1_TABLE_PATH, new SchemaInfo(DATA1_SCHEMA, 1), admin);
logFetcher =
new LogFetcher(
- DATA1_TABLE_INFO,
- null,
- null,
+ "default-scanner",
logScannerStatus,
clientConf,
metadataUpdater,
TestingScannerMetricGroup.newInstance(),
new RemoteFileDownloader(1),
- clientSchemaGetter);
+ LogRecordReadContext.SchemaResolution.TARGET);
+ tableInfo =
+ TableInfo.of(
+ DATA1_TABLE_PATH,
+ tableId,
+ 1,
+ DATA1_TABLE_DESCRIPTOR,
+ DEFAULT_REMOTE_DATA_DIR,
+ System.currentTimeMillis(),
+ System.currentTimeMillis());
+ logFetcher.registerTable(new TableScanSpec(tableInfo, null, null), clientSchemaGetter);
}
@Test
@@ -166,6 +177,15 @@ public class LogFetcherITCase extends ClientToServerITCaseBase {
logScannerStatus.assignScanBuckets(Collections.singletonMap(tb0, 0L));
LogFetcher newSchemaLogFetcher =
new LogFetcher(
+ "default-scanner",
+ logScannerStatus,
+ clientConf,
+ metadataUpdater,
+ TestingScannerMetricGroup.newInstance(),
+ new RemoteFileDownloader(1),
+ LogRecordReadContext.SchemaResolution.TARGET);
+ newSchemaLogFetcher.registerTable(
+ new TableScanSpec(
new TableInfo(
DATA1_TABLE_INFO.getTablePath(),
tableId,
@@ -181,13 +201,8 @@ public class LogFetcherITCase extends ClientToServerITCaseBase {
DATA1_TABLE_INFO.getCreatedTime(),
DATA1_TABLE_INFO.getModifiedTime()),
null,
- null,
- logScannerStatus,
- clientConf,
- metadataUpdater,
- TestingScannerMetricGroup.newInstance(),
- new RemoteFileDownloader(1),
- clientSchemaGetter);
+ null),
+ clientSchemaGetter);
newSchemaLogFetcher.sendFetches();
// The fetcher is async to fetch data, so we need to wait the result write to the
// logFetchBuffer.
@@ -279,16 +294,15 @@ public class LogFetcherITCase extends ClientToServerITCaseBase {
LogFetcher logFetcher =
new LogFetcher(
- DATA1_TABLE_INFO,
- null,
- null,
+ DATA1_TABLE_PATH.toString(),
logScannerStatus,
clientConf,
metadataUpdater,
TestingScannerMetricGroup.newInstance(),
new RemoteFileDownloader(1),
- clientSchemaGetter);
+ LogRecordReadContext.SchemaResolution.TARGET);
+ logFetcher.registerTable(new TableScanSpec(tableInfo, null, null), clientSchemaGetter);
// send fetches to fetch data, should have no available fetch.
logFetcher.sendFetches();
assertThat(logFetcher.hasAvailableFetches()).isFalse();
@@ -320,15 +334,14 @@ public class LogFetcherITCase extends ClientToServerITCaseBase {
new FlussAdmin(FLUSS_CLUSTER_EXTENSION.getRpcClient(), metadataUpdater1));
logFetcher =
new LogFetcher(
- DATA1_TABLE_INFO,
- null,
- null,
+ DATA1_TABLE_PATH.toString(),
logScannerStatus,
clientConf,
metadataUpdater1,
TestingScannerMetricGroup.newInstance(),
new RemoteFileDownloader(1),
- clientSchemaGetter);
+ LogRecordReadContext.SchemaResolution.TARGET);
+ logFetcher.registerTable(new TableScanSpec(tableInfo, null, null), clientSchemaGetter);
ExecutorService executor = Executors.newSingleThreadExecutor();
Future<?> future =
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
index 0a96b840d..564cbb06e 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
@@ -29,6 +29,7 @@ import org.apache.fluss.exception.NotLeaderOrFollowerException;
import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.rpc.entity.FetchLogResultForBucket;
import org.apache.fluss.rpc.messages.FetchLogRequest;
import org.apache.fluss.rpc.messages.FetchLogResponse;
@@ -45,6 +46,7 @@ import org.junit.jupiter.api.Test;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
@@ -81,25 +83,30 @@ public class LogFetcherTest {
}
private LogFetcher createLogFetcher(Configuration conf) {
- ClientSchemaGetter clientSchemaGetter =
- new TestingClientSchemaGetter(
- DATA1_TABLE_PATH, new SchemaInfo(DATA1_SCHEMA, 0), metadataUpdater, conf);
LogScannerStatus logScannerStatus = initializeLogScannerStatus();
- return new LogFetcher(
- DATA1_TABLE_INFO,
- null,
- null,
- logScannerStatus,
- conf,
- metadataUpdater,
- TestingScannerMetricGroup.newInstance(),
- new RemoteFileDownloader(1),
- clientSchemaGetter);
+ LogFetcher fetcher =
+ new LogFetcher(
+ "default-fetcher",
+ logScannerStatus,
+ conf,
+ metadataUpdater,
+ TestingScannerMetricGroup.newInstance(),
+ new RemoteFileDownloader(1),
+ LogRecordReadContext.SchemaResolution.TARGET);
+ fetcher.registerTable(
+ new TableScanSpec(DATA1_TABLE_INFO, null, null), createSchemaGetter(conf));
+ return fetcher;
+ }
+
+ private ClientSchemaGetter createSchemaGetter(Configuration conf) {
+ return new TestingClientSchemaGetter(
+ DATA1_TABLE_PATH, new SchemaInfo(DATA1_SCHEMA, 0), metadataUpdater, conf);
}
@Test
void sendFetchRequestWithNotLeaderOrFollowerException() {
- Map<Integer, FetchLogRequest> requestMap = logFetcher.prepareFetchLogRequests();
+ List<TableBucket> fetchable = Collections.singletonList(tb1);
+ Map<Integer, FetchLogRequest> requestMap = logFetcher.prepareFetchLogRequests(fetchable);
Set<Integer> serverSet = requestMap.keySet();
assertThat(serverSet).containsExactlyInAnyOrder(1);
@@ -120,9 +127,48 @@ public class LogFetcherTest {
assertThat(metadataUpdater.getBucketLocation(tb1)).isNotPresent();
}
+ @Test
+ void testUnregisterTableDiscardsBufferedFetches() {
+ Map<Integer, FetchLogRequest> requestMap =
+ logFetcher.prepareFetchLogRequests(Collections.singletonList(tb1));
+ logFetcher.sendFetchRequest(1, requestMap.get(1));
+ assertThat(logFetcher.getCompletedFetchesSize()).isEqualTo(1);
+
+ logFetcher.getLogScannerStatus().unassignScanBuckets(Collections.singletonList(tb1));
+ logFetcher.unregisterTable(DATA1_TABLE_ID);
+
+ assertThat(logFetcher.getCompletedFetchesSize()).isZero();
+ assertThat(logFetcher.getRegisteredTableCount()).isZero();
+ }
+
+ @Test
+ void testDiscardStaleResponseAfterTableReregistered() {
+ IOUtils.closeQuietly(logFetcher);
+ DelayedTabletServerGateway delayedGateway = new DelayedTabletServerGateway();
+ metadataUpdater = initializeMetadataUpdater(delayedGateway);
+ Configuration conf = new Configuration();
+ logFetcher = createLogFetcher(conf);
+
+ Map<Integer, FetchLogRequest> requestMap =
+ logFetcher.prepareFetchLogRequests(Collections.singletonList(tb1));
+ logFetcher.sendFetchRequest(1, requestMap.get(1));
+
+ logFetcher.getLogScannerStatus().unassignScanBuckets(Collections.singletonList(tb1));
+ logFetcher.unregisterTable(DATA1_TABLE_ID);
+ logFetcher.registerTable(
+ new TableScanSpec(DATA1_TABLE_INFO, null, null), createSchemaGetter(conf));
+ logFetcher.getLogScannerStatus().assignScanBuckets(Collections.singletonMap(tb1, 0L));
+
+ delayedGateway.completeResponse();
+
+ assertThat(logFetcher.getCompletedFetchesSize()).isZero();
+ assertThat(logFetcher.getRegisteredTableCount()).isEqualTo(1);
+ }
+
@Test
void testPrepareFetchLogRequestWithReadPreference() throws Exception {
- Map<Integer, FetchLogRequest> defaultRequestMap = logFetcher.prepareFetchLogRequests();
+ Map<Integer, FetchLogRequest> defaultRequestMap =
+ logFetcher.prepareFetchLogRequests(Collections.singletonList(tb1));
FetchLogRequest defaultRequest = defaultRequestMap.get(1);
assertThat(defaultRequest.hasReadPreference()).isTrue();
assertThat(defaultRequest.getReadPreference())
@@ -135,7 +181,7 @@ public class LogFetcherTest {
LogFetcher remoteFirstFetcher = createLogFetcher(remoteFirstConf);
try {
Map<Integer, FetchLogRequest> remoteFirstRequestMap =
- remoteFirstFetcher.prepareFetchLogRequests();
+ remoteFirstFetcher.prepareFetchLogRequests(Collections.singletonList(tb1));
FetchLogRequest remoteFirstRequest = remoteFirstRequestMap.get(1);
assertThat(remoteFirstRequest.hasReadPreference()).isTrue();
assertThat(remoteFirstRequest.getReadPreference())
@@ -178,13 +224,32 @@ public class LogFetcherTest {
}
}
+ private static class DelayedTabletServerGateway extends TestingTabletServerGateway {
+ private final CompletableFuture<FetchLogResponse> responseFuture =
+ new CompletableFuture<>();
+ private FetchLogResponse response;
+
+ @Override
+ public CompletableFuture<FetchLogResponse> fetchLog(FetchLogRequest request) {
+ response = super.fetchLog(request).join();
+ return responseFuture;
+ }
+
+ private void completeResponse() {
+ responseFuture.complete(response);
+ }
+ }
+
private TestingMetadataUpdater initializeMetadataUpdater() {
+ return initializeMetadataUpdater(new TestingTabletServerGateway());
+ }
+ private TestingMetadataUpdater initializeMetadataUpdater(TestTabletServerGateway gateway) {
return new TestingMetadataUpdater(
TestingMetadataUpdater.COORDINATOR,
Arrays.asList(NODE1, NODE2, NODE3),
Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO),
- Collections.singletonMap(1, new TestingTabletServerGateway()),
+ Collections.singletonMap(1, gateway),
new Configuration());
}
}
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerITCase.java
index e3f60e2df..7f6700acf 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerITCase.java
@@ -25,9 +25,11 @@ import org.apache.fluss.client.table.writer.UpsertWriter;
import org.apache.fluss.exception.FetchException;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.predicate.PredicateBuilder;
import org.apache.fluss.record.ArrowBatchData;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.row.GenericRow;
@@ -92,6 +94,36 @@ public class LogScannerITCase extends ClientToServerITCaseBase {
}
}
+ @Test
+ void testPollReturnsProgressOnlyRecords() throws Exception {
+ TablePath tablePath = TablePath.of("test_db_1", "test_log_scanner_progress_only");
+ TableDescriptor descriptor =
+ TableDescriptor.builder()
+ .schema(DATA1_SCHEMA)
+ .distributedBy(1)
+ .logFormat(LogFormat.ARROW)
+ .build();
+ long tableId = createTable(tablePath, descriptor, false);
+ waitAllReplicasReady(tableId, 1);
+
+ TableBucket tableBucket = new TableBucket(tableId, 0);
+ try (Table table = conn.getTable(tablePath)) {
+ appendRows(table, 1, 5);
+
+ try (LogScanner logScanner =
+ table.newScan()
+ .filter(
+ new PredicateBuilder(DATA1_SCHEMA.getRowType())
+ .greaterThan(0, 5))
+ .createLogScanner()) {
+ logScanner.subscribeFromBeginning(0);
+
+ ScanRecords scanRecords = pollUntilProgressOnly(logScanner, tableBucket);
+ assertThat(scanRecords.consumedUpToOffset(tableBucket)).isGreaterThan(0L);
+ }
+ }
+ }
+
@Test
void testPollWhileCreateTableNotReady() throws Exception {
// create one table with 30 buckets.
@@ -568,4 +600,33 @@ public class LogScannerITCase extends ClientToServerITCaseBase {
}
assertThat(count).isEqualTo(expectedRecords);
}
+
+ private static ScanRecords pollUntilProgressOnly(
+ LogScanner logScanner, TableBucket tableBucket) {
+ long deadline = System.nanoTime() + Duration.ofSeconds(30).toNanos();
+ ScanRecords scanRecords = ScanRecords.EMPTY;
+ while (System.nanoTime() < deadline) {
+ scanRecords = logScanner.poll(Duration.ofSeconds(1));
+ if (scanRecords.isEmpty()
+ && scanRecords.hasProgress()
+ && scanRecords.buckets().contains(tableBucket)) {
+ assertThat(scanRecords.count()).isEqualTo(0);
+ assertThat(scanRecords.records(tableBucket)).isEmpty();
+ return scanRecords;
+ }
+ }
+ assertThat(scanRecords.hasProgress())
+ .as("Timed out waiting for progress-only poll")
+ .isTrue();
+ return scanRecords;
+ }
+
+ private static void appendRows(Table table, int fromInclusive, int toInclusive)
+ throws Exception {
+ AppendWriter writer = table.newAppend().createWriter();
+ for (int i = fromInclusive; i <= toInclusive; i++) {
+ writer.append(row(i, "value-" + i)).get();
+ }
+ writer.flush();
+ }
}
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/MultiTableLogScannerITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/MultiTableLogScannerITCase.java
new file mode 100644
index 000000000..907c86a74
--- /dev/null
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/MultiTableLogScannerITCase.java
@@ -0,0 +1,793 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.scanner.log;
+
+import org.apache.fluss.client.Connection;
+import org.apache.fluss.client.ConnectionFactory;
+import org.apache.fluss.client.FlussConnection;
+import org.apache.fluss.client.admin.ClientToServerITCaseBase;
+import org.apache.fluss.client.metadata.ClientSchemaGetter;
+import org.apache.fluss.client.metrics.TestingScannerMetricGroup;
+import org.apache.fluss.client.table.MultiTable;
+import org.apache.fluss.client.table.Table;
+import org.apache.fluss.client.table.scanner.MultiTableRecord;
+import org.apache.fluss.client.table.writer.AppendWriter;
+import org.apache.fluss.client.table.writer.MultiTableWriteRecord;
+import org.apache.fluss.client.table.writer.MultiTableWriter;
+import org.apache.fluss.config.AutoPartitionTimeUnit;
+import org.apache.fluss.config.ConfigOptions;
+import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TableChange;
+import org.apache.fluss.metadata.TableDescriptor;
+import org.apache.fluss.metadata.TableInfo;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.predicate.PredicateBuilder;
+import org.apache.fluss.record.ChangeType;
+import org.apache.fluss.record.LogRecordReadContext;
+import org.apache.fluss.row.GenericRow;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.types.DataTypes;
+import org.apache.fluss.types.RowType;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.fluss.testutils.DataTestUtils.row;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** ITCase for {@link MultiTableLogScannerImpl}. */
+public class MultiTableLogScannerITCase extends ClientToServerITCaseBase {
+
+ private static final String TEST_DB = "test_db";
+
+ private static final Schema LOG_SCHEMA =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("name", DataTypes.STRING())
+ .build();
+
+ private static final Schema LOG_SCHEMA_2 =
+ Schema.newBuilder()
+ .column("key", DataTypes.INT())
+ .column("value", DataTypes.STRING())
+ .column("score", DataTypes.BIGINT())
+ .build();
+
+ @AfterEach
+ protected void teardown() throws Exception {
+ if (admin != null) {
+ admin.dropDatabase(TEST_DB, true, true);
+ }
+ super.teardown();
+ }
+
+ @Test
+ void testSingleTableSubscribeAndPoll() throws Exception {
+ TablePath tablePath = TablePath.of(TEST_DB, "multi_scanner_single");
+ TableDescriptor descriptor =
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build();
+ long tableId = createTable(tablePath, descriptor, false);
+ waitAllReplicasReady(tableId, 1);
+
+ int recordCount = 10;
+ List<GenericRow> expectedRows = new ArrayList<>();
+ try (Table table = conn.getTable(tablePath)) {
+ AppendWriter writer = table.newAppend().createWriter();
+ for (int i = 0; i < recordCount; i++) {
+ GenericRow r = row(i, "name-" + i);
+ expectedRows.add(r);
+ writer.append(r).get();
+ }
+ }
+
+ MultiTable multiTable = conn.getMultiTable();
+ try (MultiTableLogScanner scanner = multiTable.newMultiTableScan().createLogScanner()) {
+ scanner.subscribeFromBeginning(tablePath, 0);
+
+ List<GenericRow> actualRows = new ArrayList<>();
+ while (actualRows.size() < recordCount) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ for (MultiTableRecord record : records) {
+ assertThat(record.getTablePath()).isEqualTo(tablePath);
+ assertThat(record.getTableId()).isEqualTo(tableId);
+ assertThat(record.getSchemaId()).isGreaterThanOrEqualTo(0);
+ assertThat(record.getSchema().getRowType()).isEqualTo(LOG_SCHEMA.getRowType());
+ assertThat(record.getChangeType()).isEqualTo(ChangeType.APPEND_ONLY);
+ InternalRow r = record.getRow();
+ actualRows.add(row(r.getInt(0), r.getString(1)));
+ }
+ }
+
+ assertThat(actualRows).hasSize(recordCount);
+ assertThat(actualRows).containsExactlyInAnyOrderElementsOf(expectedRows);
+ }
+ }
+
+ @Test
+ void testPollReturnsProgressOnlyRecords() throws Exception {
+ TablePath tablePath = TablePath.of(TEST_DB, "multi_scanner_progress_only");
+ TableDescriptor descriptor =
+ TableDescriptor.builder()
+ .schema(LOG_SCHEMA)
+ .distributedBy(1)
+ .logFormat(LogFormat.ARROW)
+ .build();
+ long tableId = createTable(tablePath, descriptor, false);
+ waitAllReplicasReady(tableId, 1);
+
+ try (Table table = conn.getTable(tablePath)) {
+ appendRows(table, 1, 5);
+ }
+
+ TableInfo tableInfo = admin.getTableInfo(tablePath).get();
+ TableBucket tableBucket = new TableBucket(tableId, 0);
+ TestingScannerMetricGroup scannerMetricGroup = TestingScannerMetricGroup.newInstance();
+ LogScannerStatus logScannerStatus = new LogScannerStatus();
+ LogFetcher logFetcher =
+ createTestingLogFetcher(
+ "multi-table-progress-only-scanner", logScannerStatus, scannerMetricGroup);
+ logFetcher.registerTable(
+ new TableScanSpec(
+ tableInfo,
+ null,
+ new PredicateBuilder(LOG_SCHEMA.getRowType()).greaterThan(0, 5)),
+ new ClientSchemaGetter(tablePath, tableInfo.getSchemaInfo(), admin));
+
+ try (MultiTableLogScanner scanner =
+ createTestingScanner(
+ "multi-table-progress-only-scanner",
+ logScannerStatus,
+ logFetcher,
+ scannerMetricGroup)) {
+ scanner.subscribeFromBeginning(tablePath, 0);
+ assertThat(logFetcher.getRegisteredTableCount()).isEqualTo(1);
+
+ MultiTableRecords records = pollUntilProgressOnly(scanner, tablePath, tableBucket);
+ assertThat(records.consumedUpToOffset(tablePath, tableBucket)).isGreaterThan(0L);
+
+ scanner.unsubscribe(tablePath, 0);
+ assertThat(logFetcher.getRegisteredTableCount()).isEqualTo(0);
+ }
+ }
+
+ @Test
+ void testWrongOverloadSubscribeDoesNotRegisterTable() throws Exception {
+ TablePath partitionedPath =
+ TablePath.of(TEST_DB, "multi_scanner_wrong_overload_partitioned");
+ Schema partitionedSchema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("pt", DataTypes.STRING())
+ .build();
+ createTable(
+ partitionedPath,
+ TableDescriptor.builder()
+ .schema(partitionedSchema)
+ .distributedBy(1)
+ .partitionedBy("pt")
+ .build(),
+ false);
+
+ TablePath nonPartitionedPath =
+ TablePath.of(TEST_DB, "multi_scanner_wrong_overload_non_partitioned");
+ createTable(
+ nonPartitionedPath,
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build(),
+ true);
+
+ String scannerName = "multi-table-wrong-overload-scanner";
+ TestingScannerMetricGroup scannerMetricGroup = TestingScannerMetricGroup.newInstance();
+ LogScannerStatus logScannerStatus = new LogScannerStatus();
+ LogFetcher logFetcher =
+ createTestingLogFetcher(scannerName, logScannerStatus, scannerMetricGroup);
+
+ try (MultiTableLogScanner scanner =
+ createTestingScanner(
+ scannerName, logScannerStatus, logFetcher, scannerMetricGroup)) {
+ assertThatThrownBy(() -> scanner.subscribe(partitionedPath, 0, 0L))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("is a partitioned table");
+ assertThat(logFetcher.getRegisteredTableCount()).isEqualTo(0);
+
+ assertThatThrownBy(() -> scanner.subscribe(nonPartitionedPath, 1L, 0, 0L))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("is not a partitioned table");
+ assertThat(logFetcher.getRegisteredTableCount()).isEqualTo(0);
+ }
+ }
+
+ @Test
+ void testMultipleTableSubscribeAndPoll() throws Exception {
+ TablePath tablePath1 = TablePath.of(TEST_DB, "multi_scanner_t1");
+ TablePath tablePath2 = TablePath.of(TEST_DB, "multi_scanner_t2");
+ TableDescriptor descriptor1 =
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build();
+ TableDescriptor descriptor2 =
+ TableDescriptor.builder().schema(LOG_SCHEMA_2).distributedBy(1).build();
+
+ long tableId1 = createTable(tablePath1, descriptor1, false);
+ long tableId2 = createTable(tablePath2, descriptor2, true);
+ waitAllReplicasReady(tableId1, 1);
+ waitAllReplicasReady(tableId2, 1);
+
+ // Write data to table 1.
+ int recordCount1 = 5;
+ List<GenericRow> expectedRows1 = new ArrayList<>();
+ try (Table table1 = conn.getTable(tablePath1)) {
+ AppendWriter writer = table1.newAppend().createWriter();
+ for (int i = 0; i < recordCount1; i++) {
+ GenericRow r = row(i, "t1-" + i);
+ expectedRows1.add(r);
+ writer.append(r).get();
+ }
+ }
+
+ // Write data to table 2.
+ int recordCount2 = 5;
+ List<GenericRow> expectedRows2 = new ArrayList<>();
+ try (Table table2 = conn.getTable(tablePath2)) {
+ AppendWriter writer = table2.newAppend().createWriter();
+ for (int i = 0; i < recordCount2; i++) {
+ GenericRow r = row(100 + i, "t2-" + i, (long) (i * 10));
+ expectedRows2.add(r);
+ writer.append(r).get();
+ }
+ }
+
+ int totalRecords = recordCount1 + recordCount2;
+ MultiTable multiTable = conn.getMultiTable();
+ try (MultiTableLogScanner scanner = multiTable.newMultiTableScan().createLogScanner()) {
+ scanner.subscribeFromBeginning(tablePath1, 0);
+ scanner.subscribeFromBeginning(tablePath2, 0);
+
+ Map<TablePath, List<GenericRow>> actualRowsByTable = new HashMap<>();
+ int totalPolled = 0;
+ while (totalPolled < totalRecords) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ for (TablePath tp : records.tablePaths()) {
+ for (MultiTableRecord record : records.records(tp)) {
+ assertThat(record.getTablePath()).isEqualTo(tp);
+ assertThat(record.getSchemaId()).isGreaterThanOrEqualTo(0);
+ List<GenericRow> rows =
+ actualRowsByTable.computeIfAbsent(tp, k -> new ArrayList<>());
+ InternalRow r = record.getRow();
+ if (tp.equals(tablePath1)) {
+ rows.add(row(r.getInt(0), r.getString(1)));
+ } else {
+ rows.add(row(r.getInt(0), r.getString(1), r.getLong(2)));
+ }
+ }
+ }
+ totalPolled += records.count();
+ }
+
+ assertThat(actualRowsByTable).containsKey(tablePath1);
+ assertThat(actualRowsByTable).containsKey(tablePath2);
+ assertThat(actualRowsByTable.get(tablePath1))
+ .containsExactlyInAnyOrderElementsOf(expectedRows1);
+ assertThat(actualRowsByTable.get(tablePath2))
+ .containsExactlyInAnyOrderElementsOf(expectedRows2);
+ }
+ }
+
+ @Test
+ void testUnsubscribe() throws Exception {
+ TablePath tablePath = TablePath.of(TEST_DB, "multi_scanner_unsub");
+ TableDescriptor descriptor =
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(2).build();
+ long tableId = createTable(tablePath, descriptor, false);
+ waitAllReplicasReady(tableId, 2);
+
+ // Write data.
+ int recordCount = 10;
+ try (Table table = conn.getTable(tablePath)) {
+ AppendWriter writer = table.newAppend().createWriter();
+ for (int i = 0; i < recordCount; i++) {
+ writer.append(row(i, "v-" + i)).get();
+ }
+ }
+
+ MultiTable multiTable = conn.getMultiTable();
+ try (MultiTableLogScanner scanner = multiTable.newMultiTableScan().createLogScanner()) {
+ scanner.subscribeFromBeginning(tablePath, 0);
+ scanner.subscribeFromBeginning(tablePath, 1);
+
+ // Poll some data to ensure subscription works.
+ int polled = 0;
+ long deadline = System.currentTimeMillis() + 10_000;
+ while (polled < recordCount && System.currentTimeMillis() < deadline) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ polled += records.count();
+ }
+ assertThat(polled).isEqualTo(recordCount);
+
+ // Unsubscribe bucket 1 and verify it no longer returns records from bucket 1.
+ scanner.unsubscribe(tablePath, 1);
+
+ // Write more data.
+ try (Table table = conn.getTable(tablePath)) {
+ AppendWriter writer = table.newAppend().createWriter();
+ for (int i = 0; i < 10; i++) {
+ writer.append(row(100 + i, "new-" + i)).get();
+ }
+ }
+
+ // Poll again and collect bucket info - should only see bucket 0.
+ int extraPolled = 0;
+ deadline = System.currentTimeMillis() + 5_000;
+ while (System.currentTimeMillis() < deadline) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ for (TableBucket bucket : records.buckets(tablePath)) {
+ assertThat(bucket.getBucket()).isEqualTo(0);
+ assertThat(bucket.getTableId()).isEqualTo(tableId);
+ }
+ extraPolled += records.count();
+ }
+ // We should have polled some records only from bucket 0.
+ assertThat(extraPolled).isGreaterThan(0);
+ }
+ }
+
+ @Test
+ void testPollWithoutSubscription() throws Exception {
+ TablePath tablePath = TablePath.of(TEST_DB, "multi_scanner_no_sub");
+ TableDescriptor descriptor =
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build();
+ createTable(tablePath, descriptor, false);
+
+ MultiTable multiTable = conn.getMultiTable();
+ try (MultiTableLogScanner scanner = multiTable.newMultiTableScan().createLogScanner()) {
+ assertThatThrownBy(() -> scanner.poll(Duration.ofSeconds(1)))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("not subscribed");
+ }
+ }
+
+ @Test
+ void testPollAfterClose() throws Exception {
+ TablePath tablePath = TablePath.of(TEST_DB, "multi_scanner_closed");
+ TableDescriptor descriptor =
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build();
+ createTable(tablePath, descriptor, false);
+
+ MultiTable multiTable = conn.getMultiTable();
+ MultiTableLogScanner scanner = multiTable.newMultiTableScan().createLogScanner();
+ scanner.close();
+ assertThatThrownBy(() -> scanner.poll(Duration.ofSeconds(1)))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("closed");
+ }
+
+ @Test
+ void testPartitionedTableSubscribeAndPoll() throws Exception {
+ // Create a partitioned log table with partition key "pt".
+ TablePath tablePath = TablePath.of(TEST_DB, "multi_scanner_partitioned");
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("name", DataTypes.STRING())
+ .column("pt", DataTypes.STRING())
+ .build();
+ TableDescriptor descriptor =
+ TableDescriptor.builder()
+ .schema(schema)
+ .distributedBy(1)
+ .partitionedBy("pt")
+ .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true)
+ .property(
+ ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT,
+ AutoPartitionTimeUnit.YEAR)
+ .build();
+ long tableId = createTable(tablePath, descriptor, false);
+
+ Map<String, Long> partitionIdByNames =
+ FLUSS_CLUSTER_EXTENSION.waitUntilPartitionAllReady(tablePath);
+ assertThat(partitionIdByNames).isNotEmpty();
+
+ // Write data to each partition.
+ int recordsPerPartition = 5;
+ Map<Long, List<GenericRow>> expectedRowsByPartition = new HashMap<>();
+ try (Table table = conn.getTable(tablePath)) {
+ AppendWriter writer = table.newAppend().createWriter();
+ for (Map.Entry<String, Long> entry : partitionIdByNames.entrySet()) {
+ String partitionName = entry.getKey();
+ long partitionId = entry.getValue();
+ for (int i = 0; i < recordsPerPartition; i++) {
+ GenericRow r = row(i, "p-" + i, partitionName);
+ expectedRowsByPartition
+ .computeIfAbsent(partitionId, k -> new ArrayList<>())
+ .add(r);
+ writer.append(r).get();
+ }
+ }
+ }
+
+ int totalRecords = recordsPerPartition * partitionIdByNames.size();
+ // use a new connection to avoid sharing metadata.
+ try (Connection newConnection = ConnectionFactory.createConnection(clientConf);
+ MultiTableLogScanner scanner =
+ newConnection.getMultiTable().newMultiTableScan().createLogScanner()) {
+ // Subscribe to each partition bucket.
+ for (Long partitionId : partitionIdByNames.values()) {
+ scanner.subscribeFromBeginning(tablePath, partitionId, 0);
+ }
+
+ Map<Long, List<GenericRow>> actualRowsByPartition = new HashMap<>();
+ int totalPolled = 0;
+ while (totalPolled < totalRecords) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ for (TableBucket bucket : records.buckets(tablePath)) {
+ for (MultiTableRecord record : records.records(tablePath, bucket)) {
+ assertThat(record.getTablePath()).isEqualTo(tablePath);
+ assertThat(record.getTableId()).isEqualTo(tableId);
+ assertThat(record.getSchemaId()).isGreaterThanOrEqualTo(0);
+ assertThat(record.getSchema().getRowType()).isEqualTo(schema.getRowType());
+ assertThat(bucket.getPartitionId()).isNotNull();
+ long partitionId = bucket.getPartitionId();
+ InternalRow r = record.getRow();
+ actualRowsByPartition
+ .computeIfAbsent(partitionId, k -> new ArrayList<>())
+ .add(row(r.getInt(0), r.getString(1), r.getString(2)));
+ }
+ }
+ totalPolled += records.count();
+ }
+
+ // Verify we received records for each partition.
+ for (Map.Entry<Long, List<GenericRow>> entry : expectedRowsByPartition.entrySet()) {
+ long partitionId = entry.getKey();
+ assertThat(actualRowsByPartition).containsKey(partitionId);
+ assertThat(actualRowsByPartition.get(partitionId))
+ .containsExactlyInAnyOrderElementsOf(entry.getValue());
+ }
+ }
+ }
+
+ @Test
+ void testPartitionedAndNonPartitionedMixed() throws Exception {
+ // Table 1: partitioned log table.
+ TablePath tablePath1 = TablePath.of(TEST_DB, "multi_scanner_part_mix1");
+ Schema schema1 =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("value", DataTypes.STRING())
+ .column("pt", DataTypes.STRING())
+ .build();
+ TableDescriptor descriptor1 =
+ TableDescriptor.builder()
+ .schema(schema1)
+ .distributedBy(1)
+ .partitionedBy("pt")
+ .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true)
+ .property(
+ ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT,
+ AutoPartitionTimeUnit.YEAR)
+ .build();
+ long tableId1 = createTable(tablePath1, descriptor1, false);
+
+ Map<String, Long> partitionIdByNames =
+ FLUSS_CLUSTER_EXTENSION.waitUntilPartitionAllReady(tablePath1);
+ assertThat(partitionIdByNames).isNotEmpty();
+
+ // Table 2: non-partitioned log table.
+ TablePath tablePath2 = TablePath.of("test_db", "multi_scanner_part_mix2");
+ TableDescriptor descriptor2 =
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build();
+ long tableId2 = createTable(tablePath2, descriptor2, true);
+ waitAllReplicasReady(tableId2, 1);
+
+ // Write data to the partitioned table.
+ int recordsPerPartition = 3;
+ int partitionedRecordCount = recordsPerPartition * partitionIdByNames.size();
+ try (Table table1 = conn.getTable(tablePath1)) {
+ AppendWriter writer = table1.newAppend().createWriter();
+ for (String partitionName : partitionIdByNames.keySet()) {
+ for (int i = 0; i < recordsPerPartition; i++) {
+ writer.append(row(i, "pv-" + i, partitionName)).get();
+ }
+ }
+ }
+
+ // Write data to the non-partitioned table.
+ int nonPartitionedRecordCount = 5;
+ List<GenericRow> expectedNonPartitioned = new ArrayList<>();
+ try (Table table2 = conn.getTable(tablePath2)) {
+ AppendWriter writer = table2.newAppend().createWriter();
+ for (int i = 0; i < nonPartitionedRecordCount; i++) {
+ GenericRow r = row(i, "np-" + i);
+ expectedNonPartitioned.add(r);
+ writer.append(r).get();
+ }
+ }
+
+ int totalRecords = partitionedRecordCount + nonPartitionedRecordCount;
+ MultiTable multiTable = conn.getMultiTable();
+ try (MultiTableLogScanner scanner = multiTable.newMultiTableScan().createLogScanner()) {
+ // Subscribe to partitioned table buckets.
+ for (Long partitionId : partitionIdByNames.values()) {
+ scanner.subscribeFromBeginning(tablePath1, partitionId, 0);
+ }
+ // Subscribe to non-partitioned table bucket.
+ scanner.subscribeFromBeginning(tablePath2, 0);
+
+ List<GenericRow> actualNonPartitioned = new ArrayList<>();
+ int partitionedPolled = 0;
+ int totalPolled = 0;
+ while (totalPolled < totalRecords) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ for (MultiTableRecord record : records) {
+ long tid = record.getTableId();
+ InternalRow r = record.getRow();
+ if (tid == tableId1) {
+ partitionedPolled++;
+ } else {
+ assertThat(tid).isEqualTo(tableId2);
+ actualNonPartitioned.add(row(r.getInt(0), r.getString(1)));
+ }
+ }
+ totalPolled += records.count();
+ }
+
+ assertThat(partitionedPolled).isEqualTo(partitionedRecordCount);
+ assertThat(actualNonPartitioned)
+ .containsExactlyInAnyOrderElementsOf(expectedNonPartitioned);
+ }
+ }
+
+ @Test
+ void testMultipleBucketsOfSameTable() throws Exception {
+ TablePath tablePath = TablePath.of(TEST_DB, "multi_scanner_buckets");
+ int bucketCount = 3;
+ TableDescriptor descriptor =
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(bucketCount).build();
+ long tableId = createTable(tablePath, descriptor, false);
+ waitAllReplicasReady(tableId, bucketCount);
+
+ int recordCount = 30;
+ List<GenericRow> expectedRows = new ArrayList<>();
+ try (Table table = conn.getTable(tablePath)) {
+ AppendWriter writer = table.newAppend().createWriter();
+ for (int i = 0; i < recordCount; i++) {
+ GenericRow r = row(i, "val-" + i);
+ expectedRows.add(r);
+ writer.append(r).get();
+ }
+ }
+
+ MultiTable multiTable = conn.getMultiTable();
+ try (MultiTableLogScanner scanner = multiTable.newMultiTableScan().createLogScanner()) {
+ for (int i = 0; i < bucketCount; i++) {
+ scanner.subscribeFromBeginning(tablePath, i);
+ }
+
+ List<GenericRow> actualRows = new ArrayList<>();
+ while (actualRows.size() < recordCount) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ for (MultiTableRecord record : records) {
+ assertThat(record.getTableId()).isEqualTo(tableId);
+ InternalRow r = record.getRow();
+ actualRows.add(row(r.getInt(0), r.getString(1)));
+ }
+ }
+
+ assertThat(actualRows).hasSize(recordCount);
+ assertThat(actualRows).containsExactlyInAnyOrderElementsOf(expectedRows);
+ }
+ }
+
+ @Test
+ void testReadDataBeforeAndAfterAddColumn() throws Exception {
+ TablePath tablePath = TablePath.of(TEST_DB, "multi_scanner_add_col");
+ TableDescriptor descriptor =
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build();
+ long tableId = createTable(tablePath, descriptor, false);
+ waitAllReplicasReady(tableId, 1);
+
+ // Phase 1: Write data with initial schema (id INT, name STRING).
+ int preAddColumnCount = 5;
+ List<GenericRow> expectedPreRows = new ArrayList<>();
+ try (Table table = conn.getTable(tablePath)) {
+ AppendWriter writer = table.newAppend().createWriter();
+ for (int i = 0; i < preAddColumnCount; i++) {
+ GenericRow r = row(i, "before-" + i);
+ expectedPreRows.add(r);
+ writer.append(r).get();
+ }
+ }
+
+ // Add a new column "score BIGINT" to the table.
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "score",
+ DataTypes.BIGINT(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+
+ // Phase 2: Write data with new schema (id INT, name STRING, score BIGINT).
+ int postAddColumnCount = 5;
+ List<GenericRow> expectedPostRows = new ArrayList<>();
+ try (Table table = conn.getTable(tablePath)) {
+ AppendWriter writer = table.newAppend().createWriter();
+ for (int i = 0; i < postAddColumnCount; i++) {
+ GenericRow r = row(100 + i, "after-" + i, (long) (i * 10));
+ expectedPostRows.add(r);
+ writer.append(r).get();
+ }
+ }
+
+ // Subscribe from the beginning and read all records.
+ int totalRecords = preAddColumnCount + postAddColumnCount;
+ MultiTable multiTable = conn.getMultiTable();
+ try (MultiTableLogScanner scanner = multiTable.newMultiTableScan().createLogScanner()) {
+ scanner.subscribeFromBeginning(tablePath, 0);
+
+ List<GenericRow> actualPreRows = new ArrayList<>();
+ List<GenericRow> actualPostRows = new ArrayList<>();
+ int totalPolled = 0;
+ while (totalPolled < totalRecords) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ for (MultiTableRecord record : records) {
+ assertThat(record.getTableId()).isEqualTo(tableId);
+ InternalRow r = record.getRow();
+ RowType rowType = record.getSchema().getRowType();
+ if (rowType.getFieldCount() == 2) {
+ actualPreRows.add(row(r.getInt(0), r.getString(1)));
+ } else {
+ assertThat(rowType.getFieldCount()).isEqualTo(3);
+ actualPostRows.add(row(r.getInt(0), r.getString(1), r.getLong(2)));
+ }
+ totalPolled++;
+ }
+ }
+
+ assertThat(actualPreRows).containsExactlyInAnyOrderElementsOf(expectedPreRows);
+ assertThat(actualPostRows).containsExactlyInAnyOrderElementsOf(expectedPostRows);
+ }
+ }
+
+ @Test
+ void testSubscribeAfterDropRecreateFailsThenRecovers() throws Exception {
+ // A table that is dropped and recreated under the same TablePath gets a new tableId. A
+ // scanner that already subscribed the original table caches the old tableId; re-subscribing
+ // the same path must detect the tableId change (once the shared connection metadata learns
+ // the new id) and fail fast. Recovery: unsubscribe all stale buckets (which evicts the
+ // cached registration) and subscribe again, which re-resolves the recreated table.
+ TablePath tablePath = TablePath.of(TEST_DB, "multi_scanner_drop_recreate");
+ TableDescriptor descriptor =
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build();
+ long oldTableId = createTable(tablePath, descriptor, false);
+ waitAllReplicasReady(oldTableId, 1);
+
+ MultiTable multiTable = conn.getMultiTable();
+ try (MultiTableLogScanner scanner = multiTable.newMultiTableScan().createLogScanner()) {
+ // 1. Subscribe to the original table; the scanner caches the old tableId.
+ scanner.subscribeFromBeginning(tablePath, 0);
+
+ // 2. Drop and recreate the same path -> new tableId.
+ admin.dropTable(tablePath, false).get();
+ long newTableId = createTable(tablePath, descriptor, true);
+ waitAllReplicasReady(newTableId, 1);
+ assertThat(newTableId).isNotEqualTo(oldTableId);
+
+ // 3. Write a row to the recreated table through a writer on the SAME connection. This
+ // forces the shared connection metadata to learn the new tableId, which is the
+ // state the scanner inspects when validating its cached registration.
+ GenericRow recreatedRow = row(1, "after-recreate");
+ try (MultiTableWriter writer = multiTable.newMultiTableWrite().createWriter()) {
+ int newSid = admin.getTableInfo(tablePath).get().getSchemaId();
+ writer.write(MultiTableWriteRecord.forAppend(tablePath, recreatedRow, newSid))
+ .get();
+ writer.flush();
+ }
+
+ // 4. Re-subscribing the stale table now detects the recreate and fails fast.
+ assertThatThrownBy(() -> scanner.subscribeFromBeginning(tablePath, 0))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("has been recreated")
+ .hasMessageContaining(String.valueOf(oldTableId))
+ .hasMessageContaining(String.valueOf(newTableId));
+
+ // 5. Recovery: unsubscribe the stale bucket (evicts the cached registration), then
+ // subscribe again -> re-resolves the recreated table and reads its data.
+ scanner.unsubscribe(tablePath, 0);
+ scanner.subscribeFromBeginning(tablePath, 0);
+
+ List<GenericRow> actualRows = new ArrayList<>();
+ long deadline = System.currentTimeMillis() + 30_000;
+ while (actualRows.isEmpty() && System.currentTimeMillis() < deadline) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ for (MultiTableRecord record : records) {
+ assertThat(record.getTableId()).isEqualTo(newTableId);
+ InternalRow r = record.getRow();
+ actualRows.add(row(r.getInt(0), r.getString(1)));
+ }
+ }
+ assertThat(actualRows).containsExactly(recreatedRow);
+ }
+ }
+
+ private static MultiTableRecords pollUntilProgressOnly(
+ MultiTableLogScanner scanner, TablePath tablePath, TableBucket tableBucket) {
+ long deadline = System.nanoTime() + Duration.ofSeconds(30).toNanos();
+ while (System.nanoTime() < deadline) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ if (records.isEmpty() && records.hasProgress()) {
+ // A progress-only result must still surface the table and bucket that advanced.
+ assertThat(records.tablePaths()).contains(tablePath);
+ assertThat(records.buckets(tablePath)).contains(tableBucket);
+ assertThat(records.count()).isEqualTo(0);
+ assertThat(records.records(tablePath, tableBucket)).isEmpty();
+ return records;
+ }
+ }
+ throw new AssertionError(
+ "Timed out waiting for a progress-only poll for bucket " + tableBucket);
+ }
+
+ private LogFetcher createTestingLogFetcher(
+ String scannerName,
+ LogScannerStatus logScannerStatus,
+ TestingScannerMetricGroup scannerMetricGroup) {
+ FlussConnection flussConnection = (FlussConnection) conn;
+ return new LogFetcher(
+ scannerName,
+ logScannerStatus,
+ clientConf,
+ flussConnection.getMetadataUpdater(),
+ scannerMetricGroup,
+ flussConnection.getOrCreateRemoteFileDownloader(),
+ LogRecordReadContext.SchemaResolution.DYNAMIC);
+ }
+
+ private MultiTableLogScanner createTestingScanner(
+ String scannerName,
+ LogScannerStatus logScannerStatus,
+ LogFetcher logFetcher,
+ TestingScannerMetricGroup scannerMetricGroup) {
+ return new MultiTableLogScannerImpl(
+ scannerName,
+ ((FlussConnection) conn).getMetadataUpdater(),
+ admin,
+ logScannerStatus,
+ logFetcher,
+ scannerMetricGroup);
+ }
+
+ private static void appendRows(Table table, int fromInclusive, int toInclusive)
+ throws Exception {
+ AppendWriter writer = table.newAppend().createWriter();
+ for (int i = fromInclusive; i <= toInclusive; i++) {
+ writer.append(row(i, "value-" + i)).get();
+ }
+ writer.flush();
+ }
+}
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java
index e5fe7290c..c01ea7e2f 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java
@@ -32,6 +32,7 @@ import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.remote.RemoteLogSegment;
import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.ChunkedAllocationManager;
import org.apache.fluss.types.DataTypes;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.Projection;
@@ -298,9 +299,16 @@ class RemoteCompletedFetchTest {
Runnable recycle) {
return new RemoteCompletedFetch(
tableBucket,
+ DATA2_TABLE_PATH,
fileLogRecords,
10L,
- LogRecordReadContext.createReadContext(tableInfo, true, projection, schemaGetter),
+ LogRecordReadContext.createReadContext(
+ tableInfo,
+ true,
+ LogRecordReadContext.SchemaResolution.DYNAMIC,
+ projection,
+ schemaGetter,
+ new ChunkedAllocationManager.ChunkedFactory()),
logScannerStatus,
true,
fetchOffset,
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloaderTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloaderTest.java
index 4c2bcdaa1..77f6dc998 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloaderTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloaderTest.java
@@ -87,7 +87,11 @@ class RemoteLogDownloaderTest {
RemoteFileDownloader remoteFileDownloader = new RemoteFileDownloader(1);
RemoteLogDownloader remoteLogDownloader =
new RemoteLogDownloader(
- DATA1_TABLE_PATH, conf, remoteFileDownloader, scannerMetricGroup, 10L);
+ DATA1_TABLE_PATH.toString(),
+ conf,
+ remoteFileDownloader,
+ scannerMetricGroup,
+ 10L);
try {
// trigger auto download.
remoteLogDownloader.start();
@@ -144,6 +148,124 @@ class RemoteLogDownloaderTest {
}
}
+ @Test
+ void testDiscardQueuedDownload() throws Exception {
+ RemoteFileDownloader fileDownloader = new RemoteFileDownloader(1);
+ RemoteLogDownloader downloader =
+ new RemoteLogDownloader(
+ DATA1_TABLE_PATH.toString(), conf, fileDownloader, scannerMetricGroup, 10L);
+ try {
+ TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 0);
+ RemoteLogSegment segment =
+ buildRemoteLogSegmentList(tableBucket, DATA1_PHYSICAL_TABLE_PATH, 1, conf, 10)
+ .get(0);
+ FsPath tabletDir =
+ remoteLogTabletDir(remoteLogDir, DATA1_PHYSICAL_TABLE_PATH, tableBucket);
+
+ RemoteLogDownloadFuture future = downloader.requestRemoteLog(tabletDir, segment);
+ future.discard();
+ future.discard();
+
+ assertThat(future.isDone()).isTrue();
+ assertThat(downloader.getSizeOfSegmentsToFetch()).isZero();
+ assertThat(downloader.getPrefetchSemaphore().availablePermits()).isEqualTo(4);
+ } finally {
+ IOUtils.closeQuietly(downloader);
+ IOUtils.closeQuietly(fileDownloader);
+ }
+ }
+
+ @Test
+ void testDiscardInFlightDownload() throws Exception {
+ CountDownLatch downloadStarted = new CountDownLatch(1);
+ CountDownLatch continueDownload = new CountDownLatch(1);
+
+ class BlockingFileDownloader extends RemoteFileDownloader {
+ BlockingFileDownloader() {
+ super(1);
+ }
+
+ @Override
+ protected long downloadFile(Path targetFilePath, FsPath remoteFilePath)
+ throws IOException {
+ downloadStarted.countDown();
+ try {
+ continueDownload.await();
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted while blocking", e);
+ }
+ return super.downloadFile(targetFilePath, remoteFilePath);
+ }
+ }
+
+ BlockingFileDownloader fileDownloader = new BlockingFileDownloader();
+ RemoteLogDownloader downloader =
+ new RemoteLogDownloader(
+ DATA1_TABLE_PATH.toString(), conf, fileDownloader, scannerMetricGroup, 10L);
+ try {
+ TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 0);
+ RemoteLogSegment segment =
+ buildRemoteLogSegmentList(tableBucket, DATA1_PHYSICAL_TABLE_PATH, 1, conf, 10)
+ .get(0);
+ FsPath tabletDir =
+ remoteLogTabletDir(remoteLogDir, DATA1_PHYSICAL_TABLE_PATH, tableBucket);
+ File localFile = localFile(downloader, tabletDir, segment);
+
+ RemoteLogDownloadFuture future = downloader.requestRemoteLog(tabletDir, segment);
+ downloader.fetchOnce();
+ assertThat(downloadStarted.await(30, TimeUnit.SECONDS)).isTrue();
+ assertThat(downloader.getPrefetchSemaphore().availablePermits()).isEqualTo(3);
+
+ future.discard();
+ continueDownload.countDown();
+
+ retry(
+ Duration.ofMinutes(1),
+ () ->
+ assertThat(downloader.getPrefetchSemaphore().availablePermits())
+ .isEqualTo(4));
+ assertThat(future.isDone()).isTrue();
+ assertThat(localFile).doesNotExist();
+ } finally {
+ continueDownload.countDown();
+ IOUtils.closeQuietly(downloader);
+ IOUtils.closeQuietly(fileDownloader);
+ }
+ }
+
+ @Test
+ void testDiscardCompletedDownload() throws Exception {
+ RemoteFileDownloader fileDownloader = new RemoteFileDownloader(1);
+ RemoteLogDownloader downloader =
+ new RemoteLogDownloader(
+ DATA1_TABLE_PATH.toString(), conf, fileDownloader, scannerMetricGroup, 10L);
+ try {
+ TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 0);
+ RemoteLogSegment segment =
+ buildRemoteLogSegmentList(tableBucket, DATA1_PHYSICAL_TABLE_PATH, 1, conf, 10)
+ .get(0);
+ FsPath tabletDir =
+ remoteLogTabletDir(remoteLogDir, DATA1_PHYSICAL_TABLE_PATH, tableBucket);
+ File localFile = localFile(downloader, tabletDir, segment);
+
+ RemoteLogDownloadFuture future = downloader.requestRemoteLog(tabletDir, segment);
+ downloader.fetchOnce();
+ retry(Duration.ofMinutes(1), () -> assertThat(future.isDone()).isTrue());
+ assertThat(downloader.getPrefetchSemaphore().availablePermits()).isEqualTo(3);
+ assertThat(localFile).exists();
+
+ future.discard();
+ future.discard();
+
+ assertThat(downloader.getPrefetchSemaphore().availablePermits()).isEqualTo(4);
+ downloader.start();
+ retry(Duration.ofMinutes(1), () -> assertThat(localFile).doesNotExist());
+ } finally {
+ IOUtils.closeQuietly(downloader);
+ IOUtils.closeQuietly(fileDownloader);
+ }
+ }
+
@Test
void testDownloadLogInParallelAndInPriority() throws Exception {
class TestRemoteFileDownloader extends RemoteFileDownloader {
@@ -165,7 +287,7 @@ class RemoteLogDownloaderTest {
TestRemoteFileDownloader fileDownloader = new TestRemoteFileDownloader(4);
RemoteLogDownloader remoteLogDownloader =
new RemoteLogDownloader(
- DATA1_TABLE_PATH,
+ DATA1_TABLE_PATH.toString(),
conf, // max 4 pre-fetch num
fileDownloader,
scannerMetricGroup,
@@ -337,7 +459,7 @@ class RemoteLogDownloaderTest {
BlockingFileDownloader fileDownloader = new BlockingFileDownloader(4);
RemoteLogDownloader downloader =
new RemoteLogDownloader(
- DATA1_TABLE_PATH, conf, fileDownloader, scannerMetricGroup, 10L);
+ DATA1_TABLE_PATH.toString(), conf, fileDownloader, scannerMetricGroup, 10L);
try {
TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0);
@@ -452,4 +574,12 @@ class RemoteLogDownloaderTest {
.length())
.sum();
}
+
+ private static File localFile(
+ RemoteLogDownloader downloader, FsPath remoteLogTabletDir, RemoteLogSegment segment) {
+ return new File(
+ downloader.getLocalLogDir().toFile(),
+ RemoteLogDownloader.getFsPathAndFileName(remoteLogTabletDir, segment)
+ .getFileName());
+ }
}
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/ScanRecordsTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/ScanRecordsTest.java
index 49c6e9c42..47f1355f8 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/ScanRecordsTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/ScanRecordsTest.java
@@ -70,6 +70,7 @@ public class ScanRecordsTest {
// No records and no progress: both isEmpty() and buckets() must be empty.
ScanRecords trulyEmpty = ScanRecords.EMPTY;
assertThat(trulyEmpty.isEmpty()).isTrue();
+ assertThat(trulyEmpty.hasProgress()).isFalse();
assertThat(trulyEmpty.buckets()).isEmpty();
// Progress-only round: isEmpty() stays true (no materialized records),
@@ -83,6 +84,7 @@ public class ScanRecordsTest {
progressOffsets.put(emptyBucket, 10L);
ScanRecords progressOnly = new ScanRecords(progressRecords, progressOffsets);
assertThat(progressOnly.isEmpty()).isTrue();
+ assertThat(progressOnly.hasProgress()).isTrue();
assertThat(progressOnly.buckets()).containsExactlyInAnyOrder(tb, emptyBucket);
assertThat(progressOnly.records(emptyBucket)).isEmpty();
assertThat(progressOnly.consumedUpToOffset(tb)).isEqualTo(42L);
@@ -98,7 +100,36 @@ public class ScanRecordsTest {
new ScanRecord(0L, 1000L, ChangeType.INSERT, row(1, "a"))));
ScanRecords withRecords = new ScanRecords(matRecords);
assertThat(withRecords.isEmpty()).isFalse();
+ assertThat(withRecords.hasProgress()).isTrue();
assertThat(withRecords.buckets()).containsExactly(tb);
assertThat(withRecords.consumedUpToOffset(tb)).isNull();
}
+
+ /**
+ * Verifies the constructor surfaces progress-only buckets in buckets() even when the caller
+ * omits their (empty) record list entries.
+ */
+ @Test
+ void progressOnlyBucketsSurfacedWithoutRecordEntries() {
+ TableBucket recordBucket = new TableBucket(0L, 0);
+ TableBucket progressOnlyBucket = new TableBucket(0L, 1);
+
+ Map<TableBucket, List<ScanRecord>> records = new HashMap<>();
+ records.put(
+ recordBucket,
+ Collections.singletonList(
+ new ScanRecord(0L, 1000L, ChangeType.INSERT, row(1, "a"))));
+ Map<TableBucket, Long> offsets = new HashMap<>();
+ offsets.put(recordBucket, 1L);
+ // no records entry for this bucket, only an advanced offset
+ offsets.put(progressOnlyBucket, 10L);
+
+ ScanRecords scanRecords = new ScanRecords(records, offsets);
+ assertThat(scanRecords.buckets())
+ .containsExactlyInAnyOrder(recordBucket, progressOnlyBucket);
+ assertThat(scanRecords.records(progressOnlyBucket)).isEmpty();
+ assertThat(scanRecords.consumedUpToOffset(progressOnlyBucket)).isEqualTo(10L);
+ assertThat(scanRecords.count()).isEqualTo(1);
+ assertThat(scanRecords.hasProgress()).isTrue();
+ }
}
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/writer/MultiTableWriteRecordTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/writer/MultiTableWriteRecordTest.java
new file mode 100644
index 000000000..44a628274
--- /dev/null
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/writer/MultiTableWriteRecordTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.writer;
+
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.row.GenericRow;
+import org.apache.fluss.row.InternalRow;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Unit test for {@link MultiTableWriteRecord}. */
+class MultiTableWriteRecordTest {
+
+ private static final TablePath TP = TablePath.of("db", "t");
+
+ private static InternalRow newRow() {
+ GenericRow r = new GenericRow(1);
+ r.setField(0, 42);
+ return r;
+ }
+
+ @Test
+ void testForAppendWithExplicitSchemaId() {
+ InternalRow row = newRow();
+ MultiTableWriteRecord rec = MultiTableWriteRecord.forAppend(TP, row, 5);
+ assertThat(rec.getTablePath()).isEqualTo(TP);
+ assertThat(rec.getOperation()).isEqualTo(MultiTableWriteRecord.Operation.APPEND);
+ assertThat(rec.getRow()).isSameAs(row);
+ assertThat(rec.getSchemaId()).isEqualTo(5);
+ }
+
+ @Test
+ void testForUpsertWithExplicitSchemaId() {
+ InternalRow row = newRow();
+ MultiTableWriteRecord rec = MultiTableWriteRecord.forUpsert(TP, row, 7);
+ assertThat(rec.getOperation()).isEqualTo(MultiTableWriteRecord.Operation.UPSERT);
+ assertThat(rec.getSchemaId()).isEqualTo(7);
+ }
+
+ @Test
+ void testForDeleteWithExplicitSchemaId() {
+ InternalRow row = newRow();
+ MultiTableWriteRecord rec = MultiTableWriteRecord.forDelete(TP, row, 9);
+ assertThat(rec.getOperation()).isEqualTo(MultiTableWriteRecord.Operation.DELETE);
+ assertThat(rec.getSchemaId()).isEqualTo(9);
+ }
+
+ @Test
+ void testToStringContainsCoreFields() {
+ MultiTableWriteRecord rec = MultiTableWriteRecord.forDelete(TP, newRow(), 3);
+ assertThat(rec.toString())
+ .contains("tablePath=" + TP)
+ .contains("operation=DELETE")
+ .contains("schemaId=3");
+ }
+
+ @Test
+ void testRejectsNullTablePath() {
+ assertThatThrownBy(() -> MultiTableWriteRecord.forAppend(null, newRow(), 1))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("tablePath");
+ }
+
+ @Test
+ void testRejectsNullRow() {
+ assertThatThrownBy(() -> MultiTableWriteRecord.forAppend(TP, null, 1))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("row");
+ }
+}
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/writer/MultiTableWriterITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/writer/MultiTableWriterITCase.java
new file mode 100644
index 000000000..b45bf957c
--- /dev/null
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/writer/MultiTableWriterITCase.java
@@ -0,0 +1,949 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.writer;
+
+import org.apache.fluss.client.Connection;
+import org.apache.fluss.client.ConnectionFactory;
+import org.apache.fluss.client.FlussConnection;
+import org.apache.fluss.client.admin.Admin;
+import org.apache.fluss.client.admin.ClientToServerITCaseBase;
+import org.apache.fluss.client.lookup.Lookuper;
+import org.apache.fluss.client.table.MultiTable;
+import org.apache.fluss.client.table.Table;
+import org.apache.fluss.client.table.scanner.MultiTableRecord;
+import org.apache.fluss.client.table.scanner.ScanRecord;
+import org.apache.fluss.client.table.scanner.log.LogScanner;
+import org.apache.fluss.client.table.scanner.log.MultiTableLogScanner;
+import org.apache.fluss.client.table.scanner.log.MultiTableRecords;
+import org.apache.fluss.client.table.scanner.log.ScanRecords;
+import org.apache.fluss.client.write.WriterClient;
+import org.apache.fluss.config.AutoPartitionTimeUnit;
+import org.apache.fluss.config.ConfigOptions;
+import org.apache.fluss.config.Configuration;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TableChange;
+import org.apache.fluss.metadata.TableDescriptor;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.row.GenericRow;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.Proxy;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.fluss.testutils.DataTestUtils.row;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** ITCase for {@link MultiTableWriter} / {@link MultiTableWriterImpl}. */
+class MultiTableWriterITCase extends ClientToServerITCaseBase {
+
+ private static final Schema LOG_SCHEMA =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("name", DataTypes.STRING())
+ .build();
+
+ private static final Schema LOG_SCHEMA_2 =
+ Schema.newBuilder()
+ .column("key", DataTypes.INT())
+ .column("value", DataTypes.STRING())
+ .column("score", DataTypes.BIGINT())
+ .build();
+
+ private static final Schema PK_SCHEMA =
+ Schema.newBuilder()
+ .primaryKey("id")
+ .column("id", DataTypes.INT())
+ .column("name", DataTypes.STRING())
+ .build();
+
+ // ---------------------------------------------------------------- log
+
+ @Test
+ void testWriteSingleLogTable() throws Exception {
+ TablePath tp = TablePath.of("test_db", "mt_w_single_log");
+ TableDescriptor descriptor =
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build();
+ long tableId = createTable(tp, descriptor, false);
+ waitAllReplicasReady(tableId, 1);
+ int sid = currentSchemaId(tp);
+
+ int recordCount = 5;
+ List<GenericRow> expected = new ArrayList<>();
+ MultiTable multiTable = conn.getMultiTable();
+ try (MultiTableWriter writer = multiTable.newMultiTableWrite().createWriter()) {
+ for (int i = 0; i < recordCount; i++) {
+ GenericRow r = row(i, "v-" + i);
+ expected.add(r);
+ writer.write(MultiTableWriteRecord.forAppend(tp, r, sid)).get();
+ }
+ writer.flush();
+ }
+
+ // Verify through a log scanner bound to the table.
+ List<GenericRow> actual = pollAllAppendRows(tp, recordCount);
+ assertThat(actual).containsExactlyInAnyOrderElementsOf(expected);
+ }
+
+ @Test
+ void testWriteMultipleLogTables() throws Exception {
+ TablePath tp1 = TablePath.of("test_db", "mt_w_multi_log1");
+ TablePath tp2 = TablePath.of("test_db", "mt_w_multi_log2");
+ long id1 =
+ createTable(
+ tp1,
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build(),
+ false);
+ long id2 =
+ createTable(
+ tp2,
+ TableDescriptor.builder().schema(LOG_SCHEMA_2).distributedBy(1).build(),
+ true);
+ waitAllReplicasReady(id1, 1);
+ waitAllReplicasReady(id2, 1);
+ int sid1 = currentSchemaId(tp1);
+ int sid2 = currentSchemaId(tp2);
+
+ int n1 = 3;
+ int n2 = 4;
+ List<GenericRow> expected1 = new ArrayList<>();
+ List<GenericRow> expected2 = new ArrayList<>();
+
+ try (MultiTableWriter writer = conn.getMultiTable().newMultiTableWrite().createWriter()) {
+ for (int i = 0; i < n1; i++) {
+ GenericRow r = row(i, "t1-" + i);
+ expected1.add(r);
+ writer.write(MultiTableWriteRecord.forAppend(tp1, r, sid1)).get();
+ }
+ for (int i = 0; i < n2; i++) {
+ GenericRow r = row(100 + i, "t2-" + i, (long) (i * 10));
+ expected2.add(r);
+ writer.write(MultiTableWriteRecord.forAppend(tp2, r, sid2)).get();
+ }
+ writer.flush();
+ }
+
+ assertThat(pollAllAppendRows(tp1, n1)).containsExactlyInAnyOrderElementsOf(expected1);
+ assertThat(pollAllAppendRows(tp2, n2)).containsExactlyInAnyOrderElementsOf(expected2);
+ }
+
+ // ---------------------------------------------------------------- pk
+
+ @Test
+ void testWriteSinglePkTableUpsertAndDelete() throws Exception {
+ TablePath tp = TablePath.of("test_db", "mt_w_pk");
+ TableDescriptor descriptor =
+ TableDescriptor.builder().schema(PK_SCHEMA).distributedBy(1).build();
+ long tableId = createTable(tp, descriptor, false);
+ waitAllReplicasReady(tableId, 1);
+ int sid = currentSchemaId(tp);
+
+ try (MultiTableWriter writer = conn.getMultiTable().newMultiTableWrite().createWriter()) {
+ writer.write(MultiTableWriteRecord.forUpsert(tp, row(1, "a"), sid)).get();
+ writer.write(MultiTableWriteRecord.forUpsert(tp, row(1, "a-updated"), sid)).get();
+
+ writer.flush();
+ }
+
+ // Verify: pk=1 resolves to ("a-updated").
+ try (Table table = conn.getTable(tp)) {
+ Lookuper lookuper = table.newLookup().createLookuper();
+ InternalRow found = lookupRow(lookuper, row(1));
+ assertThat(found).isNotNull();
+ assertThat(found.getInt(0)).isEqualTo(1);
+ assertThat(found.getString(1).toString()).isEqualTo("a-updated");
+ }
+
+ // DELETE: pk=1 disappears.
+ try (MultiTableWriter writer = conn.getMultiTable().newMultiTableWrite().createWriter()) {
+ writer.write(MultiTableWriteRecord.forDelete(tp, row(1, "a-updated"), sid)).get();
+ writer.flush();
+ }
+ try (Table table = conn.getTable(tp)) {
+ Lookuper lookuper = table.newLookup().createLookuper();
+ assertThat(lookupRow(lookuper, row(1))).isNull();
+ }
+ }
+
+ // ---------------------------------------------------------------- mix
+
+ @Test
+ void testWriteMixedLogAndPkTables() throws Exception {
+ TablePath logTp = TablePath.of("test_db", "mt_w_mix_log");
+ TablePath pkTp = TablePath.of("test_db", "mt_w_mix_pk");
+ long logId =
+ createTable(
+ logTp,
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build(),
+ false);
+ long pkId =
+ createTable(
+ pkTp,
+ TableDescriptor.builder().schema(PK_SCHEMA).distributedBy(1).build(),
+ true);
+ waitAllReplicasReady(logId, 1);
+ waitAllReplicasReady(pkId, 1);
+ int logSid = currentSchemaId(logTp);
+ int pkSid = currentSchemaId(pkTp);
+
+ try (MultiTableWriter writer = conn.getMultiTable().newMultiTableWrite().createWriter()) {
+ writer.write(MultiTableWriteRecord.forAppend(logTp, row(1, "log-a"), logSid)).get();
+ writer.write(MultiTableWriteRecord.forUpsert(pkTp, row(42, "pk-a"), pkSid)).get();
+ writer.write(MultiTableWriteRecord.forUpsert(pkTp, row(42, "pk-b"), pkSid)).get();
+ writer.flush();
+ }
+
+ // Log table: one appended row.
+ List<GenericRow> logRows = pollAllAppendRows(logTp, 1);
+ assertThat(logRows).hasSize(1);
+
+ // PK table: key=42 has value "pk-b".
+ try (Table table = conn.getTable(pkTp)) {
+ Lookuper lookuper = table.newLookup().createLookuper();
+ InternalRow found = lookupRow(lookuper, row(42));
+ assertThat(found).isNotNull();
+ assertThat(found.getString(1).toString()).isEqualTo("pk-b");
+ }
+ }
+
+ // ---------------------------------------------------------------- partitioned
+
+ @Test
+ void testWritePartitionedTable() throws Exception {
+ TablePath tp = TablePath.of("test_db", "mt_w_part");
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("name", DataTypes.STRING())
+ .column("pt", DataTypes.STRING())
+ .build();
+ TableDescriptor descriptor =
+ TableDescriptor.builder()
+ .schema(schema)
+ .distributedBy(1)
+ .partitionedBy("pt")
+ .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true)
+ .property(
+ ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT,
+ AutoPartitionTimeUnit.YEAR)
+ .build();
+ long tableId = createTable(tp, descriptor, false);
+
+ Map<String, Long> partitionIdByNames =
+ FLUSS_CLUSTER_EXTENSION.waitUntilPartitionAllReady(tp);
+ assertThat(partitionIdByNames).isNotEmpty();
+
+ int perPartition = 2;
+ int partSid = currentSchemaId(tp);
+ try (MultiTableWriter writer = conn.getMultiTable().newMultiTableWrite().createWriter()) {
+ for (String partitionName : partitionIdByNames.keySet()) {
+ for (int i = 0; i < perPartition; i++) {
+ writer.write(
+ MultiTableWriteRecord.forAppend(
+ tp, row(i, "v-" + i, partitionName), partSid))
+ .get();
+ }
+ }
+ writer.flush();
+ }
+
+ int total = perPartition * partitionIdByNames.size();
+ // Scan across all partitions via a MultiTableLogScanner.
+ try (MultiTableLogScanner scanner =
+ conn.getMultiTable().newMultiTableScan().createLogScanner()) {
+ for (Long partitionId : partitionIdByNames.values()) {
+ scanner.subscribeFromBeginning(tp, partitionId, 0);
+ }
+ int polled = 0;
+ long deadline = System.currentTimeMillis() + 30_000;
+ while (polled < total && System.currentTimeMillis() < deadline) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ polled += records.count();
+ }
+ assertThat(polled).isEqualTo(total);
+ }
+ }
+
+ // ---------------------------------------------------------------- cdc mirror
+
+ @Test
+ void testCdcMirrorScanThenWrite() throws Exception {
+ TablePath src = TablePath.of("test_db", "mt_w_cdc_src");
+ TablePath dst = TablePath.of("test_db", "mt_w_cdc_dst");
+ long srcId =
+ createTable(
+ src,
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build(),
+ false);
+ long dstId =
+ createTable(
+ dst,
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build(),
+ true);
+ waitAllReplicasReady(srcId, 1);
+ waitAllReplicasReady(dstId, 1);
+
+ // 1. Seed src via the single-table AppendWriter.
+ int recordCount = 6;
+ List<GenericRow> seeded = new ArrayList<>();
+ try (Table table = conn.getTable(src)) {
+ AppendWriter append = table.newAppend().createWriter();
+ for (int i = 0; i < recordCount; i++) {
+ GenericRow r = row(i, "src-" + i);
+ seeded.add(r);
+ append.append(r).get();
+ }
+ }
+
+ // 2. Mirror src -> dst via MultiTable scanner + writer.
+ MultiTable multiTable = conn.getMultiTable();
+ try (MultiTableLogScanner scanner = multiTable.newMultiTableScan().createLogScanner();
+ MultiTableWriter writer = multiTable.newMultiTableWrite().createWriter()) {
+ scanner.subscribeFromBeginning(src, 0);
+ int mirrored = 0;
+ long deadline = System.currentTimeMillis() + 30_000;
+ while (mirrored < recordCount && System.currentTimeMillis() < deadline) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ for (MultiTableRecord scanned : records) {
+ // Retarget to dst; log tables only accept append records.
+ MultiTableWriteRecord retarget =
+ MultiTableWriteRecord.forAppend(
+ dst, scanned.getRow(), scanned.getSchemaId());
+ writer.write(retarget).get();
+ mirrored++;
+ }
+ }
+ writer.flush();
+ assertThat(mirrored).isEqualTo(recordCount);
+ }
+
+ // 3. Verify dst has the same rows as src.
+ List<GenericRow> mirroredRows = pollAllAppendRows(dst, recordCount);
+ assertThat(mirroredRows).containsExactlyInAnyOrderElementsOf(seeded);
+ }
+
+ // ---------------------------------------------------------------- errors
+
+ @Test
+ void testWriteUnknownTableFailsFuture() throws Exception {
+ TablePath unknown = TablePath.of("test_db", "mt_w_unknown_table");
+ try (MultiTableWriter writer = conn.getMultiTable().newMultiTableWrite().createWriter()) {
+ assertThatThrownBy(
+ () ->
+ writer.write(
+ MultiTableWriteRecord.forAppend(
+ unknown, row(1, "v"), 1))
+ .get())
+ .isInstanceOf(ExecutionException.class)
+ .rootCause()
+ .hasMessageContaining("Table 'test_db.mt_w_unknown_table' does not exist.");
+ }
+ }
+
+ @Test
+ void testOperationIncompatibleWithTableKind() throws Exception {
+ // DELETE on a log table must fail (log tables only accept APPEND).
+ TablePath logTp = TablePath.of("test_db", "mt_w_bad_ct_log");
+ TablePath pkTp = TablePath.of("test_db", "mt_w_bad_ct_pk");
+ long logId =
+ createTable(
+ logTp,
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build(),
+ false);
+ long pkId =
+ createTable(
+ pkTp,
+ TableDescriptor.builder().schema(PK_SCHEMA).distributedBy(1).build(),
+ true);
+ waitAllReplicasReady(logId, 1);
+ waitAllReplicasReady(pkId, 1);
+ int logSid = currentSchemaId(logTp);
+ int pkSid = currentSchemaId(pkTp);
+
+ try (MultiTableWriter writer = conn.getMultiTable().newMultiTableWrite().createWriter()) {
+ assertThatThrownBy(
+ () ->
+ writer.write(
+ MultiTableWriteRecord.forDelete(
+ logTp, row(1, "v"), logSid))
+ .get())
+ .isInstanceOf(ExecutionException.class)
+ .hasCauseInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("log table")
+ .hasMessageContaining("forAppend");
+
+ assertThatThrownBy(
+ () ->
+ writer.write(
+ MultiTableWriteRecord.forAppend(
+ pkTp, row(1, "v"), pkSid))
+ .get())
+ .isInstanceOf(ExecutionException.class)
+ .hasCauseInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("primary-key table")
+ .hasMessageContaining("forUpsert")
+ .hasMessageContaining("forDelete");
+ }
+ }
+
+ @Test
+ void testFlushAndClose() throws Exception {
+ TablePath tp = TablePath.of("test_db", "mt_w_flush_close");
+ long tableId =
+ createTable(
+ tp,
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build(),
+ false);
+ waitAllReplicasReady(tableId, 1);
+ int sid = currentSchemaId(tp);
+
+ MultiTableWriter writer = conn.getMultiTable().newMultiTableWrite().createWriter();
+ writer.write(MultiTableWriteRecord.forAppend(tp, row(1, "x"), sid)).get();
+ writer.flush();
+ writer.close();
+
+ // Further writes after close must fail synchronously.
+ assertThatThrownBy(
+ () -> writer.write(MultiTableWriteRecord.forAppend(tp, row(2, "y"), sid)))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("closed");
+
+ // Close is idempotent.
+ writer.close();
+ }
+
+ @Test
+ void testWriteAfterAddColumn() throws Exception {
+ // Verify the writer transparently picks up a new schema after an ALTER TABLE ADD
+ // COLUMN: an old-schema record tagged with the original schemaId uses the cached
+ // encoder, and a record tagged with the post-alter schemaId triggers a metadata
+ // refresh + encoder rebuild so both batches end up readable with their original
+ // shapes.
+ TablePath tp = TablePath.of("test_db", "mt_w_add_column");
+ long tableId =
+ createTable(
+ tp,
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build(),
+ false);
+ waitAllReplicasReady(tableId, 1);
+ int oldSid = currentSchemaId(tp);
+
+ int preCount = 3;
+ int postCount = 3;
+ List<GenericRow> expectedPre = new ArrayList<>();
+ List<GenericRow> expectedPost = new ArrayList<>();
+
+ try (MultiTableWriter writer = conn.getMultiTable().newMultiTableWrite().createWriter()) {
+ // Phase 1: write with the original 2-column schema.
+ for (int i = 0; i < preCount; i++) {
+ GenericRow r = row(i, "before-" + i);
+ expectedPre.add(r);
+ writer.write(MultiTableWriteRecord.forAppend(tp, r, oldSid)).get();
+ }
+
+ // ALTER TABLE: append a new column "score BIGINT".
+ admin.alterTable(
+ tp,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "score",
+ DataTypes.BIGINT(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata();
+ int newSid = waitForSchemaIdBump(tp, oldSid);
+
+ // Phase 2: write the new 3-column rows tagged with the bumped schemaId —
+ // the writer must refresh metadata and rebuild the encoder against the new schema
+ // before encoding the wider row.
+ for (int i = 0; i < postCount; i++) {
+ GenericRow r = row(100 + i, "after-" + i, (long) (i * 10));
+ expectedPost.add(r);
+ writer.write(MultiTableWriteRecord.forAppend(tp, r, newSid)).get();
+ }
+ writer.flush();
+ }
+
+ // Read back using the multi-table scanner so each record keeps its write-time schema,
+ // and split rows by their RowType field count.
+ int total = preCount + postCount;
+ List<GenericRow> actualPre = new ArrayList<>();
+ List<GenericRow> actualPost = new ArrayList<>();
+ try (MultiTableLogScanner scanner =
+ conn.getMultiTable().newMultiTableScan().createLogScanner()) {
+ scanner.subscribeFromBeginning(tp, 0);
+ long deadline = System.currentTimeMillis() + 30_000;
+ int polled = 0;
+ while (polled < total && System.currentTimeMillis() < deadline) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ for (MultiTableRecord rec : records) {
+ assertThat(rec.getTableId()).isEqualTo(tableId);
+ InternalRow r = rec.getRow();
+ int fc = rec.getSchema().getRowType().getFieldCount();
+ if (fc == 2) {
+ actualPre.add(row(r.getInt(0), r.getString(1)));
+ } else {
+ assertThat(fc).isEqualTo(3);
+ actualPost.add(row(r.getInt(0), r.getString(1), r.getLong(2)));
+ }
+ polled++;
+ }
+ }
+ }
+
+ assertThat(actualPre).containsExactlyInAnyOrderElementsOf(expectedPre);
+ assertThat(actualPost).containsExactlyInAnyOrderElementsOf(expectedPost);
+ }
+
+ @Test
+ void testWritersAreIsolatedAcrossWriterClients() throws Exception {
+ // Each MultiTableWriter owns a dedicated WriterClient, so an outstanding batch of one
+ // writer neither fails another writer's schema-bumped write nor is flushed by it.
+ TablePath tp = TablePath.of("test_db", "mt_w_isolated_writer_clients");
+ long tableId =
+ createTable(
+ tp,
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build(),
+ false);
+ waitAllReplicasReady(tableId, 1);
+ int oldSid = currentSchemaId(tp);
+
+ Configuration writerConf = new Configuration(clientConf);
+ writerConf.set(ConfigOptions.CLIENT_WRITER_BATCH_TIMEOUT, Duration.ofMinutes(1));
+
+ try (Connection sharedConn = ConnectionFactory.createConnection(writerConf);
+ MultiTableWriter oldSchemaWriter =
+ sharedConn.getMultiTable().newMultiTableWrite().createWriter();
+ MultiTableWriter newSchemaWriter =
+ sharedConn.getMultiTable().newMultiTableWrite().createWriter()) {
+ CompletableFuture<WriteResult> oldFuture =
+ oldSchemaWriter.write(
+ MultiTableWriteRecord.forAppend(tp, row(1, "old-schema"), oldSid));
+
+ admin.alterTable(
+ tp,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "score",
+ DataTypes.BIGINT(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata();
+ int newSid = waitForSchemaIdBump(tp, oldSid);
+
+ // The new-schema write goes to its own accumulator and must not collide with the
+ // other writer's outstanding old-schema batch.
+ CompletableFuture<WriteResult> newFuture =
+ newSchemaWriter.write(
+ MultiTableWriteRecord.forAppend(tp, row(2, "new-schema", 10L), newSid));
+
+ // Flushing one writer must not flush (or complete) the other writer's records.
+ newSchemaWriter.flush();
+ assertThat(newFuture.isDone()).isTrue();
+ newFuture.get();
+ assertThat(oldFuture.isDone()).isFalse();
+
+ oldSchemaWriter.flush();
+ assertThat(oldFuture.isDone()).isTrue();
+ oldFuture.get();
+ }
+
+ // Read back both records, each keeping its write-time schema shape.
+ List<GenericRow> actualOld = new ArrayList<>();
+ List<GenericRow> actualNew = new ArrayList<>();
+ pollRowsGroupedByFieldCount(tp, 2, actualOld, actualNew);
+ assertThat(actualOld).containsExactly(row(1, "old-schema"));
+ assertThat(actualNew).containsExactly(row(2, "new-schema", 10L));
+ }
+
+ @Test
+ void testSchemaSwitchRollsBatchWithoutFlush() throws Exception {
+ // A batch never mixes schemas: when a record carrying a bumped schemaId arrives while the
+ // previous schema's records are still buffered, the old-schema batch is closed and a new
+ // schema-specific batch is rolled automatically, so no explicit flush() is required at
+ // the schema boundary.
+ TablePath tp = TablePath.of("test_db", "mt_w_schema_switch_roll");
+ long tableId =
+ createTable(
+ tp,
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build(),
+ false);
+ waitAllReplicasReady(tableId, 1);
+ int oldSid = currentSchemaId(tp);
+
+ Configuration writerConf = new Configuration(clientConf);
+ writerConf.set(ConfigOptions.CLIENT_WRITER_BATCH_TIMEOUT, Duration.ofMinutes(1));
+
+ try (Connection writerConn = ConnectionFactory.createConnection(writerConf);
+ MultiTableWriter writer =
+ writerConn.getMultiTable().newMultiTableWrite().createWriter()) {
+ CompletableFuture<WriteResult> oldFuture =
+ writer.write(MultiTableWriteRecord.forAppend(tp, row(1, "old-schema"), oldSid));
+
+ admin.alterTable(
+ tp,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "score",
+ DataTypes.BIGINT(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata();
+ int newSid = waitForSchemaIdBump(tp, oldSid);
+
+ // the old-schema batch is still buffered (long batch timeout), yet the new-schema
+ // record is accepted without an intermediate flush.
+ CompletableFuture<WriteResult> newFuture =
+ writer.write(
+ MultiTableWriteRecord.forAppend(tp, row(2, "new-schema", 10L), newSid));
+
+ // a single flush drains both the rolled old-schema batch and the new-schema batch.
+ writer.flush();
+ assertThat(oldFuture.isDone()).isTrue();
+ assertThat(newFuture.isDone()).isTrue();
+ oldFuture.get();
+ newFuture.get();
+ }
+ }
+
+ @Test
+ void testHistoricalSchemaRecordsBatchWithoutPerRowRefresh() throws Exception {
+ // Regression test: consecutive records carrying the same historical schemaId must reuse
+ // the cached encoding state — a single schema fetch, no per-row metadata refresh and
+ // no per-row flush — so historical CDC records still batch normally.
+ TablePath tp = TablePath.of("test_db", "mt_w_hist_no_per_row_refresh");
+ long tableId =
+ createTable(
+ tp,
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build(),
+ false);
+ waitAllReplicasReady(tableId, 1);
+ int oldSid = currentSchemaId(tp);
+
+ admin.alterTable(
+ tp,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "score",
+ DataTypes.BIGINT(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata();
+ int newSid = waitForSchemaIdBump(tp, oldSid);
+
+ Configuration writerConf = new Configuration(clientConf);
+ writerConf.set(ConfigOptions.CLIENT_WRITER_BATCH_TIMEOUT, Duration.ofMinutes(1));
+
+ AtomicInteger schemaFetches = new AtomicInteger();
+ AtomicInteger tableInfoFetches = new AtomicInteger();
+ try (Connection writerConn = ConnectionFactory.createConnection(writerConf)) {
+ FlussConnection flussConn = (FlussConnection) writerConn;
+ Admin realAdmin = flussConn.getOrCreateAdmin();
+ Admin countingAdmin =
+ (Admin)
+ Proxy.newProxyInstance(
+ Admin.class.getClassLoader(),
+ new Class<?>[] {Admin.class},
+ (proxy, method, args) -> {
+ if ("getTableSchema".equals(method.getName())) {
+ schemaFetches.incrementAndGet();
+ } else if ("getTableInfo".equals(method.getName())) {
+ tableInfoFetches.incrementAndGet();
+ }
+ return method.invoke(realAdmin, args);
+ });
+
+ List<CompletableFuture<WriteResult>> historicalFutures = new ArrayList<>();
+ try (MultiTableWriter writer =
+ new MultiTableWriterImpl(
+ flussConn.getMetadataUpdater(),
+ countingAdmin,
+ new WriterClient(
+ flussConn.getConfiguration(),
+ flussConn.getMetadataUpdater(),
+ flussConn.getClientMetricGroup(),
+ realAdmin))) {
+ // Prime the writer with the latest schema, then flush to honor the schema-switch
+ // contract before writing historical records.
+ List<CompletableFuture<WriteResult>> newFutures = new ArrayList<>();
+ for (int i = 0; i < 2; i++) {
+ newFutures.add(
+ writer.write(
+ MultiTableWriteRecord.forAppend(
+ tp, row(100 + i, "new-" + i, (long) i), newSid)));
+ }
+ writer.flush();
+ for (CompletableFuture<WriteResult> future : newFutures) {
+ future.get();
+ }
+
+ // Consecutive historical-schema records, none awaited individually.
+ for (int i = 0; i < 5; i++) {
+ historicalFutures.add(
+ writer.write(
+ MultiTableWriteRecord.forAppend(
+ tp, row(i, "old-" + i), oldSid)));
+ }
+
+ // No per-row flush: nothing is acknowledged until the explicit flush below.
+ for (CompletableFuture<WriteResult> future : historicalFutures) {
+ assertThat(future.isDone()).isFalse();
+ }
+ // The historical schema is fetched exactly once and the latest table info at most
+ // once — never once per record.
+ assertThat(schemaFetches.get()).isEqualTo(1);
+ assertThat(tableInfoFetches.get()).isLessThanOrEqualTo(1);
+
+ writer.flush();
+ for (CompletableFuture<WriteResult> future : historicalFutures) {
+ assertThat(future.isDone()).isTrue();
+ future.get();
+ }
+ assertThat(schemaFetches.get()).isEqualTo(1);
+ }
+ }
+
+ // Read back: 5 historical rows keep the 2-column shape, 2 new rows the 3-column shape.
+ List<GenericRow> actualOld = new ArrayList<>();
+ List<GenericRow> actualNew = new ArrayList<>();
+ pollRowsGroupedByFieldCount(tp, 7, actualOld, actualNew);
+ assertThat(actualOld).hasSize(5);
+ assertThat(actualNew).hasSize(2);
+ }
+
+ @Test
+ void testWriteWithFutureSchemaIdFailsAfterRefresh() throws Exception {
+ // When the record's schemaId differs from the cached state, the writer refreshes
+ // table metadata and rebuilds encoders against the latest schema. If after the
+ // refresh the record's schemaId is still ahead of the server's schemaId (i.e., the
+ // caller is referencing a future / unknown schema), the write must fail.
+ TablePath tp = TablePath.of("test_db", "mt_w_bad_schema_id");
+ long tableId =
+ createTable(
+ tp,
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build(),
+ false);
+ waitAllReplicasReady(tableId, 1);
+ int sid = currentSchemaId(tp);
+ int futureSid = sid + 99;
+
+ try (MultiTableWriter writer = conn.getMultiTable().newMultiTableWrite().createWriter()) {
+ assertThatThrownBy(
+ () ->
+ writer.write(
+ MultiTableWriteRecord.forAppend(
+ tp, row(1, "v"), futureSid))
+ .get())
+ .isInstanceOf(ExecutionException.class)
+ .hasCauseInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("SchemaId mismatch")
+ .hasMessageContaining("record schemaId=" + futureSid)
+ .hasMessageContaining("current table schemaId=" + sid)
+ .hasMessageContaining("after metadata refresh");
+ }
+ }
+
+ @Test
+ void testWriteHistoricalSchemaAfterSchemaEvolves() throws Exception {
+ // After the table has evolved to a new schema, a record still tagged with the OLD schemaId
+ // (record.schemaId < latest schemaId) must be encoded against its historical schema and
+ // sent successfully — exercising the writer's historical-schema build path rather
+ // than
+ // the cached-latest fast path used by testWriteAfterAddColumn (which writes old rows BEFORE
+ // the alter, while the cached latest still IS the old schema).
+ TablePath tp = TablePath.of("test_db", "mt_w_historical_schema");
+ long tableId =
+ createTable(
+ tp,
+ TableDescriptor.builder().schema(LOG_SCHEMA).distributedBy(1).build(),
+ false);
+ waitAllReplicasReady(tableId, 1);
+ int oldSid = currentSchemaId(tp);
+
+ // Evolve the schema first: append "score BIGINT" so the table's latest schema is the
+ // 3-column shape before any write happens.
+ admin.alterTable(
+ tp,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "score",
+ DataTypes.BIGINT(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata();
+ int newSid = waitForSchemaIdBump(tp, oldSid);
+
+ List<GenericRow> expectedNew = new ArrayList<>();
+ List<GenericRow> expectedOld = new ArrayList<>();
+ try (MultiTableWriter writer = conn.getMultiTable().newMultiTableWrite().createWriter()) {
+ // Prime the writer with the latest (3-column) schema so its cached "latest" is newSid.
+ for (int i = 0; i < 2; i++) {
+ GenericRow r = row(100 + i, "new-" + i, (long) (i * 10));
+ expectedNew.add(r);
+ writer.write(MultiTableWriteRecord.forAppend(tp, r, newSid)).get();
+ }
+
+ // Now write 2-column rows tagged with the historical oldSid. Because oldSid is behind
+ // the cached latest newSid, the writer must fetch the historical schema and build a
+ // dedicated encoder so the narrow row is encoded & sent against schemaId=oldSid.
+ for (int i = 0; i < 3; i++) {
+ GenericRow r = row(i, "old-" + i);
+ expectedOld.add(r);
+ writer.write(MultiTableWriteRecord.forAppend(tp, r, oldSid)).get();
+ }
+ writer.flush();
+ }
+
+ // Read back via the multi-table scanner so each record keeps its write-time schema, then
+ // split rows by their RowType field count.
+ int total = expectedNew.size() + expectedOld.size();
+ List<GenericRow> actualNew = new ArrayList<>();
+ List<GenericRow> actualOld = new ArrayList<>();
+ try (MultiTableLogScanner scanner =
+ conn.getMultiTable().newMultiTableScan().createLogScanner()) {
+ scanner.subscribeFromBeginning(tp, 0);
+ long deadline = System.currentTimeMillis() + 30_000;
+ int polled = 0;
+ while (polled < total && System.currentTimeMillis() < deadline) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ for (MultiTableRecord rec : records) {
+ assertThat(rec.getTableId()).isEqualTo(tableId);
+ InternalRow r = rec.getRow();
+ int fc = rec.getSchema().getRowType().getFieldCount();
+ if (fc == 2) {
+ actualOld.add(row(r.getInt(0), r.getString(1)));
+ } else {
+ assertThat(fc).isEqualTo(3);
+ actualNew.add(row(r.getInt(0), r.getString(1), r.getLong(2)));
+ }
+ polled++;
+ }
+ }
+ }
+
+ assertThat(actualOld).containsExactlyInAnyOrderElementsOf(expectedOld);
+ assertThat(actualNew).containsExactlyInAnyOrderElementsOf(expectedNew);
+ }
+
+ // ---------------------------------------------------------------- helpers
+
+ private int currentSchemaId(TablePath tp) throws Exception {
+ return admin.getTableInfo(tp).get().getSchemaId();
+ }
+
+ /**
+ * Polls up to {@code expected} records from bucket 0 of {@code tp} via the multi-table scanner
+ * (so each record keeps its write-time schema) and splits the rows by their field count into
+ * the 2-column ({@code actualOld}) and 3-column ({@code actualNew}) shapes used by the
+ * schema-evolution tests in this class.
+ */
+ private void pollRowsGroupedByFieldCount(
+ TablePath tp, int expected, List<GenericRow> actualOld, List<GenericRow> actualNew)
+ throws Exception {
+ try (MultiTableLogScanner scanner =
+ conn.getMultiTable().newMultiTableScan().createLogScanner()) {
+ scanner.subscribeFromBeginning(tp, 0);
+ long deadline = System.currentTimeMillis() + 30_000;
+ int polled = 0;
+ while (polled < expected && System.currentTimeMillis() < deadline) {
+ MultiTableRecords records = scanner.poll(Duration.ofSeconds(1));
+ for (MultiTableRecord rec : records) {
+ InternalRow r = rec.getRow();
+ int fc = rec.getSchema().getRowType().getFieldCount();
+ if (fc == 2) {
+ actualOld.add(row(r.getInt(0), r.getString(1)));
+ } else {
+ assertThat(fc).isEqualTo(3);
+ actualNew.add(row(r.getInt(0), r.getString(1), r.getLong(2)));
+ }
+ polled++;
+ }
+ }
+ }
+ }
+
+ /**
+ * Polls {@link #currentSchemaId(TablePath)} until it advances past {@code oldSid} or the
+ * deadline expires; returns the new schemaId.
+ */
+ private int waitForSchemaIdBump(TablePath tp, int oldSid) throws Exception {
+ long deadline = System.currentTimeMillis() + 10_000;
+ int sid = currentSchemaId(tp);
+ while (sid == oldSid && System.currentTimeMillis() < deadline) {
+ Thread.sleep(100);
+ sid = currentSchemaId(tp);
+ }
+ assertThat(sid).isGreaterThan(oldSid);
+
+ return sid;
+ }
+
+ private List<GenericRow> pollAllAppendRows(TablePath tp, int expected) throws Exception {
+ List<GenericRow> result = new ArrayList<>();
+ try (Table table = conn.getTable(tp);
+ LogScanner scanner = table.newScan().createLogScanner()) {
+ subscribeFromBeginning(scanner, table);
+ int fieldCount = table.getTableInfo().getRowType().getFieldCount();
+ long deadline = System.currentTimeMillis() + 30_000;
+ while (result.size() < expected && System.currentTimeMillis() < deadline) {
+ ScanRecords records = scanner.poll(Duration.ofSeconds(1));
+ for (TableBucket bucket : records.buckets()) {
+ for (ScanRecord r : records.records(bucket)) {
+ result.add(copyRow(r.getRow(), fieldCount));
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Copies a scanner-produced row field-by-field into a detached {@link GenericRow} using the two
+ * row shapes produced by this test: {@code (int, string)} or {@code (int, string, long)}.
+ */
+ private static GenericRow copyRow(InternalRow row, int fieldCount) {
+ GenericRow out = new GenericRow(fieldCount);
+ out.setField(0, row.getInt(0));
+ out.setField(1, row.getString(1));
+ if (fieldCount == 3) {
+ out.setField(2, row.getLong(2));
+ }
+ return out;
+ }
+}
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java
index 140058eca..dc8fbce9d 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java
@@ -20,6 +20,7 @@ package org.apache.fluss.client.write;
import org.apache.fluss.memory.MemorySegment;
import org.apache.fluss.memory.PreAllocatedPagedOutputView;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.record.CompactedLogRecord;
import org.apache.fluss.record.LogRecord;
@@ -45,7 +46,9 @@ import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
import static org.apache.fluss.record.TestData.DATA1_TABLE_ID;
import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.testutils.DataTestUtils.compactedRow;
+import static org.apache.fluss.testutils.DataTestUtils.indexedRow;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -145,6 +148,41 @@ public class CompactedLogWriteBatchTest {
assertThat(appendResult).isFalse();
}
+ @Test
+ void testTryAppendRejectsIncompatibleBatchIdentity() throws Exception {
+ int bucketId = 0;
+ CompactedLogWriteBatch logProducerBatch =
+ createLogWriteBatch(new TableBucket(DATA1_TABLE_ID, bucketId));
+
+ assertThat(logProducerBatch.tryAppend(createWriteRecord(), newWriteCallback())).isTrue();
+ assertThat(
+ logProducerBatch.tryAppend(
+ createWriteRecord(
+ withTableIdAndSchemaId(
+ DATA1_TABLE_ID,
+ DATA1_TABLE_INFO.getSchemaId() + 1)),
+ newWriteCallback()))
+ .isFalse();
+ assertThat(
+ logProducerBatch.tryAppend(
+ createWriteRecord(
+ withTableIdAndSchemaId(
+ DATA1_TABLE_ID + 1,
+ DATA1_TABLE_INFO.getSchemaId())),
+ newWriteCallback()))
+ .isFalse();
+ assertThat(
+ logProducerBatch.tryAppend(
+ WriteRecord.forIndexedAppend(
+ DATA1_TABLE_INFO,
+ DATA1_PHYSICAL_TABLE_PATH,
+ indexedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}),
+ null),
+ newWriteCallback()))
+ .isFalse();
+ assertThat(logProducerBatch.getRecordCount()).isEqualTo(1);
+ }
+
@Test
void testBatchAborted() throws Exception {
int bucketId = 0;
@@ -196,8 +234,11 @@ public class CompactedLogWriteBatchTest {
}
private WriteRecord createWriteRecord() {
- return WriteRecord.forCompactedAppend(
- DATA1_TABLE_INFO, DATA1_PHYSICAL_TABLE_PATH, row, null);
+ return createWriteRecord(DATA1_TABLE_INFO);
+ }
+
+ private WriteRecord createWriteRecord(TableInfo tableInfo) {
+ return WriteRecord.forCompactedAppend(tableInfo, DATA1_PHYSICAL_TABLE_PATH, row, null);
}
private CompactedLogWriteBatch createLogWriteBatch(TableBucket tb) throws Exception {
@@ -216,13 +257,32 @@ public class CompactedLogWriteBatchTest {
System.currentTimeMillis());
}
+ private TableInfo withTableIdAndSchemaId(long tableId, int schemaId) {
+ return new TableInfo(
+ DATA1_TABLE_INFO.getTablePath(),
+ tableId,
+ schemaId,
+ DATA1_TABLE_INFO.getSchema(),
+ DATA1_TABLE_INFO.getBucketKeys(),
+ DATA1_TABLE_INFO.getPartitionKeys(),
+ DATA1_TABLE_INFO.getNumBuckets(),
+ DATA1_TABLE_INFO.getProperties(),
+ DATA1_TABLE_INFO.getCustomProperties(),
+ DATA1_TABLE_INFO.getRemoteDataDir(),
+ DATA1_TABLE_INFO.getComment().orElse(null),
+ DATA1_TABLE_INFO.getCreatedTime(),
+ DATA1_TABLE_INFO.getModifiedTime());
+ }
+
private void assertDefaultLogRecordBatchEquals(LogRecordBatch recordBatch) {
assertThat(recordBatch.getRecordCount()).isEqualTo(1);
assertThat(recordBatch.baseLogOffset()).isEqualTo(0L);
assertThat(recordBatch.schemaId()).isEqualTo((short) DATA1_TABLE_INFO.getSchemaId());
try (LogRecordReadContext readContext =
LogRecordReadContext.createCompactedRowReadContext(
- DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId());
+ DATA1_ROW_TYPE,
+ DATA1_TABLE_INFO.getSchemaId(),
+ TEST_SCHEMA_GETTER);
CloseableIterator<LogRecord> iterator = recordBatch.records(readContext)) {
assertThat(iterator.hasNext()).isTrue();
LogRecord record = iterator.next();
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
index 0e4ab9c97..82e0e35cc 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
@@ -309,6 +309,33 @@ class RecordAccumulatorTest {
}
}
+ @Test
+ void testAppendRollsNewBatchWhenSchemaIdChanges() throws Exception {
+ int batchSize = 1024;
+ IndexedRow row = indexedRow(DATA1_ROW_TYPE, new Object[] {1, "a"});
+ RecordAccumulator accum = createTestRecordAccumulator(batchSize, 10L * batchSize);
+ int oldSchemaId = DATA1_TABLE_INFO.getSchemaId();
+ int newSchemaId = oldSchemaId + 1;
+
+ accum.append(createRecord(row), writeCallback, cluster, 0, false);
+ // a record with a bumped schema id closes the old-schema batch and rolls a new one.
+ accum.append(
+ createRecord(row, withSchemaId(newSchemaId)), writeCallback, cluster, 0, false);
+
+ Deque<WriteBatch> writeBatches =
+ accum.getReadyDeque(DATA1_PHYSICAL_TABLE_PATH, tb1.getBucket());
+ assertThat(writeBatches).hasSize(2);
+ Iterator<WriteBatch> batchIterator = writeBatches.iterator();
+ WriteBatch oldBatch = batchIterator.next();
+ assertThat(oldBatch.isClosed()).isTrue();
+ assertThat(oldBatch.schemaId()).isEqualTo(oldSchemaId);
+ assertThat(oldBatch.getRecordCount()).isEqualTo(1);
+ WriteBatch newBatch = batchIterator.next();
+ assertThat(newBatch.isClosed()).isFalse();
+ assertThat(newBatch.schemaId()).isEqualTo(newSchemaId);
+ assertThat(newBatch.getRecordCount()).isEqualTo(1);
+ }
+
@Test
void testAppendLarge() throws Exception {
int batchSize = 100;
@@ -550,7 +577,28 @@ class RecordAccumulatorTest {
* format , see {@link #updateCluster(List)}.
*/
private WriteRecord createRecord(IndexedRow row) {
- return WriteRecord.forIndexedAppend(DATA1_TABLE_INFO, DATA1_PHYSICAL_TABLE_PATH, row, null);
+ return createRecord(row, DATA1_TABLE_INFO);
+ }
+
+ private WriteRecord createRecord(IndexedRow row, TableInfo tableInfo) {
+ return WriteRecord.forIndexedAppend(tableInfo, DATA1_PHYSICAL_TABLE_PATH, row, null);
+ }
+
+ private TableInfo withSchemaId(int schemaId) {
+ return new TableInfo(
+ DATA1_TABLE_INFO.getTablePath(),
+ DATA1_TABLE_INFO.getTableId(),
+ schemaId,
+ DATA1_TABLE_INFO.getSchema(),
+ DATA1_TABLE_INFO.getBucketKeys(),
+ DATA1_TABLE_INFO.getPartitionKeys(),
+ DATA1_TABLE_INFO.getNumBuckets(),
+ DATA1_TABLE_INFO.getProperties(),
+ DATA1_TABLE_INFO.getCustomProperties(),
+ DATA1_TABLE_INFO.getRemoteDataDir(),
+ DATA1_TABLE_INFO.getComment().orElse(null),
+ DATA1_TABLE_INFO.getCreatedTime(),
+ DATA1_TABLE_INFO.getModifiedTime());
}
private Cluster updateCluster(List<BucketLocation> bucketLocations) {
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java b/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
index e5e961fb1..d4bfcb9ad 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
@@ -245,20 +245,20 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
long timestamp = commitTimestamp();
LogFormat logFormat = context.getLogFormat();
RowType rowType = context.getRowType(schemaId);
+ ProjectedRow outputProjection = context.getOutputProjectedRow(schemaId);
switch (logFormat) {
case ARROW:
return columnRecordIterator(
rowType,
- context.getOutputProjectedRow(schemaId),
+ outputProjection,
context.getVectorSchemaRoot(schemaId),
context.getBufferAllocator(),
timestamp);
case INDEXED:
- return rowRecordIterator(
- rowType, context.getOutputProjectedRow(schemaId), timestamp);
+ return rowRecordIterator(rowType, outputProjection, timestamp);
case COMPACTED:
- return compactedRowRecordIterator(rowType, timestamp);
+ return compactedRowRecordIterator(rowType, outputProjection, timestamp);
default:
throw new IllegalArgumentException("Unsupported log format: " + logFormat);
}
@@ -354,7 +354,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
}
private CloseableIterator<LogRecord> compactedRowRecordIterator(
- RowType rowType, long timestamp) {
+ RowType rowType, @Nullable ProjectedRow outputProjection, long timestamp) {
DataType[] fieldTypes = rowType.getChildren().toArray(new DataType[0]);
return new LogRecordIterator() {
int position = DefaultLogRecordBatch.this.position + recordsDataOffset();
@@ -367,7 +367,15 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
segment, position, baseOffset + rowId, timestamp, fieldTypes);
rowId++;
position += logRecord.getSizeInBytes();
- return logRecord;
+ if (outputProjection == null) {
+ return logRecord;
+ } else {
+ return new GenericRecord(
+ logRecord.logOffset(),
+ logRecord.timestamp(),
+ logRecord.getChangeType(),
+ outputProjection.replaceRow(logRecord.getRow()));
+ }
}
@Override
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
index 904fbd505..cc5bcbdba 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
@@ -45,48 +45,68 @@ import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.IntStream;
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
/** A simple implementation for {@link LogRecordBatch.ReadContext}. */
@ThreadSafe
public class LogRecordReadContext
implements LogRecordBatch.ReadContext, ArrowRecordBatchContext, AutoCloseable {
- // the log format of the table
+ /**
+ * Schema resolution strategy for reading records.
+ *
+ * <p>Replaces an earlier {@code boolean readAsTargetSchema} flag that was adjacent to {@code
+ * boolean readFromRemote} in the factory signature and prone to accidental swaps at call sites.
+ */
+ public enum SchemaResolution {
+ /**
+ * Records are projected to the table's current target schema. Columns added via later
+ * schema evolutions are hidden; columns removed from the target are dropped.
+ */
+ TARGET,
+ /**
+ * Records are returned with their original write-time schema. Callers observe schema
+ * evolution: fields added in a newer schema version become visible once written with that
+ * schema.
+ */
+ DYNAMIC
+ }
+
+ private final long tableId;
private final LogFormat logFormat;
- // the schema of the date read form server or remote. (which is projected in the server side)
- private final RowType dataRowType;
- // the static schemaId of the table, should support dynamic schema evolution in the future
- private final int targetSchemaId;
- // the shaded Arrow memory buffer allocator, should be null if not ARROW log format
+ /**
+ * When present, provides fixed schema identity, data row type, and pre-built field getters.
+ * When {@code null}, the reader dynamically adapts to each batch's schema so that columns added
+ * via schema evolution are visible.
+ */
+ @Nullable private final ReadTarget target;
+
@Nullable private final BufferAllocator bufferAllocator;
// the unshaded Arrow memory buffer allocator, declared as AutoCloseable to avoid
// importing unshaded Arrow classes in this module (they are provided-scope)
@Nullable private volatile AutoCloseable unshadedBufferAllocator;
- private final FieldGetter[] selectedFieldGetters;
- // whether the projection is push downed to the server side and the returned data is pruned.
private final boolean projectionPushDowned;
private final SchemaGetter schemaGetter;
private final ConcurrentHashMap<Integer, VectorSchemaRoot> vectorSchemaRootMap =
new ConcurrentHashMap<>();
private final Object unshadedArrowResourceLock = new Object();
+ private final ConcurrentHashMap<Integer, FieldGetter[]> fieldGetterCache =
+ new ConcurrentHashMap<>();
- public static LogRecordReadContext createReadContext(
- TableInfo tableInfo,
- boolean readFromRemote,
- @Nullable Projection projection,
- SchemaGetter schemaGetter) {
- return createReadContext(
- tableInfo,
- readFromRemote,
- projection,
- schemaGetter,
- new ChunkedAllocationManager.ChunkedFactory());
- }
+ // -------------------------------------------------------------------------
+ // Main factory methods
+ // -------------------------------------------------------------------------
/**
* Creates a {@link LogRecordReadContext} with a custom {@link AllocationManager.Factory}.
*
+ * <p>When {@code schemaResolution} is {@link SchemaResolution#DYNAMIC} and {@code projection}
+ * is {@code null}, no fixed {@link ReadTarget} is created. In that mode the reader dynamically
+ * builds field getters per batch schema, so columns added via schema evolution become visible.
+ *
* @param tableInfo the table info of the table to read
* @param readFromRemote whether the data is read from remote storage
+ * @param schemaResolution how to resolve the output schema: see {@link SchemaResolution}
* @param projection the projection to apply, or null for all fields
* @param schemaGetter the schema getter to resolve schema by id
* @param allocationManagerFactory the factory for creating Arrow memory allocations
@@ -94,76 +114,89 @@ public class LogRecordReadContext
public static LogRecordReadContext createReadContext(
TableInfo tableInfo,
boolean readFromRemote,
+ SchemaResolution schemaResolution,
@Nullable Projection projection,
SchemaGetter schemaGetter,
AllocationManager.Factory allocationManagerFactory) {
+ checkNotNull(schemaResolution, "schemaResolution");
+ boolean readAsTargetSchema = schemaResolution == SchemaResolution.TARGET;
RowType rowType = tableInfo.getRowType();
LogFormat logFormat = tableInfo.getTableConfig().getLogFormat();
// only for arrow log format, the projection can be push downed to the server side
- boolean projectionPushDowned = projection != null && logFormat == LogFormat.ARROW;
- int schemaId = tableInfo.getSchemaId();
- if (projection == null) {
- // set a default dummy projection to simplify code
- projection = Projection.of(IntStream.range(0, rowType.getFieldCount()).toArray());
+ boolean projectionPushDowned =
+ logFormat == LogFormat.ARROW && !readFromRemote && projection != null;
+
+ // Compute ReadTarget when there is a fixed output configuration.
+ // When schemaResolution is DYNAMIC and no projection is set, target is null and
+ // the reader dynamically adapts to each batch's schema.
+ ReadTarget target = null;
+ if (readAsTargetSchema || projection != null) {
+ int schemaId = tableInfo.getSchemaId();
+ if (projection == null) {
+ // set a default dummy projection to simplify code
+ projection = Projection.of(IntStream.range(0, rowType.getFieldCount()).toArray());
+ }
+ RowType dataRowType;
+ int[] selectedFields;
+ if (projectionPushDowned) {
+ dataRowType = projection.projectInOrder(rowType);
+ selectedFields = projection.getReorderingIndexes();
+ } else {
+ dataRowType = rowType;
+ selectedFields = projection.getProjection();
+ }
+ target = new ReadTarget(schemaId, dataRowType, selectedFields);
}
if (logFormat == LogFormat.ARROW) {
- if (readFromRemote) {
- // currently, for remote read, arrow log doesn't support projection pushdown,
- // so set the rowType as is.
- int[] selectedFields = projection.getProjection();
- return createArrowReadContext(
- rowType,
- schemaId,
- selectedFields,
- false,
- schemaGetter,
- allocationManagerFactory);
- } else {
- // arrow data that returned from server has been projected (in order)
- RowType projectedRowType = projection.projectInOrder(rowType);
- // need to reorder the fields for final output
- int[] selectedFields = projection.getReorderingIndexes();
- return createArrowReadContext(
- projectedRowType,
- schemaId,
- selectedFields,
- projectionPushDowned,
- schemaGetter,
- allocationManagerFactory);
- }
+ return createArrowReadContext(
+ tableInfo.getTableId(),
+ target,
+ projectionPushDowned,
+ schemaGetter,
+ allocationManagerFactory);
} else if (logFormat == LogFormat.INDEXED) {
- int[] selectedFields = projection.getProjection();
- return createIndexedReadContext(rowType, schemaId, selectedFields, schemaGetter);
+ return createIndexedReadContext(tableInfo.getTableId(), target, schemaGetter);
} else if (logFormat == LogFormat.COMPACTED) {
- int[] selectedFields = projection.getProjection();
- return createCompactedRowReadContext(rowType, schemaId, selectedFields);
+ return createCompactedRowReadContext(tableInfo.getTableId(), target, schemaGetter);
} else {
throw new IllegalArgumentException("Unsupported log format: " + logFormat);
}
}
+ // -------------------------------------------------------------------------
+ // Internal factory methods
+ // -------------------------------------------------------------------------
+
private static LogRecordReadContext createArrowReadContext(
- RowType dataRowType,
- int schemaId,
- int[] selectedFields,
+ long tableId,
+ @Nullable ReadTarget target,
boolean projectionPushDowned,
SchemaGetter schemaGetter,
AllocationManager.Factory allocationManagerFactory) {
// TODO: use a more reasonable memory limit
BufferAllocator allocator =
BufferAllocatorUtil.createBufferAllocator(allocationManagerFactory);
- FieldGetter[] fieldGetters = buildProjectedFieldGetters(dataRowType, selectedFields);
return new LogRecordReadContext(
- LogFormat.ARROW,
- dataRowType,
- schemaId,
- allocator,
- fieldGetters,
- projectionPushDowned,
- schemaGetter);
+ tableId, LogFormat.ARROW, target, allocator, projectionPushDowned, schemaGetter);
+ }
+
+ private static LogRecordReadContext createIndexedReadContext(
+ long tableId, @Nullable ReadTarget target, SchemaGetter schemaGetter) {
+ return new LogRecordReadContext(
+ tableId, LogFormat.INDEXED, target, null, false, schemaGetter);
+ }
+
+ private static LogRecordReadContext createCompactedRowReadContext(
+ long tableId, @Nullable ReadTarget target, @Nullable SchemaGetter schemaGetter) {
+ return new LogRecordReadContext(
+ tableId, LogFormat.COMPACTED, target, null, false, schemaGetter);
}
+ // -------------------------------------------------------------------------
+ // @VisibleForTesting and convenience factory methods
+ // -------------------------------------------------------------------------
+
/**
* Creates a LogRecordReadContext for ARROW log format, that underlying Arrow resources are not
* reused.
@@ -176,13 +209,9 @@ public class LogRecordReadContext
public static LogRecordReadContext createArrowReadContext(
RowType rowType, int schemaId, SchemaGetter schemaGetter) {
int[] selectedFields = IntStream.range(0, rowType.getFieldCount()).toArray();
+ ReadTarget target = new ReadTarget(schemaId, rowType, selectedFields);
return createArrowReadContext(
- rowType,
- schemaId,
- selectedFields,
- false,
- schemaGetter,
- new ChunkedAllocationManager.ChunkedFactory());
+ -1L, target, false, schemaGetter, new ChunkedAllocationManager.ChunkedFactory());
}
@VisibleForTesting
@@ -192,10 +221,10 @@ public class LogRecordReadContext
SchemaGetter schemaGetter,
boolean projectionPushDowned) {
int[] selectedFields = IntStream.range(0, rowType.getFieldCount()).toArray();
+ ReadTarget target = new ReadTarget(schemaId, rowType, selectedFields);
return createArrowReadContext(
- rowType,
- schemaId,
- selectedFields,
+ -1L,
+ target,
projectionPushDowned,
schemaGetter,
new ChunkedAllocationManager.ChunkedFactory());
@@ -212,87 +241,41 @@ public class LogRecordReadContext
public static LogRecordReadContext createIndexedReadContext(
RowType rowType, int schemaId, SchemaGetter schemaGetter) {
int[] selectedFields = IntStream.range(0, rowType.getFieldCount()).toArray();
- return createIndexedReadContext(rowType, schemaId, selectedFields, schemaGetter);
+ ReadTarget target = new ReadTarget(schemaId, rowType, selectedFields);
+ return createIndexedReadContext(-1L, target, schemaGetter);
}
/** Creates a LogRecordReadContext for COMPACTED log format. */
- public static LogRecordReadContext createCompactedRowReadContext(
- RowType rowType, int schemaId) {
- int[] selectedFields = IntStream.range(0, rowType.getFieldCount()).toArray();
- return createCompactedRowReadContext(rowType, schemaId, selectedFields, null);
- }
-
- /** Creates a LogRecordReadContext for COMPACTED log format with schema evolution support. */
public static LogRecordReadContext createCompactedRowReadContext(
RowType rowType, int schemaId, SchemaGetter schemaGetter) {
int[] selectedFields = IntStream.range(0, rowType.getFieldCount()).toArray();
- return createCompactedRowReadContext(rowType, schemaId, selectedFields, schemaGetter);
- }
-
- /**
- * Creates a LogRecordReadContext for INDEXED log format.
- *
- * @param rowType the schema of the read data
- * @param schemaId the schemaId of the table
- * @param selectedFields the final selected fields of the read data
- * @param schemaGetter the schema getter of to get schema by schemaId
- */
- public static LogRecordReadContext createIndexedReadContext(
- RowType rowType, int schemaId, int[] selectedFields, SchemaGetter schemaGetter) {
- FieldGetter[] fieldGetters = buildProjectedFieldGetters(rowType, selectedFields);
- // for INDEXED log format, the projection is NEVER push downed to the server side
- return new LogRecordReadContext(
- LogFormat.INDEXED, rowType, schemaId, null, fieldGetters, false, schemaGetter);
+ ReadTarget target = new ReadTarget(schemaId, rowType, selectedFields);
+ return createCompactedRowReadContext(-1L, target, schemaGetter);
}
- /**
- * Creates a LogRecordReadContext for COMPACTED log format.
- *
- * @param rowType the schema of the read data
- * @param schemaId the schemaId of the table
- * @param selectedFields the final selected fields of the read data
- */
- public static LogRecordReadContext createCompactedRowReadContext(
- RowType rowType, int schemaId, int[] selectedFields) {
- return createCompactedRowReadContext(rowType, schemaId, selectedFields, null);
- }
-
- /**
- * Creates a LogRecordReadContext for COMPACTED log format.
- *
- * @param rowType the schema of the read data
- * @param schemaId the schemaId of the table
- * @param selectedFields the final selected fields of the read data
- * @param schemaGetter the schema getter to resolve evolved batch schemas
- */
- public static LogRecordReadContext createCompactedRowReadContext(
- RowType rowType,
- int schemaId,
- int[] selectedFields,
- @Nullable SchemaGetter schemaGetter) {
- FieldGetter[] fieldGetters = buildProjectedFieldGetters(rowType, selectedFields);
- // for COMPACTED log format, the projection is NEVER push downed to the server side
- return new LogRecordReadContext(
- LogFormat.COMPACTED, rowType, schemaId, null, fieldGetters, false, schemaGetter);
- }
+ // -------------------------------------------------------------------------
+ // Constructor
+ // -------------------------------------------------------------------------
private LogRecordReadContext(
+ long tableId,
LogFormat logFormat,
- RowType targetDataRowType,
- int targetSchemaId,
- BufferAllocator bufferAllocator,
- FieldGetter[] selectedFieldGetters,
+ @Nullable ReadTarget target,
+ @Nullable BufferAllocator bufferAllocator,
boolean projectionPushDowned,
SchemaGetter schemaGetter) {
+ this.tableId = tableId;
this.logFormat = logFormat;
- this.dataRowType = targetDataRowType;
- this.targetSchemaId = targetSchemaId;
+ this.target = target;
this.bufferAllocator = bufferAllocator;
- this.selectedFieldGetters = selectedFieldGetters;
this.projectionPushDowned = projectionPushDowned;
this.schemaGetter = schemaGetter;
}
+ // -------------------------------------------------------------------------
+ // ReadContext interface methods
+ // -------------------------------------------------------------------------
+
@Override
public LogFormat getLogFormat() {
return logFormat;
@@ -300,17 +283,30 @@ public class LogRecordReadContext
@Override
public RowType getRowType(int schemaId) {
- if (isSameRowType(schemaId)) {
- return dataRowType;
+ if (isProjectionPushDowned() || (target != null && target.schemaId == schemaId)) {
+ assert target != null;
+ return target.dataRowType;
}
-
- Schema schema = schemaGetter.getSchema(schemaId);
- return schema.getRowType();
+ return schemaGetter.getSchema(schemaId).getRowType();
}
- /** Get the selected field getters for the read data. */
- public FieldGetter[] getSelectedFieldGetters() {
- return selectedFieldGetters;
+ /**
+ * Get the selected field getters for the read data. When no fixed {@link ReadTarget} is
+ * configured, getters are built dynamically from the batch's actual schema so that columns
+ * added via schema evolution are visible.
+ */
+ public FieldGetter[] getSelectedFieldGetters(int schemaId) {
+ if (target != null) {
+ return target.fieldGetters;
+ }
+ // Dynamic: build all-field getters from the batch's actual schema.
+ return fieldGetterCache.computeIfAbsent(
+ schemaId,
+ id -> {
+ RowType batchRowType = schemaGetter.getSchema(id).getRowType();
+ int[] allFields = IntStream.range(0, batchRowType.getFieldCount()).toArray();
+ return buildProjectedFieldGetters(batchRowType, allFields);
+ });
}
/** Whether the projection is push downed to the server side and the returned data is pruned. */
@@ -359,6 +355,7 @@ public class LogRecordReadContext
if (logFormat != LogFormat.ARROW) {
throw new IllegalArgumentException("Only Arrow log format provides buffer allocator.");
}
+ checkNotNull(bufferAllocator, "The buffer allocator is not available.");
return bufferAllocator;
}
@@ -374,15 +371,14 @@ public class LogRecordReadContext
@Nullable
@Override
public ProjectedRow getOutputProjectedRow(int schemaId) {
- if (isSameRowType(schemaId)) {
- return null;
- }
- if (schemaGetter == null) {
+
+ if (target == null || isProjectionPushDowned() || schemaId == target.schemaId) {
return null;
}
+
// TODO: should we cache the projection?
Schema originSchema = schemaGetter.getSchema(schemaId);
- Schema expectedSchema = schemaGetter.getSchema(targetSchemaId);
+ Schema expectedSchema = schemaGetter.getSchema(target.schemaId);
return ProjectedRow.from(originSchema, expectedSchema);
}
@@ -452,7 +448,7 @@ public class LogRecordReadContext
if (schemaMapping != null) {
outputRoot =
UnshadedArrowReadUtils.projectVectorSchemaRoot(
- readRoot, dataRowType, schemaMapping, allocator);
+ readRoot, target.dataRowType, schemaMapping, allocator);
readRoot.close();
readRoot = null;
}
@@ -472,11 +468,10 @@ public class LogRecordReadContext
}
}
- private boolean isSameRowType(int schemaId) {
- return targetSchemaId == schemaId || isProjectionPushDowned();
- }
-
- private static FieldGetter[] buildProjectedFieldGetters(RowType rowType, int[] selectedFields) {
+ // -------------------------------------------------------------------------
+ // Private helpers
+ // -------------------------------------------------------------------------
+ static FieldGetter[] buildProjectedFieldGetters(RowType rowType, int[] selectedFields) {
List<DataType> dataTypeList = rowType.getChildren();
FieldGetter[] fieldGetters = new FieldGetter[selectedFields.length];
for (int i = 0; i < fieldGetters.length; i++) {
@@ -487,4 +482,37 @@ public class LogRecordReadContext
}
return fieldGetters;
}
+
+ // -------------------------------------------------------------------------
+ // ReadTarget
+ // -------------------------------------------------------------------------
+
+ /**
+ * Encapsulates the target output configuration for reading log records. Contains the target
+ * schema identity, data row type, selected field indices, and pre-built field getters.
+ *
+ * <p>When a {@code ReadTarget} is provided, field getters are fixed and schema evolution may be
+ * enabled. When absent ({@code null}), the reader dynamically adapts to each batch's schema so
+ * that columns added via schema evolution are visible.
+ */
+ static class ReadTarget {
+ /** The schema id this configuration was built for. */
+ final int schemaId;
+
+ /** The row type of the data (may be server-projected for ARROW format). */
+ final RowType dataRowType;
+
+ /** The field indices to select from the data row type. */
+ final int[] selectedFields;
+
+ /** Pre-built field getters corresponding to {@link #selectedFields}. */
+ final FieldGetter[] fieldGetters;
+
+ ReadTarget(int schemaId, RowType dataRowType, int[] selectedFields) {
+ this.schemaId = schemaId;
+ this.dataRowType = dataRowType;
+ this.selectedFields = selectedFields;
+ this.fieldGetters = buildProjectedFieldGetters(dataRowType, selectedFields);
+ }
+ }
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/DefaultLogRecordBatchTest.java b/fluss-common/src/test/java/org/apache/fluss/record/DefaultLogRecordBatchTest.java
index fb55be89a..2860eaece 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/DefaultLogRecordBatchTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/DefaultLogRecordBatchTest.java
@@ -90,10 +90,7 @@ public class DefaultLogRecordBatchTest extends LogTestBase {
assertThat(logRecordBatch.isValid()).isTrue();
assertThat(logRecordBatch.schemaId()).isEqualTo(schemaId);
- SchemaGetter schemaGetter =
- new TestingSchemaGetter(
- new SchemaInfo(
- Schema.newBuilder().fromRowType(allRowType).build(), schemaId));
+ SchemaGetter schemaGetter = new TestingSchemaGetter(schemaId, allRowType);
// verify record.
int i = 0;
try (LogRecordReadContext readContext =
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java b/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
index 1fb197c01..6fa2c2ad1 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
@@ -599,7 +599,8 @@ class FileLogProjectionTest {
RowType projectedType = TestData.DATA2_ROW_TYPE.project(new int[] {0, 2});
try (LogRecordReadContext context =
- createArrowReadContext(projectedType, schemaIdForData2, testingSchemaGetter)) {
+ createArrowReadContext(
+ projectedType, schemaIdForData2, testingSchemaGetter, true)) {
for (LogRecordBatch projectedBatch : projectedRecords.batches()) {
try (CloseableIterator<LogRecord> records = projectedBatch.records(context)) {
int recordCount = 0;
@@ -692,7 +693,8 @@ class FileLogProjectionTest {
RowType projectedType = TestData.DATA2_ROW_TYPE.project(new int[] {1});
try (LogRecordReadContext context =
- createArrowReadContext(projectedType, schemaIdForData2, testingSchemaGetter)) {
+ createArrowReadContext(
+ projectedType, schemaIdForData2, testingSchemaGetter, true)) {
for (LogRecordBatch projectedBatch : projectedRecords.batches()) {
try (CloseableIterator<LogRecord> records = projectedBatch.records(context)) {
int recordCount = 0;
@@ -744,7 +746,8 @@ class FileLogProjectionTest {
RowType projectedType = TestData.DATA2_ROW_TYPE.project(new int[] {0, 1, 2});
try (LogRecordReadContext context =
- createArrowReadContext(projectedType, schemaIdForData2, testingSchemaGetter)) {
+ createArrowReadContext(
+ projectedType, schemaIdForData2, testingSchemaGetter, true)) {
for (LogRecordBatch projectedBatch : projectedRecords.batches()) {
try (CloseableIterator<LogRecord> records = projectedBatch.records(context)) {
int recordCount = 0;
@@ -824,7 +827,8 @@ class FileLogProjectionTest {
RowType projectedType = TestData.DATA1_ROW_TYPE.project(new int[] {0});
try (LogRecordReadContext context =
- createArrowReadContext(projectedType, DEFAULT_SCHEMA_ID, testingSchemaGetter)) {
+ createArrowReadContext(
+ projectedType, DEFAULT_SCHEMA_ID, testingSchemaGetter, true)) {
// Verify first batch
int firstBatchCount = 0;
for (LogRecordBatch projectedBatch : firstProjectedRecords.batches()) {
@@ -881,7 +885,8 @@ class FileLogProjectionTest {
RowType projectedType = TestData.DATA2_ROW_TYPE.project(new int[] {0, 1});
try (LogRecordReadContext context =
- createArrowReadContext(projectedType, DEFAULT_SCHEMA_ID, testingSchemaGetter)) {
+ createArrowReadContext(
+ projectedType, DEFAULT_SCHEMA_ID, testingSchemaGetter, true)) {
for (LogRecordBatch projectedBatch : projectedRecords.batches()) {
// Verify that statistics are not available in projected batch
assertThat(projectedBatch.getStatistics(context)).isEmpty();
@@ -932,7 +937,8 @@ class FileLogProjectionTest {
RowType projectedType = TestData.DATA2_ROW_TYPE.project(new int[] {0, 1});
try (LogRecordReadContext context =
- createArrowReadContext(projectedType, DEFAULT_SCHEMA_ID, testingSchemaGetter)) {
+ createArrowReadContext(
+ projectedType, DEFAULT_SCHEMA_ID, testingSchemaGetter, true)) {
for (LogRecordBatch projectedBatch : projectedRecords.batches()) {
// For V0/V1, statistics should be 0 (not supported)
assertThat(projectedBatch.getStatistics(context)).isEmpty();
@@ -994,7 +1000,8 @@ class FileLogProjectionTest {
RowType projectedType = TestData.DATA2_ROW_TYPE.project(new int[] {0, 2});
try (LogRecordReadContext context =
- createArrowReadContext(projectedType, schemaIdForData2, testingSchemaGetter)) {
+ createArrowReadContext(
+ projectedType, schemaIdForData2, testingSchemaGetter, true)) {
for (LogRecordBatch projectedBatch : projectedRecords.batches()) {
// Verify that statistics are not available in projected batch
assertThat(projectedBatch.getStatistics(context)).isEmpty();
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/FileLogRecordsTest.java b/fluss-common/src/test/java/org/apache/fluss/record/FileLogRecordsTest.java
index b20505c48..f85ec23ab 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/FileLogRecordsTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/FileLogRecordsTest.java
@@ -38,6 +38,7 @@ import java.util.concurrent.Future;
import static org.apache.fluss.record.TestData.DATA1;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.shaded.guava32.com.google.common.collect.Lists.newArrayList;
import static org.apache.fluss.testutils.DataTestUtils.assertLogRecordBatchEquals;
import static org.apache.fluss.testutils.DataTestUtils.assertLogRecordsEquals;
@@ -182,7 +183,10 @@ public class FileLogRecordsTest extends LogTestBase {
for (MemoryLogRecords value : values) {
assertThat(fileLogRecordsIterator.hasNext()).isTrue();
assertLogRecordBatchEquals(
- DATA1_ROW_TYPE, fileLogRecordsIterator.next(), value.batchIterator().next());
+ DATA1_ROW_TYPE,
+ fileLogRecordsIterator.next(),
+ value.batchIterator().next(),
+ TEST_SCHEMA_GETTER);
}
assertThat(fileLogRecords.channel().position()).isEqualTo(position);
}
@@ -191,7 +195,7 @@ public class FileLogRecordsTest extends LogTestBase {
void testRead() throws Exception {
FileLogRecords read = fileLogRecords.slice(0, fileLogRecords.sizeInBytes());
assertThat(read.sizeInBytes()).isEqualTo(fileLogRecords.sizeInBytes());
- assertLogRecordsEquals(DATA1_ROW_TYPE, read, fileLogRecords);
+ assertLogRecordsEquals(DATA1_ROW_TYPE, read, fileLogRecords, TEST_SCHEMA_GETTER);
List<LogRecordBatch> items = batches(read);
LogRecordBatch first = items.get(0);
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/LogRecordBatchStatisticsTest.java b/fluss-common/src/test/java/org/apache/fluss/record/LogRecordBatchStatisticsTest.java
index dba5145b7..5e0f07972 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/LogRecordBatchStatisticsTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/LogRecordBatchStatisticsTest.java
@@ -20,6 +20,7 @@ package org.apache.fluss.record;
import org.apache.fluss.memory.MemorySegment;
import org.apache.fluss.memory.MemorySegmentOutputView;
import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.Schema;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.aligned.AlignedRow;
import org.apache.fluss.testutils.DataTestUtils;
@@ -74,7 +75,11 @@ public class LogRecordBatchStatisticsTest extends LogTestBase {
LogRecordReadContext.createArrowReadContext(
TestData.STATISTICS_WITH_BOOLEAN_ROW_TYPE,
DEFAULT_SCHEMA_ID,
- TEST_SCHEMA_GETTER)) {
+ new TestingSchemaGetter(
+ DEFAULT_SCHEMA_ID,
+ Schema.newBuilder()
+ .fromRowType(TestData.STATISTICS_WITH_BOOLEAN_ROW_TYPE)
+ .build()))) {
Optional<LogRecordBatchStatistics> statisticsOpt = batch.getStatistics(readContext);
assertThat(statisticsOpt).isPresent();
@@ -108,7 +113,11 @@ public class LogRecordBatchStatisticsTest extends LogTestBase {
LogRecordReadContext.createArrowReadContext(
TestData.STATISTICS_WITH_NULLS_ROW_TYPE,
DEFAULT_SCHEMA_ID,
- TEST_SCHEMA_GETTER)) {
+ new TestingSchemaGetter(
+ DEFAULT_SCHEMA_ID,
+ Schema.newBuilder()
+ .fromRowType(TestData.STATISTICS_WITH_NULLS_ROW_TYPE)
+ .build()))) {
LogRecordBatchStatistics statistics = batch.getStatistics(readContext).get();
assertThat(statistics.getNullCounts()[0]).isEqualTo(1);
assertThat(statistics.getNullCounts()[1]).isEqualTo(1);
@@ -178,7 +187,11 @@ public class LogRecordBatchStatisticsTest extends LogTestBase {
LogRecordReadContext.createArrowReadContext(
TestData.STATISTICS_WITH_BOOLEAN_ROW_TYPE,
DEFAULT_SCHEMA_ID,
- TEST_SCHEMA_GETTER)) {
+ new TestingSchemaGetter(
+ DEFAULT_SCHEMA_ID,
+ Schema.newBuilder()
+ .fromRowType(TestData.STATISTICS_WITH_BOOLEAN_ROW_TYPE)
+ .build()))) {
Optional<LogRecordBatchStatistics> stats1 = batch.getStatistics(readContext);
Optional<LogRecordBatchStatistics> stats2 = batch.getStatistics(readContext);
assertThat(stats2.get()).isSameAs(stats1.get());
@@ -209,7 +222,11 @@ public class LogRecordBatchStatisticsTest extends LogTestBase {
LogRecordReadContext.createArrowReadContext(
TestData.STATISTICS_WITH_BOOLEAN_ROW_TYPE,
DEFAULT_SCHEMA_ID,
- TEST_SCHEMA_GETTER)) {
+ new TestingSchemaGetter(
+ DEFAULT_SCHEMA_ID,
+ Schema.newBuilder()
+ .fromRowType(TestData.STATISTICS_WITH_BOOLEAN_ROW_TYPE)
+ .build()))) {
LogRecordBatchStatistics statistics = batch.getStatistics(readContext).get();
assertThat(statistics.getMinValues().getInt(0)).isEqualTo(1);
assertThat(statistics.getMaxValues().getInt(0)).isEqualTo(5);
@@ -232,7 +249,11 @@ public class LogRecordBatchStatisticsTest extends LogTestBase {
LogRecordReadContext.createArrowReadContext(
TestData.STATISTICS_WITH_BOOLEAN_ROW_TYPE,
DEFAULT_SCHEMA_ID,
- TEST_SCHEMA_GETTER)) {
+ new TestingSchemaGetter(
+ DEFAULT_SCHEMA_ID,
+ Schema.newBuilder()
+ .fromRowType(TestData.STATISTICS_WITH_BOOLEAN_ROW_TYPE)
+ .build()))) {
LogRecordBatchStatistics defaultStats = defaultBatch.getStatistics(readContext).get();
try (FileLogRecords fileLogRecords =
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java b/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
index 36aef5b9d..491b1fffa 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
@@ -33,6 +33,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.assertj.core.api.Assertions.assertThat;
/** base test class for Log. */
@@ -62,14 +63,18 @@ public abstract class LogTestBase {
}
public static void assertLogRecordsListEquals(
- List<MemoryLogRecords> expected, LogRecords actual, RowType rowType) {
+ List<MemoryLogRecords> expected,
+ LogRecords actual,
+ RowType rowType,
+ SchemaGetter schemaGetter) {
LogRecordsAssert.assertThatLogRecords(actual)
.withSchema(rowType)
+ .withSchemaGetter(schemaGetter)
.isEqualTo(new ListLogRecords(expected));
}
protected void assertLogRecordsListEquals(List<MemoryLogRecords> expected, LogRecords actual) {
- assertLogRecordsListEquals(expected, actual, baseRowType);
+ assertLogRecordsListEquals(expected, actual, baseRowType, TEST_SCHEMA_GETTER);
}
protected void assertIndexedLogRecordBatchAndRowEquals(
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsArrowBuilderTest.java b/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsArrowBuilderTest.java
index 63b3f52db..6c0c8e89e 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsArrowBuilderTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsArrowBuilderTest.java
@@ -24,6 +24,7 @@ import org.apache.fluss.config.Configuration;
import org.apache.fluss.config.MemorySize;
import org.apache.fluss.memory.ManagedPagedOutputView;
import org.apache.fluss.memory.TestingMemorySegmentPool;
+import org.apache.fluss.metadata.Schema;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.arrow.ArrowWriter;
import org.apache.fluss.row.arrow.ArrowWriterPool;
@@ -146,7 +147,7 @@ public class MemoryLogRecordsArrowBuilderTest {
"Tried to append a record, but MemoryLogRecordsArrowBuilder is closed for record appends");
assertThat(builder.isClosed()).isTrue();
MemoryLogRecords records = MemoryLogRecords.pointToBytesView(builder.build());
- assertLogRecordsEquals(DATA1_ROW_TYPE, records, expectedResult);
+ assertLogRecordsEquals(DATA1_ROW_TYPE, records, expectedResult, TEST_SCHEMA_GETTER);
}
@ParameterizedTest
@@ -419,7 +420,10 @@ public class MemoryLogRecordsArrowBuilderTest {
// Create read context
LogRecordReadContext readContext =
LogRecordReadContext.createArrowReadContext(
- testRowType, DEFAULT_SCHEMA_ID, TEST_SCHEMA_GETTER);
+ testRowType,
+ DEFAULT_SCHEMA_ID,
+ new TestingSchemaGetter(
+ 1, Schema.newBuilder().fromRowType(testRowType).build()));
// Test statistics reading
Optional<LogRecordBatchStatistics> statisticsOpt = batch.getStatistics(readContext);
@@ -524,7 +528,10 @@ public class MemoryLogRecordsArrowBuilderTest {
// Create read context
LogRecordReadContext readContext =
LogRecordReadContext.createArrowReadContext(
- testRowType, DEFAULT_SCHEMA_ID, TEST_SCHEMA_GETTER);
+ testRowType,
+ DEFAULT_SCHEMA_ID,
+ new TestingSchemaGetter(
+ 1, Schema.newBuilder().fromRowType(testRowType).build()));
// Test statistics reading
Optional<LogRecordBatchStatistics> statisticsOpt = batch.getStatistics(readContext);
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsCompactedBuilderTest.java b/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsCompactedBuilderTest.java
index 2a8d70c80..ce2098f1c 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsCompactedBuilderTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsCompactedBuilderTest.java
@@ -34,6 +34,7 @@ import static org.apache.fluss.record.TestData.BASE_OFFSET;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
import static org.apache.fluss.record.TestData.DEFAULT_MAGIC;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.testutils.DataTestUtils.compactedRow;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -71,7 +72,7 @@ class MemoryLogRecordsCompactedBuilderTest {
try (LogRecordReadContext ctx =
LogRecordReadContext.createCompactedRowReadContext(
- DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID);
+ DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, TEST_SCHEMA_GETTER);
CloseableIterator<LogRecord> recIt = batch.records(ctx)) {
for (CompactedRow expRow : expected) {
assertThat(recIt.hasNext()).isTrue();
@@ -127,7 +128,7 @@ class MemoryLogRecordsCompactedBuilderTest {
assertThat(batch.nextLogOffset()).isEqualTo(1);
try (LogRecordReadContext ctx =
LogRecordReadContext.createCompactedRowReadContext(
- DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID);
+ DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, TEST_SCHEMA_GETTER);
CloseableIterator<LogRecord> it = batch.records(ctx)) {
assertThat(it.hasNext()).isFalse();
}
@@ -146,7 +147,7 @@ class MemoryLogRecordsCompactedBuilderTest {
assertThat(batch.nextLogOffset()).isEqualTo(101);
try (LogRecordReadContext ctx =
LogRecordReadContext.createCompactedRowReadContext(
- DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID);
+ DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, TEST_SCHEMA_GETTER);
CloseableIterator<LogRecord> it = batch.records(ctx)) {
assertThat(it.hasNext()).isFalse();
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsTest.java b/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsTest.java
index 2b67ddfb3..fa42e4290 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsTest.java
@@ -27,6 +27,7 @@ import java.util.function.Function;
import static org.apache.fluss.record.TestData.DATA1;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
/** Tests for {@link MemoryLogRecords}. */
class MemoryLogRecordsTest {
@@ -79,6 +80,7 @@ class MemoryLogRecordsTest {
LogRecordsAssert.assertThatLogRecords(newRecords)
.hasBatchesCount(1)
.withSchema(DATA1_ROW_TYPE)
+ .withSchemaGetter(TEST_SCHEMA_GETTER)
.isEqualTo(records);
}
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/TestData.java b/fluss-common/src/test/java/org/apache/fluss/record/TestData.java
index be4e0ad78..8e202775a 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/TestData.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/TestData.java
@@ -354,5 +354,5 @@ public final class TestData {
// ------------------- Statistics test data and schemas end ------------------------
public static final TestingSchemaGetter TEST_SCHEMA_GETTER =
- new TestingSchemaGetter(DEFAULT_SCHEMA_ID, DATA2_SCHEMA);
+ new TestingSchemaGetter(DEFAULT_SCHEMA_ID, DATA1_SCHEMA);
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java b/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
index 9a9581cab..ef9470123 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
@@ -21,6 +21,7 @@ import org.apache.fluss.exception.SchemaNotExistException;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.SchemaInfo;
+import org.apache.fluss.types.RowType;
import java.util.HashMap;
import java.util.Map;
@@ -31,18 +32,16 @@ public class TestingSchemaGetter implements SchemaGetter {
private SchemaInfo schemaInfo;
private final Map<Integer, Schema> schemaCaches;
- public TestingSchemaGetter() {
- this.schemaInfo = null;
- this.schemaCaches = new HashMap<>();
- }
-
- // todo: remove it(简化测试代码）
public TestingSchemaGetter(SchemaInfo schemaInfo) {
this.schemaInfo = schemaInfo;
this.schemaCaches = new HashMap<>();
this.schemaCaches.put(schemaInfo.getSchemaId(), schemaInfo.getSchema());
}
+ public TestingSchemaGetter(int schemaId, RowType rowType) {
+ this(schemaId, Schema.newBuilder().fromRowType(rowType).build());
+ }
+
public TestingSchemaGetter(int schemaId, Schema schema) {
this.schemaInfo = new SchemaInfo(schema, schemaId);
this.schemaCaches = new HashMap<>();
diff --git a/fluss-common/src/test/java/org/apache/fluss/testutils/DataTestUtils.java b/fluss-common/src/test/java/org/apache/fluss/testutils/DataTestUtils.java
index aaa449d46..a3f05785c 100644
--- a/fluss-common/src/test/java/org/apache/fluss/testutils/DataTestUtils.java
+++ b/fluss-common/src/test/java/org/apache/fluss/testutils/DataTestUtils.java
@@ -749,11 +749,11 @@ public class DataTestUtils {
}
public static void assertLogRecordsEquals(LogRecords actual, LogRecords expected) {
- assertLogRecordsEquals(DATA1_ROW_TYPE, actual, expected);
+ assertLogRecordsEquals(DATA1_ROW_TYPE, actual, expected, TEST_SCHEMA_GETTER);
}
public static void assertLogRecordsEquals(
- RowType rowType, LogRecords actual, LogRecords expected) {
+ RowType rowType, LogRecords actual, LogRecords expected, SchemaGetter schemaGetter) {
Iterator<LogRecordBatch> actualIterator = actual.batches().iterator();
Iterator<LogRecordBatch> expectedIterator = expected.batches().iterator();
@@ -761,14 +761,20 @@ public class DataTestUtils {
assertThat(expectedIterator.hasNext()).isTrue();
LogRecordBatch actualBatch = actualIterator.next();
LogRecordBatch expectedBatch = expectedIterator.next();
- assertLogRecordBatchEquals(rowType, actualBatch, expectedBatch);
+ assertLogRecordBatchEquals(rowType, actualBatch, expectedBatch, schemaGetter);
}
assertThat(expectedIterator.hasNext()).isFalse();
}
public static void assertLogRecordBatchEquals(
- RowType rowType, LogRecordBatch actual, LogRecordBatch expected) {
- assertThatLogRecordBatch(actual).withSchema(rowType).isEqualTo(expected);
+ RowType rowType,
+ LogRecordBatch actual,
+ LogRecordBatch expected,
+ SchemaGetter schemaGetter) {
+ assertThatLogRecordBatch(actual)
+ .withSchema(rowType)
+ .withSchemaGetter(schemaGetter)
+ .isEqualTo(expected);
}
private static void assertLogRecordsEqualsWithRowKind(
diff --git a/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordsAssert.java b/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordsAssert.java
index 28df40e34..66efdce5d 100644
--- a/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordsAssert.java
+++ b/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordsAssert.java
@@ -20,7 +20,6 @@ package org.apache.fluss.testutils;
import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.record.LogRecordBatch;
import org.apache.fluss.record.LogRecords;
-import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.types.RowType;
import org.assertj.core.api.AbstractAssert;
@@ -40,7 +39,7 @@ public class LogRecordsAssert extends AbstractAssert<LogRecordsAssert, LogRecord
private RowType rowType;
private boolean assertCheckSum = true;
- private SchemaGetter schemaGetter = new TestingSchemaGetter();
+ private SchemaGetter schemaGetter;
private LogRecordsAssert(LogRecords actual) {
super(actual, LogRecordsAssert.class);
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
index f78a258ec..29c6cae8c 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
@@ -289,7 +289,7 @@ public class KvRecoverHelper {
currentRowType, currentSchemaId, schemaGetter);
} else if (logFormat == LogFormat.COMPACTED) {
return LogRecordReadContext.createCompactedRowReadContext(
- currentRowType, currentSchemaId);
+ currentRowType, currentSchemaId, schemaGetter);
} else {
throw new UnsupportedOperationException("Unsupported log format: " + logFormat);
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletMergeModeTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletMergeModeTest.java
index 97ae1d9cc..d5fe4b8b4 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletMergeModeTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletMergeModeTest.java
@@ -209,6 +209,7 @@ class KvTabletMergeModeTest {
assertThatLogRecords(actualLogRecords)
.withSchema(AGG_ROW_TYPE)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
}
@@ -248,6 +249,7 @@ class KvTabletMergeModeTest {
assertThatLogRecords(actualLogRecords)
.withSchema(AGG_ROW_TYPE)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
@@ -299,6 +301,7 @@ class KvTabletMergeModeTest {
assertThatLogRecords(actualLogRecords)
.withSchema(AGG_ROW_TYPE)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
}
@@ -321,6 +324,7 @@ class KvTabletMergeModeTest {
assertThatLogRecords(actualLogRecords)
.withSchema(AGG_ROW_TYPE)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
}
@@ -351,6 +355,7 @@ class KvTabletMergeModeTest {
assertThatLogRecords(actualLogRecords)
.withSchema(AGG_ROW_TYPE)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
}
@@ -404,6 +409,7 @@ class KvTabletMergeModeTest {
assertThatLogRecords(actualLogRecords)
.withSchema(AGG_ROW_TYPE)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
}
@@ -442,6 +448,7 @@ class KvTabletMergeModeTest {
assertThatLogRecords(actualLogRecords)
.withSchema(AGG_ROW_TYPE)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
}
@@ -489,6 +496,7 @@ class KvTabletMergeModeTest {
assertThatLogRecords(actualLogRecords)
.withSchema(AGG_ROW_TYPE)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
}
@@ -528,6 +536,7 @@ class KvTabletMergeModeTest {
assertThatLogRecords(actualLogRecords)
.withSchema(AGG_ROW_TYPE)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletSchemaEvolutionTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletSchemaEvolutionTest.java
index 130352170..f3658e9b0 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletSchemaEvolutionTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletSchemaEvolutionTest.java
@@ -198,6 +198,7 @@ class KvTabletSchemaEvolutionTest {
assertThatLogRecords(actualLogRecords)
.withSchema(ROW_TYPE_V1)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
}
@@ -242,6 +243,7 @@ class KvTabletSchemaEvolutionTest {
assertThatLogRecords(actualLogRecords)
.withSchema(ROW_TYPE_V1)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java
index 3425fa75c..874ff268d 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java
@@ -841,6 +841,7 @@ class KvTabletTest {
: Arrays.asList(new Object[] {1, "v11"}, new Object[] {2, "v21"}));
assertThatLogRecords(actualLogRecords)
.withSchema(readLogRowType)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(!doProjection)
.isEqualTo(expectedLogs);
@@ -875,6 +876,7 @@ class KvTabletTest {
actualLogRecords = readLogRecords(logTablet, 0, logProjection);
assertThatLogRecords(actualLogRecords)
.withSchema(readLogRowType)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(!doProjection)
.isEqualTo(expectedLogs);
}
@@ -898,6 +900,7 @@ class KvTabletTest {
actualLogRecords = readLogRecords(logTablet, endOffset, logProjection);
assertThatLogRecords(actualLogRecords)
.withSchema(readLogRowType)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(!doProjection)
.isEqualTo(expectedLogs);
@@ -933,6 +936,7 @@ class KvTabletTest {
actualLogRecords = readLogRecords(logTablet, endOffset, logProjection);
assertThatLogRecords(actualLogRecords)
.withSchema(readLogRowType)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(!doProjection)
.isEqualTo(expectedLogs);
@@ -954,6 +958,7 @@ class KvTabletTest {
actualLogRecords = readLogRecords(logTablet, endOffset, logProjection);
assertThatLogRecords(actualLogRecords)
.withSchema(readLogRowType)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(!doProjection)
.isEqualTo(expectedLogs);
}
@@ -1031,6 +1036,7 @@ class KvTabletTest {
new Object[] {2, 1001}));
assertThatLogRecords(actualLogRecords)
.withSchema(readLogRowType)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(!doProjection)
.isEqualTo(expectedLogs);
@@ -1065,6 +1071,7 @@ class KvTabletTest {
actualLogRecords = readLogRecords(logTablet, 0, logProjection);
assertThatLogRecords(actualLogRecords)
.withSchema(readLogRowType)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(!doProjection)
.isEqualTo(expectedLogs);
@@ -1097,6 +1104,7 @@ class KvTabletTest {
actualLogRecords = readLogRecords(logTablet, endOffset, logProjection);
assertThatLogRecords(actualLogRecords)
.withSchema(readLogRowType)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(!doProjection)
.isEqualTo(expectedLogs);
@@ -1136,6 +1144,7 @@ class KvTabletTest {
actualLogRecords = readLogRecords(logTablet, endOffset, logProjection);
assertThatLogRecords(actualLogRecords)
.withSchema(newSchemaLogRowType)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(!doProjection)
.isEqualTo(expectedLogs);
@@ -1158,6 +1167,7 @@ class KvTabletTest {
actualLogRecords = readLogRecords(logTablet, endOffset, logProjection);
assertThatLogRecords(actualLogRecords)
.withSchema(newSchemaLogRowType)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(!doProjection)
.isEqualTo(expectedLogs);
}
@@ -1205,6 +1215,7 @@ class KvTabletTest {
new Object[] {2, 20L, 200, "Bob"}));
assertThatLogRecords(actualLogRecords)
.withSchema(rowType)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
@@ -1238,6 +1249,7 @@ class KvTabletTest {
actualLogRecords = readLogRecords(logTablet, endOffset, null);
assertThatLogRecords(actualLogRecords)
.withSchema(rowType)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
@@ -1259,6 +1271,7 @@ class KvTabletTest {
actualLogRecords = readLogRecords(logTablet, endOffset, null);
assertThatLogRecords(actualLogRecords)
.withSchema(rowType)
+ .withSchemaGetter(schemaGetter)
.assertCheckSum(true)
.isEqualTo(expectedLogs);
}
@@ -1485,11 +1498,13 @@ class KvTabletTest {
private void checkEqual(
LogRecords actaulLogRecords, List<MemoryLogRecords> expectedLogs, RowType rowType) {
- LogTestBase.assertLogRecordsListEquals(expectedLogs, actaulLogRecords, rowType);
+ LogTestBase.assertLogRecordsListEquals(
+ expectedLogs, actaulLogRecords, rowType, schemaGetter);
}
private void checkEqual(LogRecords actaulLogRecords, List<MemoryLogRecords> expectedLogs) {
- LogTestBase.assertLogRecordsListEquals(expectedLogs, actaulLogRecords, baseRowType);
+ LogTestBase.assertLogRecordsListEquals(
+ expectedLogs, actaulLogRecords, baseRowType, schemaGetter);
}
private Value valueOf(BinaryRow row) {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/AggregateRowMergerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/AggregateRowMergerTest.java
index 387fb95bb..bd1bd3217 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/AggregateRowMergerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/AggregateRowMergerTest.java
@@ -270,8 +270,7 @@ class AggregateRowMergerTest {
// Create schema getter with both schemas
// Initialize with newSchema as latest schema for AggregateRowMerger constructor
- TestingSchemaGetter schemaGetter =
- new TestingSchemaGetter(new SchemaInfo(newSchema, newSchemaId));
+ TestingSchemaGetter schemaGetter = new TestingSchemaGetter(newSchemaId, newSchema);
// Add old schema to cache
schemaGetter.updateLatestSchemaInfo(new SchemaInfo(oldSchema, oldSchemaId));
// Ensure newSchema is latest
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/LogSegmentTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/LogSegmentTest.java
index a82dd9953..1859c6a5b 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/log/LogSegmentTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/log/LogSegmentTest.java
@@ -788,7 +788,7 @@ final class LogSegmentTest extends LogTestBase {
org.apache.fluss.types.RowType projectedType = DATA1_ROW_TYPE.project(new int[] {0});
try (LogRecordReadContext projectedContext =
LogRecordReadContext.createArrowReadContext(
- projectedType, DEFAULT_SCHEMA_ID, TEST_SCHEMA_GETTER)) {
+ projectedType, DEFAULT_SCHEMA_ID, TEST_SCHEMA_GETTER, true)) {
int recordCount = 0;
for (LogRecordBatch b : read.getRecords().batches()) {
try (CloseableIterator<LogRecord> iter = b.records(projectedContext)) {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/PredicateSchemaResolverTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/PredicateSchemaResolverTest.java
index 08a68c697..33f70a5f4 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/log/PredicateSchemaResolverTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/log/PredicateSchemaResolverTest.java
@@ -110,7 +110,7 @@ class PredicateSchemaResolverTest {
@Test
void testSchemaGetterThrowsInConstructor() {
// SchemaGetter that throws for the predicate schema ID
- TestingSchemaGetter schemaGetter = new TestingSchemaGetter();
+ TestingSchemaGetter schemaGetter = new TestingSchemaGetter(SCHEMA_ID_1, ROW_TYPE_V1);
Predicate predicate = new PredicateBuilder(ROW_TYPE_V1).equal(0, 42);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
index d37e91dc6..8074b4f1c 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
@@ -313,6 +313,7 @@ final class ReplicaTest extends ReplicaTestBase {
new Object[] {3, "b1"}));
assertThatLogRecords(fetchRecords(kvReplica))
.withSchema(DATA1_ROW_TYPE)
+ .withSchemaGetter(kvReplica.getSchemaGetter())
.isEqualTo(expected);
int currentOffset = 4;
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherITCase.java
index f94db7184..36d3ef567 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherITCase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherITCase.java
@@ -57,6 +57,7 @@ import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH;
import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH_PK;
import static org.apache.fluss.record.TestData.DATA_1_WITH_KEY_AND_VALUE;
import static org.apache.fluss.record.TestData.EXPECTED_LOG_RESULTS_FOR_DATA_1_WITH_PK;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.server.testutils.KvTestUtils.assertLookupResponse;
import static org.apache.fluss.server.testutils.RpcMessageTestUtils.assertFetchLogResponse;
import static org.apache.fluss.server.testutils.RpcMessageTestUtils.assertFetchLogResponseWithRowKind;
@@ -165,7 +166,7 @@ public class ReplicaFetcherITCase {
assertThat(resultForBucket.getTableBucket()).isEqualTo(tb);
LogRecords records = resultForBucket.records();
assertThat(records).isNotNull();
- assertLogRecordsEquals(DATA1_ROW_TYPE, records, DATA1);
+ assertLogRecordsEquals(DATA1_ROW_TYPE, records, DATA1, TEST_SCHEMA_GETTER);
}
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServiceITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServiceITCase.java
index 5d2dd1d71..6190f566d 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServiceITCase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServiceITCase.java
@@ -24,7 +24,6 @@ import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TablePath;
@@ -229,7 +228,6 @@ public class TabletServiceITCase {
long tableId =
createTable(FLUSS_CLUSTER_EXTENSION, DATA1_TABLE_PATH, DATA1_TABLE_DESCRIPTOR);
TableBucket tb = new TableBucket(tableId, 0);
- SchemaGetter schemaGetter = new TestingSchemaGetter(1, DATA1_SCHEMA);
FLUSS_CLUSTER_EXTENSION.waitUntilAllReplicaReady(tb);
@@ -308,7 +306,7 @@ public class TabletServiceITCase {
.fetchLog(newFetchLogRequest(-1, tableId, 0, 10L, new int[] {0}))
.get(),
DATA1_ROW_TYPE.project(new int[] {0}),
- schemaGetter,
+ TEST_SCHEMA_GETTER,
tableId,
0,
20L,
@@ -324,7 +322,7 @@ public class TabletServiceITCase {
.fetchLog(newFetchLogRequest(-1, tableId, 0, 15L, new int[] {1}))
.get(),
DATA1_ROW_TYPE.project(new int[] {1}),
- schemaGetter,
+ TEST_SCHEMA_GETTER,
tableId,
0,
20L,
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
index a51fd2148..bc833e9c0 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
@@ -337,6 +337,7 @@ public final class FlussClusterExtension
tabletServerConf.setString(
ConfigOptions.ZOOKEEPER_ADDRESS, zooKeeperServer.getConnectString());
tabletServerConf.setString(ConfigOptions.BIND_LISTENERS, tabletServerListeners);
+ tabletServerConf.setDouble(ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO, 1.0);
if (overwriteConfig != null) {
tabletServerConf.addAll(overwriteConfig);
}